diff options
| author | Igor Pavlov <ipavlov@users.sourceforge.net> | 2021-07-23 01:00:14 +0300 |
|---|---|---|
| committer | Kornel <kornel@geekhood.net> | 2021-07-23 01:00:14 +0300 |
| commit | 585698650f7257d2cefa6a3a2a49d5bbe84fd9b2 (patch) | |
| tree | 8900be42e892a440bbd1063804b0557288c2f97f | |
| parent | 4a960640a340a848a2d2c27f19b339c2c3d3f734 (diff) | |
21.0221.02
619 files changed, 34731 insertions, 10686 deletions
diff --git a/Asm/arm64/7zAsm.S b/Asm/arm64/7zAsm.S new file mode 100644 index 00000000..12e950b4 --- /dev/null +++ b/Asm/arm64/7zAsm.S @@ -0,0 +1,181 @@ +// 7zAsm.S -- ASM macros for arm64 +// 2021-04-25 : Igor Pavlov : Public domain + +#define r0 x0 +#define r1 x1 +#define r2 x2 +#define r3 x3 +#define r4 x4 +#define r5 x5 +#define r6 x6 +#define r7 x7 +#define r8 x8 +#define r9 x9 +#define r10 x10 +#define r11 x11 +#define r12 x12 +#define r13 x13 +#define r14 x14 +#define r15 x15 +#define r16 x16 +#define r17 x17 +#define r18 x18 +#define r19 x19 +#define r20 x20 +#define r21 x21 +#define r22 x22 +#define r23 x23 +#define r24 x24 +#define r25 x25 +#define r26 x26 +#define r27 x27 +#define r28 x28 +#define r29 x29 +#define r30 x30 + +#define REG_ABI_PARAM_0 r0 +#define REG_ABI_PARAM_1 r1 +#define REG_ABI_PARAM_2 r2 + + +.macro p2_add reg:req, param:req + add \reg, \reg, \param +.endm + +.macro p2_sub reg:req, param:req + sub \reg, \reg, \param +.endm + +.macro p2_sub_s reg:req, param:req + subs \reg, \reg, \param +.endm + +.macro p2_and reg:req, param:req + and \reg, \reg, \param +.endm + +.macro xor reg:req, param:req + eor \reg, \reg, \param +.endm + +.macro or reg:req, param:req + orr \reg, \reg, \param +.endm + +.macro shl reg:req, param:req + lsl \reg, \reg, \param +.endm + +.macro shr reg:req, param:req + lsr \reg, \reg, \param +.endm + +.macro sar reg:req, param:req + asr \reg, \reg, \param +.endm + +.macro p1_neg reg:req + neg \reg, \reg +.endm + +.macro dec reg:req + sub \reg, \reg, 1 +.endm + +.macro dec_s reg:req + subs \reg, \reg, 1 +.endm + +.macro inc reg:req + add \reg, \reg, 1 +.endm + +.macro inc_s reg:req + adds \reg, \reg, 1 +.endm + + +.macro imul reg:req, param:req + mul \reg, \reg, \param +.endm + +/* +arm64 and arm use reverted c flag after subs/cmp instructions: + arm64-arm : x86 + b.lo / b.cc : jb / jc + b.hs / b.cs : jae / jnc +*/ + +.macro jmp lab:req + b \lab +.endm + +.macro je lab:req + b.eq \lab +.endm + +.macro jz lab:req + b.eq \lab +.endm + +.macro jnz lab:req + b.ne \lab +.endm + +.macro jne lab:req + b.ne \lab +.endm + +.macro jb lab:req + b.lo \lab +.endm + +.macro jbe lab:req + b.ls \lab +.endm + +.macro ja lab:req + b.hi \lab +.endm + +.macro jae lab:req + b.hs \lab +.endm + + +.macro cmove dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, eq +.endm + +.macro cmovne dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, ne +.endm + +.macro cmovs dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, mi +.endm + +.macro cmovns dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, pl +.endm + +.macro cmovb dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, lo +.endm + +.macro cmovae dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, hs +.endm + + +.macro MY_ALIGN_16 macro + .p2align 4,, (1 << 4) - 1 +.endm + +.macro MY_ALIGN_32 macro + .p2align 5,, (1 << 5) - 1 +.endm + +.macro MY_ALIGN_64 macro + .p2align 6,, (1 << 6) - 1 +.endm diff --git a/Asm/arm64/LzmaDecOpt.S b/Asm/arm64/LzmaDecOpt.S new file mode 100644 index 00000000..10dc4735 --- /dev/null +++ b/Asm/arm64/LzmaDecOpt.S @@ -0,0 +1,1487 @@ +// LzmaDecOpt.S -- ARM64-ASM version of LzmaDec_DecodeReal_3() function +// 2021-04-25 : Igor Pavlov : Public domain + +/* +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). +*/ + + +#include "7zAsm.S" + + // .arch armv8-a + // .file "LzmaDecOpt.c" + .text + .align 2 + .p2align 4,,15 +#ifdef __APPLE__ + .globl _LzmaDec_DecodeReal_3 +#else + .global LzmaDec_DecodeReal_3 +#endif + // .type LzmaDec_DecodeReal_3, %function + +// #define _LZMA_SIZE_OPT 1 + +#define LZMA_USE_4BYTES_FILL 1 +// #define LZMA_USE_2BYTES_COPY 1 +// #define LZMA_USE_CMOV_LZ_WRAP 1 +// #define _LZMA_PROB32 1 + +#define MY_ALIGN_FOR_ENTRY MY_ALIGN_32 +#define MY_ALIGN_FOR_LOOP MY_ALIGN_32 +#define MY_ALIGN_FOR_LOOP_16 MY_ALIGN_16 + +#ifdef _LZMA_PROB32 + .equ PSHIFT , 2 + .macro PLOAD dest:req, mem:req + ldr \dest, [\mem] + .endm + .macro PLOAD_PREINDEXED dest:req, mem:req, offset:req + ldr \dest, [\mem, \offset]! + .endm + .macro PLOAD_2 dest:req, mem1:req, mem2:req + ldr \dest, [\mem1, \mem2] + .endm + .macro PLOAD_LSL dest:req, mem1:req, mem2:req + ldr \dest, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE src:req, mem:req + str \src, [\mem] + .endm + .macro PSTORE_2 src:req, mem1:req, mem2:req + str \src, [\mem1, \mem2] + .endm + .macro PSTORE_LSL src:req, mem1:req, mem2:req + str \src, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE_LSL_M1 src:req, mem1:req, mem2:req, temp_reg:req + // you must check that temp_reg is free register when macro is used + add \temp_reg, \mem1, \mem2 + str \src, [\temp_reg, \mem2] + .endm +#else + // .equ PSHIFT , 1 + #define PSHIFT 1 + .macro PLOAD dest:req, mem:req + ldrh \dest, [\mem] + .endm + .macro PLOAD_PREINDEXED dest:req, mem:req, offset:req + ldrh \dest, [\mem, \offset]! + .endm + .macro PLOAD_2 dest:req, mem1:req, mem2:req + ldrh \dest, [\mem1, \mem2] + .endm + .macro PLOAD_LSL dest:req, mem1:req, mem2:req + ldrh \dest, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE src:req, mem:req + strh \src, [\mem] + .endm + .macro PSTORE_2 src:req, mem1:req, mem2:req + strh \src, [\mem1, \mem2] + .endm + .macro PSTORE_LSL src:req, mem1:req, mem2:req + strh \src, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE_LSL_M1 src:req, mem1:req, mem2:req, temp_reg:req + strh \src, [\mem1, \mem2] + .endm +#endif + +.equ PMULT , (1 << PSHIFT) +.equ PMULT_2 , (2 << PSHIFT) + +.equ kMatchSpecLen_Error_Data , (1 << 9) + +# x7 t0 : NORM_CALC : prob2 (IF_BIT_1) +# x6 t1 : NORM_CALC : probs_state +# x8 t2 : (LITM) temp : (TREE) temp +# x4 t3 : (LITM) bit : (TREE) temp : UPDATE_0/UPDATE_0 temp +# x10 t4 : (LITM) offs : (TREE) probs_PMULT : numBits +# x9 t5 : (LITM) match : sym2 (ShortDist) +# x1 t6 : (LITM) litm_prob : (TREE) prob_reg : pbPos +# x2 t7 : (LITM) prm : probBranch : cnt +# x3 sym : dist +# x12 len +# x0 range +# x5 cod + + +#define range w0 + +// t6 +#define pbPos w1 +#define pbPos_R r1 +#define prob_reg w1 +#define litm_prob prob_reg + +// t7 +#define probBranch w2 +#define cnt w2 +#define cnt_R r2 +#define prm r2 + +#define sym w3 +#define sym_R r3 +#define dist sym + +#define t3 w4 +#define bit w4 +#define bit_R r4 +#define update_temp_reg r4 + +#define cod w5 + +#define t1 w6 +#define t1_R r6 +#define probs_state t1_R + +#define t0 w7 +#define t0_R r7 +#define prob2 t0 + +#define t2 w8 +#define t2_R r8 + +// t5 +#define match w9 +#define sym2 w9 +#define sym2_R r9 + +#define t4 w10 +#define t4_R r10 + +#define offs w10 +#define offs_R r10 + +#define probs r11 + +#define len w12 +#define len_R x12 + +#define state w13 +#define state_R r13 + +#define dicPos r14 +#define buf r15 +#define bufLimit r16 +#define dicBufSize r17 + +#define limit r19 +#define rep0 w20 +#define rep0_R r20 +#define rep1 w21 +#define rep2 w22 +#define rep3 w23 +#define dic r24 +#define probs_IsMatch r25 +#define probs_Spec r26 +#define checkDicSize w27 +#define processedPos w28 +#define pbMask w29 +#define lc2_lpMask w30 + + +.equ kNumBitModelTotalBits , 11 +.equ kBitModelTotal , (1 << kNumBitModelTotalBits) +.equ kNumMoveBits , 5 +.equ kBitModelOffset , (kBitModelTotal - (1 << kNumMoveBits) + 1) + +.macro NORM_2 macro + ldrb t0, [buf], 1 + shl range, 8 + orr cod, t0, cod, lsl 8 + /* + mov t0, cod + ldrb cod, [buf], 1 + shl range, 8 + bfi cod, t0, #8, #24 + */ +.endm + +.macro TEST_HIGH_BYTE_range macro + tst range, 0xFF000000 +.endm + +.macro NORM macro + TEST_HIGH_BYTE_range + jnz 1f + NORM_2 +1: +.endm + + +# ---------- Branch MACROS ---------- + +.macro UPDATE_0__0 + sub prob2, probBranch, kBitModelOffset +.endm + +.macro UPDATE_0__1 + sub probBranch, probBranch, prob2, asr #(kNumMoveBits) +.endm + +.macro UPDATE_0__2 probsArray:req, probOffset:req, probDisp:req + .if \probDisp == 0 + PSTORE_2 probBranch, \probsArray, \probOffset + .elseif \probOffset == 0 + PSTORE_2 probBranch, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + // add update_temp_reg, \probsArray, \probOffset + PSTORE_2 probBranch, update_temp_reg, \probDisp * PMULT + .endif +.endm + +.macro UPDATE_0 probsArray:req, probOffset:req, probDisp:req + UPDATE_0__0 + UPDATE_0__1 + UPDATE_0__2 \probsArray, \probOffset, \probDisp +.endm + + +.macro UPDATE_1 probsArray:req, probOffset:req, probDisp:req + // sub cod, cod, prob2 + // sub range, range, prob2 + p2_sub cod, range + sub range, prob2, range + sub prob2, probBranch, probBranch, lsr #(kNumMoveBits) + .if \probDisp == 0 + PSTORE_2 prob2, \probsArray, \probOffset + .elseif \probOffset == 0 + PSTORE_2 prob2, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + // add update_temp_reg, \probsArray, \probOffset + PSTORE_2 prob2, update_temp_reg, \probDisp * PMULT + .endif +.endm + + +.macro CMP_COD_BASE + NORM + // lsr prob2, range, kNumBitModelTotalBits + // imul prob2, probBranch + // cmp cod, prob2 + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +.endm + +.macro CMP_COD_1 probsArray:req + PLOAD probBranch, \probsArray + CMP_COD_BASE +.endm + +.macro CMP_COD_3 probsArray:req, probOffset:req, probDisp:req + .if \probDisp == 0 + PLOAD_2 probBranch, \probsArray, \probOffset + .elseif \probOffset == 0 + PLOAD_2 probBranch, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + add update_temp_reg, \probsArray, \probOffset + PLOAD_2 probBranch, update_temp_reg, \probDisp * PMULT + .endif + CMP_COD_BASE +.endm + + +.macro IF_BIT_1_NOUP probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD_3 \probsArray, \probOffset, \probDisp + jae \toLabel +.endm + + +.macro IF_BIT_1 probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP \probsArray, \probOffset, \probDisp, \toLabel + UPDATE_0 \probsArray, \probOffset, \probDisp +.endm + + +.macro IF_BIT_0_NOUP probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD_3 \probsArray, \probOffset, \probDisp + jb \toLabel +.endm + +.macro IF_BIT_0_NOUP_1 probsArray:req, toLabel:req + CMP_COD_1 \probsArray + jb \toLabel +.endm + + +# ---------- CMOV MACROS ---------- + +.macro NORM_LSR + NORM + lsr t0, range, #kNumBitModelTotalBits +.endm + +.macro COD_RANGE_SUB + subs t1, cod, t0 + p2_sub range, t0 +.endm + +.macro RANGE_IMUL prob:req + imul t0, \prob +.endm + +.macro NORM_CALC prob:req + NORM_LSR + RANGE_IMUL \prob + COD_RANGE_SUB +.endm + +.macro CMOV_range + cmovb range, t0 +.endm + +.macro CMOV_code + cmovae cod, t1 +.endm + +.macro CMOV_code_Model_Pre prob:req + sub t0, \prob, kBitModelOffset + CMOV_code + cmovae t0, \prob +.endm + + +.macro PUP_BASE_2 prob:req, dest_reg:req + # only sar works for both 16/32 bit prob modes + sub \dest_reg, \prob, \dest_reg, asr #(kNumMoveBits) +.endm + +.macro PUP prob:req, probPtr:req, mem2:req + PUP_BASE_2 \prob, t0 + PSTORE_2 t0, \probPtr, \mem2 +.endm + + + +#define probs_PMULT t4_R + +.macro BIT_01 + add probs_PMULT, probs, PMULT +.endm + + +.macro BIT_0_R prob:req + PLOAD_2 \prob, probs, 1 * PMULT + NORM_LSR + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + PLOAD_2 t2, probs, 1 * PMULT_2 + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + PLOAD_2 t0, probs, 1 * PMULT_2 + PMULT + PUP_BASE_2 \prob, t3 + csel \prob, t2, t0, lo + CMOV_code + mov sym, 2 + PSTORE_2 t3, probs, 1 * PMULT + adc sym, sym, wzr + BIT_01 +.endm + +.macro BIT_1_R prob:req + NORM_LSR + p2_add sym, sym + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + PLOAD_LSL t2, probs, sym_R + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + PLOAD_LSL t0, probs_PMULT, sym_R + PUP_BASE_2 \prob, t3 + csel \prob, t2, t0, lo + CMOV_code + PSTORE_LSL_M1 t3, probs, sym_R, t2_R + adc sym, sym, wzr +.endm + + +.macro BIT_2_R prob:req + NORM_LSR + p2_add sym, sym + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + CMOV_code + PUP_BASE_2 \prob, t3 + PSTORE_LSL_M1 t3, probs, sym_R, t2_R + adc sym, sym, wzr +.endm + + +# ---------- MATCHED LITERAL ---------- + +.macro LITM_0 macro + shl match, (PSHIFT + 1) + and bit, match, 256 * PMULT + add prm, probs, 256 * PMULT + 1 * PMULT + p2_add match, match + p2_add prm, bit_R + eor offs, bit, 256 * PMULT + PLOAD litm_prob, prm + + NORM_LSR + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + cmovae offs, bit + CMOV_range + and bit, match, offs + cmovae t2, litm_prob + CMOV_code + mov sym, 2 + PUP_BASE_2 litm_prob, t2 + PSTORE t2, prm + add prm, probs, offs_R + adc sym, sym, wzr +.endm + +.macro LITM macro + p2_add prm, bit_R + xor offs, bit + PLOAD_LSL litm_prob, prm, sym_R + + NORM_LSR + p2_add match, match + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + cmovae offs, bit + CMOV_range + and bit, match, offs + cmovae t2, litm_prob + CMOV_code + PUP_BASE_2 litm_prob, t2 + PSTORE_LSL t2, prm, sym_R + add prm, probs, offs_R + adc sym, sym, sym +.endm + + +.macro LITM_2 macro + p2_add prm, bit_R + PLOAD_LSL litm_prob, prm, sym_R + + NORM_LSR + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + CMOV_range + cmovae t2, litm_prob + CMOV_code + PUP_BASE_2 litm_prob, t2 + PSTORE_LSL t2, prm, sym_R + adc sym, sym, sym +.endm + + +# ---------- REVERSE BITS ---------- + +.macro REV_0 prob:req + NORM_CALC \prob + CMOV_range + PLOAD t2, sym2_R + PLOAD_2 t3, probs, 3 * PMULT + CMOV_code_Model_Pre \prob + add t1_R, probs, 3 * PMULT + cmovae sym2_R, t1_R + PUP \prob, probs, 1 * PMULT + csel \prob, t2, t3, lo +.endm + + +.macro REV_1 prob:req, step:req + NORM_LSR + PLOAD_PREINDEXED t2, sym2_R, (\step * PMULT) + RANGE_IMUL \prob + COD_RANGE_SUB + CMOV_range + PLOAD_2 t3, sym2_R, (\step * PMULT) + sub t0, \prob, kBitModelOffset + CMOV_code + add t1_R, sym2_R, \step * PMULT + cmovae t0, \prob + cmovae sym2_R, t1_R + PUP_BASE_2 \prob, t0 + csel \prob, t2, t3, lo + PSTORE_2 t0, t1_R, 0 - \step * PMULT_2 +.endm + + +.macro REV_2 prob:req, step:req + sub t1_R, sym2_R, probs + NORM_LSR + orr sym, sym, t1, lsr #PSHIFT + RANGE_IMUL \prob + COD_RANGE_SUB + sub t2, sym, \step + CMOV_range + cmovb sym, t2 + CMOV_code_Model_Pre \prob + PUP \prob, sym2_R, 0 +.endm + + +.macro REV_1_VAR prob:req + PLOAD \prob, sym_R + mov probs, sym_R + p2_add sym_R, sym2_R + NORM_LSR + add t2_R, sym_R, sym2_R + RANGE_IMUL \prob + COD_RANGE_SUB + cmovae sym_R, t2_R + CMOV_range + CMOV_code_Model_Pre \prob + p2_add sym2, sym2 + PUP \prob, probs, 0 +.endm + + +.macro add_big dest:req, src:req, param:req + .if (\param) < (1 << 12) + add \dest, \src, \param + .else + #ifndef _LZMA_PROB32 + .error "unexpcted add_big expansion" + #endif + add \dest, \src, (\param) / 2 + add \dest, \dest, (\param) - (\param) / 2 + .endif +.endm + +.macro sub_big dest:req, src:req, param:req + .if (\param) < (1 << 12) + sub \dest, \src, \param + .else + #ifndef _LZMA_PROB32 + .error "unexpcted sub_big expansion" + #endif + sub \dest, \src, (\param) / 2 + sub \dest, \dest, (\param) - (\param) / 2 + .endif +.endm + + +.macro SET_probs offset:req + // add_big probs, probs_Spec, (\offset) * PMULT + add probs, probs_IsMatch, ((\offset) - IsMatch) * PMULT +.endm + + +.macro LIT_PROBS + add sym, sym, processedPos, lsl 8 + inc processedPos + UPDATE_0__0 + shl sym, lc2_lpMask + SET_probs Literal + p2_and sym, lc2_lpMask + // p2_add probs_state, pbPos_R + p2_add probs, sym_R + UPDATE_0__1 + add probs, probs, sym_R, lsl 1 + UPDATE_0__2 probs_state, pbPos_R, 0 +.endm + + + +.equ kNumPosBitsMax , 4 +.equ kNumPosStatesMax , (1 << kNumPosBitsMax) + +.equ kLenNumLowBits , 3 +.equ kLenNumLowSymbols , (1 << kLenNumLowBits) +.equ kLenNumHighBits , 8 +.equ kLenNumHighSymbols , (1 << kLenNumHighBits) +.equ kNumLenProbs , (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +.equ LenLow , 0 +.equ LenChoice , LenLow +.equ LenChoice2 , (LenLow + kLenNumLowSymbols) +.equ LenHigh , (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +.equ kNumStates , 12 +.equ kNumStates2 , 16 +.equ kNumLitStates , 7 + +.equ kStartPosModelIndex , 4 +.equ kEndPosModelIndex , 14 +.equ kNumFullDistances , (1 << (kEndPosModelIndex >> 1)) + +.equ kNumPosSlotBits , 6 +.equ kNumLenToPosStates , 4 + +.equ kNumAlignBits , 4 +.equ kAlignTableSize , (1 << kNumAlignBits) + +.equ kMatchMinLen , 2 +.equ kMatchSpecLenStart , (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +// .equ kStartOffset , 1408 +.equ kStartOffset , 0 +.equ SpecPos , (-kStartOffset) +.equ IsRep0Long , (SpecPos + kNumFullDistances) +.equ RepLenCoder , (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +.equ LenCoder , (RepLenCoder + kNumLenProbs) +.equ IsMatch , (LenCoder + kNumLenProbs) +.equ kAlign , (IsMatch + (kNumStates2 << kNumPosBitsMax)) +.equ IsRep , (kAlign + kAlignTableSize) +.equ IsRepG0 , (IsRep + kNumStates) +.equ IsRepG1 , (IsRepG0 + kNumStates) +.equ IsRepG2 , (IsRepG1 + kNumStates) +.equ PosSlot , (IsRepG2 + kNumStates) +.equ Literal , (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +.equ NUM_BASE_PROBS , (Literal + kStartOffset) + +.if kStartOffset != 0 // && IsMatch != 0 + .error "Stop_Compiling_Bad_StartOffset" +.endif + +.if NUM_BASE_PROBS != 1984 + .error "Stop_Compiling_Bad_LZMA_PROBS" +.endif + +.equ offset_lc , 0 +.equ offset_lp , 1 +.equ offset_pb , 2 +.equ offset_dicSize , 4 +.equ offset_probs , 4 + offset_dicSize +.equ offset_probs_1664 , 8 + offset_probs +.equ offset_dic , 8 + offset_probs_1664 +.equ offset_dicBufSize , 8 + offset_dic +.equ offset_dicPos , 8 + offset_dicBufSize +.equ offset_buf , 8 + offset_dicPos +.equ offset_range , 8 + offset_buf +.equ offset_code , 4 + offset_range +.equ offset_processedPos , 4 + offset_code +.equ offset_checkDicSize , 4 + offset_processedPos +.equ offset_rep0 , 4 + offset_checkDicSize +.equ offset_rep1 , 4 + offset_rep0 +.equ offset_rep2 , 4 + offset_rep1 +.equ offset_rep3 , 4 + offset_rep2 +.equ offset_state , 4 + offset_rep3 +.equ offset_remainLen , 4 + offset_state +.equ offset_TOTAL_SIZE , 4 + offset_remainLen + +.if offset_TOTAL_SIZE != 96 + .error "Incorrect offset_TOTAL_SIZE" +.endif + + +.macro IsMatchBranch_Pre + # prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + and pbPos, pbMask, processedPos, lsl #(kLenNumLowBits + 1 + PSHIFT) + add probs_state, probs_IsMatch, state_R +.endm + + +/* +.macro IsMatchBranch + IsMatchBranch_Pre + IF_BIT_1 probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label +.endm +*/ + +.macro CheckLimits + cmp buf, bufLimit + jae fin_OK + cmp dicPos, limit + jae fin_OK +.endm + +#define CheckLimits_lit CheckLimits +/* +.macro CheckLimits_lit + cmp buf, bufLimit + jae fin_OK_lit + cmp dicPos, limit + jae fin_OK_lit +.endm +*/ + + +#define PARAM_lzma REG_ABI_PARAM_0 +#define PARAM_limit REG_ABI_PARAM_1 +#define PARAM_bufLimit REG_ABI_PARAM_2 + + +.macro LOAD_LZMA_VAR reg:req, struct_offs:req + ldr \reg, [PARAM_lzma, \struct_offs] +.endm + +.macro LOAD_LZMA_BYTE reg:req, struct_offs:req + ldrb \reg, [PARAM_lzma, \struct_offs] +.endm + +.macro LOAD_LZMA_PAIR reg0:req, reg1:req, struct_offs:req + ldp \reg0, \reg1, [PARAM_lzma, \struct_offs] +.endm + + +LzmaDec_DecodeReal_3: +_LzmaDec_DecodeReal_3: +/* +.LFB0: + .cfi_startproc +*/ + + stp x19, x20, [sp, -128]! + stp x21, x22, [sp, 16] + stp x23, x24, [sp, 32] + stp x25, x26, [sp, 48] + stp x27, x28, [sp, 64] + stp x29, x30, [sp, 80] + + str PARAM_lzma, [sp, 120] + + mov bufLimit, PARAM_bufLimit + mov limit, PARAM_limit + + LOAD_LZMA_PAIR dic, dicBufSize, offset_dic + LOAD_LZMA_PAIR dicPos, buf, offset_dicPos + LOAD_LZMA_PAIR rep0, rep1, offset_rep0 + LOAD_LZMA_PAIR rep2, rep3, offset_rep2 + + mov t0, 1 << (kLenNumLowBits + 1 + PSHIFT) + LOAD_LZMA_BYTE pbMask, offset_pb + p2_add limit, dic + mov len, wzr // we can set it in all requiread branches instead + lsl pbMask, t0, pbMask + p2_add dicPos, dic + p2_sub pbMask, t0 + + LOAD_LZMA_BYTE lc2_lpMask, offset_lc + mov t0, 256 << PSHIFT + LOAD_LZMA_BYTE t1, offset_lp + p2_add t1, lc2_lpMask + p2_sub lc2_lpMask, (256 << PSHIFT) - PSHIFT + shl t0, t1 + p2_add lc2_lpMask, t0 + + LOAD_LZMA_VAR probs_Spec, offset_probs + LOAD_LZMA_VAR checkDicSize, offset_checkDicSize + LOAD_LZMA_VAR processedPos, offset_processedPos + LOAD_LZMA_VAR state, offset_state + // range is r0 : this load must be last don't move + LOAD_LZMA_PAIR range, cod, offset_range + mov sym, wzr + shl state, PSHIFT + + add_big probs_IsMatch, probs_Spec, ((IsMatch - SpecPos) << PSHIFT) + + // if (processedPos != 0 || checkDicSize != 0) + orr t0, checkDicSize, processedPos + cbz t0, 1f + add t0_R, dicBufSize, dic + cmp dicPos, dic + cmovne t0_R, dicPos + ldrb sym, [t0_R, -1] +1: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + +#define BIT_0 BIT_0_R prob_reg +#define BIT_1 BIT_1_R prob_reg +#define BIT_2 BIT_2_R prob_reg + +# ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + mov state, wzr +lit_start_2: + LIT_PROBS + + #ifdef _LZMA_SIZE_OPT + + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +lit_loop: + BIT_1 + tbz sym, 7, lit_loop + + #else + + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + + #endif + + BIT_2 + IsMatchBranch_Pre + strb sym, [dicPos], 1 + p2_and sym, 255 + + CheckLimits_lit +lit_end: + IF_BIT_0_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), lit_start + + # jmp IsMatch_label + + +#define FLAG_STATE_BITS (4 + PSHIFT) + +# ---------- MATCHES ---------- +# MY_ALIGN_FOR_ENTRY +IsMatch_label: + UPDATE_1 probs_state, pbPos_R, (IsMatch - IsMatch) + IF_BIT_1 probs_state, 0, (IsRep - IsMatch), IsRep_label + + SET_probs LenCoder + or state, (1 << FLAG_STATE_BITS) + +# ---------- LEN DECODE ---------- +len_decode: + mov len, 8 - kMatchMinLen + IF_BIT_0_NOUP_1 probs, len_mid_0 + UPDATE_1 probs, 0, 0 + p2_add probs, (1 << (kLenNumLowBits + PSHIFT)) + mov len, 0 - kMatchMinLen + IF_BIT_0_NOUP_1 probs, len_mid_0 + UPDATE_1 probs, 0, 0 + p2_add probs, LenHigh * PMULT - (1 << (kLenNumLowBits + PSHIFT)) + + #if 0 == 1 + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + #else + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +len8_loop: + BIT_1 + tbz sym, 6, len8_loop + #endif + + mov len, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - kMatchMinLen + jmp len_mid_2 + +MY_ALIGN_FOR_ENTRY +len_mid_0: + UPDATE_0 probs, 0, 0 + p2_add probs, pbPos_R + BIT_0 +len_mid_2: + BIT_1 + BIT_2 + sub len, sym, len + tbz state, FLAG_STATE_BITS, copy_match + +# ---------- DECODE DISTANCE ---------- + // probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp len, 3 + kMatchMinLen + cmovb t0, len + SET_probs PosSlot - (kMatchMinLen << (kNumPosSlotBits)) + add probs, probs, t0_R, lsl #(kNumPosSlotBits + PSHIFT) + + #ifdef _LZMA_SIZE_OPT + + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +slot_loop: + BIT_1 + tbz sym, 5, slot_loop + + #else + + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + + #endif + + #define numBits t4 + mov numBits, sym + BIT_2 + // we need only low bits + p2_and sym, 3 + cmp numBits, 32 + kEndPosModelIndex / 2 + jb short_dist + + SET_probs kAlign + + # unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + p2_sub numBits, (32 + 1 + kNumAlignBits) + # distance = (2 | (distance & 1)); + or sym, 2 + PLOAD_2 prob_reg, probs, 1 * PMULT + add sym2_R, probs, 2 * PMULT + +# ---------- DIRECT DISTANCE ---------- + +.macro DIRECT_1 + shr range, 1 + subs t0, cod, range + p2_add sym, sym + // add t1, sym, 1 + csel cod, cod, t0, mi + csinc sym, sym, sym, mi + // csel sym, t1, sym, pl + // adc sym, sym, sym // not 100% compatible for "corruptued-allowed" LZMA streams + dec_s numBits + je direct_end +.endm + + #ifdef _LZMA_SIZE_OPT + + jmp direct_norm +MY_ALIGN_FOR_ENTRY +direct_loop: + DIRECT_1 +direct_norm: + TEST_HIGH_BYTE_range + jnz direct_loop + NORM_2 + jmp direct_loop + + #else + +.macro DIRECT_2 + TEST_HIGH_BYTE_range + jz direct_unroll + DIRECT_1 +.endm + + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + +direct_unroll: + NORM_2 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + jmp direct_unroll + + #endif + +MY_ALIGN_FOR_ENTRY +direct_end: + shl sym, kNumAlignBits + REV_0 prob_reg + REV_1 prob_reg, 2 + REV_1 prob_reg, 4 + REV_2 prob_reg, 8 + +decode_dist_end: + + // if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + tst checkDicSize, checkDicSize + csel t0, processedPos, checkDicSize, eq + cmp sym, t0 + jae end_of_payload + // jmp end_of_payload # for debug + + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + add rep0, sym, 1 + +.macro STATE_UPDATE_FOR_MATCH + // state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + // cmp state, (kNumStates + kNumLitStates) * PMULT + cmp state, kNumLitStates * PMULT + (1 << FLAG_STATE_BITS) + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 +.endm + STATE_UPDATE_FOR_MATCH + +# ---------- COPY MATCH ---------- +copy_match: + + // if ((rem = limit - dicPos) == 0) break // return SZ_ERROR_DATA; + subs cnt_R, limit, dicPos + // jz fin_dicPos_LIMIT + jz fin_OK + + // curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, len_R + cmovae cnt, len + + sub t0_R, dicPos, dic + p2_add dicPos, cnt_R + p2_add processedPos, cnt + p2_sub len, cnt + + // pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + p2_sub_s t0_R, rep0_R + jae 1f + + cmn t0_R, cnt_R + p2_add t0_R, dicBufSize + ja copy_match_cross +1: +# ---------- COPY MATCH FAST ---------- + # t0_R : src_pos + p2_add t0_R, dic + ldrb sym, [t0_R] + p2_add t0_R, cnt_R + p1_neg cnt_R + +copy_common: + dec dicPos + + # dicPos : (ptr_to_last_dest_BYTE) + # t0_R : (src_lim) + # cnt_R : (-curLen) + + IsMatchBranch_Pre + + inc_s cnt_R + jz copy_end + + cmp rep0, 1 + je copy_match_0 + + #ifdef LZMA_USE_2BYTES_COPY + strb sym, [dicPos, cnt_R] + dec dicPos + # dicPos : (ptr_to_last_dest_16bitWORD) + p2_and cnt_R, -2 + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jz 2f +MY_ALIGN_FOR_LOOP +1: + /* + strh sym, [dicPos, cnt_R] + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jz 2f + */ + + strh sym, [dicPos, cnt_R] + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jnz 1b +2: + + /* + // for universal little/big endian code, but slow + strh sym, [dicPos] + inc dicPos + ldrb sym, [t0_R, -1] + */ + + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + // we must improve big-endian detection for another compilers + // for big-endian we need to revert bytes + rev16 sym, sym + #endif + + // (sym) must represent as little-endian here: + strb sym, [dicPos], 1 + shr sym, 8 + + #else + +MY_ALIGN_FOR_LOOP +1: + strb sym, [dicPos, cnt_R] + ldrb sym, [t0_R, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + ldrb sym, [t0_R, cnt_R] + inc_s cnt_R + jnz 1b + #endif + +copy_end: +lz_end_match: + strb sym, [dicPos], 1 + + # IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label + + + +# ---------- LITERAL MATCHED ---------- + + LIT_PROBS + + // matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + + sub t0_R, dicPos, dic + p2_sub_s t0_R, rep0_R + + #ifdef LZMA_USE_CMOV_LZ_WRAP + add t1_R, t0_R, dicBufSize + cmovb t0_R, t1_R + #else + jae 1f + p2_add t0_R, dicBufSize +1: + #endif + + ldrb match, [dic, t0_R] + + // state -= (state < 10) ? 3 : 6; + sub sym, state, 6 * PMULT + cmp state, 10 * PMULT + p2_sub state, 3 * PMULT + cmovae state, sym + + #ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov sym, 1 + and bit, match, offs + add prm, probs, offs_R + +MY_ALIGN_FOR_LOOP +litm_loop: + LITM + tbz sym, 8, litm_loop + + #else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + #endif + + IsMatchBranch_Pre + strb sym, [dicPos], 1 + p2_and sym, 255 + + // mov len, wzr // LITM uses same regisetr (len / offs). So we clear it + CheckLimits_lit +lit_matched_end: + IF_BIT_1_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label + # IsMatchBranch + p2_sub state, 3 * PMULT + jmp lit_start_2 + + + +# ---------- REP 0 LITERAL ---------- +MY_ALIGN_FOR_ENTRY +IsRep0Short_label: + UPDATE_0 probs_state, pbPos_R, 0 + + // dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + sub t0_R, dicPos, dic + + // state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + + # the caller doesn't allow (dicPos >= limit) case for REP_SHORT + # so we don't need the following (dicPos == limit) check here: + # cmp dicPos, limit + # jae fin_dicPos_LIMIT_REP_SHORT + # // jmp fin_dicPos_LIMIT_REP_SHORT // for testing/debug puposes + + inc processedPos + + IsMatchBranch_Pre + + p2_sub_s t0_R, rep0_R + #ifdef LZMA_USE_CMOV_LZ_WRAP + add sym_R, t0_R, dicBufSize + cmovb t0_R, sym_R + #else + jae 1f + p2_add t0_R, dicBufSize +1: + #endif + + ldrb sym, [dic, t0_R] + // mov len, wzr + jmp lz_end_match + +MY_ALIGN_FOR_ENTRY +IsRep_label: + UPDATE_1 probs_state, 0, (IsRep - IsMatch) + + # The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + # So we don't check it here. + + # mov t0, processedPos + # or t0, checkDicSize + # jz fin_ERROR_2 + + // state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + SET_probs RepLenCoder + + IF_BIT_1 probs_state, 0, (IsRepG0 - IsMatch), IsRepG0_label + sub_big probs_state, probs_state, (IsMatch - IsRep0Long) << PSHIFT + IF_BIT_0_NOUP probs_state, pbPos_R, 0, IsRep0Short_label + UPDATE_1 probs_state, pbPos_R, 0 + jmp len_decode + +MY_ALIGN_FOR_ENTRY +IsRepG0_label: + UPDATE_1 probs_state, 0, (IsRepG0 - IsMatch) + IF_BIT_1 probs_state, 0, (IsRepG1 - IsMatch), IsRepG1_label + mov dist, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + +# MY_ALIGN_FOR_ENTRY +IsRepG1_label: + UPDATE_1 probs_state, 0, (IsRepG1 - IsMatch) + IF_BIT_1 probs_state, 0, (IsRepG2 - IsMatch), IsRepG2_label + mov dist, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + +# MY_ALIGN_FOR_ENTRY +IsRepG2_label: + UPDATE_1 probs_state, 0, (IsRepG2 - IsMatch) + mov dist, rep3 + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + + + +# ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_FOR_ENTRY +short_dist: + p2_sub_s numBits, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, numBits + add sym_R, probs_Spec, sym_R, lsl #PSHIFT + p2_add sym_R, SpecPos * PMULT + 1 * PMULT + mov sym2, PMULT // # step +MY_ALIGN_FOR_LOOP +spec_loop: + REV_1_VAR prob_reg + dec_s numBits + jnz spec_loop + + p2_add sym2_R, probs_Spec + .if SpecPos != 0 + p2_add sym2_R, SpecPos * PMULT + .endif + p2_sub sym_R, sym2_R + shr sym, PSHIFT + + jmp decode_dist_end + + + +# ---------- COPY MATCH 0 ---------- +MY_ALIGN_FOR_ENTRY +copy_match_0: + #ifdef LZMA_USE_4BYTES_FILL + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + orr t3, sym, sym, lsl 8 + p2_and cnt_R, -4 + orr t3, t3, t3, lsl 16 +MY_ALIGN_FOR_LOOP_16 +1: + /* + str t3, [dicPos, cnt_R] + adds cnt_R, cnt_R, 4 + jz 2f + */ + + str t3, [dicPos, cnt_R] + adds cnt_R, cnt_R, 4 + jnz 1b +2: + // p2_and sym, 255 + #else + +MY_ALIGN_FOR_LOOP +1: + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jnz 1b + #endif + + jmp copy_end + + +# ---------- COPY MATCH CROSS ---------- +copy_match_cross: + # t0_R - src pos + # cnt_R - total copy len + + p1_neg cnt_R +1: + ldrb sym, [dic, t0_R] + inc t0_R + strb sym, [dicPos, cnt_R] + inc cnt_R + cmp t0_R, dicBufSize + jne 1b + + ldrb sym, [dic] + sub t0_R, dic, cnt_R + jmp copy_common + + + + +/* +fin_dicPos_LIMIT_REP_SHORT: + mov len, 1 + jmp fin_OK +*/ + +/* +fin_dicPos_LIMIT: + jmp fin_OK + # For more strict mode we can stop decoding with error + # mov sym, 1 + # jmp fin +*/ + +fin_ERROR_MATCH_DIST: + # rep0 = distance + 1; + p2_add len, kMatchSpecLen_Error_Data + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, sym + STATE_UPDATE_FOR_MATCH + # jmp fin_OK + mov sym, 1 + jmp fin + +end_of_payload: + inc_s sym + jnz fin_ERROR_MATCH_DIST + + mov len, kMatchSpecLenStart + xor state, (1 << FLAG_STATE_BITS) + jmp fin_OK + +/* +fin_OK_lit: + mov len, wzr +*/ + +fin_OK: + mov sym, wzr + +fin: + NORM + + #define fin_lzma_reg t0_R + + .macro STORE_LZMA_VAR reg:req, struct_offs:req + str \reg, [fin_lzma_reg, \struct_offs] + .endm + + .macro STORE_LZMA_PAIR reg0:req, reg1:req, struct_offs:req + stp \reg0, \reg1, [fin_lzma_reg, \struct_offs] + .endm + + ldr fin_lzma_reg, [sp, 120] + p2_sub dicPos, dic + shr state, PSHIFT + + STORE_LZMA_PAIR dicPos, buf, offset_dicPos + STORE_LZMA_PAIR range, cod, offset_range + STORE_LZMA_VAR processedPos, offset_processedPos + STORE_LZMA_PAIR rep0, rep1, offset_rep0 + STORE_LZMA_PAIR rep2, rep3, offset_rep2 + STORE_LZMA_PAIR state, len, offset_state + + mov w0, sym + + ldp x29, x30, [sp, 80] + ldp x27, x28, [sp, 64] + ldp x25, x26, [sp, 48] + ldp x23, x24, [sp, 32] + ldp x21, x22, [sp, 16] + ldp x19, x20, [sp], 128 + + ret +/* + .cfi_endproc +.LFE0: + .size LzmaDec_DecodeReal_3, .-LzmaDec_DecodeReal_3 + .ident "TAG_LZMA" + .section .note.GNU-stack,"",@progbits +*/ diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm index 067db822..dde40da6 100644 --- a/Asm/x86/7zAsm.asm +++ b/Asm/x86/7zAsm.asm @@ -1,5 +1,32 @@ ; 7zAsm.asm -- ASM macros -; 2018-02-03 : Igor Pavlov : Public domain +; 2021-02-07 : Igor Pavlov : Public domain + +ifdef RAX + x64 equ 1 +endif + +ifdef x64 + IS_X64 equ 1 +else + IS_X64 equ 0 +endif + +ifdef ABI_LINUX + IS_LINUX equ 1 +else + IS_LINUX equ 0 +endif + +ifndef x64 +; Use ABI_CDECL for x86 (32-bit) only +; if ABI_CDECL is not defined, we use fastcall abi +ifdef ABI_CDECL + IS_CDECL equ 1 +else + IS_CDECL equ 0 +endif +endif + MY_ASM_START macro ifdef x64 @@ -14,8 +41,12 @@ endm MY_PROC macro name:req, numParams:req align 16 proc_numParams = numParams - ifdef x64 + if (IS_X64 gt 0) proc_name equ name + elseif (IS_LINUX gt 0) + proc_name equ name + elseif (IS_CDECL gt 0) + proc_name equ @CatStr(_,name) else proc_name equ @CatStr(@,name,@, %numParams * 4) endif @@ -23,18 +54,19 @@ MY_PROC macro name:req, numParams:req endm MY_ENDP macro - ifdef x64 - ret - else - if proc_numParams LT 3 - ret + if (IS_X64 gt 0) + ret + elseif (IS_CDECL gt 0) + ret + elseif (proc_numParams LT 3) + ret else - ret (proc_numParams - 2) * 4 + ret (proc_numParams - 2) * 4 endif - endif proc_name ENDP endm + ifdef x64 REG_SIZE equ 8 REG_LOGAR_SIZE equ 3 @@ -103,6 +135,24 @@ else r7 equ x7 endif + +ifdef x64 +ifdef ABI_LINUX + +MY_PUSH_2_REGS macro + push r3 + push r5 +endm + +MY_POP_2_REGS macro + pop r5 + pop r3 +endm + +endif +endif + + MY_PUSH_4_REGS macro push r3 push r5 @@ -118,30 +168,91 @@ MY_POP_4_REGS macro endm -ifdef x64 +; for fastcall and for WIN-x64 +REG_PARAM_0_x equ x1 +REG_PARAM_0 equ r1 +REG_PARAM_1 equ r2 -; for WIN64-x64 ABI: +ifndef x64 +; for x86-fastcall -REG_PARAM_0 equ r1 -REG_PARAM_1 equ r2 +REG_ABI_PARAM_0_x equ REG_PARAM_0_x +REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1 equ REG_PARAM_1 + +else +; x64 + +if (IS_LINUX eq 0) + +; for WIN-x64: REG_PARAM_2 equ r8 REG_PARAM_3 equ r9 -MY_PUSH_PRESERVED_REGS macro - MY_PUSH_4_REGS - push r12 - push r13 - push r14 - push r15 +REG_ABI_PARAM_0_x equ REG_PARAM_0_x +REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1 equ REG_PARAM_1 +REG_ABI_PARAM_2 equ REG_PARAM_2 +REG_ABI_PARAM_3 equ REG_PARAM_3 + +else +; for LINUX-x64: +REG_LINUX_PARAM_0_x equ x7 +REG_LINUX_PARAM_0 equ r7 +REG_LINUX_PARAM_1 equ r6 +REG_LINUX_PARAM_2 equ r2 +REG_LINUX_PARAM_3 equ r1 + +REG_ABI_PARAM_0_x equ REG_LINUX_PARAM_0_x +REG_ABI_PARAM_0 equ REG_LINUX_PARAM_0 +REG_ABI_PARAM_1 equ REG_LINUX_PARAM_1 +REG_ABI_PARAM_2 equ REG_LINUX_PARAM_2 +REG_ABI_PARAM_3 equ REG_LINUX_PARAM_3 + +MY_ABI_LINUX_TO_WIN_2 macro + mov r2, r6 + mov r1, r7 endm +MY_ABI_LINUX_TO_WIN_3 macro + mov r8, r2 + mov r2, r6 + mov r1, r7 +endm -MY_POP_PRESERVED_REGS macro - pop r15 - pop r14 - pop r13 - pop r12 - MY_POP_4_REGS +MY_ABI_LINUX_TO_WIN_4 macro + mov r9, r1 + mov r8, r2 + mov r2, r6 + mov r1, r7 endm -endif +endif ; IS_LINUX + + +MY_PUSH_PRESERVED_ABI_REGS macro + if (IS_LINUX gt 0) + MY_PUSH_2_REGS + else + MY_PUSH_4_REGS + endif + push r12 + push r13 + push r14 + push r15 +endm + + +MY_POP_PRESERVED_ABI_REGS macro + pop r15 + pop r14 + pop r13 + pop r12 + if (IS_LINUX gt 0) + MY_POP_2_REGS + else + MY_POP_4_REGS + endif +endm + +endif ; x64 diff --git a/Asm/x86/7zCrcOpt.asm b/Asm/x86/7zCrcOpt.asm index 724b9b2c..0fee2064 100644 --- a/Asm/x86/7zCrcOpt.asm +++ b/Asm/x86/7zCrcOpt.asm @@ -1,5 +1,5 @@ ; 7zCrcOpt.asm -- CRC32 calculation : optimized version -; 2009-12-12 : Igor Pavlov : Public domain +; 2021-02-07 : Igor Pavlov : Public domain include 7zAsm.asm @@ -7,21 +7,28 @@ MY_ASM_START rD equ r2 rN equ r7 +rT equ r5 ifdef x64 num_VAR equ r8 table_VAR equ r9 else - data_size equ (REG_SIZE * 5) - crc_table equ (REG_SIZE + data_size) - num_VAR equ [r4 + data_size] - table_VAR equ [r4 + crc_table] + if (IS_CDECL gt 0) + crc_OFFS equ (REG_SIZE * 5) + data_OFFS equ (REG_SIZE + crc_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + else + size_OFFS equ (REG_SIZE * 5) + endif + table_OFFS equ (REG_SIZE + size_OFFS) + num_VAR equ [r4 + size_OFFS] + table_VAR equ [r4 + table_OFFS] endif -SRCDAT equ rN + rD + 4 * +SRCDAT equ rD + rN * 1 + 4 * CRC macro op:req, dest:req, src:req, t:req - op dest, DWORD PTR [r5 + src * 4 + 0400h * t] + op dest, DWORD PTR [rT + src * 4 + 0400h * t] endm CRC_XOR macro dest:req, src:req, t:req @@ -43,11 +50,33 @@ CRC1b macro endm MY_PROLOG macro crc_end:req - MY_PUSH_4_REGS + + ifdef x64 + if (IS_LINUX gt 0) + MY_PUSH_2_REGS + mov x0, REG_ABI_PARAM_0_x ; x0 = x7 + mov rT, REG_ABI_PARAM_3 ; r5 = r1 + mov rN, REG_ABI_PARAM_2 ; r7 = r2 + mov rD, REG_ABI_PARAM_1 ; r2 = r6 + else + MY_PUSH_4_REGS + mov x0, REG_ABI_PARAM_0_x ; x0 = x1 + mov rT, REG_ABI_PARAM_3 ; r5 = r9 + mov rN, REG_ABI_PARAM_2 ; r7 = r8 + ; mov rD, REG_ABI_PARAM_1 ; r2 = r2 + endif + else + MY_PUSH_4_REGS + if (IS_CDECL gt 0) + mov x0, [r4 + crc_OFFS] + mov rD, [r4 + data_OFFS] + else + mov x0, REG_ABI_PARAM_0_x + endif + mov rN, num_VAR + mov rT, table_VAR + endif - mov x0, x1 - mov rN, num_VAR - mov r5, table_VAR test rN, rN jz crc_end @@: @@ -77,7 +106,11 @@ MY_EPILOG macro crc_end:req CRC1b jmp crc_end @@: - MY_POP_4_REGS + if (IS_X64 gt 0) and (IS_LINUX gt 0) + MY_POP_2_REGS + else + MY_POP_4_REGS + endif endm MY_PROC CrcUpdateT8, 4 diff --git a/Asm/x86/AesOpt.asm b/Asm/x86/AesOpt.asm index 21466712..62c1ba2d 100644 --- a/Asm/x86/AesOpt.asm +++ b/Asm/x86/AesOpt.asm @@ -1,237 +1,734 @@ -; AesOpt.asm -- Intel's AES. -; 2009-12-12 : Igor Pavlov : Public domain +; AesOpt.asm -- AES optimized code for x86 AES hardware instructions +; 2021-03-10 : Igor Pavlov : Public domain include 7zAsm.asm +ifdef ymm0 + use_vaes_256 equ 1 + ECHO "++ VAES 256" +else + ECHO "-- NO VAES 256" +endif + +ifdef x64 + ECHO "x86-64" +else + ECHO "x86" +if (IS_CDECL gt 0) + ECHO "ABI : CDECL" +else + ECHO "ABI : no CDECL : FASTCALL" +endif +endif + +if (IS_LINUX gt 0) + ECHO "ABI : LINUX" +else + ECHO "ABI : WINDOWS" +endif + MY_ASM_START ifndef x64 + .686 .xmm endif + +; MY_ALIGN EQU ALIGN(64) +MY_ALIGN EQU + +SEG_ALIGN EQU MY_ALIGN + +MY_SEG_PROC macro name:req, numParams:req + ; seg_name equ @CatStr(_TEXT$, name) + ; seg_name SEGMENT SEG_ALIGN 'CODE' + MY_PROC name, numParams +endm + +MY_SEG_ENDP macro + ; seg_name ENDS +endm + + +NUM_AES_KEYS_MAX equ 15 + +; the number of push operators in function PROLOG +if (IS_LINUX eq 0) or (IS_X64 eq 0) +num_regs_push equ 2 +stack_param_offset equ (REG_SIZE * (1 + num_regs_push)) +endif + ifdef x64 - num equ r8 + num_param equ REG_ABI_PARAM_2 else - num equ [r4 + REG_SIZE * 4] + if (IS_CDECL gt 0) + ; size_t size + ; void * data + ; UInt32 * aes + ; ret-ip <- (r4) + aes_OFFS equ (stack_param_offset) + data_OFFS equ (REG_SIZE + aes_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + num_param equ [r4 + size_OFFS] + else + num_param equ [r4 + stack_param_offset] + endif endif -rD equ r2 -rN equ r0 +keys equ REG_PARAM_0 ; r1 +rD equ REG_PARAM_1 ; r2 +rN equ r0 -MY_PROLOG macro reg:req - ifdef x64 - movdqa [r4 + 8], xmm6 - movdqa [r4 + 8 + 16], xmm7 - endif +koffs_x equ x7 +koffs_r equ r7 + +ksize_x equ x6 +ksize_r equ r6 + +keys2 equ r3 + +state equ xmm0 +key equ xmm0 +key_ymm equ ymm0 +key_ymm_n equ 0 + +ifdef x64 + ways = 11 +else + ways = 4 +endif + +ways_start_reg equ 1 - push r3 - push r5 - push r6 +iv equ @CatStr(xmm, %(ways_start_reg + ways)) +iv_ymm equ @CatStr(ymm, %(ways_start_reg + ways)) - mov rN, num - mov x6, [r1 + 16] - shl x6, 5 - movdqa reg, [r1] - add r1, 32 +WOP macro op, op2 + i = 0 + rept ways + op @CatStr(xmm, %(ways_start_reg + i)), op2 + i = i + 1 + endm endm -MY_EPILOG macro - pop r6 - pop r5 - pop r3 +ifndef ABI_LINUX +ifdef x64 + +; we use 32 bytes of home space in stack in WIN64-x64 +NUM_HOME_MM_REGS equ (32 / 16) +; we preserve xmm registers starting from xmm6 in WIN64-x64 +MM_START_SAVE_REG equ 6 + +SAVE_XMM macro num_used_mm_regs:req + num_save_mm_regs = num_used_mm_regs - MM_START_SAVE_REG + if num_save_mm_regs GT 0 + num_save_mm_regs2 = num_save_mm_regs - NUM_HOME_MM_REGS + ; RSP is (16*x + 8) after entering the function in WIN64-x64 + stack_offset = 16 * num_save_mm_regs2 + (stack_param_offset mod 16) + + i = 0 + rept num_save_mm_regs + + if i eq NUM_HOME_MM_REGS + sub r4, stack_offset + endif + + if i lt NUM_HOME_MM_REGS + movdqa [r4 + stack_param_offset + i * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) + else + movdqa [r4 + (i - NUM_HOME_MM_REGS) * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) + endif + + i = i + 1 + endm + endif +endm + +RESTORE_XMM macro num_used_mm_regs:req + if num_save_mm_regs GT 0 + i = 0 + if num_save_mm_regs2 GT 0 + rept num_save_mm_regs2 + movdqa @CatStr(xmm, %(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i)), [r4 + i * 16] + i = i + 1 + endm + add r4, stack_offset + endif + + num_low_regs = num_save_mm_regs - i + i = 0 + rept num_low_regs + movdqa @CatStr(xmm, %(MM_START_SAVE_REG + i)), [r4 + stack_param_offset + i * 16] + i = i + 1 + endm + endif +endm + +endif ; x64 +endif ; ABI_LINUX + + +MY_PROLOG macro num_used_mm_regs:req + ; num_regs_push: must be equal to the number of push operators + ; push r3 + ; push r5 + if (IS_LINUX eq 0) or (IS_X64 eq 0) + push r6 + push r7 + endif + + mov rN, num_param ; don't move it; num_param can use stack pointer (r4) + + if (IS_X64 eq 0) + if (IS_CDECL gt 0) + mov rD, [r4 + data_OFFS] + mov keys, [r4 + aes_OFFS] + endif + elseif (IS_LINUX gt 0) + MY_ABI_LINUX_TO_WIN_2 + endif + + + ifndef ABI_LINUX ifdef x64 - movdqa xmm6, [r4 + 8] - movdqa xmm7, [r4 + 8 + 16] + SAVE_XMM num_used_mm_regs + endif endif + + mov ksize_x, [keys + 16] + shl ksize_x, 5 +endm + +MY_EPILOG macro + ifndef ABI_LINUX + ifdef x64 + RESTORE_XMM num_save_mm_regs + endif + endif + + if (IS_LINUX eq 0) or (IS_X64 eq 0) + pop r7 + pop r6 + endif + ; pop r5 + ; pop r3 MY_ENDP endm -ways equ 4 -ways16 equ (ways * 16) -OP_W macro op, op2 +OP_KEY macro op:req, offs:req + op state, [keys + offs] +endm + + +WOP_KEY macro op:req, offs:req + movdqa key, [keys + offs] + WOP op, key +endm + + +; ---------- AES-CBC Decode ---------- + + +XOR_WITH_DATA macro reg, _ppp_ + pxor reg, [rD + i * 16] +endm + +WRITE_TO_DATA macro reg, _ppp_ + movdqa [rD + i * 16], reg +endm + + +; state0 equ @CatStr(xmm, %(ways_start_reg)) + +key0 equ @CatStr(xmm, %(ways_start_reg + ways + 1)) +key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) + +key_last equ @CatStr(xmm, %(ways_start_reg + ways + 2)) +key_last_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) +key_last_ymm_n equ (ways_start_reg + ways + 2) + +NUM_CBC_REGS equ (ways_start_reg + ways + 3) + + +MY_SEG_PROC AesCbc_Decode_HW, 3 + + AesCbc_Decode_HW_start:: + MY_PROLOG NUM_CBC_REGS + + AesCbc_Decode_HW_start_2:: + movdqa iv, [keys] + add keys, 32 + + movdqa key0, [keys + 1 * ksize_r] + movdqa key_last, [keys] + sub ksize_x, 16 + + jmp check2 + align 16 + nextBlocks2: + WOP movdqa, [rD + i * 16] + mov koffs_x, ksize_x + ; WOP_KEY pxor, ksize_r + 16 + WOP pxor, key0 + ; align 16 + @@: + WOP_KEY aesdec, 1 * koffs_r + sub koffs_r, 16 + jnz @B + ; WOP_KEY aesdeclast, 0 + WOP aesdeclast, key_last + + pxor @CatStr(xmm, %(ways_start_reg)), iv + i = 1 + rept ways - 1 + pxor @CatStr(xmm, %(ways_start_reg + i)), [rD + i * 16 - 16] + i = i + 1 + endm + movdqa iv, [rD + ways * 16 - 16] + WOP WRITE_TO_DATA + + add rD, ways * 16 + AesCbc_Decode_HW_start_3:: + check2: + sub rN, ways + jnc nextBlocks2 + add rN, ways + + sub ksize_x, 16 + + jmp check + nextBlock: + movdqa state, [rD] + mov koffs_x, ksize_x + ; OP_KEY pxor, 1 * ksize_r + 32 + pxor state, key0 + ; movdqa state0, [rD] + ; movdqa state, key0 + ; pxor state, state0 + @@: + OP_KEY aesdec, 1 * koffs_r + 16 + OP_KEY aesdec, 1 * koffs_r + sub koffs_r, 32 + jnz @B + OP_KEY aesdec, 16 + ; OP_KEY aesdeclast, 0 + aesdeclast state, key_last + + pxor state, iv + movdqa iv, [rD] + ; movdqa iv, state0 + movdqa [rD], state + + add rD, 16 + check: + sub rN, 1 + jnc nextBlock + + movdqa [keys - 32], iv +MY_EPILOG + + + + +; ---------- AVX ---------- + + +AVX__WOP_n macro op i = 0 rept ways - op @CatStr(xmm,%i), op2 - i = i + 1 + op (ways_start_reg + i) + i = i + 1 endm endm -LOAD_OP macro op:req, offs:req - op xmm0, [r1 + r3 offs] +AVX__WOP macro op + i = 0 + rept ways + op @CatStr(ymm, %(ways_start_reg + i)) + i = i + 1 + endm endm - -LOAD_OP_W macro op:req, offs:req - movdqa xmm7, [r1 + r3 offs] - OP_W op, xmm7 + + +AVX__WOP_KEY macro op:req, offs:req + vmovdqa key_ymm, ymmword ptr [keys2 + offs] + AVX__WOP_n op endm -; ---------- AES-CBC Decode ---------- +AVX__CBC_START macro reg + ; vpxor reg, key_ymm, ymmword ptr [rD + 32 * i] + vpxor reg, key0_ymm, ymmword ptr [rD + 32 * i] +endm -CBC_DEC_UPDATE macro reg, offs - pxor reg, xmm6 - movdqa xmm6, [rD + offs] - movdqa [rD + offs], reg -endm - -DECODE macro op:req - op aesdec, +16 - @@: - op aesdec, +0 - op aesdec, -16 - sub x3, 32 - jnz @B - op aesdeclast, +0 -endm - -MY_PROC AesCbc_Decode_Intel, 3 - MY_PROLOG xmm6 - - sub x6, 32 - - jmp check2 - - align 16 - nextBlocks2: - mov x3, x6 - OP_W movdqa, [rD + i * 16] - LOAD_OP_W pxor, +32 - DECODE LOAD_OP_W - OP_W CBC_DEC_UPDATE, i * 16 - add rD, ways16 - check2: - sub rN, ways - jnc nextBlocks2 - - add rN, ways - jmp check - - nextBlock: - mov x3, x6 - movdqa xmm1, [rD] - LOAD_OP movdqa, +32 - pxor xmm0, xmm1 - DECODE LOAD_OP - pxor xmm0, xmm6 - movdqa [rD], xmm0 - movdqa xmm6, xmm1 - add rD, 16 - check: - sub rN, 1 - jnc nextBlock - - movdqa [r1 - 32], xmm6 - MY_EPILOG +AVX__CBC_END macro reg + if i eq 0 + vpxor reg, reg, iv_ymm + else + vpxor reg, reg, ymmword ptr [rD + i * 32 - 16] + endif +endm -; ---------- AES-CBC Encode ---------- +AVX__WRITE_TO_DATA macro reg + vmovdqu ymmword ptr [rD + 32 * i], reg +endm -ENCODE macro op:req - op aesenc, -16 - @@: - op aesenc, +0 - op aesenc, +16 - add r3, 32 - jnz @B - op aesenclast, +0 +AVX__XOR_WITH_DATA macro reg + vpxor reg, reg, ymmword ptr [rD + 32 * i] endm -MY_PROC AesCbc_Encode_Intel, 3 - MY_PROLOG xmm0 +AVX__CTR_START macro reg + vpaddq iv_ymm, iv_ymm, one_ymm + ; vpxor reg, iv_ymm, key_ymm + vpxor reg, iv_ymm, key0_ymm +endm - add r1, r6 - neg r6 - add r6, 32 - jmp check_e +MY_VAES_INSTR_2 macro cmd, dest, a1, a2 + db 0c4H + db 2 + 040H + 020h * (1 - (a2) / 8) + 080h * (1 - (dest) / 8) + db 5 + 8 * ((not (a1)) and 15) + db cmd + db 0c0H + 8 * ((dest) and 7) + ((a2) and 7) +endm - align 16 - nextBlock_e: - mov r3, r6 - pxor xmm0, [rD] - pxor xmm0, [r1 + r3 - 32] - ENCODE LOAD_OP - movdqa [rD], xmm0 - add rD, 16 - check_e: - sub rN, 1 - jnc nextBlock_e +MY_VAES_INSTR macro cmd, dest, a + MY_VAES_INSTR_2 cmd, dest, dest, a +endm - movdqa [r1 + r6 - 64], xmm0 - MY_EPILOG +MY_vaesenc macro dest, a + MY_VAES_INSTR 0dcH, dest, a +endm +MY_vaesenclast macro dest, a + MY_VAES_INSTR 0ddH, dest, a +endm +MY_vaesdec macro dest, a + MY_VAES_INSTR 0deH, dest, a +endm +MY_vaesdeclast macro dest, a + MY_VAES_INSTR 0dfH, dest, a +endm -; ---------- AES-CTR ---------- +AVX__VAES_DEC macro reg + MY_vaesdec reg, key_ymm_n +endm + +AVX__VAES_DEC_LAST_key_last macro reg + ; MY_vaesdeclast reg, key_ymm_n + MY_vaesdeclast reg, key_last_ymm_n +endm + +AVX__VAES_ENC macro reg + MY_vaesenc reg, key_ymm_n +endm -XOR_UPD_1 macro reg, offs - pxor reg, [rD + offs] +AVX__VAES_ENC_LAST macro reg + MY_vaesenclast reg, key_ymm_n endm -XOR_UPD_2 macro reg, offs - movdqa [rD + offs], reg +AVX__vinserti128_TO_HIGH macro dest, src + vinserti128 dest, dest, src, 1 endm -MY_PROC AesCtr_Code_Intel, 3 - MY_PROLOG xmm6 - mov r5, r4 - shr r5, 4 - dec r5 - shl r5, 4 +MY_PROC AesCbc_Decode_HW_256, 3 + ifdef use_vaes_256 + MY_PROLOG NUM_CBC_REGS + + cmp rN, ways * 2 + jb AesCbc_Decode_HW_start_2 + + vmovdqa iv, xmmword ptr [keys] + add keys, 32 + + vbroadcasti128 key0_ymm, xmmword ptr [keys + 1 * ksize_r] + vbroadcasti128 key_last_ymm, xmmword ptr [keys] + sub ksize_x, 16 + mov koffs_x, ksize_x + add ksize_x, ksize_x + + AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 2) * 32) + push keys2 + sub r4, AVX_STACK_SUB + ; sub r4, 32 + ; sub r4, ksize_r + ; lea keys2, [r4 + 32] + mov keys2, r4 + and keys2, -32 + broad: + vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] + vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + sub koffs_r, 16 + ; jnc broad + jnz broad + + sub rN, ways * 2 + + align 16 + avx_cbcdec_nextBlock2: + mov koffs_x, ksize_x + ; AVX__WOP_KEY AVX__CBC_START, 1 * koffs_r + 32 + AVX__WOP AVX__CBC_START + @@: + AVX__WOP_KEY AVX__VAES_DEC, 1 * koffs_r + sub koffs_r, 32 + jnz @B + ; AVX__WOP_KEY AVX__VAES_DEC_LAST, 0 + AVX__WOP_n AVX__VAES_DEC_LAST_key_last + + AVX__vinserti128_TO_HIGH iv_ymm, xmmword ptr [rD] + AVX__WOP AVX__CBC_END + + vmovdqa iv, xmmword ptr [rD + ways * 32 - 16] + AVX__WOP AVX__WRITE_TO_DATA + + add rD, ways * 32 + sub rN, ways * 2 + jnc avx_cbcdec_nextBlock2 + add rN, ways * 2 + + shr ksize_x, 1 + + ; lea r4, [r4 + 1 * ksize_r + 32] + add r4, AVX_STACK_SUB + pop keys2 + + vzeroupper + jmp AesCbc_Decode_HW_start_3 + else + jmp AesCbc_Decode_HW_start + endif +MY_ENDP +MY_SEG_ENDP + + - mov DWORD PTR [r5], 1 - mov DWORD PTR [r5 + 4], 0 - mov DWORD PTR [r5 + 8], 0 - mov DWORD PTR [r5 + 12], 0 - add r1, r6 - neg r6 - add r6, 32 +; ---------- AES-CBC Encode ---------- - jmp check2_c +e0 equ xmm1 - align 16 - nextBlocks2_c: - movdqa xmm7, [r5] +CENC_START_KEY equ 2 +CENC_NUM_REG_KEYS equ (3 * 2) +; last_key equ @CatStr(xmm, %(CENC_START_KEY + CENC_NUM_REG_KEYS)) +MY_SEG_PROC AesCbc_Encode_HW, 3 + MY_PROLOG (CENC_START_KEY + CENC_NUM_REG_KEYS + 0) + + movdqa state, [keys] + add keys, 32 + i = 0 - rept ways - paddq xmm6, xmm7 - movdqa @CatStr(xmm,%i), xmm6 - i = i + 1 + rept CENC_NUM_REG_KEYS + movdqa @CatStr(xmm, %(CENC_START_KEY + i)), [keys + i * 16] + i = i + 1 endm + + add keys, ksize_r + neg ksize_r + add ksize_r, (16 * CENC_NUM_REG_KEYS) + ; movdqa last_key, [keys] + jmp check_e + + align 16 + nextBlock_e: + movdqa e0, [rD] + mov koffs_r, ksize_r + pxor e0, @CatStr(xmm, %(CENC_START_KEY)) + pxor state, e0 + + i = 1 + rept (CENC_NUM_REG_KEYS - 1) + aesenc state, @CatStr(xmm, %(CENC_START_KEY + i)) + i = i + 1 + endm + + @@: + OP_KEY aesenc, 1 * koffs_r + OP_KEY aesenc, 1 * koffs_r + 16 + add koffs_r, 32 + jnz @B + OP_KEY aesenclast, 0 + ; aesenclast state, last_key + + movdqa [rD], state + add rD, 16 + check_e: + sub rN, 1 + jnc nextBlock_e + + ; movdqa [keys - 32], state + movdqa [keys + 1 * ksize_r - (16 * CENC_NUM_REG_KEYS) - 32], state +MY_EPILOG +MY_SEG_ENDP + + + +; ---------- AES-CTR ---------- + +ifdef x64 + ; ways = 11 +endif + + +one equ @CatStr(xmm, %(ways_start_reg + ways + 1)) +one_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) +key0 equ @CatStr(xmm, %(ways_start_reg + ways + 2)) +key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) +NUM_CTR_REGS equ (ways_start_reg + ways + 3) + +INIT_CTR macro reg, _ppp_ + paddq iv, one + movdqa reg, iv +endm + - mov r3, r6 - LOAD_OP_W pxor, -32 - ENCODE LOAD_OP_W - OP_W XOR_UPD_1, i * 16 - OP_W XOR_UPD_2, i * 16 - add rD, ways16 - check2_c: - sub rN, ways - jnc nextBlocks2_c - - add rN, ways - jmp check_c - - nextBlock_c: - paddq xmm6, [r5] - mov r3, r6 - movdqa xmm0, [r1 + r3 - 32] - pxor xmm0, xmm6 - ENCODE LOAD_OP - XOR_UPD_1 xmm0, 0 - XOR_UPD_2 xmm0, 0 - add rD, 16 - check_c: - sub rN, 1 - jnc nextBlock_c - - movdqa [r1 + r6 - 64], xmm6 - MY_EPILOG +MY_SEG_PROC AesCtr_Code_HW, 3 + Ctr_start:: + MY_PROLOG NUM_CTR_REGS + + Ctr_start_2:: + movdqa iv, [keys] + add keys, 32 + movdqa key0, [keys] + + add keys, ksize_r + neg ksize_r + add ksize_r, 16 + + Ctr_start_3:: + mov koffs_x, 1 + movd one, koffs_x + jmp check2_c + + align 16 + nextBlocks2_c: + WOP INIT_CTR, 0 + mov koffs_r, ksize_r + ; WOP_KEY pxor, 1 * koffs_r -16 + WOP pxor, key0 + @@: + WOP_KEY aesenc, 1 * koffs_r + add koffs_r, 16 + jnz @B + WOP_KEY aesenclast, 0 + + WOP XOR_WITH_DATA + WOP WRITE_TO_DATA + add rD, ways * 16 + check2_c: + sub rN, ways + jnc nextBlocks2_c + add rN, ways + + sub keys, 16 + add ksize_r, 16 + + jmp check_c + + ; align 16 + nextBlock_c: + paddq iv, one + ; movdqa state, [keys + 1 * koffs_r - 16] + movdqa state, key0 + mov koffs_r, ksize_r + pxor state, iv + + @@: + OP_KEY aesenc, 1 * koffs_r + OP_KEY aesenc, 1 * koffs_r + 16 + add koffs_r, 32 + jnz @B + OP_KEY aesenc, 0 + OP_KEY aesenclast, 16 + + pxor state, [rD] + movdqa [rD], state + add rD, 16 + check_c: + sub rN, 1 + jnc nextBlock_c + + ; movdqa [keys - 32], iv + movdqa [keys + 1 * ksize_r - 16 - 32], iv +MY_EPILOG + + +MY_PROC AesCtr_Code_HW_256, 3 + ifdef use_vaes_256 + MY_PROLOG NUM_CTR_REGS + + cmp rN, ways * 2 + jb Ctr_start_2 + + vbroadcasti128 iv_ymm, xmmword ptr [keys] + add keys, 32 + vbroadcasti128 key0_ymm, xmmword ptr [keys] + mov koffs_x, 1 + vmovd one, koffs_x + vpsubq iv_ymm, iv_ymm, one_ymm + vpaddq one, one, one + AVX__vinserti128_TO_HIGH one_ymm, one + + add keys, ksize_r + sub ksize_x, 16 + neg ksize_r + mov koffs_r, ksize_r + add ksize_r, ksize_r + + AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 1) * 32) + push keys2 + lea keys2, [r4 - 32] + sub r4, AVX_STACK_SUB + and keys2, -32 + vbroadcasti128 key_ymm, xmmword ptr [keys] + vmovdqa ymmword ptr [keys2], key_ymm + @@: + vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] + vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + add koffs_r, 16 + jnz @B + + sub rN, ways * 2 + + align 16 + avx_ctr_nextBlock2: + mov koffs_r, ksize_r + AVX__WOP AVX__CTR_START + ; AVX__WOP_KEY AVX__CTR_START, 1 * koffs_r - 32 + @@: + AVX__WOP_KEY AVX__VAES_ENC, 1 * koffs_r + add koffs_r, 32 + jnz @B + AVX__WOP_KEY AVX__VAES_ENC_LAST, 0 + + AVX__WOP AVX__XOR_WITH_DATA + AVX__WOP AVX__WRITE_TO_DATA + + add rD, ways * 32 + sub rN, ways * 2 + jnc avx_ctr_nextBlock2 + add rN, ways * 2 + + vextracti128 iv, iv_ymm, 1 + sar ksize_r, 1 + + add r4, AVX_STACK_SUB + pop keys2 + + vzeroupper + jmp Ctr_start_3 + else + jmp Ctr_start + endif +MY_ENDP +MY_SEG_ENDP end diff --git a/Asm/x86/LzmaDecOpt.asm b/Asm/x86/LzmaDecOpt.asm index 8ebbc5f2..f2818e77 100644 --- a/Asm/x86/LzmaDecOpt.asm +++ b/Asm/x86/LzmaDecOpt.asm @@ -1,5 +1,5 @@ ; LzmaDecOpt.asm -- ASM version of LzmaDec_DecodeReal_3() function -; 2018-02-06: Igor Pavlov : Public domain +; 2021-02-23: Igor Pavlov : Public domain ; ; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() ; function for check at link time. @@ -62,6 +62,7 @@ PMULT equ (1 SHL PSHIFT) PMULT_HALF equ (1 SHL (PSHIFT - 1)) PMULT_2 equ (1 SHL (PSHIFT + 1)) +kMatchSpecLen_Error_Data equ (1 SHL 9) ; x0 range ; x1 pbPos / (prob) TREE @@ -416,7 +417,7 @@ REV_1_VAR macro prob:req NORM_CALC prob cmovae range, t0 - lea t0_R, [sym_R + sym2_R] + lea t0_R, [sym_R + 1 * sym2_R] cmovae sym_R, t0_R mov t0, kBitModelOffset cmovb cod, t1 @@ -583,7 +584,7 @@ IsMatchBranch_Pre macro reg mov pbPos, LOC pbMask and pbPos, processedPos shl pbPos, (kLenNumLowBits + 1 + PSHIFT) - lea probs_state_R, [probs + state_R] + lea probs_state_R, [probs + 1 * state_R] endm @@ -605,13 +606,13 @@ endm ; RSP is (16x + 8) bytes aligned in WIN64-x64 ; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) -PARAM_lzma equ REG_PARAM_0 -PARAM_limit equ REG_PARAM_1 -PARAM_bufLimit equ REG_PARAM_2 +PARAM_lzma equ REG_ABI_PARAM_0 +PARAM_limit equ REG_ABI_PARAM_1 +PARAM_bufLimit equ REG_ABI_PARAM_2 ; MY_ALIGN_64 MY_PROC LzmaDec_DecodeReal_3, 3 -MY_PUSH_PRESERVED_REGS +MY_PUSH_PRESERVED_ABI_REGS lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] and r0, -128 @@ -777,7 +778,7 @@ len8_loop: jb len8_loop mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen - jmp len_mid_2 + jmp short len_mid_2 ; we use short here for MASM that doesn't optimize that code as another assembler programs MY_ALIGN_32 len_mid_0: @@ -890,11 +891,16 @@ decode_dist_end: ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + mov t1, LOC rep0 + mov x1, LOC rep1 + mov x2, LOC rep2 + mov t0, LOC checkDicSize test t0, t0 cmove t0, processedPos cmp sym, t0 jae end_of_payload + ; jmp end_of_payload ; for debug ; rep3 = rep2; ; rep2 = rep1; @@ -902,15 +908,12 @@ decode_dist_end: ; rep0 = distance + 1; inc sym - mov t0, LOC rep0 - mov t1, LOC rep1 - mov x1, LOC rep2 mov LOC rep0, sym ; mov sym, LOC remainLen mov sym, len_temp - mov LOC rep1, t0 - mov LOC rep2, t1 - mov LOC rep3, x1 + mov LOC rep1, t1 + mov LOC rep2, x1 + mov LOC rep3, x2 ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; cmp state, (kNumStates + kNumLitStates) * PMULT @@ -932,7 +935,7 @@ copy_match: ; } mov cnt_R, LOC limit sub cnt_R, dicPos - jz fin_ERROR + jz fin_dicPos_LIMIT ; curLen = ((rem < len) ? (unsigned)rem : len); cmp cnt_R, sym_R @@ -1091,11 +1094,23 @@ IsRep0Short_label: sub t0_R, dic sub probs, RepLenCoder * PMULT - inc processedPos + ; state = state < kNumLitStates ? 9 : 11; or state, 1 * PMULT + + ; the caller doesn't allow (dicPos >= limit) case for REP_SHORT + ; so we don't need the following (dicPos == limit) check here: + ; cmp dicPos, LOC limit + ; jae fin_dicPos_LIMIT_REP_SHORT + + inc processedPos + IsMatchBranch_Pre +; xor sym, sym +; sub t0_R, probBranch_R +; cmovb sym_R, LOC dicBufSize +; add t0_R, sym_R sub t0_R, probBranch_R jae @f add t0_R, LOC dicBufSize @@ -1210,15 +1225,45 @@ copy_match_cross: -fin_ERROR: +; fin_dicPos_LIMIT_REP_SHORT: + ; mov sym, 1 + +fin_dicPos_LIMIT: + mov LOC remainLen, sym + jmp fin_OK + ; For more strict mode we can stop decoding with error + ; mov sym, 1 + ; jmp fin + + +fin_ERROR_MATCH_DIST: + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + add len_temp, kMatchSpecLen_Error_Data mov LOC remainLen, len_temp -; fin_ERROR_2: + + mov LOC rep0, sym + mov LOC rep1, t1 + mov LOC rep2, x1 + mov LOC rep3, x2 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + ; jmp fin_OK mov sym, 1 jmp fin end_of_payload: - cmp sym, 0FFFFFFFFh ; -1 - jne fin_ERROR + inc sym + jnz fin_ERROR_MATCH_DIST mov LOC remainLen, kMatchSpecLenStart sub state, kNumStates * PMULT @@ -1250,7 +1295,7 @@ fin: mov RSP, LOC Old_RSP -MY_POP_PRESERVED_REGS +MY_POP_PRESERVED_ABI_REGS MY_ENDP _TEXT$LZMADECOPT ENDS diff --git a/Asm/x86/Sha1Opt.asm b/Asm/x86/Sha1Opt.asm new file mode 100644 index 00000000..3495fd16 --- /dev/null +++ b/Asm/x86/Sha1Opt.asm @@ -0,0 +1,263 @@ +; Sha1Opt.asm -- SHA-1 optimized code for SHA-1 x86 hardware instructions +; 2021-03-10 : Igor Pavlov : Public domain + +include 7zAsm.asm + +MY_ASM_START + + + + + + + + + + + + + + + + +CONST SEGMENT + +align 16 +Reverse_Endian_Mask db 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0 + + + + + + + + + + + + + + + + + + + + + + +CONST ENDS + +; _TEXT$SHA1OPT SEGMENT 'CODE' + +ifndef x64 + .686 + .xmm +endif + +ifdef x64 + rNum equ REG_ABI_PARAM_2 + if (IS_LINUX eq 0) + LOCAL_SIZE equ (16 * 2) + endif +else + rNum equ r0 + LOCAL_SIZE equ (16 * 1) +endif + +rState equ REG_ABI_PARAM_0 +rData equ REG_ABI_PARAM_1 + + +MY_sha1rnds4 macro a1, a2, imm + db 0fH, 03aH, 0ccH, (0c0H + a1 * 8 + a2), imm +endm + +MY_SHA_INSTR macro cmd, a1, a2 + db 0fH, 038H, cmd, (0c0H + a1 * 8 + a2) +endm + +cmd_sha1nexte equ 0c8H +cmd_sha1msg1 equ 0c9H +cmd_sha1msg2 equ 0caH + +MY_sha1nexte macro a1, a2 + MY_SHA_INSTR cmd_sha1nexte, a1, a2 +endm + +MY_sha1msg1 macro a1, a2 + MY_SHA_INSTR cmd_sha1msg1, a1, a2 +endm + +MY_sha1msg2 macro a1, a2 + MY_SHA_INSTR cmd_sha1msg2, a1, a2 +endm + +MY_PROLOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa [r4 + 8], xmm6 + movdqa [r4 + 8 + 16], xmm7 + sub r4, LOCAL_SIZE + 8 + movdqa [r4 ], xmm8 + movdqa [r4 + 16], xmm9 + endif + else ; x86 + if (IS_CDECL gt 0) + mov rState, [r4 + REG_SIZE * 1] + mov rData, [r4 + REG_SIZE * 2] + mov rNum, [r4 + REG_SIZE * 3] + else ; fastcall + mov rNum, [r4 + REG_SIZE * 1] + endif + push r5 + mov r5, r4 + and r4, -16 + sub r4, LOCAL_SIZE + endif +endm + +MY_EPILOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa xmm8, [r4] + movdqa xmm9, [r4 + 16] + add r4, LOCAL_SIZE + 8 + movdqa xmm6, [r4 + 8] + movdqa xmm7, [r4 + 8 + 16] + endif + else ; x86 + mov r4, r5 + pop r5 + endif + MY_ENDP +endm + + +e0_N equ 0 +e1_N equ 1 +abcd_N equ 2 +e0_save_N equ 3 +w_regs equ 4 + +e0 equ @CatStr(xmm, %e0_N) +e1 equ @CatStr(xmm, %e1_N) +abcd equ @CatStr(xmm, %abcd_N) +e0_save equ @CatStr(xmm, %e0_save_N) + + +ifdef x64 + abcd_save equ xmm8 + mask2 equ xmm9 +else + abcd_save equ [r4] + mask2 equ e1 +endif + +LOAD_MASK macro + movdqa mask2, XMMWORD PTR Reverse_Endian_Mask +endm + +LOAD_W macro k:req + movdqu @CatStr(xmm, %(w_regs + k)), [rData + (16 * (k))] + pshufb @CatStr(xmm, %(w_regs + k)), mask2 +endm + + +; pre2 can be 2 or 3 (recommended) +pre2 equ 3 +pre1 equ (pre2 + 1) + +NUM_ROUNDS4 equ 20 + +RND4 macro k + movdqa @CatStr(xmm, %(e0_N + ((k + 1) mod 2))), abcd + MY_sha1rnds4 abcd_N, (e0_N + (k mod 2)), k / 5 + + nextM = (w_regs + ((k + 1) mod 4)) + + if (k EQ NUM_ROUNDS4 - 1) + nextM = e0_save_N + endif + + MY_sha1nexte (e0_N + ((k + 1) mod 2)), nextM + + if (k GE (4 - pre2)) AND (k LT (NUM_ROUNDS4 - pre2)) + pxor @CatStr(xmm, %(w_regs + ((k + pre2) mod 4))), @CatStr(xmm, %(w_regs + ((k + pre2 - 2) mod 4))) + endif + + if (k GE (4 - pre1)) AND (k LT (NUM_ROUNDS4 - pre1)) + MY_sha1msg1 (w_regs + ((k + pre1) mod 4)), (w_regs + ((k + pre1 - 3) mod 4)) + endif + + if (k GE (4 - pre2)) AND (k LT (NUM_ROUNDS4 - pre2)) + MY_sha1msg2 (w_regs + ((k + pre2) mod 4)), (w_regs + ((k + pre2 - 1) mod 4)) + endif +endm + + +REVERSE_STATE macro + ; abcd ; dcba + ; e0 ; 000e + pshufd abcd, abcd, 01bH ; abcd + pshufd e0, e0, 01bH ; e000 +endm + + + + + +MY_PROC Sha1_UpdateBlocks_HW, 3 + MY_PROLOG + + cmp rNum, 0 + je end_c + + movdqu abcd, [rState] ; dcba + movd e0, dword ptr [rState + 16] ; 000e + + REVERSE_STATE + + ifdef x64 + LOAD_MASK + endif + + align 16 + nextBlock: + movdqa abcd_save, abcd + movdqa e0_save, e0 + + ifndef x64 + LOAD_MASK + endif + + LOAD_W 0 + LOAD_W 1 + LOAD_W 2 + LOAD_W 3 + + paddd e0, @CatStr(xmm, %(w_regs)) + k = 0 + rept NUM_ROUNDS4 + RND4 k + k = k + 1 + endm + + paddd abcd, abcd_save + + + add rData, 64 + sub rNum, 1 + jnz nextBlock + + REVERSE_STATE + + movdqu [rState], abcd + movd dword ptr [rState + 16], e0 + + end_c: +MY_EPILOG + +; _TEXT$SHA1OPT ENDS + +end diff --git a/Asm/x86/Sha256Opt.asm b/Asm/x86/Sha256Opt.asm new file mode 100644 index 00000000..5d02c90a --- /dev/null +++ b/Asm/x86/Sha256Opt.asm @@ -0,0 +1,263 @@ +; Sha256Opt.asm -- SHA-256 optimized code for SHA-256 x86 hardware instructions +; 2021-03-10 : Igor Pavlov : Public domain + +include 7zAsm.asm + +MY_ASM_START + +; .data +; public K + +; we can use external SHA256_K_ARRAY defined in Sha256.c +; but we must guarantee that SHA256_K_ARRAY is aligned for 16-bytes + +COMMENT @ +ifdef x64 +K_CONST equ SHA256_K_ARRAY +else +K_CONST equ _SHA256_K_ARRAY +endif +EXTRN K_CONST:xmmword +@ + +CONST SEGMENT + +align 16 +Reverse_Endian_Mask db 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +; COMMENT @ +align 16 +K_CONST \ +DD 0428a2f98H, 071374491H, 0b5c0fbcfH, 0e9b5dba5H +DD 03956c25bH, 059f111f1H, 0923f82a4H, 0ab1c5ed5H +DD 0d807aa98H, 012835b01H, 0243185beH, 0550c7dc3H +DD 072be5d74H, 080deb1feH, 09bdc06a7H, 0c19bf174H +DD 0e49b69c1H, 0efbe4786H, 00fc19dc6H, 0240ca1ccH +DD 02de92c6fH, 04a7484aaH, 05cb0a9dcH, 076f988daH +DD 0983e5152H, 0a831c66dH, 0b00327c8H, 0bf597fc7H +DD 0c6e00bf3H, 0d5a79147H, 006ca6351H, 014292967H +DD 027b70a85H, 02e1b2138H, 04d2c6dfcH, 053380d13H +DD 0650a7354H, 0766a0abbH, 081c2c92eH, 092722c85H +DD 0a2bfe8a1H, 0a81a664bH, 0c24b8b70H, 0c76c51a3H +DD 0d192e819H, 0d6990624H, 0f40e3585H, 0106aa070H +DD 019a4c116H, 01e376c08H, 02748774cH, 034b0bcb5H +DD 0391c0cb3H, 04ed8aa4aH, 05b9cca4fH, 0682e6ff3H +DD 0748f82eeH, 078a5636fH, 084c87814H, 08cc70208H +DD 090befffaH, 0a4506cebH, 0bef9a3f7H, 0c67178f2H +; @ + +CONST ENDS + +; _TEXT$SHA256OPT SEGMENT 'CODE' + +ifndef x64 + .686 + .xmm +endif + +ifdef x64 + rNum equ REG_ABI_PARAM_2 + if (IS_LINUX eq 0) + LOCAL_SIZE equ (16 * 2) + endif +else + rNum equ r0 + LOCAL_SIZE equ (16 * 1) +endif + +rState equ REG_ABI_PARAM_0 +rData equ REG_ABI_PARAM_1 + + + + + + +MY_SHA_INSTR macro cmd, a1, a2 + db 0fH, 038H, cmd, (0c0H + a1 * 8 + a2) +endm + +cmd_sha256rnds2 equ 0cbH +cmd_sha256msg1 equ 0ccH +cmd_sha256msg2 equ 0cdH + +MY_sha256rnds2 macro a1, a2 + MY_SHA_INSTR cmd_sha256rnds2, a1, a2 +endm + +MY_sha256msg1 macro a1, a2 + MY_SHA_INSTR cmd_sha256msg1, a1, a2 +endm + +MY_sha256msg2 macro a1, a2 + MY_SHA_INSTR cmd_sha256msg2, a1, a2 +endm + +MY_PROLOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa [r4 + 8], xmm6 + movdqa [r4 + 8 + 16], xmm7 + sub r4, LOCAL_SIZE + 8 + movdqa [r4 ], xmm8 + movdqa [r4 + 16], xmm9 + endif + else ; x86 + if (IS_CDECL gt 0) + mov rState, [r4 + REG_SIZE * 1] + mov rData, [r4 + REG_SIZE * 2] + mov rNum, [r4 + REG_SIZE * 3] + else ; fastcall + mov rNum, [r4 + REG_SIZE * 1] + endif + push r5 + mov r5, r4 + and r4, -16 + sub r4, LOCAL_SIZE + endif +endm + +MY_EPILOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa xmm8, [r4] + movdqa xmm9, [r4 + 16] + add r4, LOCAL_SIZE + 8 + movdqa xmm6, [r4 + 8] + movdqa xmm7, [r4 + 8 + 16] + endif + else ; x86 + mov r4, r5 + pop r5 + endif + MY_ENDP +endm + + +msg equ xmm0 +tmp equ xmm0 +state0_N equ 2 +state1_N equ 3 +w_regs equ 4 + + +state1_save equ xmm1 +state0 equ @CatStr(xmm, %state0_N) +state1 equ @CatStr(xmm, %state1_N) + + +ifdef x64 + state0_save equ xmm8 + mask2 equ xmm9 +else + state0_save equ [r4] + mask2 equ xmm0 +endif + +LOAD_MASK macro + movdqa mask2, XMMWORD PTR Reverse_Endian_Mask +endm + +LOAD_W macro k:req + movdqu @CatStr(xmm, %(w_regs + k)), [rData + (16 * (k))] + pshufb @CatStr(xmm, %(w_regs + k)), mask2 +endm + + +; pre1 <= 4 && pre2 >= 1 && pre1 > pre2 && (pre1 - pre2) <= 1 +pre1 equ 3 +pre2 equ 2 + + + +RND4 macro k + movdqa msg, xmmword ptr [K_CONST + (k) * 16] + paddd msg, @CatStr(xmm, %(w_regs + ((k + 0) mod 4))) + MY_sha256rnds2 state0_N, state1_N + pshufd msg, msg, 0eH + + if (k GE (4 - pre1)) AND (k LT (16 - pre1)) + ; w4[0] = msg1(w4[-4], w4[-3]) + MY_sha256msg1 (w_regs + ((k + pre1) mod 4)), (w_regs + ((k + pre1 - 3) mod 4)) + endif + + MY_sha256rnds2 state1_N, state0_N + + if (k GE (4 - pre2)) AND (k LT (16 - pre2)) + movdqa tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 1) mod 4))) + palignr tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 2) mod 4))), 4 + paddd @CatStr(xmm, %(w_regs + ((k + pre2) mod 4))), tmp + ; w4[0] = msg2(w4[0], w4[-1]) + MY_sha256msg2 %(w_regs + ((k + pre2) mod 4)), %(w_regs + ((k + pre2 - 1) mod 4)) + endif +endm + + + + + +REVERSE_STATE macro + ; state0 ; dcba + ; state1 ; hgfe + pshufd tmp, state0, 01bH ; abcd + pshufd state0, state1, 01bH ; efgh + movdqa state1, state0 ; efgh + punpcklqdq state0, tmp ; cdgh + punpckhqdq state1, tmp ; abef +endm + + +MY_PROC Sha256_UpdateBlocks_HW, 3 + MY_PROLOG + + cmp rNum, 0 + je end_c + + movdqu state0, [rState] ; dcba + movdqu state1, [rState + 16] ; hgfe + + REVERSE_STATE + + ifdef x64 + LOAD_MASK + endif + + align 16 + nextBlock: + movdqa state0_save, state0 + movdqa state1_save, state1 + + ifndef x64 + LOAD_MASK + endif + + LOAD_W 0 + LOAD_W 1 + LOAD_W 2 + LOAD_W 3 + + + k = 0 + rept 16 + RND4 k + k = k + 1 + endm + + paddd state0, state0_save + paddd state1, state1_save + + add rData, 64 + sub rNum, 1 + jnz nextBlock + + REVERSE_STATE + + movdqu [rState], state0 + movdqu [rState + 16], state1 + + end_c: +MY_EPILOG + +; _TEXT$SHA256OPT ENDS + +end diff --git a/Asm/x86/XzCrc64Opt.asm b/Asm/x86/XzCrc64Opt.asm index 734ca5c4..ad22cc2f 100644 --- a/Asm/x86/XzCrc64Opt.asm +++ b/Asm/x86/XzCrc64Opt.asm @@ -1,5 +1,5 @@ ; XzCrc64Opt.asm -- CRC64 calculation : optimized version -; 2011-06-28 : Igor Pavlov : Public domain +; 2021-02-06 : Igor Pavlov : Public domain include 7zAsm.asm @@ -7,16 +7,15 @@ MY_ASM_START ifdef x64 - rD equ r9 - rN equ r10 - - num_VAR equ r8 - table_VAR equ r9 - - SRCDAT equ rN + rD +rD equ r9 +rN equ r10 +rT equ r5 +num_VAR equ r8 +SRCDAT4 equ dword ptr [rD + rN * 1] + CRC_XOR macro dest:req, src:req, t:req - xor dest, QWORD PTR [r5 + src * 8 + 0800h * t] + xor dest, QWORD PTR [rT + src * 8 + 0800h * t] endm CRC1b macro @@ -30,12 +29,15 @@ CRC1b macro endm MY_PROLOG macro crc_end:req + ifdef ABI_LINUX + MY_PUSH_2_REGS + else MY_PUSH_4_REGS - - mov r0, r1 - mov rN, num_VAR - mov r5, table_VAR - mov rD, r2 + endif + mov r0, REG_ABI_PARAM_0 + mov rN, REG_ABI_PARAM_2 + mov rT, REG_ABI_PARAM_3 + mov rD, REG_ABI_PARAM_1 test rN, rN jz crc_end @@: @@ -51,14 +53,14 @@ MY_PROLOG macro crc_end:req sub rN, 4 and rN, NOT 3 sub rD, rN - mov x1, [SRCDAT] + mov x1, SRCDAT4 xor r0, r1 add rN, 4 endm MY_EPILOG macro crc_end:req sub rN, 4 - mov x1, [SRCDAT] + mov x1, SRCDAT4 xor r0, r1 mov rD, rN mov rN, num_VAR @@ -69,14 +71,18 @@ MY_EPILOG macro crc_end:req CRC1b jmp crc_end @@: + ifdef ABI_LINUX + MY_POP_2_REGS + else MY_POP_4_REGS + endif endm MY_PROC XzCrc64UpdateT4, 4 MY_PROLOG crc_end_4 align 16 main_loop_4: - mov x1, [SRCDAT] + mov x1, SRCDAT4 movzx x2, x0_L movzx x3, x0_H shr r0, 16 @@ -96,21 +102,43 @@ MY_PROC XzCrc64UpdateT4, 4 MY_ENDP else +; x86 (32-bit) + +rD equ r1 +rN equ r7 +rT equ r5 + +crc_OFFS equ (REG_SIZE * 5) + +if (IS_CDECL gt 0) or (IS_LINUX gt 0) + ; cdecl or (GNU fastcall) stack: + ; (UInt32 *) table + ; size_t size + ; void * data + ; (UInt64) crc + ; ret-ip <-(r4) + data_OFFS equ (8 + crc_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + table_OFFS equ (REG_SIZE + size_OFFS) + num_VAR equ [r4 + size_OFFS] + table_VAR equ [r4 + table_OFFS] +else + ; Windows fastcall: + ; r1 = data, r2 = size + ; stack: + ; (UInt32 *) table + ; (UInt64) crc + ; ret-ip <-(r4) + table_OFFS equ (8 + crc_OFFS) + table_VAR equ [r4 + table_OFFS] + num_VAR equ table_VAR +endif - rD equ r1 - rN equ r7 - - crc_val equ (REG_SIZE * 5) - crc_table equ (8 + crc_val) - table_VAR equ [r4 + crc_table] - num_VAR equ table_VAR - - - SRCDAT equ rN + rD +SRCDAT4 equ dword ptr [rD + rN * 1] CRC macro op0:req, op1:req, dest0:req, dest1:req, src:req, t:req - op0 dest0, DWORD PTR [r5 + src * 8 + 0800h * t] - op1 dest1, DWORD PTR [r5 + src * 8 + 0800h * t + 4] + op0 dest0, DWORD PTR [rT + src * 8 + 0800h * t] + op1 dest1, DWORD PTR [rT + src * 8 + 0800h * t + 4] endm CRC_XOR macro dest0:req, dest1:req, src:req, t:req @@ -131,12 +159,18 @@ endm MY_PROLOG macro crc_end:req MY_PUSH_4_REGS - + + if (IS_CDECL gt 0) or (IS_LINUX gt 0) + proc_numParams = proc_numParams + 2 ; for ABI_LINUX + mov rN, [r4 + size_OFFS] + mov rD, [r4 + data_OFFS] + else mov rN, r2 + endif - mov x0, [r4 + crc_val] - mov x2, [r4 + crc_val + 4] - mov r5, table_VAR + mov x0, [r4 + crc_OFFS] + mov x2, [r4 + crc_OFFS + 4] + mov rT, table_VAR test rN, rN jz crc_end @@: @@ -154,13 +188,13 @@ MY_PROLOG macro crc_end:req sub rN, 4 and rN, NOT 3 sub rD, rN - xor r0, [SRCDAT] + xor r0, SRCDAT4 add rN, 4 endm MY_EPILOG macro crc_end:req sub rN, 4 - xor r0, [SRCDAT] + xor r0, SRCDAT4 mov rD, rN mov rN, num_VAR @@ -179,7 +213,7 @@ MY_PROC XzCrc64UpdateT4, 5 movzx x6, x0_L align 16 main_loop_4: - mov r3, [SRCDAT] + mov r3, SRCDAT4 xor r3, r2 CRC xor, mov, r3, r2, r6, 3 @@ -200,6 +234,6 @@ MY_PROC XzCrc64UpdateT4, 5 MY_EPILOG crc_end_4 MY_ENDP -endif +endif ; ! x64 end @@ -1,5 +1,5 @@ /* 7z.h -- 7z interface -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-02 : Igor Pavlov : Public domain */ #ifndef __7Z_H #define __7Z_H @@ -91,6 +91,8 @@ typedef struct UInt64 *CoderUnpackSizes; // for all coders in all folders Byte *CodersData; + + UInt64 RangeLimit; } CSzAr; UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); diff --git a/C/7zArcIn.c b/C/7zArcIn.c index f74d0fad..0d9dec41 100644 --- a/C/7zArcIn.c +++ b/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2018-12-31 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -75,7 +75,7 @@ static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) return SZ_OK; } -void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) +static void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; @@ -83,7 +83,7 @@ void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) #define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) +static void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; @@ -105,6 +105,8 @@ static void SzAr_Init(CSzAr *p) p->CoderUnpackSizes = NULL; p->CodersData = NULL; + + p->RangeLimit = 0; } static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) @@ -502,7 +504,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) return SZ_ERROR_ARCHIVE; if (propsSize >= 0x80) return SZ_ERROR_UNSUPPORTED; - coder->PropsOffset = sd->Data - dataStart; + coder->PropsOffset = (size_t)(sd->Data - dataStart); coder->PropsSize = (Byte)propsSize; sd->Data += (size_t)propsSize; sd->Size -= (size_t)propsSize; @@ -677,7 +679,7 @@ static SRes ReadUnpackInfo(CSzAr *p, { UInt32 numCoders, ci, numInStreams = 0; - p->FoCodersOffsets[fo] = sd.Data - startBufPtr; + p->FoCodersOffsets[fo] = (size_t)(sd.Data - startBufPtr); RINOK(SzReadNumber32(&sd, &numCoders)); if (numCoders == 0 || numCoders > k_Scan_NumCoders_MAX) @@ -797,7 +799,7 @@ static SRes ReadUnpackInfo(CSzAr *p, p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; { - size_t dataSize = sd.Data - startBufPtr; + const size_t dataSize = (size_t)(sd.Data - startBufPtr); p->FoStartPackStreamIndex[fo] = packStreamIndex; p->FoCodersOffsets[fo] = dataSize; MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc); @@ -885,7 +887,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) if (numStreams != 1 || !SzBitWithVals_Check(&p->FolderCRCs, i)) numSubDigests += numStreams; } - ssi->sdNumSubStreams.Size = sd->Data - ssi->sdNumSubStreams.Data; + ssi->sdNumSubStreams.Size = (size_t)(sd->Data - ssi->sdNumSubStreams.Data); continue; } if (type == k7zIdCRC || type == k7zIdSize || type == k7zIdEnd) @@ -907,7 +909,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) { ssi->sdSizes.Data = sd->Data; RINOK(SkipNumbers(sd, numUnpackSizesInData)); - ssi->sdSizes.Size = sd->Data - ssi->sdSizes.Data; + ssi->sdSizes.Size = (size_t)(sd->Data - ssi->sdSizes.Data); RINOK(ReadID(sd, &type)); } @@ -919,7 +921,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) { ssi->sdCRCs.Data = sd->Data; RINOK(SkipBitUi32s(sd, numSubDigests)); - ssi->sdCRCs.Size = sd->Data - ssi->sdCRCs.Data; + ssi->sdCRCs.Size = (size_t)(sd->Data - ssi->sdCRCs.Data); } else { @@ -947,7 +949,11 @@ static SRes SzReadStreamsInfo(CSzAr *p, if (type == k7zIdPackInfo) { RINOK(ReadNumber(sd, dataOffset)); + if (*dataOffset > p->RangeLimit) + return SZ_ERROR_ARCHIVE; RINOK(ReadPackInfo(p, sd, alloc)); + if (p->PackPositions[p->NumPackStreams] > p->RangeLimit - *dataOffset) + return SZ_ERROR_ARCHIVE; RINOK(ReadID(sd, &type)); } if (type == k7zIdUnpackInfo) @@ -1028,12 +1034,12 @@ static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size return SZ_ERROR_ARCHIVE; for (p = data + pos; #ifdef _WIN32 - *(const UInt16 *)p != 0 + *(const UInt16 *)(const void *)p != 0 #else p[0] != 0 || p[1] != 0 #endif ; p += 2); - pos = p - data + 2; + pos = (size_t)(p - data) + 2; *offsets++ = (pos >> 1); } while (--numFiles); @@ -1133,6 +1139,8 @@ static SRes SzReadHeader2( SRes res; SzAr_Init(&tempAr); + tempAr.RangeLimit = p->db.RangeLimit; + res = SzReadAndDecodePackedStreams(inStream, sd, tempBufs, NUM_ADDITIONAL_STREAMS_MAX, p->startPosAfterHeader, &tempAr, allocTemp); *numTempBufs = tempAr.NumFolders; @@ -1526,11 +1534,13 @@ static SRes SzArEx_Open2( nextHeaderSize = GetUi64(header + 20); nextHeaderCRC = GetUi32(header + 28); - p->startPosAfterHeader = startArcPos + k7zStartHeaderSize; + p->startPosAfterHeader = (UInt64)startArcPos + k7zStartHeaderSize; if (CrcCalc(header + 12, 20) != GetUi32(header + 8)) return SZ_ERROR_CRC; + p->db.RangeLimit = nextHeaderOffset; + nextHeaderSizeT = (size_t)nextHeaderSize; if (nextHeaderSizeT != nextHeaderSize) return SZ_ERROR_MEM; @@ -1543,13 +1553,13 @@ static SRes SzArEx_Open2( { Int64 pos = 0; RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); - if ((UInt64)pos < startArcPos + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) + if ((UInt64)pos < (UInt64)startArcPos + nextHeaderOffset || + (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset || + (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) return SZ_ERROR_INPUT_EOF; } - RINOK(LookInStream_SeekTo(inStream, startArcPos + k7zStartHeaderSize + nextHeaderOffset)); + RINOK(LookInStream_SeekTo(inStream, (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset)); if (!Buf_Create(&buf, nextHeaderSizeT, allocTemp)) return SZ_ERROR_MEM; @@ -1575,6 +1585,8 @@ static SRes SzArEx_Open2( Buf_Init(&tempBuf); SzAr_Init(&tempAr); + tempAr.RangeLimit = p->db.RangeLimit; + res = SzReadAndDecodePackedStreams(inStream, &sd, &tempBuf, 1, p->startPosAfterHeader, &tempAr, allocTemp); SzAr_Free(&tempAr, allocTemp); @@ -1,5 +1,5 @@ /* 7zCrc.c -- CRC32 init -2017-06-06 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -26,8 +26,20 @@ typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); +extern CRC_FUNC g_CrcUpdateT4; +CRC_FUNC g_CrcUpdateT4; +extern +CRC_FUNC g_CrcUpdateT8; CRC_FUNC g_CrcUpdateT8; +extern +CRC_FUNC g_CrcUpdateT0_32; +CRC_FUNC g_CrcUpdateT0_32; +extern +CRC_FUNC g_CrcUpdateT0_64; +CRC_FUNC g_CrcUpdateT0_64; +extern +CRC_FUNC g_CrcUpdate; CRC_FUNC g_CrcUpdate; UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; @@ -44,6 +56,7 @@ UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -53,6 +66,166 @@ UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const U return v; } + +/* ---------- hardware CRC ---------- */ + +#ifdef MY_CPU_LE + +#if defined(MY_CPU_ARM_OR_ARM64) + +// #pragma message("ARM*") + + #if defined(_MSC_VER) + #if defined(MY_CPU_ARM64) + #if (_MSC_VER >= 1910) + #define USE_ARM64_CRC + #endif + #endif + #elif (defined(__clang__) && (__clang_major__ >= 3)) \ + || (defined(__GNUC__) && (__GNUC__ > 4)) + #if !defined(__ARM_FEATURE_CRC32) + #define __ARM_FEATURE_CRC32 1 + #if (!defined(__clang__) || (__clang_major__ > 3)) // fix these numbers + #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) + #endif + #endif + #if defined(__ARM_FEATURE_CRC32) + #define USE_ARM64_CRC + #include <arm_acle.h> + #endif + #endif + +#else + +// no hardware CRC + +// #define USE_CRC_EMU + +#ifdef USE_CRC_EMU + +#pragma message("ARM64 CRC emulation") + +MY_FORCE_INLINE +UInt32 __crc32b(UInt32 v, UInt32 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); + return v; +} + +MY_FORCE_INLINE +UInt32 __crc32w(UInt32 v, UInt32 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + return v; +} + +MY_FORCE_INLINE +UInt32 __crc32d(UInt32 v, UInt64 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + return v; +} + +#endif // USE_CRC_EMU + +#endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE) + + + +#if defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) + +#define T0_32_UNROLL_BYTES (4 * 4) +#define T0_64_UNROLL_BYTES (4 * 8) + +#ifndef ATTRIB_CRC +#define ATTRIB_CRC +#endif +// #pragma message("USE ARM HW CRC") + +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table); +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + UNUSED_VAR(table); + + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--) + v = __crc32b(v, *p++); + + if (size >= T0_32_UNROLL_BYTES) + { + const Byte *lim = p + size; + size &= (T0_32_UNROLL_BYTES - 1); + lim -= size; + do + { + v = __crc32w(v, *(const UInt32 *)(const void *)(p)); + v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + v = __crc32w(v, *(const UInt32 *)(const void *)(p)); + v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + } + while (p != lim); + } + + for (; size != 0; size--) + v = __crc32b(v, *p++); + + return v; +} + +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table); +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + UNUSED_VAR(table); + + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--) + v = __crc32b(v, *p++); + + if (size >= T0_64_UNROLL_BYTES) + { + const Byte *lim = p + size; + size &= (T0_64_UNROLL_BYTES - 1); + lim -= size; + do + { + v = __crc32d(v, *(const UInt64 *)(const void *)(p)); + v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; + v = __crc32d(v, *(const UInt64 *)(const void *)(p)); + v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; + } + while (p != lim); + } + + for (; size != 0; size--) + v = __crc32b(v, *p++); + + return v; +} + +#endif // defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) + +#endif // MY_CPU_LE + + + + void MY_FAST_CALL CrcGenerateTable() { UInt32 i; @@ -123,6 +296,27 @@ void MY_FAST_CALL CrcGenerateTable() } } #endif + #endif + #ifdef MY_CPU_LE + #ifdef USE_ARM64_CRC + if (CPU_IsSupported_CRC32()) + { + g_CrcUpdateT0_32 = CrcUpdateT0_32; + g_CrcUpdateT0_64 = CrcUpdateT0_64; + g_CrcUpdate = + #if defined(MY_CPU_ARM) + CrcUpdateT0_32; + #else + CrcUpdateT0_64; + #endif + } + #endif + + #ifdef USE_CRC_EMU + g_CrcUpdateT0_32 = CrcUpdateT0_32; + g_CrcUpdateT0_64 = CrcUpdateT0_64; + g_CrcUpdate = CrcUpdateT0_64; + #endif #endif } diff --git a/C/7zCrcOpt.c b/C/7zCrcOpt.c index 73beba29..69fad9ca 100644 --- a/C/7zCrcOpt.c +++ b/C/7zCrcOpt.c @@ -1,5 +1,5 @@ /* 7zCrcOpt.c -- CRC32 calculation -2017-04-03 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,7 @@ #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -16,7 +17,7 @@ UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const U v = CRC_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = (table + 0x300)[((v ) & 0xFF)] ^ (table + 0x200)[((v >> 8) & 0xFF)] @@ -28,6 +29,7 @@ UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const U return v; } +UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -36,13 +38,13 @@ UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const U for (; size >= 8; size -= 8, p += 8) { UInt32 d; - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = (table + 0x700)[((v ) & 0xFF)] ^ (table + 0x600)[((v >> 8) & 0xFF)] ^ (table + 0x500)[((v >> 16) & 0xFF)] ^ (table + 0x400)[((v >> 24))]; - d = *((const UInt32 *)p + 1); + d = *((const UInt32 *)(const void *)p + 1); v ^= (table + 0x300)[((d ) & 0xFF)] ^ (table + 0x200)[((d >> 8) & 0xFF)] @@ -72,7 +74,7 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, co v = CRC_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = (table + 0x000)[((v ) & 0xFF)] ^ (table + 0x100)[((v >> 8) & 0xFF)] @@ -94,13 +96,13 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, co for (; size >= 8; size -= 8, p += 8) { UInt32 d; - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = (table + 0x400)[((v ) & 0xFF)] ^ (table + 0x500)[((v >> 8) & 0xFF)] ^ (table + 0x600)[((v >> 16) & 0xFF)] ^ (table + 0x700)[((v >> 24))]; - d = *((const UInt32 *)p + 1); + d = *((const UInt32 *)(const void *)p + 1); v ^= (table + 0x000)[((d ) & 0xFF)] ^ (table + 0x100)[((d >> 8) & 0xFF)] @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2019-02-02 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -21,17 +21,20 @@ #endif #define k_Copy 0 -#define k_Delta 3 +#ifndef _7Z_NO_METHOD_LZMA2 #define k_LZMA2 0x21 +#endif #define k_LZMA 0x30101 -#define k_BCJ 0x3030103 #define k_BCJ2 0x303011B +#ifndef _7Z_NO_METHODS_FILTERS +#define k_Delta 3 +#define k_BCJ 0x3030103 #define k_PPC 0x3030205 #define k_IA64 0x3030401 #define k_ARM 0x3030501 #define k_ARMT 0x3030701 #define k_SPARC 0x3030805 - +#endif #ifdef _7ZIP_PPMD_SUPPPORT @@ -56,7 +59,7 @@ static Byte ReadByte(const IByteIn *pp) return *p->cur++; if (p->res == SZ_OK) { - size_t size = p->cur - p->begin; + size_t size = (size_t)(p->cur - p->begin); p->processed += size; p->res = ILookInStream_Skip(p->inStream, size); size = (1 << 25); @@ -101,28 +104,32 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, c Ppmd7_Init(&ppmd, order); } { - CPpmd7z_RangeDec rc; - Ppmd7z_RangeDec_CreateVTable(&rc); - rc.Stream = &s.vt; - if (!Ppmd7z_RangeDec_Init(&rc)) + ppmd.rc.dec.Stream = &s.vt; + if (!Ppmd7z_RangeDec_Init(&ppmd.rc.dec)) res = SZ_ERROR_DATA; - else if (s.extra) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else + else if (!s.extra) { - SizeT i; - for (i = 0; i < outSize; i++) + Byte *buf = outBuffer; + const Byte *lim = buf + outSize; + for (; buf != lim; buf++) { - int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt); + int sym = Ppmd7z_DecodeSymbol(&ppmd); if (s.extra || sym < 0) break; - outBuffer[i] = (Byte)sym; + *buf = (Byte)sym; } - if (i != outSize) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else if (s.processed + (s.cur - s.begin) != inSize || !Ppmd7z_RangeDec_IsFinishedOK(&rc)) + if (buf != lim) + res = SZ_ERROR_DATA; + else if (!Ppmd7z_RangeDec_IsFinishedOK(&ppmd.rc.dec)) + { + /* if (Ppmd7z_DecodeSymbol(&ppmd) != PPMD7_SYM_END || !Ppmd7z_RangeDec_IsFinishedOK(&ppmd.rc.dec)) */ res = SZ_ERROR_DATA; + } } + if (s.extra) + res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); + else if (s.processed + (size_t)(s.cur - s.begin) != inSize) + res = SZ_ERROR_DATA; } Ppmd7_Free(&ppmd, allocMain); return res; @@ -365,7 +372,9 @@ static SRes CheckSupportedFolder(const CSzFolder *f) return SZ_ERROR_UNSUPPORTED; } +#ifndef _7Z_NO_METHODS_FILTERS #define CASE_BRA_CONV(isa) case k_ ## isa: isa ## _Convert(outBuffer, outSize, 0, 0); break; +#endif static SRes SzFolder_Decode2(const CSzFolder *folder, const Byte *propsData, @@ -1,5 +1,5 @@ /* 7zFile.c -- File IO -2017-04-03 : Igor Pavlov : Public domain */ +2021-04-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,9 +7,19 @@ #ifndef USE_WINDOWS_FILE -#ifndef UNDER_CE -#include <errno.h> -#endif + #include <errno.h> + + #ifndef USE_FOPEN + #include <stdio.h> + #include <fcntl.h> + #ifdef _WIN32 + #include <io.h> + typedef int ssize_t; + typedef int off_t; + #else + #include <unistd.h> + #endif + #endif #else @@ -23,30 +33,36 @@ And message can be "Network connection was lost" */ -#define kChunkSizeMax (1 << 22) - #endif +#define kChunkSizeMax (1 << 22) + void File_Construct(CSzFile *p) { #ifdef USE_WINDOWS_FILE p->handle = INVALID_HANDLE_VALUE; - #else + #elif defined(USE_FOPEN) p->file = NULL; + #else + p->fd = -1; #endif } #if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) + static WRes File_Open(CSzFile *p, const char *name, int writeMode) { #ifdef USE_WINDOWS_FILE + p->handle = CreateFileA(name, writeMode ? GENERIC_WRITE : GENERIC_READ, FILE_SHARE_READ, NULL, writeMode ? CREATE_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); - #else + + #elif defined(USE_FOPEN) + p->file = fopen(name, writeMode ? "wb+" : "rb"); return (p->file != 0) ? 0 : #ifdef UNDER_CE @@ -54,13 +70,34 @@ static WRes File_Open(CSzFile *p, const char *name, int writeMode) #else errno; #endif + + #else + + int flags = (writeMode ? (O_CREAT | O_EXCL | O_WRONLY) : O_RDONLY); + #ifdef O_BINARY + flags |= O_BINARY; + #endif + p->fd = open(name, flags, 0666); + return (p->fd != -1) ? 0 : errno; + #endif } WRes InFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 0); } -WRes OutFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 1); } + +WRes OutFile_Open(CSzFile *p, const char *name) +{ + #if defined(USE_WINDOWS_FILE) || defined(USE_FOPEN) + return File_Open(p, name, 1); + #else + p->fd = creat(name, 0666); + return (p->fd != -1) ? 0 : errno; + #endif +} + #endif + #ifdef USE_WINDOWS_FILE static WRes File_OpenW(CSzFile *p, const WCHAR *name, int writeMode) { @@ -78,74 +115,124 @@ WRes OutFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 1 WRes File_Close(CSzFile *p) { #ifdef USE_WINDOWS_FILE + if (p->handle != INVALID_HANDLE_VALUE) { if (!CloseHandle(p->handle)) return GetLastError(); p->handle = INVALID_HANDLE_VALUE; } - #else + + #elif defined(USE_FOPEN) + if (p->file != NULL) { int res = fclose(p->file); if (res != 0) + { + if (res == EOF) + return errno; return res; + } p->file = NULL; } + + #else + + if (p->fd != -1) + { + if (close(p->fd) != 0) + return errno; + p->fd = -1; + } + #endif + return 0; } + WRes File_Read(CSzFile *p, void *data, size_t *size) { size_t originalSize = *size; + *size = 0; if (originalSize == 0) return 0; #ifdef USE_WINDOWS_FILE - *size = 0; do { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + const DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; DWORD processed = 0; - BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); + const BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); data = (void *)((Byte *)data + processed); originalSize -= processed; *size += processed; if (!res) return GetLastError(); + // debug : we can break here for partial reading mode + if (processed == 0) + break; + } + while (originalSize > 0); + + #elif defined(USE_FOPEN) + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const size_t processed = fread(data, 1, curSize, p->file); + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= processed; + *size += processed; + if (processed != curSize) + return ferror(p->file); + // debug : we can break here for partial reading mode if (processed == 0) break; } while (originalSize > 0); - return 0; #else - - *size = fread(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const ssize_t processed = read(p->fd, data, curSize); + if (processed == -1) + return errno; + if (processed == 0) + break; + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= (size_t)processed; + *size += (size_t)processed; + // debug : we can break here for partial reading mode + // break; + } + while (originalSize > 0); + #endif + + return 0; } + WRes File_Write(CSzFile *p, const void *data, size_t *size) { size_t originalSize = *size; + *size = 0; if (originalSize == 0) return 0; #ifdef USE_WINDOWS_FILE - *size = 0; do { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + const DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; DWORD processed = 0; - BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); + const BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); + data = (const void *)((const Byte *)data + processed); originalSize -= processed; *size += processed; if (!res) @@ -154,26 +241,52 @@ WRes File_Write(CSzFile *p, const void *data, size_t *size) break; } while (originalSize > 0); - return 0; + + #elif defined(USE_FOPEN) + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const size_t processed = fwrite(data, 1, curSize, p->file); + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= processed; + *size += processed; + if (processed != curSize) + return ferror(p->file); + if (processed == 0) + break; + } + while (originalSize > 0); #else - *size = fwrite(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const ssize_t processed = write(p->fd, data, curSize); + if (processed == -1) + return errno; + if (processed == 0) + break; + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= (size_t)processed; + *size += (size_t)processed; + } + while (originalSize > 0); + #endif + + return 0; } + WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) { #ifdef USE_WINDOWS_FILE - LARGE_INTEGER value; DWORD moveMethod; - value.LowPart = (DWORD)*pos; - value.HighPart = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ + UInt32 low = (UInt32)*pos; + LONG high = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ switch (origin) { case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; @@ -181,34 +294,52 @@ WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) case SZ_SEEK_END: moveMethod = FILE_END; break; default: return ERROR_INVALID_PARAMETER; } - value.LowPart = SetFilePointer(p->handle, value.LowPart, &value.HighPart, moveMethod); - if (value.LowPart == 0xFFFFFFFF) + low = SetFilePointer(p->handle, (LONG)low, &high, moveMethod); + if (low == (UInt32)0xFFFFFFFF) { WRes res = GetLastError(); if (res != NO_ERROR) return res; } - *pos = ((Int64)value.HighPart << 32) | value.LowPart; + *pos = ((Int64)high << 32) | low; return 0; #else - int moveMethod; - int res; + int moveMethod; // = origin; + switch (origin) { case SZ_SEEK_SET: moveMethod = SEEK_SET; break; case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; case SZ_SEEK_END: moveMethod = SEEK_END; break; - default: return 1; + default: return EINVAL; } - res = fseek(p->file, (long)*pos, moveMethod); - *pos = ftell(p->file); - return res; - #endif + #if defined(USE_FOPEN) + { + int res = fseek(p->file, (long)*pos, moveMethod); + if (res == -1) + return errno; + *pos = ftell(p->file); + if (*pos == -1) + return errno; + return 0; + } + #else + { + off_t res = lseek(p->fd, (off_t)*pos, moveMethod); + if (res == -1) + return errno; + *pos = res; + return 0; + } + + #endif // USE_FOPEN + #endif // USE_WINDOWS_FILE } + WRes File_GetLength(CSzFile *p, UInt64 *length) { #ifdef USE_WINDOWS_FILE @@ -224,13 +355,31 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) *length = (((UInt64)sizeHigh) << 32) + sizeLow; return 0; - #else + #elif defined(USE_FOPEN) long pos = ftell(p->file); int res = fseek(p->file, 0, SEEK_END); *length = ftell(p->file); fseek(p->file, pos, SEEK_SET); return res; + + #else + + off_t pos; + *length = 0; + pos = lseek(p->fd, 0, SEEK_CUR); + if (pos != -1) + { + const off_t len2 = lseek(p->fd, 0, SEEK_END); + const off_t res2 = lseek(p->fd, pos, SEEK_SET); + if (len2 != -1) + { + *length = (UInt64)len2; + if (res2 != -1) + return 0; + } + } + return errno; #endif } @@ -241,7 +390,9 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) { CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); - return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; + WRes wres = File_Read(&p->file, buf, size); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } void FileSeqInStream_CreateVTable(CFileSeqInStream *p) @@ -255,13 +406,17 @@ void FileSeqInStream_CreateVTable(CFileSeqInStream *p) static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) { CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; + WRes wres = File_Read(&p->file, buf, size); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) { CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - return File_Seek(&p->file, pos, origin); + WRes wres = File_Seek(&p->file, pos, origin); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } void FileInStream_CreateVTable(CFileInStream *p) @@ -276,7 +431,8 @@ void FileInStream_CreateVTable(CFileInStream *p) static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); - File_Write(&p->file, data, &size); + WRes wres = File_Write(&p->file, data, &size); + p->wres = wres; return size; } @@ -1,17 +1,20 @@ /* 7zFile.h -- File IO -2017-04-03 : Igor Pavlov : Public domain */ +2021-02-15 : Igor Pavlov : Public domain */ #ifndef __7Z_FILE_H #define __7Z_FILE_H #ifdef _WIN32 #define USE_WINDOWS_FILE +// #include <windows.h> #endif #ifdef USE_WINDOWS_FILE #include <windows.h> #else -#include <stdio.h> +// note: USE_FOPEN mode is limited to 32-bit file size +// #define USE_FOPEN +// #include <stdio.h> #endif #include "7zTypes.h" @@ -24,8 +27,10 @@ typedef struct { #ifdef USE_WINDOWS_FILE HANDLE handle; - #else + #elif defined(USE_FOPEN) FILE *file; + #else + int fd; #endif } CSzFile; @@ -56,6 +61,7 @@ typedef struct { ISeqInStream vt; CSzFile file; + WRes wres; } CFileSeqInStream; void FileSeqInStream_CreateVTable(CFileSeqInStream *p); @@ -65,6 +71,7 @@ typedef struct { ISeekInStream vt; CSzFile file; + WRes wres; } CFileInStream; void FileInStream_CreateVTable(CFileInStream *p); @@ -74,6 +81,7 @@ typedef struct { ISeqOutStream vt; CSzFile file; + WRes wres; } CFileOutStream; void FileOutStream_CreateVTable(CFileOutStream *p); diff --git a/C/7zStream.c b/C/7zStream.c index 6b5aa162..28a14604 100644 --- a/C/7zStream.c +++ b/C/7zStream.c @@ -1,5 +1,5 @@ /* 7zStream.c -- 7z Stream functions -2017-04-03 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -37,7 +37,7 @@ SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) { - Int64 t = offset; + Int64 t = (Int64)offset; return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); } diff --git a/C/7zTypes.h b/C/7zTypes.h index 65b3af63..f817b7f5 100644 --- a/C/7zTypes.h +++ b/C/7zTypes.h @@ -1,11 +1,13 @@ /* 7zTypes.h -- Basic types -2018-08-04 : Igor Pavlov : Public domain */ +2021-04-25 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H #ifdef _WIN32 /* #include <windows.h> */ +#else +#include <errno.h> #endif #include <stddef.h> @@ -43,18 +45,112 @@ EXTERN_C_BEGIN typedef int SRes; +#ifdef _MSC_VER + #if _MSC_VER > 1200 + #define MY_ALIGN(n) __declspec(align(n)) + #else + #define MY_ALIGN(n) + #endif +#else + #define MY_ALIGN(n) __attribute__ ((aligned(n))) +#endif + + #ifdef _WIN32 /* typedef DWORD WRes; */ typedef unsigned WRes; #define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) -#else +#else // _WIN32 +// #define ENV_HAVE_LSTAT typedef int WRes; -#define MY__FACILITY_WIN32 7 -#define MY__FACILITY__WRes MY__FACILITY_WIN32 -#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000))) + +// (FACILITY_ERRNO = 0x800) is 7zip's FACILITY constant to represent (errno) errors in HRESULT +#define MY__FACILITY_ERRNO 0x800 +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_ERRNO + +#define MY_HRESULT_FROM_errno_CONST_ERROR(x) ((HRESULT)( \ + ( (HRESULT)(x) & 0x0000FFFF) \ + | (MY__FACILITY__WRes << 16) \ + | (HRESULT)0x80000000 )) + +#define MY_SRes_HRESULT_FROM_WRes(x) \ + ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : MY_HRESULT_FROM_errno_CONST_ERROR(x)) + +// we call macro HRESULT_FROM_WIN32 for system errors (WRes) that are (errno) +#define HRESULT_FROM_WIN32(x) MY_SRes_HRESULT_FROM_WRes(x) + +/* +#define ERROR_FILE_NOT_FOUND 2L +#define ERROR_ACCESS_DENIED 5L +#define ERROR_NO_MORE_FILES 18L +#define ERROR_LOCK_VIOLATION 33L +#define ERROR_FILE_EXISTS 80L +#define ERROR_DISK_FULL 112L +#define ERROR_NEGATIVE_SEEK 131L +#define ERROR_ALREADY_EXISTS 183L +#define ERROR_DIRECTORY 267L +#define ERROR_TOO_MANY_POSTS 298L + +#define ERROR_INVALID_REPARSE_DATA 4392L +#define ERROR_REPARSE_TAG_INVALID 4393L +#define ERROR_REPARSE_TAG_MISMATCH 4394L +*/ + +// we use errno equivalents for some WIN32 errors: + +#define ERROR_INVALID_FUNCTION EINVAL +#define ERROR_ALREADY_EXISTS EEXIST +#define ERROR_FILE_EXISTS EEXIST +#define ERROR_PATH_NOT_FOUND ENOENT +#define ERROR_FILE_NOT_FOUND ENOENT +#define ERROR_DISK_FULL ENOSPC +// #define ERROR_INVALID_HANDLE EBADF + +// we use FACILITY_WIN32 for errors that has no errno equivalent +// Too many posts were made to a semaphore. +#define ERROR_TOO_MANY_POSTS ((HRESULT)0x8007012AL) +#define ERROR_INVALID_REPARSE_DATA ((HRESULT)0x80071128L) +#define ERROR_REPARSE_TAG_INVALID ((HRESULT)0x80071129L) + +// if (MY__FACILITY__WRes != FACILITY_WIN32), +// we use FACILITY_WIN32 for COM errors: +#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) +#define E_INVALIDARG ((HRESULT)0x80070057L) +#define MY__E_ERROR_NEGATIVE_SEEK ((HRESULT)0x80070083L) + +/* +// we can use FACILITY_ERRNO for some COM errors, that have errno equivalents: +#define E_OUTOFMEMORY MY_HRESULT_FROM_errno_CONST_ERROR(ENOMEM) +#define E_INVALIDARG MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +#define MY__E_ERROR_NEGATIVE_SEEK MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +*/ + +// gcc / clang : (sizeof(long) == sizeof(void*)) in 32/64 bits +typedef long INT_PTR; +typedef unsigned long UINT_PTR; + +#define TEXT(quote) quote + +#define FILE_ATTRIBUTE_READONLY 0x0001 +#define FILE_ATTRIBUTE_HIDDEN 0x0002 +#define FILE_ATTRIBUTE_SYSTEM 0x0004 +#define FILE_ATTRIBUTE_DIRECTORY 0x0010 +#define FILE_ATTRIBUTE_ARCHIVE 0x0020 +#define FILE_ATTRIBUTE_DEVICE 0x0040 +#define FILE_ATTRIBUTE_NORMAL 0x0080 +#define FILE_ATTRIBUTE_TEMPORARY 0x0100 +#define FILE_ATTRIBUTE_SPARSE_FILE 0x0200 +#define FILE_ATTRIBUTE_REPARSE_POINT 0x0400 +#define FILE_ATTRIBUTE_COMPRESSED 0x0800 +#define FILE_ATTRIBUTE_OFFLINE 0x1000 +#define FILE_ATTRIBUTE_NOT_CONTENT_INDEXED 0x2000 +#define FILE_ATTRIBUTE_ENCRYPTED 0x4000 + +#define FILE_ATTRIBUTE_UNIX_EXTENSION 0x8000 /* trick for Unix */ #endif @@ -63,6 +159,10 @@ typedef int WRes; #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } #endif +#ifndef RINOK_WRes +#define RINOK_WRes(x) { WRes __result__ = (x); if (__result__ != 0) return __result__; } +#endif + typedef unsigned char Byte; typedef short Int16; typedef unsigned short UInt16; @@ -75,6 +175,38 @@ typedef int Int32; typedef unsigned int UInt32; #endif + +#ifndef _WIN32 + +typedef int INT; +typedef Int32 INT32; +typedef unsigned int UINT; +typedef UInt32 UINT32; +typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit for _WIN32 compatibility +typedef UINT32 ULONG; + +#undef DWORD +typedef UINT32 DWORD; + +#define VOID void + +#define HRESULT LONG + +typedef void *LPVOID; +// typedef void VOID; +// typedef ULONG_PTR DWORD_PTR, *PDWORD_PTR; +// gcc / clang on Unix : sizeof(long==sizeof(void*) in 32 or 64 bits) +typedef long INT_PTR; +typedef unsigned long UINT_PTR; +typedef long LONG_PTR; +typedef unsigned long DWORD_PTR; + +typedef size_t SIZE_T; + +#endif // _WIN32 + + + #ifdef _SZ_NO_INT_64 /* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. @@ -128,25 +260,37 @@ typedef int BoolInt; #define MY_CDECL __cdecl #define MY_FAST_CALL __fastcall -#else +#else // _MSC_VER +#if (defined(__GNUC__) && (__GNUC__ >= 4)) \ + || (defined(__clang__) && (__clang_major__ >= 4)) \ + || defined(__INTEL_COMPILER) \ + || defined(__xlC__) +#define MY_NO_INLINE __attribute__((noinline)) +// #define MY_FORCE_INLINE __attribute__((always_inline)) inline +#else #define MY_NO_INLINE +#endif + #define MY_FORCE_INLINE -#define MY_CDECL -#define MY_FAST_CALL -/* inline keyword : for C++ / C99 */ -/* GCC, clang: */ -/* -#if defined (__GNUC__) && (__GNUC__ >= 4) -#define MY_FORCE_INLINE __attribute__((always_inline)) -#define MY_NO_INLINE __attribute__((noinline)) -#endif -*/ +#define MY_CDECL +#if defined(_M_IX86) \ + || defined(__i386__) +// #define MY_FAST_CALL __attribute__((fastcall)) +// #define MY_FAST_CALL __attribute__((cdecl)) +#define MY_FAST_CALL +#elif defined(MY_CPU_AMD64) +// #define MY_FAST_CALL __attribute__((ms_abi)) +#define MY_FAST_CALL +#else +#define MY_FAST_CALL #endif +#endif // _MSC_VER + /* The following interfaces use first parameter as pointer to structure */ @@ -335,12 +479,11 @@ struct ISzAlloc GCC 4.8.1 : classes with non-public variable members" */ -#define MY_container_of(ptr, type, m) ((type *)((char *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) - +#define MY_container_of(ptr, type, m) ((type *)(void *)((char *)(void *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) #endif -#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(ptr)) +#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(void *)(ptr)) /* #define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) @@ -353,6 +496,7 @@ struct ISzAlloc */ +#define MY_memset_0_ARRAY(a) memset((a), 0, sizeof(a)) #ifdef _WIN32 diff --git a/C/7zVersion.h b/C/7zVersion.h index c176823a..57c4c205 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 19 -#define MY_VER_MINOR 00 +#define MY_VER_MAJOR 21 +#define MY_VER_MINOR 02 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION_NUMBERS "21.02 alpha" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,12 +10,12 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2019-02-21" +#define MY_DATE "2021-05-06" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2021 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/C/7zip_gcc_c.mak b/C/7zip_gcc_c.mak new file mode 100644 index 00000000..e98d6761 --- /dev/null +++ b/C/7zip_gcc_c.mak @@ -0,0 +1,301 @@ + +MY_ARCH_2 = $(MY_ARCH) + +MY_ASM = jwasm +MY_ASM = asmc + +PROGPATH = $(O)/$(PROG) + + +# for object file +CFLAGS_BASE_LIST = -c +# for ASM file +# CFLAGS_BASE_LIST = -S +CFLAGS_BASE = $(MY_ARCH_2) -O2 $(CFLAGS_BASE_LIST) -Wall -Werror -Wextra $(CFLAGS_WARN) \ + -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + + +LDFLAGS_STATIC = -DNDEBUG +# -static + +ifdef SystemDrive +IS_MINGW = 1 +endif + +ifdef DEF_FILE + + +ifdef IS_MINGW +SHARED_EXT=.dll +LDFLAGS = -shared -DEF $(DEF_FILE) $(LDFLAGS_STATIC) +else +SHARED_EXT=.so +LDFLAGS = -shared -fPIC $(LDFLAGS_STATIC) +CC_SHARED=-fPIC +endif + + +else + +LDFLAGS = $(LDFLAGS_STATIC) +# -s is not required for clang, do we need it for GGC ??? +# -s + +#-static -static-libgcc -static-libstdc++ + +ifdef IS_MINGW +SHARED_EXT=.exe +else +SHARED_EXT= +endif + +endif + + +PROGPATH = $(O)/$(PROG)$(SHARED_EXT) + + +ifndef O +O=_o +endif + +ifdef IS_MINGW + +RM = del +MY_MKDIR=mkdir +LIB2 = -loleaut32 -luuid -ladvapi32 -lUser32 + + +CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE +# -Wno-delete-non-virtual-dtor + +DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll + +else + +RM = rm -f +MY_MKDIR=mkdir -p +# CFLAGS_BASE := $(CFLAGS_BASE) -D_7ZIP_ST +# CXXFLAGS_EXTRA = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + +# LOCAL_LIBS=-lpthread +# LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl +LIB2 = -lpthread -ldl + +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(OBJS) + +endif + + + +CFLAGS = $(LOCAL_FLAGS) $(CFLAGS_BASE2) $(CFLAGS_BASE) $(CC_SHARED) -o $@ + + +ifdef IS_X64 +AFLAGS_ABI = -elf64 -DABI_LINUX +else +AFLAGS_ABI = -elf -DABI_LINUX -DABI_CDECL +# -DABI_CDECL +# -DABI_LINUX +# -DABI_CDECL +endif +AFLAGS = $(AFLAGS_ABI) -Fo$(O)/ + + +CXX_WARN_FLAGS = +#-Wno-invalid-offsetof +#-Wno-reorder + +CXXFLAGS = $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(CXXFLAGS_EXTRA) $(CC_SHARED) -o $@ $(CXX_WARN_FLAGS) + +all: $(O) $(PROGPATH) + +$(O): + $(MY_MKDIR) $(O) + +$(PROGPATH): $(OBJS) + $(CXX) -s -o $(PROGPATH) $(MY_ARCH_2) $(LDFLAGS) $(OBJS) $(MY_LIBS) $(LIB2) + + + +ifndef NO_DEFAULT_RES +$O/resource.o: resource.rc + windres.exe $(RFLAGS) resource.rc $O/resource.o +endif + + + +$O/7zAlloc.o: ../../../C/7zAlloc.c + $(CC) $(CFLAGS) $< +$O/7zArcIn.o: ../../../C/7zArcIn.c + $(CC) $(CFLAGS) $< +$O/7zBuf.o: ../../../C/7zBuf.c + $(CC) $(CFLAGS) $< +$O/7zBuf2.o: ../../../C/7zBuf2.c + $(CC) $(CFLAGS) $< +$O/7zCrc.o: ../../../C/7zCrc.c + $(CC) $(CFLAGS) $< +$O/7zDec.o: ../../../C/7zDec.c + $(CC) $(CFLAGS) $< +$O/7zFile.o: ../../../C/7zFile.c + $(CC) $(CFLAGS) $< +$O/7zStream.o: ../../../C/7zStream.c + $(CC) $(CFLAGS) $< +$O/Aes.o: ../../../C/Aes.c + $(CC) $(CFLAGS) $< +$O/Alloc.o: ../../../C/Alloc.c + $(CC) $(CFLAGS) $< +$O/Bcj2.o: ../../../C/Bcj2.c + $(CC) $(CFLAGS) $< +$O/Bcj2Enc.o: ../../../C/Bcj2Enc.c + $(CC) $(CFLAGS) $< +$O/Blake2s.o: ../../../C/Blake2s.c + $(CC) $(CFLAGS) $< +$O/Bra.o: ../../../C/Bra.c + $(CC) $(CFLAGS) $< +$O/Bra86.o: ../../../C/Bra86.c + $(CC) $(CFLAGS) $< +$O/BraIA64.o: ../../../C/BraIA64.c + $(CC) $(CFLAGS) $< +$O/BwtSort.o: ../../../C/BwtSort.c + $(CC) $(CFLAGS) $< + +$O/CpuArch.o: ../../../C/CpuArch.c + $(CC) $(CFLAGS) $< +$O/Delta.o: ../../../C/Delta.c + $(CC) $(CFLAGS) $< +$O/DllSecur.o: ../../../C/DllSecur.c + $(CC) $(CFLAGS) $< +$O/HuffEnc.o: ../../../C/HuffEnc.c + $(CC) $(CFLAGS) $< +$O/LzFind.o: ../../../C/LzFind.c + $(CC) $(CFLAGS) $< + +# ifdef MT_FILES +$O/LzFindMt.o: ../../../C/LzFindMt.c + $(CC) $(CFLAGS) $< + +$O/Threads.o: ../../../C/Threads.c + $(CC) $(CFLAGS) $< +# endif + +$O/LzmaEnc.o: ../../../C/LzmaEnc.c + $(CC) $(CFLAGS) $< +$O/Lzma86Dec.o: ../../../C/Lzma86Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma86Enc.o: ../../../C/Lzma86Enc.c + $(CC) $(CFLAGS) $< +$O/Lzma2Dec.o: ../../../C/Lzma2Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma2DecMt.o: ../../../C/Lzma2DecMt.c + $(CC) $(CFLAGS) $< +$O/Lzma2Enc.o: ../../../C/Lzma2Enc.c + $(CC) $(CFLAGS) $< +$O/LzmaLib.o: ../../../C/LzmaLib.c + $(CC) $(CFLAGS) $< +$O/MtCoder.o: ../../../C/MtCoder.c + $(CC) $(CFLAGS) $< +$O/MtDec.o: ../../../C/MtDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7.o: ../../../C/Ppmd7.c + $(CC) $(CFLAGS) $< +$O/Ppmd7aDec.o: ../../../C/Ppmd7aDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Dec.o: ../../../C/Ppmd7Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Enc.o: ../../../C/Ppmd7Enc.c + $(CC) $(CFLAGS) $< +$O/Ppmd8.o: ../../../C/Ppmd8.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Dec.o: ../../../C/Ppmd8Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Enc.o: ../../../C/Ppmd8Enc.c + $(CC) $(CFLAGS) $< +$O/Sha1.o: ../../../C/Sha1.c + $(CC) $(CFLAGS) $< +$O/Sha256.o: ../../../C/Sha256.c + $(CC) $(CFLAGS) $< +$O/Sort.o: ../../../C/Sort.c + $(CC) $(CFLAGS) $< +$O/Xz.o: ../../../C/Xz.c + $(CC) $(CFLAGS) $< +$O/XzCrc64.o: ../../../C/XzCrc64.c + $(CC) $(CFLAGS) $< + + +ifdef USE_ASM +ifdef IS_X64 +USE_X86_ASM=1 +else +ifdef IS_X86 +USE_X86_ASM=1 +endif +endif +endif + +ifdef USE_X86_ASM +$O/7zCrcOpt.o: ../../../Asm/x86/7zCrcOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/XzCrc64Opt.o: ../../../Asm/x86/XzCrc64Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/AesOpt.o: ../../../Asm/x86/AesOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha1Opt.o: ../../../Asm/x86/Sha1Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha256Opt.o: ../../../Asm/x86/Sha256Opt.asm + $(MY_ASM) $(AFLAGS) $< +else +$O/7zCrcOpt.o: ../../7zCrcOpt.c + $(CC) $(CFLAGS) $< +$O/XzCrc64Opt.o: ../../XzCrc64Opt.c + $(CC) $(CFLAGS) $< +$O/Sha1Opt.o: ../../Sha1Opt.c + $(CC) $(CFLAGS) $< +$O/Sha256Opt.o: ../../Sha256Opt.c + $(CC) $(CFLAGS) $< +$O/AesOpt.o: ../../AesOpt.c + $(CC) $(CFLAGS) $< +endif + + +ifdef USE_LZMA_DEC_ASM + +ifdef IS_X64 +$O/LzmaDecOpt.o: ../../../Asm/x86/LzmaDecOpt.asm + $(MY_ASM) $(AFLAGS) $< +endif + +ifdef IS_ARM64 +$O/LzmaDecOpt.o: ../../../Asm/arm64/LzmaDecOpt.S ../../../Asm/arm64/7zAsm.S + $(CC) $(CFLAGS) $< +endif + +$O/LzmaDec.o: ../../LzmaDec.c + $(CC) $(CFLAGS) -D_LZMA_DEC_OPT $< + +else + +$O/LzmaDec.o: ../../LzmaDec.c + $(CC) $(CFLAGS) $< + +endif + + + +$O/XzDec.o: ../../../C/XzDec.c + $(CC) $(CFLAGS) $< +$O/XzEnc.o: ../../../C/XzEnc.c + $(CC) $(CFLAGS) $< +$O/XzIn.o: ../../../C/XzIn.c + $(CC) $(CFLAGS) $< + + +$O/7zMain.o: ../../../C/Util/7z/7zMain.c + $(CC) $(CFLAGS) $< +$O/LzmaUtil.o: ../../../C/Util/Lzma/LzmaUtil.c + $(CC) $(CFLAGS) $< + + + +clean: + -$(DEL_OBJ_EXE) @@ -1,10 +1,17 @@ /* Aes.c -- AES encryption / decryption -2017-01-24 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "Aes.h" #include "CpuArch.h" +#include "Aes.h" + +AES_CODE_FUNC g_AesCbc_Decode; +#ifndef _SFX +AES_CODE_FUNC g_AesCbc_Encode; +AES_CODE_FUNC g_AesCtr_Code; +UInt32 g_Aes_SupportedFunctions_Flags; +#endif static UInt32 T[256 * 4]; static const Byte Sbox[256] = { @@ -25,23 +32,10 @@ static const Byte Sbox[256] = { 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); - -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); - -AES_CODE_FUNC g_AesCbc_Encode; -AES_CODE_FUNC g_AesCbc_Decode; -AES_CODE_FUNC g_AesCtr_Code; static UInt32 D[256 * 4]; static Byte InvS[256]; -static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; - #define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) #define Ui32(a0, a1, a2, a3) ((UInt32)(a0) | ((UInt32)(a1) << 8) | ((UInt32)(a2) << 16) | ((UInt32)(a3) << 24)) @@ -57,6 +51,36 @@ static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0 #define DD(x) (D + (x << 8)) +// #define _SHOW_AES_STATUS + +#ifdef MY_CPU_X86_OR_AMD64 + #define USE_HW_AES +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif +#endif + +#ifdef USE_HW_AES +#ifdef _SHOW_AES_STATUS +#include <stdio.h> +#define _PRF(x) x +#else +#define _PRF(x) +#endif +#endif + + void AesGenTables(void) { unsigned i; @@ -90,18 +114,48 @@ void AesGenTables(void) } } - g_AesCbc_Encode = AesCbc_Encode; - g_AesCbc_Decode = AesCbc_Decode; - g_AesCtr_Code = AesCtr_Code; + { + AES_CODE_FUNC d = AesCbc_Decode; + #ifndef _SFX + AES_CODE_FUNC e = AesCbc_Encode; + AES_CODE_FUNC c = AesCtr_Code; + UInt32 flags = 0; + #endif - #ifdef MY_CPU_X86_OR_AMD64 - if (CPU_Is_Aes_Supported()) + #ifdef USE_HW_AES + if (CPU_IsSupported_AES()) { - g_AesCbc_Encode = AesCbc_Encode_Intel; - g_AesCbc_Decode = AesCbc_Decode_Intel; - g_AesCtr_Code = AesCtr_Code_Intel; + // #pragma message ("AES HW") + _PRF(printf("\n===AES HW\n")); + d = AesCbc_Decode_HW; + + #ifndef _SFX + e = AesCbc_Encode_HW; + c = AesCtr_Code_HW; + flags = k_Aes_SupportedFunctions_HW; + #endif + + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_VAES_AVX2()) + { + _PRF(printf("\n===vaes avx2\n")); + d = AesCbc_Decode_HW_256; + #ifndef _SFX + c = AesCtr_Code_HW_256; + flags |= k_Aes_SupportedFunctions_HW_256; + #endif + } + #endif } #endif + + g_AesCbc_Decode = d; + #ifndef _SFX + g_AesCbc_Encode = e; + g_AesCtr_Code = c; + g_Aes_SupportedFunctions_Flags = flags; + #endif + } } @@ -142,8 +196,11 @@ void AesGenTables(void) void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) { - unsigned i, wSize; - wSize = keySize + 28; + unsigned i, m; + const UInt32 *wLim; + UInt32 t; + UInt32 rcon = 1; + keySize /= 4; w[0] = ((UInt32)keySize / 2) + 3; w += 4; @@ -151,16 +208,26 @@ void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) for (i = 0; i < keySize; i++, key += 4) w[i] = GetUi32(key); - for (; i < wSize; i++) + t = w[(size_t)keySize - 1]; + wLim = w + (size_t)keySize * 3 + 28; + m = 0; + do { - UInt32 t = w[(size_t)i - 1]; - unsigned rem = i % keySize; - if (rem == 0) - t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); - else if (keySize > 6 && rem == 4) + if (m == 0) + { + t = Ui32(Sbox[gb1(t)] ^ rcon, Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); + rcon <<= 1; + if (rcon & 0x100) + rcon = 0x1b; + m = keySize; + } + else if (m == 4 && keySize > 6) t = Ui32(Sbox[gb0(t)], Sbox[gb1(t)], Sbox[gb2(t)], Sbox[gb3(t)]); - w[i] = w[i - keySize] ^ t; + m--; + t ^= w[0]; + w[keySize] = t; } + while (++w != wLim); } void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) @@ -184,6 +251,7 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) src and dest are pointers to 4 UInt32 words. src and dest can point to same block */ +// MY_FORCE_INLINE static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -207,6 +275,7 @@ static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) FT4(0); FT4(1); FT4(2); FT4(3); } +MY_FORCE_INLINE static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -294,7 +363,7 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) UInt32 t = temp[i]; #ifdef MY_CPU_LE_UNALIGN - *((UInt32 *)data) ^= t; + *((UInt32 *)(void *)data) ^= t; #else data[0] ^= (t & 0xFF); data[1] ^= ((t >> 8) & 0xFF); @@ -1,5 +1,5 @@ /* Aes.h -- AES encryption / decryption -2013-01-18 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #ifndef __AES_H #define __AES_H @@ -26,12 +26,34 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize) /* ivAes - 16-byte aligned pointer to iv+keyMode+roundKeys sequence: UInt32[AES_NUM_IVMRK_WORDS] */ void AesCbc_Init(UInt32 *ivAes, const Byte *iv); /* iv size is AES_BLOCK_SIZE */ + /* data - 16-byte aligned pointer to data */ /* numBlocks - the number of 16-byte blocks in data array */ typedef void (MY_FAST_CALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); -extern AES_CODE_FUNC g_AesCbc_Encode; + extern AES_CODE_FUNC g_AesCbc_Decode; +#ifndef _SFX +extern AES_CODE_FUNC g_AesCbc_Encode; extern AES_CODE_FUNC g_AesCtr_Code; +#define k_Aes_SupportedFunctions_HW (1 << 2) +#define k_Aes_SupportedFunctions_HW_256 (1 << 3) +extern UInt32 g_Aes_SupportedFunctions_Flags; +#endif + + +#define DECLARE__AES_CODE_FUNC(funcName) \ + void MY_FAST_CALL funcName(UInt32 *ivAes, Byte *data, size_t numBlocks); + +DECLARE__AES_CODE_FUNC (AesCbc_Encode) +DECLARE__AES_CODE_FUNC (AesCbc_Decode) +DECLARE__AES_CODE_FUNC (AesCtr_Code) + +DECLARE__AES_CODE_FUNC (AesCbc_Encode_HW) +DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW) +DECLARE__AES_CODE_FUNC (AesCtr_Code_HW) + +DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW_256) +DECLARE__AES_CODE_FUNC (AesCtr_Code_HW_256) EXTERN_C_END @@ -1,184 +1,776 @@ -/* AesOpt.c -- Intel's AES -2017-06-08 : Igor Pavlov : Public domain */ +/* AesOpt.c -- AES optimized code for x86 AES hardware instructions +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) -#define USE_INTEL_AES + + #if defined(__clang__) + #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 8) + #define USE_INTEL_AES + #define ATTRIB_AES __attribute__((__target__("aes"))) + #if (__clang_major__ >= 8) + #define USE_INTEL_VAES + #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) + #endif + #endif + #elif defined(__GNUC__) + #if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) + #define USE_INTEL_AES + #ifndef __AES__ + #define ATTRIB_AES __attribute__((__target__("aes"))) + #endif + #if (__GNUC__ >= 8) + #define USE_INTEL_VAES + #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) + #endif + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1110) + #define USE_INTEL_AES + #if (__INTEL_COMPILER >= 1900) + #define USE_INTEL_VAES + #endif + #endif + #elif defined(_MSC_VER) + #if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) + #define USE_INTEL_AES + #if (_MSC_VER >= 1910) + #define USE_INTEL_VAES + #endif + #endif + #endif + +#ifndef ATTRIB_AES + #define ATTRIB_AES #endif +#ifndef ATTRIB_VAES + #define ATTRIB_VAES #endif + #ifdef USE_INTEL_AES #include <wmmintrin.h> -void MY_FAST_CALL AesCbc_Encode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +#ifndef USE_INTEL_VAES +#define AES_TYPE_keys __m128i +#define AES_TYPE_data __m128i +#endif + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(__m128i *p, __m128i *data, size_t numBlocks) + +#define AES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_AES \ +AES_FUNC_START (name) + +#define MM_OP(op, dest, src) dest = op(dest, src); +#define MM_OP_m(op, src) MM_OP(op, m, src); + +#define MM_XOR( dest, src) MM_OP(_mm_xor_si128, dest, src); +#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src); + + +AES_FUNC_START2 (AesCbc_Encode_HW) { __m128i m = *p; + const __m128i k0 = p[2]; + const __m128i k1 = p[3]; + const UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; for (; numBlocks != 0; numBlocks--, data++) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p + 3; - m = _mm_xor_si128(m, *data); - m = _mm_xor_si128(m, p[2]); + UInt32 r = numRounds2; + const __m128i *w = p + 4; + __m128i temp = *data; + MM_XOR (temp, k0); + MM_XOR (m, temp); + MM_OP_m (_mm_aesenc_si128, k1); do { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); w += 2; } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); + while (--r); + MM_OP_m (_mm_aesenclast_si128, w[0]); *data = m; } *p = m; } -#define NUM_WAYS 3 -#define AES_OP_W(op, n) { \ - const __m128i t = w[n]; \ - m0 = op(m0, t); \ - m1 = op(m1, t); \ - m2 = op(m2, t); \ - } +#define WOP_1(op) +#define WOP_2(op) WOP_1 (op) op (m1, 1); +#define WOP_3(op) WOP_2 (op) op (m2, 2); +#define WOP_4(op) WOP_3 (op) op (m3, 3); +#ifdef MY_CPU_AMD64 +#define WOP_5(op) WOP_4 (op) op (m4, 4); +#define WOP_6(op) WOP_5 (op) op (m5, 5); +#define WOP_7(op) WOP_6 (op) op (m6, 6); +#define WOP_8(op) WOP_7 (op) op (m7, 7); +#endif +/* +#define WOP_9(op) WOP_8 (op) op (m8, 8); +#define WOP_10(op) WOP_9 (op) op (m9, 9); +#define WOP_11(op) WOP_10(op) op (m10, 10); +#define WOP_12(op) WOP_11(op) op (m11, 11); +#define WOP_13(op) WOP_12(op) op (m12, 12); +#define WOP_14(op) WOP_13(op) op (m13, 13); +*/ + +#ifdef MY_CPU_AMD64 + #define NUM_WAYS 8 + #define WOP_M1 WOP_8 +#else + #define NUM_WAYS 4 + #define WOP_M1 WOP_4 +#endif + +#define WOP(op) op (m0, 0); WOP_M1(op) + + +#define DECLARE_VAR(reg, ii) __m128i reg +#define LOAD_data( reg, ii) reg = data[ii]; +#define STORE_data( reg, ii) data[ii] = reg; +#if (NUM_WAYS > 1) +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#endif + +#define AVX__DECLARE_VAR(reg, ii) __m256i reg +#define AVX__LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; +#define AVX__STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; +#define AVX__XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])); + +#define MM_OP_key(op, reg) MM_OP(op, reg, key); + +#define AES_DEC( reg, ii) MM_OP_key (_mm_aesdec_si128, reg) +#define AES_DEC_LAST( reg, ii) MM_OP_key (_mm_aesdeclast_si128, reg) +#define AES_ENC( reg, ii) MM_OP_key (_mm_aesenc_si128, reg) +#define AES_ENC_LAST( reg, ii) MM_OP_key (_mm_aesenclast_si128, reg) +#define AES_XOR( reg, ii) MM_OP_key (_mm_xor_si128, reg) + -#define AES_DEC(n) AES_OP_W(_mm_aesdec_si128, n) -#define AES_DEC_LAST(n) AES_OP_W(_mm_aesdeclast_si128, n) -#define AES_ENC(n) AES_OP_W(_mm_aesenc_si128, n) -#define AES_ENC_LAST(n) AES_OP_W(_mm_aesenclast_si128, n) +#define AVX__AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) +#define AVX__AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) +#define AVX__AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) +#define AVX__AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) +#define AVX__AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) -void MY_FAST_CALL AesCbc_Decode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +#define CTR_START(reg, ii) MM_OP (_mm_add_epi64, ctr, one); reg = ctr; +#define CTR_END( reg, ii) MM_XOR (data[ii], reg); + +#define AVX__CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two); reg = _mm256_xor_si256(ctr2, key); +#define AVX__CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg); + +#define WOP_KEY(op, n) { \ + const __m128i key = w[n]; \ + WOP(op); } + +#define AVX__WOP_KEY(op, n) { \ + const __m256i key = w[n]; \ + WOP(op); } + + +#define WIDE_LOOP_START \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS) \ + { dataEnd -= NUM_WAYS; do { \ + + +#define WIDE_LOOP_END \ + data += NUM_WAYS; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS; } \ + + +#define SINGLE_LOOP \ + for (; data < dataEnd; data++) + + +#define NUM_AES_KEYS_MAX 15 + +#define WIDE_LOOP_START_AVX(OP) \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS * 2) \ + { __m256i keys[NUM_AES_KEYS_MAX]; \ + UInt32 ii; \ + OP \ + for (ii = 0; ii < numRounds; ii++) \ + keys[ii] = _mm256_broadcastsi128_si256(p[ii]); \ + dataEnd -= NUM_WAYS * 2; do { \ + + +#define WIDE_LOOP_END_AVX(OP) \ + data += NUM_WAYS * 2; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS * 2; \ + OP \ + _mm256_zeroupper(); \ + } \ + +/* MSVC for x86: If we don't call _mm256_zeroupper(), and -arch:IA32 is not specified, + MSVC still can insert vzeroupper instruction. */ + + +AES_FUNC_START2 (AesCbc_Decode_HW) { __m128i iv = *p; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + const __m128i *wStart = p + *(const UInt32 *)(p + 1) * 2 + 2 - 1; + const __m128i *dataEnd; + p += 2; + + WIDE_LOOP_START { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m0, m1, m2; + const __m128i *w = wStart; + + WOP (DECLARE_VAR) + WOP (LOAD_data); + WOP_KEY (AES_XOR, 1) + + do { - const __m128i t = w[2]; - m0 = _mm_xor_si128(t, data[0]); - m1 = _mm_xor_si128(t, data[1]); - m2 = _mm_xor_si128(t, data[2]); + WOP_KEY (AES_DEC, 0) + w--; } - numRounds2--; + while (w != p); + WOP_KEY (AES_DEC_LAST, 0) + + MM_XOR (m0, iv); + WOP_M1 (XOR_data_M1) + iv = data[NUM_WAYS - 1]; + WOP (STORE_data); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const __m128i *w = wStart - 1; + __m128i m = _mm_xor_si128 (w[2], *data); do { - AES_DEC(1) - AES_DEC(0) + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdec_si128, w[0]); w -= 2; } - while (--numRounds2 != 0); - AES_DEC(1) - AES_DEC_LAST(0) + while (w != p); + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdeclast_si128, w[0]); + MM_XOR (m, iv); + iv = *data; + *data = m; + } + + p[-2] = iv; +} + + +AES_FUNC_START2 (AesCtr_Code_HW) +{ + __m128i ctr = *p; + UInt32 numRoundsMinus2 = *(const UInt32 *)(p + 1) * 2 - 1; + const __m128i *dataEnd; + __m128i one = _mm_cvtsi32_si128(1); + + p += 2; + + WIDE_LOOP_START + { + const __m128i *w = p; + UInt32 r = numRoundsMinus2; + WOP (DECLARE_VAR) + WOP (CTR_START); + WOP_KEY (AES_XOR, 0) + w += 1; + do { - __m128i t; - t = _mm_xor_si128(m0, iv); iv = data[0]; data[0] = t; - t = _mm_xor_si128(m1, iv); iv = data[1]; data[1] = t; - t = _mm_xor_si128(m2, iv); iv = data[2]; data[2] = t; + WOP_KEY (AES_ENC, 0) + w += 1; } + while (--r); + WOP_KEY (AES_ENC_LAST, 0) + + WOP (CTR_END); } - for (; numBlocks != 0; numBlocks--, data++) + WIDE_LOOP_END + + SINGLE_LOOP + { + UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; + const __m128i *w = p; + __m128i m; + MM_OP (_mm_add_epi64, ctr, one); + m = _mm_xor_si128 (ctr, p[0]); + w += 1; + do + { + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); + w += 2; + } + while (--numRounds2); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenclast_si128, w[1]); + MM_XOR (*data, m); + } + + p[-2] = ctr; +} + + + +#ifdef USE_INTEL_VAES + +#if defined(__clang__) && defined(_MSC_VER) +#define __SSE4_2__ +#define __AES__ +#define __AVX__ +#define __AVX2__ +#define __VAES__ +#define __AVX512F__ +#define __AVX512VL__ +#endif + +#include <immintrin.h> + +#define VAES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_VAES \ +AES_FUNC_START (name) + +VAES_FUNC_START2 (AesCbc_Decode_HW_256) +{ + __m128i iv = *p; + const __m128i *dataEnd; + UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; + p += 2; + + WIDE_LOOP_START_AVX(;) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m = _mm_xor_si128(w[2], *data); - numRounds2--; + const __m256i *w = keys + numRounds - 2; + + WOP (AVX__DECLARE_VAR) + WOP (AVX__LOAD_data); + AVX__WOP_KEY (AVX__AES_XOR, 1) + do { - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdec_si128(m, w[0]); + AVX__WOP_KEY (AVX__AES_DEC, 0) + w--; + } + while (w != keys); + AVX__WOP_KEY (AVX__AES_DEC_LAST, 0) + + AVX_XOR (m0, _mm256_setr_m128i(iv, data[0])); + WOP_M1 (AVX__XOR_data_M1) + iv = data[NUM_WAYS * 2 - 1]; + WOP (AVX__STORE_data); + } + WIDE_LOOP_END_AVX(;) + + SINGLE_LOOP + { + const __m128i *w = p + *(const UInt32 *)(p + 1 - 2) * 2 + 1 - 3; + __m128i m = _mm_xor_si128 (w[2], *data); + do + { + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdec_si128, w[0]); w -= 2; } - while (--numRounds2 != 0); - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdeclast_si128(m, w[0]); + while (w != p); + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdeclast_si128, w[0]); - m = _mm_xor_si128(m, iv); + MM_XOR (m, iv); iv = *data; *data = m; } - *p = iv; + + p[-2] = iv; } -void MY_FAST_CALL AesCtr_Code_Intel(__m128i *p, __m128i *data, size_t numBlocks) + +/* +SSE2: _mm_cvtsi32_si128 : movd +AVX: _mm256_setr_m128i : vinsertf128 +AVX2: _mm256_add_epi64 : vpaddq ymm, ymm, ymm + _mm256_extracti128_si256 : vextracti128 + _mm256_broadcastsi128_si256 : vbroadcasti128 +*/ + +#define AVX__CTR_LOOP_START \ + ctr2 = _mm256_setr_m128i(_mm_sub_epi64(ctr, one), ctr); \ + two = _mm256_setr_m128i(one, one); \ + two = _mm256_add_epi64(two, two); \ + +// two = _mm256_setr_epi64x(2, 0, 2, 0); + +#define AVX__CTR_LOOP_ENC \ + ctr = _mm256_extracti128_si256 (ctr2, 1); \ + +VAES_FUNC_START2 (AesCtr_Code_HW_256) { __m128i ctr = *p; - __m128i one; - one.m128i_u64[0] = 1; - one.m128i_u64[1] = 0; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; + const __m128i *dataEnd; + __m128i one = _mm_cvtsi32_si128(1); + __m256i ctr2, two; + p += 2; + + WIDE_LOOP_START_AVX (AVX__CTR_LOOP_START) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p; - __m128i m0, m1, m2; - { - const __m128i t = w[2]; - ctr = _mm_add_epi64(ctr, one); m0 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m1 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m2 = _mm_xor_si128(ctr, t); - } - w += 3; + const __m256i *w = keys; + UInt32 r = numRounds - 2; + WOP (AVX__DECLARE_VAR) + AVX__WOP_KEY (AVX__CTR_START, 0); + + w += 1; do { - AES_ENC(0) - AES_ENC(1) - w += 2; + AVX__WOP_KEY (AVX__AES_ENC, 0) + w += 1; } - while (--numRounds2 != 0); - AES_ENC(0) - AES_ENC_LAST(1) - data[0] = _mm_xor_si128(data[0], m0); - data[1] = _mm_xor_si128(data[1], m1); - data[2] = _mm_xor_si128(data[2], m2); + while (--r); + AVX__WOP_KEY (AVX__AES_ENC_LAST, 0) + + WOP (AVX__CTR_END); } - for (; numBlocks != 0; numBlocks--, data++) + WIDE_LOOP_END_AVX (AVX__CTR_LOOP_ENC) + + SINGLE_LOOP { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; + UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; const __m128i *w = p; __m128i m; - ctr = _mm_add_epi64(ctr, one); - m = _mm_xor_si128(ctr, p[2]); - w += 3; + MM_OP (_mm_add_epi64, ctr, one); + m = _mm_xor_si128 (ctr, p[0]); + w += 1; do { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); w += 2; } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); - *data = _mm_xor_si128(*data, m); + while (--numRounds2); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenclast_si128, w[1]); + MM_XOR (*data, m); } - *p = ctr; + + p[-2] = ctr; } +#endif // USE_INTEL_VAES + +#else // USE_INTEL_AES + +/* no USE_INTEL_AES */ + +#pragma message("AES HW_SW stub was used") + +#define AES_TYPE_keys UInt32 +#define AES_TYPE_data Byte + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(UInt32 *p, Byte *data, size_t numBlocks) \ + +#define AES_COMPAT_STUB(name) \ + AES_FUNC_START(name); \ + AES_FUNC_START(name ## _HW) \ + { name(p, data, numBlocks); } + +AES_COMPAT_STUB (AesCbc_Encode) +AES_COMPAT_STUB (AesCbc_Decode) +AES_COMPAT_STUB (AesCtr_Code) + +#endif // USE_INTEL_AES + + +#ifndef USE_INTEL_VAES + +#pragma message("VAES HW_SW stub was used") + +#define VAES_COMPAT_STUB(name) \ + void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \ + void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \ + { name((AES_TYPE_keys *)(void *)p, (AES_TYPE_data *)(void *)data, numBlocks); } + +VAES_COMPAT_STUB (AesCbc_Decode_HW) +VAES_COMPAT_STUB (AesCtr_Code_HW) + +#endif // ! USE_INTEL_VAES + + +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif + +#ifdef USE_HW_AES + +// #pragma message("=== AES HW === ") + +#if defined(__clang__) || defined(__GNUC__) + #ifdef MY_CPU_ARM64 + #define ATTRIB_AES __attribute__((__target__("+crypto"))) + #else + #define ATTRIB_AES __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#ifndef ATTRIB_AES + #define ATTRIB_AES +#endif + +#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#include <arm64_neon.h> #else +#include <arm_neon.h> +#endif + +typedef uint8x16_t v128; + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(v128 *p, v128 *data, size_t numBlocks) + +#define AES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_AES \ +AES_FUNC_START (name) + +#define MM_OP(op, dest, src) dest = op(dest, src); +#define MM_OP_m(op, src) MM_OP(op, m, src); +#define MM_OP1_m(op) m = op(m); -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); +#define MM_XOR( dest, src) MM_OP(veorq_u8, dest, src); +#define MM_XOR_m( src) MM_XOR(m, src); -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *p, Byte *data, size_t numBlocks) +#define AES_E_m(k) MM_OP_m (vaeseq_u8, k); +#define AES_E_MC_m(k) AES_E_m (k); MM_OP1_m(vaesmcq_u8); + + +AES_FUNC_START2 (AesCbc_Encode_HW) { - AesCbc_Encode(p, data, numBlocks); + v128 m = *p; + const v128 k0 = p[2]; + const v128 k1 = p[3]; + const v128 k2 = p[4]; + const v128 k3 = p[5]; + const v128 k4 = p[6]; + const v128 k5 = p[7]; + const v128 k6 = p[8]; + const v128 k7 = p[9]; + const v128 k8 = p[10]; + const v128 k9 = p[11]; + const UInt32 numRounds2 = *(const UInt32 *)(p + 1); + const v128 *w = p + ((size_t)numRounds2 * 2); + const v128 k_z1 = w[1]; + const v128 k_z0 = w[2]; + for (; numBlocks != 0; numBlocks--, data++) + { + MM_XOR_m (*data); + AES_E_MC_m (k0) + AES_E_MC_m (k1) + AES_E_MC_m (k2) + AES_E_MC_m (k3) + AES_E_MC_m (k4) + AES_E_MC_m (k5) + AES_E_MC_m (k6) + AES_E_MC_m (k7) + AES_E_MC_m (k8) + if (numRounds2 >= 6) + { + AES_E_MC_m (k9) + AES_E_MC_m (p[12]) + if (numRounds2 != 6) + { + AES_E_MC_m (p[13]) + AES_E_MC_m (p[14]) + } + } + AES_E_m (k_z1); + MM_XOR_m (k_z0); + *data = m; + } + *p = m; } -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *p, Byte *data, size_t numBlocks) + +#define WOP_1(op) +#define WOP_2(op) WOP_1 (op) op (m1, 1); +#define WOP_3(op) WOP_2 (op) op (m2, 2); +#define WOP_4(op) WOP_3 (op) op (m3, 3); +#define WOP_5(op) WOP_4 (op) op (m4, 4); +#define WOP_6(op) WOP_5 (op) op (m5, 5); +#define WOP_7(op) WOP_6 (op) op (m6, 6); +#define WOP_8(op) WOP_7 (op) op (m7, 7); + + #define NUM_WAYS 8 + #define WOP_M1 WOP_8 + +#define WOP(op) op (m0, 0); WOP_M1(op) + +#define DECLARE_VAR(reg, ii) v128 reg +#define LOAD_data( reg, ii) reg = data[ii]; +#define STORE_data( reg, ii) data[ii] = reg; +#if (NUM_WAYS > 1) +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#endif + +#define MM_OP_key(op, reg) MM_OP (op, reg, key); + +#define AES_D_m(k) MM_OP_m (vaesdq_u8, k); +#define AES_D_IMC_m(k) AES_D_m (k); MM_OP1_m (vaesimcq_u8); + +#define AES_XOR( reg, ii) MM_OP_key (veorq_u8, reg) +#define AES_D( reg, ii) MM_OP_key (vaesdq_u8, reg) +#define AES_E( reg, ii) MM_OP_key (vaeseq_u8, reg) + +#define AES_D_IMC( reg, ii) AES_D (reg, ii); reg = vaesimcq_u8(reg) +#define AES_E_MC( reg, ii) AES_E (reg, ii); reg = vaesmcq_u8(reg) + +#define CTR_START(reg, ii) MM_OP (vaddq_u64, ctr, one); reg = vreinterpretq_u8_u64(ctr); +#define CTR_END( reg, ii) MM_XOR (data[ii], reg); + +#define WOP_KEY(op, n) { \ + const v128 key = w[n]; \ + WOP(op); } + +#define WIDE_LOOP_START \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS) \ + { dataEnd -= NUM_WAYS; do { \ + +#define WIDE_LOOP_END \ + data += NUM_WAYS; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS; } \ + +#define SINGLE_LOOP \ + for (; data < dataEnd; data++) + + +AES_FUNC_START2 (AesCbc_Decode_HW) { - AesCbc_Decode(p, data, numBlocks); + v128 iv = *p; + const v128 *wStart = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; + const v128 *dataEnd; + p += 2; + + WIDE_LOOP_START + { + const v128 *w = wStart; + WOP (DECLARE_VAR) + WOP (LOAD_data); + WOP_KEY (AES_D_IMC, 2) + do + { + WOP_KEY (AES_D_IMC, 1) + WOP_KEY (AES_D_IMC, 0) + w -= 2; + } + while (w != p); + WOP_KEY (AES_D, 1) + WOP_KEY (AES_XOR, 0) + MM_XOR (m0, iv); + WOP_M1 (XOR_data_M1) + iv = data[NUM_WAYS - 1]; + WOP (STORE_data); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const v128 *w = wStart; + v128 m = *data; + AES_D_IMC_m (w[2]) + do + { + AES_D_IMC_m (w[1]); + AES_D_IMC_m (w[0]); + w -= 2; + } + while (w != p); + AES_D_m (w[1]); + MM_XOR_m (w[0]); + MM_XOR_m (iv); + iv = *data; + *data = m; + } + + p[-2] = iv; } -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *p, Byte *data, size_t numBlocks) + +AES_FUNC_START2 (AesCtr_Code_HW) { - AesCtr_Code(p, data, numBlocks); + uint64x2_t ctr = vreinterpretq_u64_u8(*p); + const v128 *wEnd = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; + const v128 *dataEnd; + uint64x2_t one = vdupq_n_u64(0); + one = vsetq_lane_u64(1, one, 0); + p += 2; + + WIDE_LOOP_START + { + const v128 *w = p; + WOP (DECLARE_VAR) + WOP (CTR_START); + do + { + WOP_KEY (AES_E_MC, 0) + WOP_KEY (AES_E_MC, 1) + w += 2; + } + while (w != wEnd); + WOP_KEY (AES_E_MC, 0) + WOP_KEY (AES_E, 1) + WOP_KEY (AES_XOR, 2) + WOP (CTR_END); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const v128 *w = p; + v128 m; + CTR_START (m, 0); + do + { + AES_E_MC_m (w[0]); + AES_E_MC_m (w[1]); + w += 2; + } + while (w != wEnd); + AES_E_MC_m (w[0]); + AES_E_m (w[1]); + MM_XOR_m (w[2]); + CTR_END (m, 0); + } + + p[-2] = vreinterpretq_u8_u64(ctr); } -#endif +#endif // USE_HW_AES + +#endif // MY_CPU_ARM_OR_ARM64 @@ -1,12 +1,12 @@ /* Alloc.c -- Memory allocation functions -2018-04-27 : Igor Pavlov : Public domain */ +2020-10-29 : Igor Pavlov : Public domain */ #include "Precomp.h" #include <stdio.h> #ifdef _WIN32 -#include <windows.h> +#include <Windows.h> #endif #include <stdlib.h> @@ -122,7 +122,6 @@ static void PrintAddr(void *p) #define Print(s) #define PrintLn() #define PrintHex(v, align) -#define PrintDec(v, align) #define PrintAddr(p) #endif @@ -133,10 +132,11 @@ void *MyAlloc(size_t size) { if (size == 0) return NULL; + PRINT_ALLOC("Alloc ", g_allocCount, size, NULL); #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - PRINT_ALLOC("Alloc ", g_allocCount, size, p); + // PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -172,14 +172,20 @@ void MidFree(void *address) VirtualFree(address, 0, MEM_RELEASE); } -#ifndef MEM_LARGE_PAGES -#undef _7ZIP_LARGE_PAGES +#ifdef _7ZIP_LARGE_PAGES + +#ifdef MEM_LARGE_PAGES + #define MY__MEM_LARGE_PAGES MEM_LARGE_PAGES +#else + #define MY__MEM_LARGE_PAGES 0x20000000 #endif -#ifdef _7ZIP_LARGE_PAGES +extern +SIZE_T g_LargePageSize; SIZE_T g_LargePageSize = 0; -typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); -#endif +typedef SIZE_T (WINAPI *GetLargePageMinimumP)(VOID); + +#endif // _7ZIP_LARGE_PAGES void SetLargePageSize() { @@ -214,7 +220,7 @@ void *BigAlloc(size_t size) size2 = (size + ps) & ~ps; if (size2 >= size) { - void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MY__MEM_LARGE_PAGES, PAGE_READWRITE); if (res) return res; } @@ -280,13 +286,15 @@ const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; */ #define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) -#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) - -#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) +#if !defined(_WIN32) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) #define USE_posix_memalign #endif +#ifndef USE_posix_memalign +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) +#endif + /* This posix_memalign() is for test purposes only. We also need special Free() function instead of free(), @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2018-02-19 : Igor Pavlov : Public domain */ +2021-02-08 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -13,7 +13,7 @@ void MyFree(void *address); #ifdef _WIN32 -void SetLargePageSize(); +void SetLargePageSize(void); void *MidAlloc(size_t size); void MidFree(void *address); @@ -1,5 +1,5 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2018-04-28 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -123,7 +123,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) const Byte *src = p->bufs[BCJ2_STREAM_MAIN]; const Byte *srcLim; Byte *dest; - SizeT num = p->lims[BCJ2_STREAM_MAIN] - src; + SizeT num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - src); if (num == 0) { @@ -134,7 +134,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) dest = p->dest; if (num > (SizeT)(p->destLim - dest)) { - num = p->destLim - dest; + num = (SizeT)(p->destLim - dest); if (num == 0) { p->state = BCJ2_DEC_STATE_ORIG; @@ -168,7 +168,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) break; } - num = src - p->bufs[BCJ2_STREAM_MAIN]; + num = (SizeT)(src - p->bufs[BCJ2_STREAM_MAIN]); if (src == srcLim) { @@ -228,7 +228,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) p->ip += 4; val -= p->ip; dest = p->dest; - rem = p->destLim - dest; + rem = (SizeT)(p->destLim - dest); if (rem < 4) { diff --git a/C/Bcj2Enc.c b/C/Bcj2Enc.c index bfbeb8e4..682362a1 100644 --- a/C/Bcj2Enc.c +++ b/C/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2019-02-02 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -104,7 +104,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) const Byte *src = p->src; const Byte *srcLim; Byte *dest; - SizeT num = p->srcLim - src; + SizeT num = (SizeT)(p->srcLim - src); if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) { @@ -118,7 +118,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) dest = p->bufs[BCJ2_STREAM_MAIN]; if (num > (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest)) { - num = p->lims[BCJ2_STREAM_MAIN] - dest; + num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest); if (num == 0) { p->state = BCJ2_STREAM_MAIN; @@ -152,7 +152,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) break; } - num = src - p->src; + num = (SizeT)(src - p->src); if (src == srcLim) { diff --git a/C/Blake2s.c b/C/Blake2s.c index 6527415e..3c56a8b8 100644 --- a/C/Blake2s.c +++ b/C/Blake2s.c @@ -1,5 +1,5 @@ /* Blake2s.c -- BLAKE2s and BLAKE2sp Hash -2015-06-30 : Igor Pavlov : Public domain +2021-02-09 : Igor Pavlov : Public domain 2015 : Samuel Neves : Public domain */ #include <string.h> @@ -34,7 +34,7 @@ static const Byte k_Blake2s_Sigma[BLAKE2S_NUM_ROUNDS][16] = }; -void Blake2s_Init0(CBlake2s *p) +static void Blake2s_Init0(CBlake2s *p) { unsigned i; for (i = 0; i < 8; i++) @@ -1,5 +1,5 @@ /* Bra.c -- Converters for RISC code -2017-04-04 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -22,7 +22,7 @@ SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) for (;;) { if (p >= lim) - return p - data; + return (SizeT)(p - data); p += 4; if (p[-1] == 0xEB) break; @@ -43,7 +43,7 @@ SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) for (;;) { if (p >= lim) - return p - data; + return (SizeT)(p - data); p += 4; if (p[-1] == 0xEB) break; @@ -78,7 +78,7 @@ SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { UInt32 b3; if (p > lim) - return p - data; + return (SizeT)(p - data); b1 = p[1]; b3 = p[3]; p += 2; @@ -113,7 +113,7 @@ SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { UInt32 b3; if (p > lim) - return p - data; + return (SizeT)(p - data); b1 = p[1]; b3 = p[3]; p += 2; @@ -162,7 +162,7 @@ SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) for (;;) { if (p >= lim) - return p - data; + return (SizeT)(p - data); p += 4; /* if ((v & 0xFC000003) == 0x48000001) */ if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) @@ -196,7 +196,7 @@ SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) for (;;) { if (p >= lim) - return p - data; + return (SizeT)(p - data); /* v = GetBe32(p); p += 4; @@ -1,5 +1,5 @@ /* Bra86.c -- Converter for x86 code (BCJ) -2017-04-03 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -25,7 +25,7 @@ SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding break; { - SizeT d = (SizeT)(p - data - pos); + SizeT d = (SizeT)(p - data) - pos; pos = (SizeT)(p - data); if (p >= limit) { diff --git a/C/BwtSort.c b/C/BwtSort.c index cc2f4b29..3eb57efa 100644 --- a/C/BwtSort.c +++ b/C/BwtSort.c @@ -1,5 +1,5 @@ /* BwtSort.c -- BWT block sorting -2018-07-04 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -60,7 +60,7 @@ SortGroup - is recursive Range-Sort function with HeapSort optimization for smal returns: 1 - if there are groups, 0 - no more groups */ -UInt32 NO_INLINE SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 groupSize, int NumRefBits, UInt32 *Indices +static UInt32 NO_INLINE SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 groupSize, int NumRefBits, UInt32 *Indices #ifndef BLOCK_SORT_USE_HEAP_SORT , UInt32 left, UInt32 range #endif @@ -116,7 +116,7 @@ UInt32 NO_INLINE SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 group } HeapSort(temp, groupSize); - mask = ((1 << NumRefBits) - 1); + mask = (((UInt32)1 << NumRefBits) - 1); thereAreGroups = 0; group = groupOffset; diff --git a/C/Compiler.h b/C/Compiler.h index 0cc409d8..a9816fa5 100644 --- a/C/Compiler.h +++ b/C/Compiler.h @@ -1,9 +1,13 @@ /* Compiler.h -2017-04-03 : Igor Pavlov : Public domain */ +2021-01-05 : Igor Pavlov : Public domain */ #ifndef __7Z_COMPILER_H #define __7Z_COMPILER_H + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wunused-private-field" + #endif + #ifdef _MSC_VER #ifdef UNDER_CE @@ -25,6 +29,12 @@ #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information #endif + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wdeprecated-declarations" + #pragma clang diagnostic ignored "-Wmicrosoft-exception-spec" + // #pragma clang diagnostic ignored "-Wreserved-id-macro" + #endif + #endif #define UNUSED_VAR(x) (void)x; diff --git a/C/CpuArch.c b/C/CpuArch.c index 02e482e0..1d761416 100644 --- a/C/CpuArch.c +++ b/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2018-02-18: Igor Pavlov : Public domain */ +2021-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -55,6 +55,47 @@ static UInt32 CheckFlag(UInt32 flag) #define CHECK_CPUID_IS_SUPPORTED #endif +#ifndef USE_ASM + #ifdef _MSC_VER + #if _MSC_VER >= 1600 + #define MY__cpuidex __cpuidex + #else + +/* + __cpuid (function == 4) requires subfunction number in ECX. + MSDN: The __cpuid intrinsic clears the ECX register before calling the cpuid instruction. + __cpuid() in new MSVC clears ECX. + __cpuid() in old MSVC (14.00) doesn't clear ECX + We still can use __cpuid for low (function) values that don't require ECX, + but __cpuid() in old MSVC will be incorrect for some function values: (function == 4). + So here we use the hack for old MSVC to send (subFunction) in ECX register to cpuid instruction, + where ECX value is first parameter for FAST_CALL / NO_INLINE function, + So the caller of MY__cpuidex_HACK() sets ECX as subFunction, and + old MSVC for __cpuid() doesn't change ECX and cpuid instruction gets (subFunction) value. + + DON'T remove MY_NO_INLINE and MY_FAST_CALL for MY__cpuidex_HACK() !!! +*/ + +static +MY_NO_INLINE +void MY_FAST_CALL MY__cpuidex_HACK(UInt32 subFunction, int *CPUInfo, UInt32 function) +{ + UNUSED_VAR(subFunction); + __cpuid(CPUInfo, function); +} + + #define MY__cpuidex(info, func, func2) MY__cpuidex_HACK(func2, info, func) + #pragma message("======== MY__cpuidex_HACK WAS USED ========") + #endif + #else + #define MY__cpuidex(info, func, func2) __cpuid(info, func) + #pragma message("======== (INCORRECT ?) cpuid WAS USED ========") + #endif +#endif + + + + void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) { #ifdef USE_ASM @@ -99,18 +140,20 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #endif "=c" (*c) , "=d" (*d) - : "0" (function)) ; + : "0" (function), "c"(0) ) ; #endif #else int CPUInfo[4]; - __cpuid(CPUInfo, function); - *a = CPUInfo[0]; - *b = CPUInfo[1]; - *c = CPUInfo[2]; - *d = CPUInfo[3]; + + MY__cpuidex(CPUInfo, (int)function, 0); + + *a = (UInt32)CPUInfo[0]; + *b = (UInt32)CPUInfo[1]; + *c = (UInt32)CPUInfo[2]; + *d = (UInt32)CPUInfo[3]; #endif } @@ -174,7 +217,7 @@ BoolInt CPU_Is_InOrder() } #if !defined(MY_CPU_AMD64) && defined(_WIN32) -#include <windows.h> +#include <Windows.h> static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; @@ -188,13 +231,77 @@ static BoolInt CPU_Sys_Is_SSE_Supported() #define CHECK_SYS_SSE_SUPPORT #endif -BoolInt CPU_Is_Aes_Supported() + +static UInt32 X86_CPUID_ECX_Get_Flags() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT if (!x86cpuid_CheckAndRead(&p)) + return 0; + return p.c; +} + +BoolInt CPU_IsSupported_AES() +{ + return (X86_CPUID_ECX_Get_Flags() >> 25) & 1; +} + +BoolInt CPU_IsSupported_SSSE3() +{ + return (X86_CPUID_ECX_Get_Flags() >> 9) & 1; +} + +BoolInt CPU_IsSupported_SSE41() +{ + return (X86_CPUID_ECX_Get_Flags() >> 19) & 1; +} + +BoolInt CPU_IsSupported_SHA() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_CheckAndRead(&p)) + return False; + + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + return (d[1] >> 29) & 1; + } +} + +// #include <stdio.h> + +#ifdef _WIN32 +#include <Windows.h> +#endif + +BoolInt CPU_IsSupported_VAES_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) return False; - return (p.c >> 25) & 1; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5) // avx2 + // & (d[1] >> 31) // avx512vl + & (d[2] >> 9); // vaes // VEX-256/EVEX + } } BoolInt CPU_IsSupported_PageGB() @@ -215,4 +322,117 @@ BoolInt CPU_IsSupported_PageGB() } } + +#elif defined(MY_CPU_ARM_OR_ARM64) + +#ifdef _WIN32 + +#include <Windows.h> + +BoolInt CPU_IsSupported_CRC32() + { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRYPTO() + { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } + +#else + +#if defined(__APPLE__) + +/* +#include <stdio.h> +#include <string.h> +static void Print_sysctlbyname(const char *name) +{ + size_t bufSize = 256; + char buf[256]; + int res = sysctlbyname(name, &buf, &bufSize, NULL, 0); + { + int i; + printf("\nres = %d : %s : '%s' : bufSize = %d, numeric", res, name, buf, (unsigned)bufSize); + for (i = 0; i < 20; i++) + printf(" %2x", (unsigned)(Byte)buf[i]); + + } +} +*/ + +BoolInt CPU_IsSupported_CRC32(void) +{ + /* + Print_sysctlbyname("hw.pagesize"); + Print_sysctlbyname("machdep.cpu.brand_string"); + */ + + UInt32 val = 0; + if (My_sysctlbyname_Get_UInt32("hw.optional.armv8_crc32", &val) == 0 && val == 1) + return 1; + return 0; +} + +#ifdef MY_CPU_ARM64 +#define APPLE_CRYPTO_SUPPORT_VAL 1 +#else +#define APPLE_CRYPTO_SUPPORT_VAL 0 +#endif + +BoolInt CPU_IsSupported_SHA1(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_SHA2(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } + + +#else // __APPLE__ + +#include <sys/auxv.h> + +#define USE_HWCAP + +#ifdef USE_HWCAP + +#include <asm/hwcap.h> + +#ifdef MY_CPU_ARM64 + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name)) ? 1 : 0; } +#elif defined(MY_CPU_ARM) + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } +#endif + +#else // USE_HWCAP + + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return 0; } + +#endif // USE_HWCAP + +MY_HWCAP_CHECK_FUNC (CRC32) +MY_HWCAP_CHECK_FUNC (SHA1) +MY_HWCAP_CHECK_FUNC (SHA2) +MY_HWCAP_CHECK_FUNC (AES) + +#endif // __APPLE__ +#endif // _WIN32 + +#endif // MY_CPU_ARM_OR_ARM64 + + + +#ifdef __APPLE__ + +#include <sys/sysctl.h> + +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize) +{ + return sysctlbyname(name, buf, bufSize, NULL, 0); +} + +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val) +{ + size_t bufSize = sizeof(*val); + int res = My_sysctlbyname_Get(name, val, &bufSize); + if (res == 0 && bufSize != sizeof(*val)) + return EFAULT; + return res; +} + #endif diff --git a/C/CpuArch.h b/C/CpuArch.h index bd429388..6c4ab404 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2018-02-18 : Igor Pavlov : Public domain */ +2021-04-25 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -14,6 +14,10 @@ MY_CPU_BE means that CPU is BIG ENDIAN. If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of platform. MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. + +MY_CPU_64BIT means that processor can work with 64-bit registers. + MY_CPU_64BIT can be used to select fast code branch + MY_CPU_64BIT doesn't mean that (sizeof(void *) == 8) */ #if defined(_M_X64) \ @@ -24,8 +28,10 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define MY_CPU_AMD64 #ifdef __ILP32__ #define MY_CPU_NAME "x32" + #define MY_CPU_SIZEOF_POINTER 4 #else #define MY_CPU_NAME "x64" + #define MY_CPU_SIZEOF_POINTER 8 #endif #define MY_CPU_64BIT #endif @@ -35,7 +41,8 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem || defined(__i386__) #define MY_CPU_X86 #define MY_CPU_NAME "x86" - #define MY_CPU_32BIT + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 #endif @@ -59,8 +66,14 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem || defined(__THUMBEL__) \ || defined(__THUMBEB__) #define MY_CPU_ARM - #define MY_CPU_NAME "arm" - #define MY_CPU_32BIT + + #if defined(__thumb__) || defined(__THUMBEL__) || defined(_M_ARMT) + #define MY_CPU_NAME "armt" + #else + #define MY_CPU_NAME "arm" + #endif + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 #endif @@ -84,17 +97,29 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #if defined(__ppc64__) \ - || defined(__powerpc64__) + || defined(__powerpc64__) \ + || defined(__ppc__) \ + || defined(__powerpc__) \ + || defined(__PPC__) \ + || defined(_POWER) + +#if defined(__ppc64__) \ + || defined(__powerpc64__) \ + || defined(_LP64) \ + || defined(__64BIT__) #ifdef __ILP32__ #define MY_CPU_NAME "ppc64-32" + #define MY_CPU_SIZEOF_POINTER 4 #else #define MY_CPU_NAME "ppc64" + #define MY_CPU_SIZEOF_POINTER 8 #endif #define MY_CPU_64BIT -#elif defined(__ppc__) \ - || defined(__powerpc__) +#else #define MY_CPU_NAME "ppc" - #define MY_CPU_32BIT + #define MY_CPU_SIZEOF_POINTER 4 + /* #define MY_CPU_32BIT */ +#endif #endif @@ -111,6 +136,10 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define MY_CPU_X86_OR_AMD64 #endif +#if defined(MY_CPU_ARM) || defined(MY_CPU_ARM64) +#define MY_CPU_ARM_OR_ARM64 +#endif + #ifdef _WIN32 @@ -170,6 +199,41 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #error Stop_Compiling_Bad_32_64_BIT #endif +#ifdef __SIZEOF_POINTER__ + #ifdef MY_CPU_SIZEOF_POINTER + #if MY_CPU_SIZEOF_POINTER != __SIZEOF_POINTER__ + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE + #endif + #else + #define MY_CPU_SIZEOF_POINTER __SIZEOF_POINTER__ + #endif +#endif + +#if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) +#if defined (_LP64) + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE +#endif +#endif + +#ifdef _MSC_VER + #if _MSC_VER >= 1300 + #define MY_CPU_pragma_pack_push_1 __pragma(pack(push, 1)) + #define MY_CPU_pragma_pop __pragma(pack(pop)) + #else + #define MY_CPU_pragma_pack_push_1 + #define MY_CPU_pragma_pop + #endif +#else + #ifdef __xlC__ + // for XLC compiler: + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(1)") + #define MY_CPU_pragma_pop _Pragma("pack()") + #else + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(push, 1)") + #define MY_CPU_pragma_pop _Pragma("pack(pop)") + #endif +#endif + #ifndef MY_CPU_NAME #ifdef MY_CPU_LE @@ -202,9 +266,9 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define GetUi32(p) (*(const UInt32 *)(const void *)(p)) #define GetUi64(p) (*(const UInt64 *)(const void *)(p)) -#define SetUi16(p, v) { *(UInt16 *)(p) = (v); } -#define SetUi32(p, v) { *(UInt32 *)(p) = (v); } -#define SetUi64(p, v) { *(UInt64 *)(p) = (v); } +#define SetUi16(p, v) { *(UInt16 *)(void *)(p) = (v); } +#define SetUi32(p, v) { *(UInt32 *)(void *)(p) = (v); } +#define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } #else @@ -242,7 +306,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define MY__has_builtin(x) 0 #endif -#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) +#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ defined(_MSC_VER) && (_MSC_VER >= 1300) /* Note: we use bswap instruction, that is unsupported in 386 cpu */ @@ -253,8 +317,8 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #pragma intrinsic(_byteswap_uint64) /* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ -#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) +#define GetBe32(p) _byteswap_ulong (*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const void *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) @@ -262,9 +326,9 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) -/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ -#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const void *)(p)) */ +#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const void *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) @@ -325,10 +389,35 @@ int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -BoolInt CPU_Is_InOrder(); -BoolInt CPU_Is_Aes_Supported(); -BoolInt CPU_IsSupported_PageGB(); +BoolInt CPU_Is_InOrder(void); + +BoolInt CPU_IsSupported_AES(void); +BoolInt CPU_IsSupported_VAES_AVX2(void); +BoolInt CPU_IsSupported_SSSE3(void); +BoolInt CPU_IsSupported_SSE41(void); +BoolInt CPU_IsSupported_SHA(void); +BoolInt CPU_IsSupported_PageGB(void); + +#elif defined(MY_CPU_ARM_OR_ARM64) + +BoolInt CPU_IsSupported_CRC32(void); + +#if defined(_WIN32) +BoolInt CPU_IsSupported_CRYPTO(void); +#define CPU_IsSupported_SHA1 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_SHA2 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_AES CPU_IsSupported_CRYPTO +#else +BoolInt CPU_IsSupported_SHA1(void); +BoolInt CPU_IsSupported_SHA2(void); +BoolInt CPU_IsSupported_AES(void); +#endif + +#endif +#if defined(__APPLE__) +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize); +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val); #endif EXTERN_C_END @@ -1,5 +1,5 @@ /* Delta.c -- Delta converter -2009-05-26 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,53 +12,158 @@ void Delta_Init(Byte *state) state[i] = 0; } -static void MyMemCpy(Byte *dest, const Byte *src, unsigned size) -{ - unsigned i; - for (i = 0; i < size; i++) - dest[i] = src[i]; -} void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size) { - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); + Byte temp[DELTA_STATE_SIZE]; + + if (size == 0) + return; + + { + unsigned i = 0; + do + temp[i] = state[i]; + while (++i != delta); + } + + if (size <= delta) + { + unsigned i = 0, k; + do + { + Byte b = *data; + *data++ = (Byte)(b - temp[i]); + temp[i] = b; + } + while (++i != size); + + k = 0; + + do + { + if (i == delta) + i = 0; + state[k] = temp[i++]; + } + while (++k != delta); + + return; + } + { - SizeT i; - for (i = 0; i < size;) + Byte *p = data + size - delta; + { + unsigned i = 0; + do + state[i] = *p++; + while (++i != delta); + } { - for (j = 0; j < delta && i < size; i++, j++) + const Byte *lim = data + delta; + ptrdiff_t dif = -(ptrdiff_t)delta; + + if (((ptrdiff_t)size + dif) & 1) { - Byte b = data[i]; - data[i] = (Byte)(b - buf[j]); - buf[j] = b; + --p; *p = (Byte)(*p - p[dif]); } + + while (p != lim) + { + --p; *p = (Byte)(*p - p[dif]); + --p; *p = (Byte)(*p - p[dif]); + } + + dif = -dif; + + do + { + --p; *p = (Byte)(*p - temp[--dif]); + } + while (dif != 0); } } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); } + void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size) { - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); + unsigned i; + const Byte *lim; + + if (size == 0) + return; + + i = 0; + lim = data + size; + + if (size <= delta) + { + do + *data = (Byte)(*data + state[i++]); + while (++data != lim); + + for (; delta != i; state++, delta--) + *state = state[i]; + data -= i; + } + else { - SizeT i; - for (i = 0; i < size;) + /* + #define B(n) b ## n + #define I(n) Byte B(n) = state[n]; + #define U(n) { B(n) = (Byte)((B(n)) + *data++); data[-1] = (B(n)); } + #define F(n) if (data != lim) { U(n) } + + if (delta == 1) + { + I(0) + if ((lim - data) & 1) { U(0) } + while (data != lim) { U(0) U(0) } + data -= 1; + } + else if (delta == 2) { - for (j = 0; j < delta && i < size; i++, j++) + I(0) I(1) + lim -= 1; while (data < lim) { U(0) U(1) } + lim += 1; F(0) + data -= 2; + } + else if (delta == 3) + { + I(0) I(1) I(2) + lim -= 2; while (data < lim) { U(0) U(1) U(2) } + lim += 2; F(0) F(1) + data -= 3; + } + else if (delta == 4) + { + I(0) I(1) I(2) I(3) + lim -= 3; while (data < lim) { U(0) U(1) U(2) U(3) } + lim += 3; F(0) F(1) F(2) + data -= 4; + } + else + */ + { + do + { + *data = (Byte)(*data + state[i++]); + data++; + } + while (i != delta); + { - buf[j] = data[i] = (Byte)(buf[j] + data[i]); + ptrdiff_t dif = -(ptrdiff_t)delta; + do + *data = (Byte)(*data + data[dif]); + while (++data != lim); + data += dif; } } } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); + + do + *state++ = *data; + while (++data != lim); } diff --git a/C/DllSecur.c b/C/DllSecur.c index 5ea108ab..4b2c521d 100644 --- a/C/DllSecur.c +++ b/C/DllSecur.c @@ -5,7 +5,7 @@ #ifdef _WIN32 -#include <windows.h> +#include <Windows.h> #include "DllSecur.h" diff --git a/C/DllSecur.h b/C/DllSecur.h index e2a049ad..64ff26cd 100644 --- a/C/DllSecur.h +++ b/C/DllSecur.h @@ -10,8 +10,8 @@ EXTERN_C_BEGIN #ifdef _WIN32 -void My_SetDefaultDllDirectories(); -void LoadSecurityDlls(); +void My_SetDefaultDllDirectories(void); +void LoadSecurityDlls(void); #endif diff --git a/C/HuffEnc.c b/C/HuffEnc.c index a54b3d87..f3c2996d 100644 --- a/C/HuffEnc.c +++ b/C/HuffEnc.c @@ -1,5 +1,5 @@ /* HuffEnc.c -- functions for Huffman encoding -2017-04-03 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,7 +8,7 @@ #define kMaxLen 16 #define NUM_BITS 10 -#define MASK ((1 << NUM_BITS) - 1) +#define MASK (((unsigned)1 << NUM_BITS) - 1) #define NUM_COUNTERS 64 @@ -1,10 +1,11 @@ /* LzFind.c -- Match finder for LZ algorithms -2018-07-08 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include <string.h> +#include "CpuArch.h" #include "LzFind.h" #include "LzHash.h" @@ -14,7 +15,45 @@ #define kNormalizeMask (~(UInt32)(kNormalizeStepMin - 1)) #define kMaxHistorySize ((UInt32)7 << 29) -#define kStartMaxLen 3 +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) +#define kFix5HashSize kFix4HashSize + +/* + HASH2_CALC: + if (hv) match, then cur[0] and cur[1] also match +*/ +#define HASH2_CALC hv = GetUi16(cur); + +// (crc[0 ... 255] & 0xFF) provides one-to-one correspondence to [0 ... 255] + +/* + HASH3_CALC: + if (cur[0]) and (h2) match, then cur[1] also match + if (cur[0]) and (hv) match, then cur[1] and cur[2] also match +*/ +#define HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } + +#define HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + hv = (temp ^ (p->crc[cur[3]] << kLzHash_CrcShift_1)) & p->hashMask; } + +#define HASH5_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + temp ^= (p->crc[cur[3]] << kLzHash_CrcShift_1); \ + /* h4 = temp & p->hash4Mask; */ /* (kHash4Size - 1); */ \ + hv = (temp ^ (p->crc[cur[4]] << kLzHash_CrcShift_2)) & p->hashMask; } + +#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; + static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { @@ -44,9 +83,9 @@ static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr return (p->bufferBase != NULL); } -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } +static Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } +static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) { @@ -77,7 +116,7 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) for (;;) { Byte *dest = p->buffer + (p->streamPos - p->pos); - size_t size = (p->bufferBase + p->blockSize - dest); + size_t size = (size_t)(p->bufferBase + p->blockSize - dest); if (size == 0) return; @@ -204,10 +243,10 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 hs; p->matchMaxLen = matchMaxLen; { + // UInt32 hs4; p->fixedHashSize = 0; - if (p->numHashBytes == 2) - hs = (1 << 16) - 1; - else + hs = (1 << 16) - 1; + if (p->numHashBytes != 2) { hs = historySize; if (hs > p->expectedDataSize) @@ -218,9 +257,9 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, hs |= (hs >> 2); hs |= (hs >> 4); hs |= (hs >> 8); + // we propagated 16 bits in (hs). Low 16 bits must be set later hs >>= 1; - hs |= 0xFFFF; /* don't change it! It's required for Deflate */ - if (hs > (1 << 24)) + if (hs >= (1 << 24)) { if (p->numHashBytes == 3) hs = (1 << 24) - 1; @@ -228,12 +267,30 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, hs >>= 1; /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ } + + // hs = ((UInt32)1 << 25) - 1; // for test + + // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) + hs |= (1 << 16) - 1; /* don't change it! */ + + // bt5: we adjust the size with recommended minimum size + if (p->numHashBytes >= 5) + hs |= (256 << kLzHash_CrcShift_2) - 1; } p->hashMask = hs; hs++; + + /* + hs4 = (1 << 20); + if (hs4 > hs) + hs4 = hs; + // hs4 = (1 << 16); // for test + p->hash4Mask = hs4 - 1; + */ + if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; + // if (p->numHashBytes > 4) p->fixedHashSize += hs4; // kHash4Size; hs += p->fixedHashSize; } @@ -249,6 +306,10 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, numSons <<= 1; newSize = hs + numSons; + // aligned size is not required here, but it can be better for some loops + #define NUM_REFS_ALIGN_MASK 0xF + newSize = (newSize + NUM_REFS_ALIGN_MASK) & ~(size_t)NUM_REFS_ALIGN_MASK; + if (p->hash && p->numRefs == newSize) return 1; @@ -349,15 +410,23 @@ static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) { - size_t i; - for (i = 0; i < numItems; i++) + if (numItems == 0) + return; { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; - else - value -= subValue; - items[i] = value; + const CLzRef *lim = items + numItems - 1; + for (; items < lim; items += 2) + { + UInt32 v, m; + v = items[0]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; + v = items[1]; items[0] = m; m = v - subValue; if (v < subValue) m = kEmptyHashValue; + items[1] = m; + } + if (items == lim) + { + UInt32 v, m; + v = items[0]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; + items[0] = m; + } } } @@ -429,8 +498,8 @@ static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos { ptrdiff_t diff; curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; - diff = (ptrdiff_t)0 - delta; - if (cur[maxLen] == cur[maxLen + diff]) + diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + if (cur[maxLen] == cur[(ptrdiff_t)maxLen + diff]) { const Byte *c = cur; while (*c == c[diff]) @@ -588,15 +657,21 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue -#define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; +#define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ + offset = (unsigned)(func((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)(_maxLen_)) - distances); MOVE_POS_RET; + +#define GET_MATCHES_FOOTER_BT(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) + +#define GET_MATCHES_FOOTER_HC(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, Hc_GetMatchesSpec) #define SKIP_FOOTER \ SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ - ptrdiff_t diff = (ptrdiff_t)0 - d2; \ + ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ @@ -610,7 +685,7 @@ static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) curMatch = p->hash[hv]; p->hash[hv] = p->pos; offset = 0; - GET_MATCHES_FOOTER(offset, 1) + GET_MATCHES_FOOTER_BT(1) } UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) @@ -621,7 +696,7 @@ UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) curMatch = p->hash[hv]; p->hash[hv] = p->pos; offset = 0; - GET_MATCHES_FOOTER(offset, 2) + GET_MATCHES_FOOTER_BT(2) } static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) @@ -659,9 +734,10 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } } - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } + static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { UInt32 h2, h3, d2, d3, pos; @@ -676,53 +752,61 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = (hash + kFix4HashSize)[hv]; hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; - maxLen = 0; + maxLen = 3; offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - maxLen = 2; - distances[0] = 2; - distances[1] = d2 - 1; - offset = 2; - } - - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + for (;;) { - maxLen = 3; - distances[(size_t)offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[0] = 3; + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + d2 = d3; + distances[2 + 1] = d3 - 1; + offset = 4; + } + else + break; + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + offset = 2; + } + else + break; - if (offset != 0) - { UPDATE_maxLen distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; + MOVE_POS_RET } + break; } - if (maxLen < 3) - maxLen = 3; - - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } -/* + static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, maxLen, offset, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -733,53 +817,49 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; - d4 = pos - (hash + kFix4HashSize)[h4]; + // d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[h4] = pos; + // (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; - maxLen = 0; + maxLen = 4; offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + for (;;) { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[3] = d3 - 1; + offset = 4; + d2 = d3; + } + else + break; + } else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + offset = 2; d2 = d3; } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[(size_t)offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { + else + break; + + distances[(size_t)offset - 2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; UPDATE_maxLen distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) @@ -787,14 +867,12 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } + break; } - - if (maxLen < 4) - maxLen = 4; - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } -*/ + static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { @@ -816,27 +894,38 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; - maxLen = 0; + maxLen = 3; offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - maxLen = 2; - distances[0] = 2; - distances[1] = d2 - 1; - offset = 2; - } - - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - maxLen = 3; - distances[(size_t)offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } - - if (offset != 0) + for (;;) { + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[0] = 3; + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + d2 = d3; + distances[2 + 1] = d3 - 1; + offset = 4; + } + else + break; + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + offset = 2; + } + else + break; + UPDATE_maxLen distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) @@ -844,20 +933,16 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) p->son[p->cyclicBufferPos] = curMatch; MOVE_POS_RET; } + break; } - if (maxLen < 3) - maxLen = 3; - - offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET + GET_MATCHES_FOOTER_HC(maxLen); } -/* + static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos + UInt32 h2, h3, d2, d3, maxLen, offset, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -865,56 +950,52 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; - d4 = pos - (hash + kFix4HashSize)[h4]; + // d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[h4] = pos; + // (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; - maxLen = 0; + maxLen = 4; offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + for (;;) { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[3] = d3 - 1; + offset = 4; + d2 = d3; + } + else + break; + } else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + offset = 2; d2 = d3; } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[(size_t)offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { + else + break; + + distances[(size_t)offset - 2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; UPDATE_maxLen distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) @@ -922,16 +1003,12 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) p->son[p->cyclicBufferPos] = curMatch; MOVE_POS_RET; } + break; } - if (maxLen < 4) - maxLen = 4; - - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET + GET_MATCHES_FOOTER_HC(maxLen); } -*/ + UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { @@ -940,11 +1017,11 @@ UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances, 2) - (distances)); - MOVE_POS_RET + offset = 0; + GET_MATCHES_FOOTER_HC(2) } + static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { do @@ -1006,12 +1083,11 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) while (--num != 0); } -/* static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { do { - UInt32 h2, h3, h4; + UInt32 h2, h3; UInt32 *hash; SKIP_HEADER(5) HASH5_CALC; @@ -1019,13 +1095,12 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[h4] = + // (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0); } -*/ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { @@ -1046,27 +1121,26 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) while (--num != 0); } -/* static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { do { - UInt32 h2, h3, h4; + UInt32 h2, h3; UInt32 *hash; SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = hash + kFix5HashSize)[hv]; + curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[h4] = + // (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; MOVE_POS } while (--num != 0); } -*/ + void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { @@ -1089,18 +1163,16 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; if (!p->btMode) { - /* if (p->numHashBytes <= 4) */ + if (p->numHashBytes <= 4) { vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; } - /* else { vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; } - */ } else if (p->numHashBytes == 2) { @@ -1112,16 +1184,14 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; } - else /* if (p->numHashBytes == 4) */ + else if (p->numHashBytes == 4) { vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; } - /* else { vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; } - */ } @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H @@ -61,7 +61,7 @@ typedef struct _CMatchFinder && (!(p)->directInput || (p)->directInputRem == 0)) int MatchFinder_NeedMove(CMatchFinder *p); -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); +// Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); void MatchFinder_MoveBlock(CMatchFinder *p); void MatchFinder_ReadIfRequired(CMatchFinder *p); diff --git a/C/LzFindMt.c b/C/LzFindMt.c index bb0f42c3..49369ec0 100644 --- a/C/LzFindMt.c +++ b/C/LzFindMt.c @@ -1,12 +1,64 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2018-12-29 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "LzHash.h" +#include "CpuArch.h" +#include "LzHash.h" #include "LzFindMt.h" +// #define LOG_ITERS + +#ifdef LOG_ITERS +#include <stdio.h> +static UInt64 g_NumIters_Tree; +static UInt64 g_NumIters_Loop; +#define LOG_ITER(x) x +#else +#define LOG_ITER(x) +#endif + +#define kMtHashBlockSize (1 << 17) +#define kMtHashNumBlocks (1 << 1) +#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1) + +#define kMtBtBlockSize (1 << 16) +#define kMtBtNumBlocks (1 << 4) +#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1) + +/* + HASH functions: + We use raw 8/16 bits from a[1] and a[2], + xored with crc(a[0]) and crc(a[3]). + We check a[0], a[3] only. We don't need to compare a[1] and a[2] in matches. + our crc() function provides one-to-one correspondence for low 8-bit values: + (crc[0...0xFF] & 0xFF) <-> [0...0xFF] +*/ + +#define MT_HASH2_CALC \ + h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); + +#define MT_HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } + +/* +#define MT_HASH3_CALC__NO_2 { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } + +#define __MT_HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + h4 = (temp ^ (p->crc[cur[3]] << kLzHash_CrcShift_1)) & p->hash4Mask; } + // (kHash4Size - 1); +*/ + + static void MtSync_Construct(CMtSync *p) { p->wasCreated = False; @@ -18,8 +70,11 @@ static void MtSync_Construct(CMtSync *p) Event_Construct(&p->wasStopped); Semaphore_Construct(&p->freeSemaphore); Semaphore_Construct(&p->filledSemaphore); + p->affinity = 0; } + +MY_NO_INLINE static void MtSync_GetNextBlock(CMtSync *p) { if (p->needStart) @@ -81,8 +136,7 @@ static void MtSync_Destruct(CMtSync *p) p->exit = True; if (p->needStart) Event_Set(&p->canStart); - Thread_Wait(&p->thread); - Thread_Close(&p->thread); + Thread_Wait_Close(&p->thread); } if (p->csWasInitialized) { @@ -103,6 +157,7 @@ static void MtSync_Destruct(CMtSync *p) static SRes MtSync_Create2(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks) { + WRes wres; if (p->wasCreated) return SZ_OK; @@ -117,8 +172,12 @@ static SRes MtSync_Create2(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks)); p->needStart = True; - - RINOK_THREAD(Thread_Create(&p->thread, startAddress, obj)); + + if (p->affinity != 0) + wres = Thread_Create_With_Affinity(&p->thread, startAddress, obj, (CAffinityMask)p->affinity); + else + wres = Thread_Create(&p->thread, startAddress, obj); + RINOK_THREAD(wres); p->wasCreated = True; return SZ_OK; } @@ -131,23 +190,161 @@ static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, return res; } -void MtSync_Init(CMtSync *p) { p->needStart = True; } +// static void MtSync_Init(CMtSync *p) { p->needStart = True; } #define kMtMaxValForNormalize 0xFFFFFFFF +// #define kMtMaxValForNormalize ((1 << 25) + (1 << 20)) -#define DEF_GetHeads2(name, v, action) \ - static void GetHeads ## name(const Byte *p, UInt32 pos, \ - UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) \ - { action; for (; numHeads != 0; numHeads--) { \ - const UInt32 value = (v); p++; *heads++ = pos - hash[value]; hash[value] = pos++; } } +#ifdef MY_CPU_LE_UNALIGN + #define GetUi24hi_from32(p) ((UInt32)GetUi32(p) >> 8) +#else + #define GetUi24hi_from32(p) ((p)[1] ^ ((UInt32)(p)[2] << 8) ^ ((UInt32)(p)[3] << 16)) +#endif + +#define GetHeads_DECL(name) \ + static void GetHeads ## name(const Byte *p, UInt32 pos, \ + UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) + +#define GetHeads_LOOP(v) \ + for (; numHeads != 0; numHeads--) { \ + const UInt32 value = (v); \ + p++; \ + *heads++ = pos - hash[value]; \ + hash[value] = pos++; } + +#define DEF_GetHeads2(name, v, action) \ + GetHeads_DECL(name) { action \ + GetHeads_LOOP(v) } + #define DEF_GetHeads(name, v) DEF_GetHeads2(name, v, ;) -DEF_GetHeads2(2, (p[0] | ((UInt32)p[1] << 8)), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) -DEF_GetHeads(3, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8)) & hashMask) -DEF_GetHeads(4, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5)) & hashMask) -DEF_GetHeads(4b, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ ((UInt32)p[3] << 16)) & hashMask) -/* DEF_GetHeads(5, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5) ^ (crc[p[4]] << 3)) & hashMask) */ +DEF_GetHeads2(2, GetUi16(p), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) +DEF_GetHeads(3, (crc[p[0]] ^ GetUi16(p + 1)) & hashMask) +DEF_GetHeads2(3b, GetUi16(p) ^ ((UInt32)(p)[2] << 16), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) +// BT3 is not good for crc collisions for big hashMask values. + +/* +GetHeads_DECL(3b) +{ + UNUSED_VAR(hashMask); + UNUSED_VAR(crc); + { + const Byte *pLim = p + numHeads; + if (numHeads == 0) + return; + pLim--; + while (p < pLim) + { + UInt32 v1 = GetUi32(p); + UInt32 v0 = v1 & 0xFFFFFF; + UInt32 h0, h1; + p += 2; + v1 >>= 8; + h0 = hash[v0]; hash[v0] = pos; heads[0] = pos - h0; pos++; + h1 = hash[v1]; hash[v1] = pos; heads[1] = pos - h1; pos++; + heads += 2; + } + if (p == pLim) + { + UInt32 v0 = GetUi16(p) ^ ((UInt32)(p)[2] << 16); + *heads = pos - hash[v0]; + hash[v0] = pos; + } + } +} +*/ + +/* +GetHeads_DECL(4) +{ + unsigned sh = 0; + UNUSED_VAR(crc) + while ((hashMask & 0x80000000) == 0) + { + hashMask <<= 1; + sh++; + } + GetHeads_LOOP((GetUi32(p) * 0xa54a1) >> sh) +} +#define GetHeads4b GetHeads4 +*/ + +#define USE_GetHeads_LOCAL_CRC + +#ifdef USE_GetHeads_LOCAL_CRC + +GetHeads_DECL(4) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + // crc1[i] = rotlFixed(v, 8) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[3]] ^ (UInt32)GetUi16(p+1)) +} + +GetHeads_DECL(4b) +{ + UInt32 crc0[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + crc0[i] = crc[i] & hashMask; + } + GetHeads_LOOP(crc0[p[0]] ^ GetUi24hi_from32(p)) +} + +GetHeads_DECL(5) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + UInt32 crc2[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + crc2[i] = (v << kLzHash_CrcShift_2) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[3]] ^ crc2[p[4]] ^ (UInt32)GetUi16(p+1)) +} + +GetHeads_DECL(5b) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[4]] ^ GetUi24hi_from32(p)) +} + +#else + +DEF_GetHeads(4, (crc[p[0]] ^ (crc[p[3]] << kLzHash_CrcShift_1) ^ (UInt32)GetUi16(p+1)) & hashMask) +DEF_GetHeads(4b, (crc[p[0]] ^ GetUi24hi_from32(p)) & hashMask) +DEF_GetHeads(5, (crc[p[0]] ^ (crc[p[3]] << kLzHash_CrcShift_1) ^ (crc[p[4]] << kLzHash_CrcShift_2) ^ (UInt32)GetUi16(p + 1)) & hashMask) +DEF_GetHeads(5b, (crc[p[0]] ^ (crc[p[4]] << kLzHash_CrcShift_1) ^ GetUi24hi_from32(p)) & hashMask) + +#endif + static void HashThreadFunc(CMatchFinderMt *mt) { @@ -244,11 +441,11 @@ static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) MY_NO_INLINE static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) + UInt32 *d, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) { do { - UInt32 *_distances = ++distances; + UInt32 *_distances = ++d; UInt32 delta = *hash++; CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; @@ -258,14 +455,15 @@ static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLz unsigned maxLen = (unsigned)_maxLen; /* - if (size > 1) + #define PREF_STEP 1 + if (size > PREF_STEP) { - UInt32 delta = *hash; + UInt32 delta = hash[PREF_STEP - 1]; if (delta < _cyclicBufferSize) { - UInt32 cyc1 = _cyclicBufferPos + 1; + size_t cyc1 = _cyclicBufferPos + PREF_STEP; CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); - Byte b = *(cur + 1 - delta); + Byte b = *(cur + PREF_STEP - delta); _distances[0] = pair[0]; _distances[1] = b; } @@ -276,8 +474,9 @@ static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLz *ptr0 = *ptr1 = kEmptyHashValue; } else - for(;;) + for (LOG_ITER(g_NumIters_Tree++);;) { + LOG_ITER(g_NumIters_Loop++); { CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; @@ -292,8 +491,8 @@ static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLz if (maxLen < len) { maxLen = len; - *distances++ = (UInt32)len; - *distances++ = delta - 1; + *d++ = (UInt32)len; + *d++ = delta - 1; if (len == lenLimit) { UInt32 pair1 = pair[1]; @@ -333,39 +532,39 @@ static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLz _cyclicBufferPos++; cur++; { - UInt32 num = (UInt32)(distances - _distances); + UInt32 num = (UInt32)(d - _distances); _distances[-1] = num; } } - while (distances < limit && --size != 0); + while (d < limit && --size != 0); *posRes = pos; - return distances; + return d; } #endif -static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) +static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) { UInt32 numProcessed = 0; UInt32 curPos = 2; UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 - distances[1] = p->hashNumAvail; + d[1] = p->hashNumAvail; while (curPos < limit) { if (p->hashBufPos == p->hashBufPosLimit) { MatchFinderMt_GetNextBlock_Hash(p); - distances[1] = numProcessed + p->hashNumAvail; + d[1] = numProcessed + p->hashNumAvail; if (p->hashNumAvail >= p->numHashBytes) continue; - distances[0] = curPos + p->hashNumAvail; - distances += curPos; + d[0] = curPos + p->hashNumAvail; + d += curPos; for (; p->hashNumAvail != 0; p->hashNumAvail--) - *distances++ = 0; + *d++ = 0; return; } { @@ -387,7 +586,7 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) #ifndef MFMT_GM_INLINE while (curPos < limit && size-- != 0) { - UInt32 *startDistances = distances + curPos; + UInt32 *startDistances = d + curPos; UInt32 num = (UInt32)(GetMatchesSpec1(lenLimit, pos - p->hashBuf[p->hashBufPos++], pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, startDistances + 1, p->numHashBytes - 1) - startDistances); @@ -401,9 +600,9 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) { UInt32 posRes; curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, - distances + limit, - size, &posRes) - distances); + d + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + d + limit, + size, &posRes) - d); p->hashBufPos += posRes - pos; cyclicBufferPos += posRes - pos; p->buffer += posRes - pos; @@ -420,7 +619,7 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) } } - distances[0] = curPos; + d[0] = curPos; } static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) @@ -448,7 +647,7 @@ static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) } } -void BtThreadFunc(CMatchFinderMt *mt) +static void BtThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->btSync; for (;;) @@ -491,6 +690,14 @@ void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc) { MtSync_Destruct(&p->hashSync); MtSync_Destruct(&p->btSync); + + LOG_ITER( + printf("\nTree %9d * %7d iter = %9d sum \n", + (UInt32)(g_NumIters_Tree / 1000), + (UInt32)(((UInt64)g_NumIters_Loop * 1000) / (g_NumIters_Tree + 1)), + (UInt32)(g_NumIters_Loop / 1000) + )); + MatchFinderMt_FreeMem(p, alloc); } @@ -553,6 +760,7 @@ static void MatchFinderMt_Init(CMatchFinderMt *p) p->hash = mf->hash; p->fixedHashSize = mf->fixedHashSize; + // p->hash4Mask = mf->hash4Mask; p->crc = mf->crc; p->son = mf->son; @@ -572,22 +780,24 @@ void MatchFinderMt_ReleaseStream(CMatchFinderMt *p) /* p->MatchFinder->ReleaseStream(); */ } -static void MatchFinderMt_Normalize(CMatchFinderMt *p) -{ - MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize); - p->lzPos = p->historySize + 1; -} +MY_NO_INLINE static void MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) { - UInt32 blockIndex; + UInt32 blockIndex, k; + MtSync_GetNextBlock(&p->btSync); + blockIndex = ((p->btSync.numProcessedBlocks - 1) & kMtBtNumBlocksMask); - p->btBufPosLimit = p->btBufPos = blockIndex * kMtBtBlockSize; - p->btBufPosLimit += p->btBuf[p->btBufPos++]; - p->btNumAvailBytes = p->btBuf[p->btBufPos++]; + k = blockIndex * kMtBtBlockSize; + p->btBufPosLimit = k + p->btBuf[k]; + p->btNumAvailBytes = p->btBuf[k + 1]; + p->btBufPos = k + 2; if (p->lzPos >= kMtMaxValForNormalize - kMtBtBlockSize) - MatchFinderMt_Normalize(p); + { + MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize); + p->lzPos = p->historySize + 1; + } } static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) @@ -603,170 +813,289 @@ static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) return p->btNumAvailBytes; } -static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) +static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { - UInt32 h2, curMatch2; + UInt32 h2, c2; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; + UInt32 m = p->lzPos; MT_HASH2_CALC - curMatch2 = hash[h2]; - hash[h2] = lzPos; + c2 = hash[h2]; + hash[h2] = m; - if (curMatch2 >= matchMinPos) - if (cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + if (c2 >= matchMinPos) + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - *distances++ = 2; - *distances++ = lzPos - curMatch2 - 1; + *d++ = 2; + *d++ = m - c2 - 1; } - return distances; + return d; } -static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) +static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { - UInt32 h2, h3, curMatch2, curMatch3; + UInt32 h2, h3, c2, c3; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; + UInt32 m = p->lzPos; MT_HASH3_CALC - curMatch2 = hash[ h2]; - curMatch3 = (hash + kFix3HashSize)[h3]; + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; - hash[ h2] = lzPos; - (hash + kFix3HashSize)[h3] = lzPos; + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) { - distances[0] = 3; - return distances + 2; + d[0] = 3; + return d + 2; } - distances[0] = 2; - distances += 2; + d[0] = 2; + d += 2; } - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) + if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) { - *distances++ = 3; - *distances++ = lzPos - curMatch3 - 1; + *d++ = 3; + *d++ = m - c3 - 1; } - return distances; + return d; } + +#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++; + /* -static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) +static +UInt32 MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) { - UInt32 h2, h3, h4, curMatch2, curMatch3, curMatch4; + UInt32 pos = p->btBufPos; + const UInt32 *bt = p->btBuf + pos; + UInt32 len = *bt++; + UInt32 matchMinPos; + const UInt32 *d_base = d; + UInt32 avail = p->btNumAvailBytes - 1; + p->btBufPos = pos + 1 + len; + + { + UInt32 temp1 = p->historySize; + p->btNumAvailBytes = avail; + + #define BT_HASH_BYTES_MAX 5 + + if (len != 0) + temp1 = bt[1]; + else if (avail < (BT_HASH_BYTES_MAX - 2)) + { + INCREASE_LZ_POS + return 0; + } + matchMinPos = p->lzPos - temp1; + } + + for (;;) + { + + UInt32 h2, h3, c2, c3; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; - MT_HASH4_CALC - - curMatch2 = hash[ h2]; - curMatch3 = (hash + kFix3HashSize)[h3]; - curMatch4 = (hash + kFix4HashSize)[h4]; + UInt32 m = p->lzPos; + MT_HASH3_CALC + + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; + + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; + + if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) + { + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) + { + d[0] = 3; + d += 2; + break; + } + // else + { + d[0] = 2; + d += 2; + } + } + if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) + { + *d++ = 3; + *d++ = m - c3 - 1; + } + break; + } + + if (len != 0) + { + do + { + UInt32 v0 = bt[0]; + UInt32 v1 = bt[1]; + bt += 2; + d[0] = v0; + d[1] = v1; + d += 2; + } + while ((len -= 2) != 0); + } + INCREASE_LZ_POS + return (UInt32)(d - d_base); +} +*/ + + +static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) +{ + UInt32 h2, h3, /* h4, */ c2, c3 /* , c4 */; + UInt32 *hash = p->hash; + const Byte *cur = p->pointerToCurPos; + UInt32 m = p->lzPos; + MT_HASH3_CALC + // MT_HASH4_CALC + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; + // c4 = (hash + kFix4HashSize)[h4]; - hash[ h2] = lzPos; - (hash + kFix3HashSize)[h3] = lzPos; - (hash + kFix4HashSize)[h4] = lzPos; + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; + // (hash + kFix4HashSize)[h4] = m; + + #define _USE_H2 - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + #ifdef _USE_H2 + if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) { - distances[0] = (cur[(ptrdiff_t)curMatch2 - lzPos + 3] == cur[3]) ? 4 : 3; - return distances + 2; + // d[0] = (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 3] == cur[3]) ? 4 : 3; + // return d + 2; + + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 3] == cur[3]) + { + d[0] = 4; + return d + 2; + } + d[0] = 3; + d += 2; + + #ifdef _USE_H4 + if (c4 >= matchMinPos) + if ( + cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && + cur[(ptrdiff_t)c4 - (ptrdiff_t)m + 3] == cur[3] + ) + { + *d++ = 4; + *d++ = m - c4 - 1; + } + #endif + return d; } - distances[0] = 2; - distances += 2; + d[0] = 2; + d += 2; } + #endif - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) + if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) { - distances[1] = lzPos - curMatch3 - 1; - if (cur[(ptrdiff_t)curMatch3 - lzPos + 3] == cur[3]) + d[1] = m - c3 - 1; + if (cur[(ptrdiff_t)c3 - (ptrdiff_t)m + 3] == cur[3]) { - distances[0] = 4; - return distances + 2; + d[0] = 4; + return d + 2; } - distances[0] = 3; - distances += 2; + d[0] = 3; + d += 2; } - if (curMatch4 >= matchMinPos) + #ifdef _USE_H4 + if (c4 >= matchMinPos) if ( - cur[(ptrdiff_t)curMatch4 - lzPos] == cur[0] && - cur[(ptrdiff_t)curMatch4 - lzPos + 3] == cur[3] + cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && + cur[(ptrdiff_t)c4 - (ptrdiff_t)m + 3] == cur[3] ) { - *distances++ = 4; - *distances++ = lzPos - curMatch4 - 1; + *d++ = 4; + *d++ = m - c4 - 1; } + #endif - return distances; + return d; } -*/ -#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++; -static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances) +static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) { - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; + const UInt32 *bt = p->btBuf + p->btBufPos; + UInt32 len = *bt++; p->btBufPos += 1 + len; p->btNumAvailBytes--; { UInt32 i; for (i = 0; i < len; i += 2) { - UInt32 v0 = btBuf[0]; - UInt32 v1 = btBuf[1]; - btBuf += 2; - distances[0] = v0; - distances[1] = v1; - distances += 2; + UInt32 v0 = bt[0]; + UInt32 v1 = bt[1]; + bt += 2; + d[0] = v0; + d[1] = v1; + d += 2; } } INCREASE_LZ_POS return len; } -static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) -{ - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; - p->btBufPos += 1 + len; + +static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) +{ + UInt32 pos = p->btBufPos; + const UInt32 *bt = p->btBuf + pos; + UInt32 len = *bt++; + UInt32 avail = p->btNumAvailBytes - 1; + p->btNumAvailBytes = avail; + p->btBufPos = pos + 1 + len; if (len == 0) { - /* change for bt5 ! */ - if (p->btNumAvailBytes-- >= 4) - len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, distances) - (distances)); + #define BT_HASH_BYTES_MAX 5 + if (avail >= (BT_HASH_BYTES_MAX - 1) - 1) + len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, d) - d); } else { - /* Condition: there are matches in btBuf with length < p->numHashBytes */ - UInt32 *distances2; - p->btNumAvailBytes--; - distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); + /* + first match pair from BinTree: (match_len, match_dist), + (match_len >= numHashBytes). + MixMatchesFunc() inserts only hash matches that are nearer than (match_dist) + */ + UInt32 *d2; + d2 = p->MixMatchesFunc(p, p->lzPos - bt[1], d); do { - UInt32 v0 = btBuf[0]; - UInt32 v1 = btBuf[1]; - btBuf += 2; - distances2[0] = v0; - distances2[1] = v1; - distances2 += 2; + UInt32 v0 = bt[0]; + UInt32 v1 = bt[1]; + bt += 2; + d2[0] = v0; + d2[1] = v1; + d2 += 2; } while ((len -= 2) != 0); - len = (UInt32)(distances2 - (distances)); + len = (UInt32)(d2 - d); } INCREASE_LZ_POS return len; @@ -802,19 +1131,18 @@ static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) SKIP_FOOTER_MT } -/* static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) { SKIP_HEADER_MT(4) - UInt32 h2, h3, h4; - MT_HASH4_CALC - (hash + kFix4HashSize)[h4] = + UInt32 h2, h3 /*, h4 */; + MT_HASH3_CALC + // MT_HASH4_CALC + // (hash + kFix4HashSize)[h4] = (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT } -*/ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) { @@ -832,22 +1160,23 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break; case 3: - p->GetHeadsFunc = GetHeads3; + p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads3b : GetHeads3; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; break; - default: - /* case 4: */ + case 4: p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads4b : GetHeads4; + + // it's fast inline version of GetMatches() + // vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches_Bt4; + p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3; vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; break; - /* default: - p->GetHeadsFunc = GetHeads5; + p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads5b : GetHeads5; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; vTable->Skip = (Mf_Skip_Func)MatchFinderMt4_Skip; break; - */ } } diff --git a/C/LzFindMt.h b/C/LzFindMt.h index ef431e3f..05297ca3 100644 --- a/C/LzFindMt.h +++ b/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2018-07-04 : Igor Pavlov : Public domain */ +2019-11-05 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -9,14 +9,6 @@ EXTERN_C_BEGIN -#define kMtHashBlockSize (1 << 13) -#define kMtHashNumBlocks (1 << 3) -#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1) - -#define kMtBtBlockSize (1 << 14) -#define kMtBtNumBlocks (1 << 6) -#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1) - typedef struct _CMtSync { BoolInt wasCreated; @@ -34,6 +26,7 @@ typedef struct _CMtSync BoolInt csWasEntered; CCriticalSection cs; UInt32 numProcessedBlocks; + UInt64 affinity; } CMtSync; typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); @@ -56,11 +49,12 @@ typedef struct _CMatchFinderMt UInt32 *hash; UInt32 fixedHashSize; + // UInt32 hash4Mask; UInt32 historySize; const UInt32 *crc; Mf_Mix_Matches MixMatchesFunc; - + /* LZ + BT */ CMtSync btSync; Byte btDummy[kMtCacheLineDummy]; @@ -1,57 +1,34 @@ /* LzHash.h -- HASH functions for LZ algorithms -2015-04-12 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZ_HASH_H #define __LZ_HASH_H +/* + (kHash2Size >= (1 << 8)) : Required + (kHash3Size >= (1 << 16)) : Required +*/ + #define kHash2Size (1 << 10) #define kHash3Size (1 << 16) -#define kHash4Size (1 << 20) +// #define kHash4Size (1 << 20) #define kFix3HashSize (kHash2Size) #define kFix4HashSize (kHash2Size + kHash3Size) -#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) - -#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8); - -#define HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } - -#define HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; } - -#define HASH5_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - temp ^= (p->crc[cur[3]] << 5); \ - h4 = temp & (kHash4Size - 1); \ - hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; } - -/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ -#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; - - -#define MT_HASH2_CALC \ - h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); - -#define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } - -#define MT_HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) + +/* + We use up to 3 crc values for hash: + crc0 + crc1 << Shift_1 + crc2 << Shift_2 + (Shift_1 = 5) and (Shift_2 = 10) is good tradeoff. + Small values for Shift are not good for collision rate. + Big value for Shift_2 increases the minimum size + of hash table, that will be slow for small files. +*/ + +#define kLzHash_CrcShift_1 5 +#define kLzHash_CrcShift_2 10 #endif diff --git a/C/Lzma2Dec.c b/C/Lzma2Dec.c index 4e138a4a..ac970a84 100644 --- a/C/Lzma2Dec.c +++ b/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2019-02-02 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -93,7 +93,8 @@ void Lzma2Dec_Init(CLzma2Dec *p) LzmaDec_Init(&p->decoder); } -static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) +// ELzma2State +static unsigned Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) { switch (p->state) { diff --git a/C/Lzma2DecMt.c b/C/Lzma2DecMt.c index 988643d9..9f1dc52b 100644 --- a/C/Lzma2DecMt.c +++ b/C/Lzma2DecMt.c @@ -1,25 +1,25 @@ /* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread -2019-02-02 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" // #define SHOW_DEBUG_INFO +// #define _7ZIP_ST + #ifdef SHOW_DEBUG_INFO #include <stdio.h> #endif +#ifndef _7ZIP_ST #ifdef SHOW_DEBUG_INFO #define PRF(x) x #else #define PRF(x) #endif - #define PRF_STR(s) PRF(printf("\n" s "\n")) -#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) #define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) - -// #define _7ZIP_ST +#endif #include "Alloc.h" @@ -28,10 +28,10 @@ #ifndef _7ZIP_ST #include "MtDec.h" -#endif - #define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) +#endif + void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) { @@ -255,7 +255,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa const unsigned kNumAlignBits = 12; const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ t->alloc.numAlignBits = kNumAlignBits; - t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.offset = ((UInt32)coderIndex * (((unsigned)1 << 11) + (1 << 8) + (1 << 6))) & (((unsigned)1 << kNumAlignBits) - ((unsigned)1 << kNumCacheLineBits)); t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; } } @@ -527,7 +527,7 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, BoolInt needWriteToStream, - const Byte *src, size_t srcSize, + const Byte *src, size_t srcSize, BoolInt isCross, BoolInt *needContinue, BoolInt *canRecode) { CLzma2DecMt *me = (CLzma2DecMt *)pp; @@ -536,12 +536,14 @@ static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, const Byte *data = t->outBuf; BoolInt needContinue2 = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + UNUSED_VAR(isCross) + PRF_STR_INT_2("Write", coderIndex, srcSize); *needContinue = False; *canRecode = True; - UNUSED_VAR(src) - UNUSED_VAR(srcSize) if ( // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK @@ -696,7 +698,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p inPos = 0; inLim = p->inBufSize; inData = p->inBuf; - p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readRes = ISeqInStream_Read(p->inStream, (void *)(p->inBuf), &inLim); // p->readProcessed += inLim; // inLim -= 5; p->readWasFinished = True; // for test if (inLim == 0 || p->readRes != SZ_OK) @@ -838,6 +840,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, p->inProcessed = 0; p->readWasFinished = False; + p->readRes = SZ_OK; *isMT = False; @@ -856,7 +859,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, if (p->props.numThreads > 1) { - IMtDecCallback vt; + IMtDecCallback2 vt; Lzma2DecMt_FreeSt(p); @@ -955,7 +958,12 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, *inProcessed = p->inProcessed; // res = SZ_OK; // for test - if (res == SZ_OK && p->readRes != SZ_OK) + if (res == SZ_ERROR_INPUT_EOF) + { + if (p->readRes != SZ_OK) + res = p->readRes; + } + else if (res == SZ_OK && p->readRes != SZ_OK) res = p->readRes; /* diff --git a/C/Lzma2Enc.c b/C/Lzma2Enc.c index 5c1ad493..e61a5dfe 100644 --- a/C/Lzma2Enc.c +++ b/C/Lzma2Enc.c @@ -1,5 +1,5 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2018-07-04 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -330,7 +330,7 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) numBlocks++; if (numBlocks < (unsigned)t2) { - t2r = (unsigned)numBlocks; + t2r = (int)numBlocks; if (t2r == 0) t2r = 1; t3 = t1 * t2r; @@ -632,15 +632,15 @@ static SRes Lzma2Enc_EncodeMt1( { if (outBuf) { - size_t destPos = *outBufSize; + const size_t destPos = *outBufSize; if (destPos >= outLim) return SZ_ERROR_OUTPUT_EOF; - outBuf[destPos] = 0; + outBuf[destPos] = LZMA2_CONTROL_EOF; // 0 *outBufSize = destPos + 1; } else { - Byte b = 0; + const Byte b = LZMA2_CONTROL_EOF; // 0; if (ISeqOutStream_Write(outStream, &b, 1) != 1) return SZ_ERROR_WRITE; } @@ -780,13 +780,13 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, p->outBufSize = destBlockSize; } - p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; + p->mtCoder.numThreadsMax = (unsigned)p->props.numBlockThreads_Max; p->mtCoder.expectedDataSize = p->expectedDataSize; { SRes res = MtCoder_Code(&p->mtCoder); if (!outStream) - *outBufSize = p->outBuf - outBuf; + *outBufSize = (size_t)(p->outBuf - outBuf); return res; } } diff --git a/C/Lzma86Enc.c b/C/Lzma86Enc.c index 2617bab8..14fcd65c 100644 --- a/C/Lzma86Enc.c +++ b/C/Lzma86Enc.c @@ -11,8 +11,6 @@ #include "Bra.h" #include "LzmaEnc.h" -#define SZE_OUT_OVERFLOW SZE_DATA_ERROR - int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, int level, UInt32 dictSize, int filterMode) { diff --git a/C/LzmaDec.c b/C/LzmaDec.c index ba3e1dd5..d6742e5a 100644 --- a/C/LzmaDec.c +++ b/C/LzmaDec.c @@ -1,5 +1,5 @@ /* LzmaDec.c -- LZMA Decoder -2018-07-04 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -13,10 +13,12 @@ #define kNumBitModelTotalBits 11 #define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 #define RC_INIT_SIZE 5 +#ifndef _LZMA_DEC_OPT + +#define kNumMoveBits 5 #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } #define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) @@ -62,9 +64,10 @@ probLit = prob + (offs + bit + symbol); \ GET_BIT2(probLit, symbol, offs ^= bit; , ;) +#endif // _LZMA_DEC_OPT -#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_INPUT_EOF; range <<= 8; code = (code << 8) | (*buf++); } #define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; @@ -114,6 +117,9 @@ #define kMatchMinLen 2 #define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) +#define kMatchSpecLen_Error_Data (1 << 9) +#define kMatchSpecLen_Error_Fail (kMatchSpecLen_Error_Data - 1) + /* External ASM code needs same CLzmaProb array layout. So don't change it. */ /* (probs_1664) is faster and better for code size at some platforms */ @@ -166,10 +172,12 @@ /* p->remainLen : shows status of LZMA decoder: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : need init range coder - = kMatchSpecLenStart + 2 : need init range coder and state + < kMatchSpecLenStart : the number of bytes to be copied with (p->rep0) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state + = kMatchSpecLen_Error_Fail : Internal Code Failure + = kMatchSpecLen_Error_Data + [0 ... 273] : LZMA Data Error */ /* ---------- LZMA_DECODE_REAL ---------- */ @@ -188,23 +196,31 @@ In: { LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol - is not END_OF_PAYALOAD_MARKER, then function returns error code. + is not END_OF_PAYALOAD_MARKER, then the function doesn't write any byte to dictionary, + the function returns SZ_OK, and the caller can use (p->remainLen) and (p->reps[0]) later. } Processing: - first LZMA symbol will be decoded in any case - All checks for limits are at the end of main loop, - It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + The first LZMA symbol will be decoded in any case. + All main checks for limits are at the end of main loop, + It decodes additional LZMA-symbols while (p->buf < bufLimit && dicPos < limit), RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + But if (p->buf < bufLimit), the caller provided at least (LZMA_REQUIRED_INPUT_MAX + 1) bytes for + next iteration before limit (bufLimit + LZMA_REQUIRED_INPUT_MAX), + that is enough for worst case LZMA symbol with one additional RangeCoder normalization for one bit. + So that function never reads bufLimit [LZMA_REQUIRED_INPUT_MAX] byte. Out: RangeCoder is normalized Result: SZ_OK - OK - SZ_ERROR_DATA - Error - p->remainLen: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished + p->remainLen: + < kMatchSpecLenStart : the number of bytes to be copied with (p->reps[0]) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + + SZ_ERROR_DATA - error, when the MATCH-Symbol refers out of dictionary + p->remainLen : undefined + p->reps[*] : undefined */ @@ -316,11 +332,6 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit else { UPDATE_1(prob); - /* - // that case was checked before with kBadRepCode - if (checkDicSize == 0 && processedPos == 0) - return SZ_ERROR_DATA; - */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { @@ -329,6 +340,13 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit IF_BIT_0(prob) { UPDATE_0(prob); + + // that case was checked before with kBadRepCode + // if (checkDicSize == 0 && processedPos == 0) { len = kMatchSpecLen_Error_Data + 1; break; } + // The caller doesn't allow (dicPos == limit) case here + // so we don't need the following check: + // if (dicPos == limit) { state = state < kNumLitStates ? 9 : 11; len = 1; break; } + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; dicPos++; processedPos++; @@ -518,8 +536,10 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { - p->dicPos = dicPos; - return SZ_ERROR_DATA; + len += kMatchSpecLen_Error_Data + kMatchMinLen; + // len = kMatchSpecLen_Error_Data; + // len += kMatchMinLen; + break; } } @@ -532,8 +552,13 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit if ((rem = limit - dicPos) == 0) { - p->dicPos = dicPos; - return SZ_ERROR_DATA; + /* + We stop decoding and return SZ_OK, and we can resume decoding later. + Any error conditions can be tested later in caller code. + For more strict mode we can stop decoding with error + // len += kMatchSpecLen_Error_Data; + */ + break; } curLen = ((rem < len) ? (unsigned)rem : len); @@ -572,7 +597,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit p->buf = buf; p->range = range; p->code = code; - p->remainLen = (UInt32)len; + p->remainLen = (UInt32)len; // & (kMatchSpecLen_Error_Data - 1); // we can write real length for error matches too. p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; @@ -580,40 +605,61 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit p->reps[2] = rep2; p->reps[3] = rep3; p->state = (UInt32)state; - + if (len >= kMatchSpecLen_Error_Data) + return SZ_ERROR_DATA; return SZ_OK; } #endif + + static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { - if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + unsigned len = (unsigned)p->remainLen; + if (len == 0 /* || len >= kMatchSpecLenStart */) + return; { - Byte *dic = p->dic; SizeT dicPos = p->dicPos; - SizeT dicBufSize = p->dicBufSize; - unsigned len = (unsigned)p->remainLen; - SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ - SizeT rem = limit - dicPos; - if (rem < len) - len = (unsigned)(rem); + Byte *dic; + SizeT dicBufSize; + SizeT rep0; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ + { + SizeT rem = limit - dicPos; + if (rem < len) + { + len = (unsigned)(rem); + if (len == 0) + return; + } + } if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; p->processedPos += (UInt32)len; p->remainLen -= (UInt32)len; - while (len != 0) + dic = p->dic; + rep0 = p->reps[0]; + dicBufSize = p->dicBufSize; + do { - len--; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; dicPos++; } + while (--len); p->dicPos = dicPos; } } +/* +At staring of new stream we have one of the following symbols: + - Literal - is allowed + - Non-Rep-Match - is allowed only if it's end marker symbol + - Rep-Match - is not allowed +We use early check of (RangeCoder:Code) over kBadRepCode to simplify main decoding code +*/ + #define kRange0 0xFFFFFFFF #define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) #define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) @@ -621,69 +667,77 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) #error Stop_Compiling_Bad_LZMA_Check #endif + +/* +LzmaDec_DecodeReal2(): + It calls LZMA_DECODE_REAL() and it adjusts limit according (p->checkDicSize). + +We correct (p->checkDicSize) after LZMA_DECODE_REAL() and in LzmaDec_WriteRem(), +and we support the following state of (p->checkDicSize): + if (total_processed < p->prop.dicSize) then + { + (total_processed == p->processedPos) + (p->checkDicSize == 0) + } + else + (p->checkDicSize == p->prop.dicSize) +*/ + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { - do + if (p->checkDicSize == 0) { - SizeT limit2 = limit; - if (p->checkDicSize == 0) - { - UInt32 rem = p->prop.dicSize - p->processedPos; - if (limit - p->dicPos > rem) - limit2 = p->dicPos + rem; - - if (p->processedPos == 0) - if (p->code >= kBadRepCode) - return SZ_ERROR_DATA; - } - - RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); - + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit = p->dicPos + rem; + } + { + int res = LZMA_DECODE_REAL(p, limit, bufLimit); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; - - LzmaDec_WriteRem(p, limit); + return res; } - while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - - return 0; } + + typedef enum { - DUMMY_ERROR, /* unexpected end of input stream */ + DUMMY_INPUT_EOF, /* need more input data */ DUMMY_LIT, DUMMY_MATCH, DUMMY_REP } ELzmaDummy; -static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) + +#define IS_DUMMY_END_MARKER_POSSIBLE(dummyRes) ((dummyRes) == DUMMY_MATCH) + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byte **bufOut) { UInt32 range = p->range; UInt32 code = p->code; - const Byte *bufLimit = buf + inSize; + const Byte *bufLimit = *bufOut; const CLzmaProb *probs = GET_PROBS; unsigned state = (unsigned)p->state; ELzmaDummy res; + for (;;) { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, ((unsigned)1 << p->prop.pb) - 1); prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK - /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ - prob = probs + Literal; if (p->checkDicSize != 0 || p->processedPos != 0) prob += ((UInt32)LZMA_LIT_SIZE * - ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + - (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + ((((p->processedPos) & (((unsigned)1 << (p->prop.lp)) - 1)) << p->prop.lc) + + ((unsigned)p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); if (state < kNumLitStates) { @@ -735,8 +789,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - NORMALIZE_CHECK; - return DUMMY_REP; + break; } else { @@ -812,8 +865,6 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned numDirectBits = ((posSlot >> 1) - 1); - /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ - if (posSlot < kEndPosModelIndex) { prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); @@ -844,12 +895,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS } } } + break; } NORMALIZE_CHECK; + + *bufOut = buf; return res; } - +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { p->remainLen = kMatchSpecLenStart + 1; @@ -872,16 +926,41 @@ void LzmaDec_Init(CLzmaDec *p) } +/* +LZMA supports optional end_marker. +So the decoder can lookahead for one additional LZMA-Symbol to check end_marker. +That additional LZMA-Symbol can require up to LZMA_REQUIRED_INPUT_MAX bytes in input stream. +When the decoder reaches dicLimit, it looks (finishMode) parameter: + if (finishMode == LZMA_FINISH_ANY), the decoder doesn't lookahead + if (finishMode != LZMA_FINISH_ANY), the decoder lookahead, if end_marker is possible for current position + +When the decoder lookahead, and the lookahead symbol is not end_marker, we have two ways: + 1) Strict mode (default) : the decoder returns SZ_ERROR_DATA. + 2) The relaxed mode (alternative mode) : we could return SZ_OK, and the caller + must check (status) value. The caller can show the error, + if the end of stream is expected, and the (status) is noit + LZMA_STATUS_FINISHED_WITH_MARK or LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK. +*/ + + +#define RETURN__NOT_FINISHED__FOR_FINISH \ + *status = LZMA_STATUS_NOT_FINISHED; \ + return SZ_ERROR_DATA; // for strict mode + // return SZ_OK; // for relaxed mode + + SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - *status = LZMA_STATUS_NOT_SPECIFIED; if (p->remainLen > kMatchSpecLenStart) { + if (p->remainLen > kMatchSpecLenStart + 2) + return p->remainLen == kMatchSpecLen_Error_Fail ? SZ_ERROR_FAIL : SZ_ERROR_DATA; + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) p->tempBuf[p->tempBufSize++] = *src++; if (p->tempBufSize != 0 && p->tempBuf[0] != 0) @@ -896,6 +975,12 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr | ((UInt32)p->tempBuf[2] << 16) | ((UInt32)p->tempBuf[3] << 8) | ((UInt32)p->tempBuf[4]); + + if (p->checkDicSize == 0 + && p->processedPos == 0 + && p->code >= kBadRepCode) + return SZ_ERROR_DATA; + p->range = 0xFFFFFFFF; p->tempBufSize = 0; @@ -913,10 +998,21 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->remainLen = 0; } - LzmaDec_WriteRem(p, dicLimit); - - while (p->remainLen != kMatchSpecLenStart) + for (;;) { + if (p->remainLen == kMatchSpecLenStart) + { + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } + + LzmaDec_WriteRem(p, dicLimit); + + { + // (p->remainLen == 0 || p->dicPos == dicLimit) + int checkEndMarkNow = 0; if (p->dicPos >= dicLimit) @@ -933,92 +1029,174 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr } if (p->remainLen != 0) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + RETURN__NOT_FINISHED__FOR_FINISH; } checkEndMarkNow = 1; } + // (p->remainLen == 0) + if (p->tempBufSize == 0) { - SizeT processed; const Byte *bufLimit; + int dummyProcessed = -1; + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, src, inSize); - if (dummyRes == DUMMY_ERROR) + const Byte *bufOut = src + inSize; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, src, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) { - memcpy(p->tempBuf, src, inSize); - p->tempBufSize = (unsigned)inSize; + size_t i; + if (inSize >= LZMA_REQUIRED_INPUT_MAX) + break; (*srcLen) += inSize; + p->tempBufSize = (unsigned)inSize; + for (i = 0; i < inSize; i++) + p->tempBuf[i] = src[i]; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + + dummyProcessed = (int)(bufOut - src); + if ((unsigned)dummyProcessed > LZMA_REQUIRED_INPUT_MAX) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + unsigned i; + (*srcLen) += (unsigned)dummyProcessed; + p->tempBufSize = (unsigned)dummyProcessed; + for (i = 0; i < (unsigned)dummyProcessed; i++) + p->tempBuf[i] = src[i]; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; } + bufLimit = src; + // we will decode only one iteration } else bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + p->buf = src; - if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) - return SZ_ERROR_DATA; - processed = (SizeT)(p->buf - src); - (*srcLen) += processed; - src += processed; - inSize -= processed; + + { + int res = LzmaDec_DecodeReal2(p, dicLimit, bufLimit); + + SizeT processed = (SizeT)(p->buf - src); + + if (dummyProcessed < 0) + { + if (processed > inSize) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } + } + continue; } - else + { - unsigned rem = p->tempBufSize, lookAhead = 0; - while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) - p->tempBuf[rem++] = src[lookAhead++]; - p->tempBufSize = rem; + // we have some data in (p->tempBuf) + // in strict mode: tempBufSize is not enough for one Symbol decoding. + // in relaxed mode: tempBufSize not larger than required for one Symbol decoding. + + unsigned rem = p->tempBufSize; + unsigned ahead = 0; + int dummyProcessed = -1; + + while (rem < LZMA_REQUIRED_INPUT_MAX && ahead < inSize) + p->tempBuf[rem++] = src[ahead++]; + + // ahead - the size of new data copied from (src) to (p->tempBuf) + // rem - the size of temp buffer including new data from (src) + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); - if (dummyRes == DUMMY_ERROR) + const Byte *bufOut = p->tempBuf + rem; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, p->tempBuf, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) { - (*srcLen) += (SizeT)lookAhead; + if (rem >= LZMA_REQUIRED_INPUT_MAX) + break; + p->tempBufSize = rem; + (*srcLen) += (SizeT)ahead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + + dummyProcessed = (int)(bufOut - p->tempBuf); + + if ((unsigned)dummyProcessed < p->tempBufSize) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + (*srcLen) += (unsigned)dummyProcessed - p->tempBufSize; + p->tempBufSize = (unsigned)dummyProcessed; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; } } + p->buf = p->tempBuf; - if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) - return SZ_ERROR_DATA; { - unsigned kkk = (unsigned)(p->buf - p->tempBuf); - if (rem < kkk) - return SZ_ERROR_FAIL; /* some internal error */ - rem -= kkk; - if (lookAhead < rem) - return SZ_ERROR_FAIL; /* some internal error */ - lookAhead -= rem; + // we decode one symbol from (p->tempBuf) here, so the (bufLimit) is equal to (p->buf) + int res = LzmaDec_DecodeReal2(p, dicLimit, p->buf); + + SizeT processed = (SizeT)(p->buf - p->tempBuf); + rem = p->tempBufSize; + + if (dummyProcessed < 0) + { + if (processed > LZMA_REQUIRED_INPUT_MAX) + break; + if (processed < rem) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + processed -= rem; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + p->tempBufSize = 0; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } } - (*srcLen) += (SizeT)lookAhead; - src += lookAhead; - inSize -= (SizeT)lookAhead; - p->tempBufSize = 0; } + } } - - if (p->code != 0) - return SZ_ERROR_DATA; - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + + /* Some unexpected error: internal error of code, memory corruption or hardware failure */ + p->remainLen = kMatchSpecLen_Error_Fail; + return SZ_ERROR_FAIL; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; diff --git a/C/LzmaDec.h b/C/LzmaDec.h index 1f0927ab..6f129625 100644 --- a/C/LzmaDec.h +++ b/C/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2018-04-21 : Igor Pavlov : Public domain */ +2020-03-19 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -181,6 +181,7 @@ Returns: LZMA_STATUS_NEEDS_MORE_INPUT LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK SZ_ERROR_DATA - Data error + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, @@ -223,6 +224,7 @@ Returns: SZ_ERROR_MEM - Memory allocation error SZ_ERROR_UNSUPPORTED - Unsupported properties SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index 46a0db00..377e20c3 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2019-01-10: Igor Pavlov : Public domain */ +2021-04-01: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -19,6 +19,19 @@ #include "LzFindMt.h" #endif +/* the following LzmaEnc_* declarations is internal LZMA interface for LZMA2 encoder */ + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); +void LzmaEnc_Finish(CLzmaEncHandle pp); +void LzmaEnc_SaveState(CLzmaEncHandle pp); +void LzmaEnc_RestoreState(CLzmaEncHandle pp); + #ifdef SHOW_STAT static unsigned g_STAT_OFFSET = 0; #endif @@ -36,7 +49,7 @@ static unsigned g_STAT_OFFSET = 0; #define kNumMoveReducingBits 4 #define kNumBitPriceShiftBits 4 -#define kBitPrice (1 << kNumBitPriceShiftBits) +// #define kBitPrice (1 << kNumBitPriceShiftBits) #define REP_LEN_COUNT 64 @@ -47,6 +60,7 @@ void LzmaEncProps_Init(CLzmaEncProps *p) p->reduceSize = (UInt64)(Int64)-1; p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; p->writeEndMark = 0; + p->affinity = 0; } void LzmaEncProps_Normalize(CLzmaEncProps *p) @@ -55,7 +69,13 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (level < 0) level = 5; p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); + if (p->dictSize == 0) + p->dictSize = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) + ))); + if (p->dictSize > p->reduceSize) { unsigned i; @@ -74,8 +94,8 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); - if (p->numHashBytes < 0) p->numHashBytes = 4; - if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); + if (p->numHashBytes < 0) p->numHashBytes = (p->btMode ? 4 : 5); + if (p->mc == 0) p->mc = (16 + ((unsigned)p->fb >> 1)) >> (p->btMode ? 0 : 1); if (p->numThreads < 0) p->numThreads = @@ -93,7 +113,7 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) return props.dictSize; } -#if (_MSC_VER >= 1400) +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* BSR code is fast for some new CPUs */ /* #define LZMA_LOG_BSR */ #endif @@ -193,7 +213,7 @@ typedef struct #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 -#define kDicLogSizeMin 0 +// #define kDicLogSizeMin 0 #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) @@ -462,16 +482,16 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->dictSize = props.dictSize; { - unsigned fb = props.fb; + unsigned fb = (unsigned)props.fb; if (fb < 5) fb = 5; if (fb > LZMA_MATCH_LEN_MAX) fb = LZMA_MATCH_LEN_MAX; p->numFastBytes = fb; } - p->lc = props.lc; - p->lp = props.lp; - p->pb = props.pb; + p->lc = (unsigned)props.lc; + p->lp = (unsigned)props.lp; + p->pb = (unsigned)props.pb; p->fastMode = (props.algo == 0); // p->_maxMode = True; p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); @@ -479,17 +499,17 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) unsigned numHashBytes = 4; if (props.btMode) { - if (props.numHashBytes < 2) - numHashBytes = 2; - else if (props.numHashBytes < 4) - numHashBytes = props.numHashBytes; + if (props.numHashBytes < 2) numHashBytes = 2; + else if (props.numHashBytes < 4) numHashBytes = (unsigned)props.numHashBytes; } + if (props.numHashBytes >= 5) numHashBytes = 5; + p->matchFinderBase.numHashBytes = numHashBytes; } p->matchFinderBase.cutValue = props.mc; - p->writeEndMark = props.writeEndMark; + p->writeEndMark = (BoolInt)props.writeEndMark; #ifndef _7ZIP_ST /* @@ -500,6 +520,8 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) } */ p->multiThread = (props.numThreads > 1); + p->matchFinderMt.btSync.affinity = + p->matchFinderMt.hashSync.affinity = props.affinity; #endif return SZ_OK; @@ -536,8 +558,8 @@ static void RangeEnc_Construct(CRangeEnc *p) p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) -#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) +#define RangeEnc_GetProcessed(p) ( (p)->processed + (size_t)((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + (size_t)((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) @@ -578,7 +600,7 @@ MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) size_t num; if (p->res != SZ_OK) return; - num = p->buf - p->bufBase; + num = (size_t)(p->buf - p->bufBase); if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) p->res = SZ_ERROR_WRITE; p->processed += num; @@ -656,7 +678,7 @@ static void RangeEnc_FlushData(CRangeEnc *p) range += newBound & mask; \ mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ mask += ((1 << kNumMoveBits) - 1); \ - ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + ttt += (UInt32)((Int32)(mask - ttt) >> kNumMoveBits); \ *(prob) = (CLzmaProb)ttt; \ RC_NORM(p) \ } @@ -749,7 +771,7 @@ static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) bitCount++; } } - ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)(((unsigned)kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); // printf("\n%3d: %5d", i, ProbPrices[i]); } } @@ -1011,7 +1033,7 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; const Byte *p2 = p1 + len; - ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; + ptrdiff_t dif = (ptrdiff_t)-1 - (ptrdiff_t)p->matches[(size_t)numPairs - 1]; const Byte *lim = p1 + numAvail; for (; p2 != lim && *p2 == p2[dif]; p2++) {} @@ -2198,7 +2220,7 @@ MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) -void LzmaEnc_Construct(CLzmaEnc *p) +static void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); MatchFinder_Construct(&p->matchFinderBase); @@ -2233,7 +2255,7 @@ CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) return p; } -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) +static void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->litProbs); ISzAlloc_Free(alloc, p->saveState.litProbs); @@ -2241,7 +2263,7 @@ void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) p->saveState.litProbs = NULL; } -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { #ifndef _7ZIP_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); @@ -2259,6 +2281,7 @@ void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) } +MY_NO_INLINE static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; @@ -2521,12 +2544,12 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa // { int y; for (y = 0; y < 100; y++) { FillDistancesPrices(p); // }} - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); } if (p->repLenEncCounter <= 0) { p->repLenEncCounter = REP_LEN_COUNT; - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); } } @@ -2611,7 +2634,7 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, return SZ_OK; } -void LzmaEnc_Init(CLzmaEnc *p) +static void LzmaEnc_Init(CLzmaEnc *p) { unsigned i; p->state = 0; @@ -2675,12 +2698,12 @@ void LzmaEnc_Init(CLzmaEnc *p) p->additionalOffset = 0; - p->pbMask = (1 << p->pb) - 1; + p->pbMask = ((unsigned)1 << p->pb) - 1; p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } -void LzmaEnc_InitPrices(CLzmaEnc *p) +static void LzmaEnc_InitPrices(CLzmaEnc *p) { if (!p->fastMode) { @@ -2694,8 +2717,8 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->repLenEncCounter = REP_LEN_COUNT; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); } static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) @@ -2788,12 +2811,13 @@ static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, s } +/* UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) { const CLzmaEnc *p = (CLzmaEnc *)pp; return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); } - +*/ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) { @@ -2841,6 +2865,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, } +MY_NO_INLINE static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) { SRes res = SZ_OK; @@ -2899,14 +2924,14 @@ SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) if (dictSize >= ((UInt32)1 << 22)) { - UInt32 kDictMask = ((UInt32)1 << 20) - 1; + const UInt32 kDictMask = ((UInt32)1 << 20) - 1; if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) dictSize = (dictSize + kDictMask) & ~kDictMask; } else for (i = 11; i <= 30; i++) { - if (dictSize <= ((UInt32)2 << i)) { dictSize = (2 << i); break; } - if (dictSize <= ((UInt32)3 << i)) { dictSize = (3 << i); break; } + if (dictSize <= ((UInt32)2 << i)) { dictSize = ((UInt32)2 << i); break; } + if (dictSize <= ((UInt32)3 << i)) { dictSize = ((UInt32)3 << i); break; } } for (i = 0; i < 4; i++) @@ -2917,7 +2942,7 @@ SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) { - return ((CLzmaEnc *)pp)->writeEndMark; + return (unsigned)((CLzmaEnc *)pp)->writeEndMark; } @@ -2974,3 +2999,15 @@ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, LzmaEnc_Destroy(p, alloc, allocBig); return res; } + + +/* +#ifndef _7ZIP_ST +void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + lz_threads[0] = p->matchFinderMt.hashSync.thread; + lz_threads[1] = p->matchFinderMt.btSync.thread; +} +#endif +*/ diff --git a/C/LzmaEnc.h b/C/LzmaEnc.h index 9194ee57..bc2ed504 100644 --- a/C/LzmaEnc.h +++ b/C/LzmaEnc.h @@ -1,5 +1,5 @@ /* LzmaEnc.h -- LZMA Encoder -2017-07-27 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H @@ -29,6 +29,8 @@ typedef struct _CLzmaEncProps UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. Encoder uses this value to reduce dictionary size */ + + UInt64 affinity; } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p); diff --git a/C/LzmaLib.h b/C/LzmaLib.h index 88fa87d3..c343a859 100644 --- a/C/LzmaLib.h +++ b/C/LzmaLib.h @@ -1,5 +1,5 @@ /* LzmaLib.h -- LZMA library interface -2013-01-18 : Igor Pavlov : Public domain */ +2021-04-03 : Igor Pavlov : Public domain */ #ifndef __LZMA_LIB_H #define __LZMA_LIB_H @@ -40,14 +40,16 @@ outPropsSize - level - compression level: 0 <= level <= 9; level dictSize algo fb - 0: 16 KB 0 32 - 1: 64 KB 0 32 - 2: 256 KB 0 32 - 3: 1 MB 0 32 - 4: 4 MB 0 32 + 0: 64 KB 0 32 + 1: 256 KB 0 32 + 2: 1 MB 0 32 + 3: 4 MB 0 32 + 4: 16 MB 0 32 5: 16 MB 1 32 6: 32 MB 1 32 - 7+: 64 MB 1 64 + 7: 32 MB 1 64 + 8: 64 MB 1 64 + 9: 64 MB 1 64 The default value for "level" is 5. @@ -83,6 +85,11 @@ fb - Word size (the number of fast bytes). numThreads - The number of thereads. 1 or 2. The default value is 2. Fast mode (algo = 0) can use only 1 thread. +In: + dest - output data buffer + destLen - output data buffer size + src - input data + srcLen - input data size Out: destLen - processed output size Returns: @@ -108,8 +115,8 @@ MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char LzmaUncompress -------------- In: - dest - output data - destLen - output data size + dest - output data buffer + destLen - output data buffer size src - input data srcLen - input data size Out: diff --git a/C/MtCoder.c b/C/MtCoder.c index 95359857..17e33182 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,5 +1,5 @@ /* MtCoder.c -- Multi-thread Coder -2018-07-04 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,7 +7,7 @@ #ifndef _7ZIP_ST -SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) { CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); UInt64 inSize2 = 0; @@ -70,8 +70,7 @@ static void MtCoderThread_Destruct(CMtCoderThread *t) { t->stop = 1; Event_Set(&t->startEvent); - Thread_Wait(&t->thread); - Thread_Close(&t->thread); + Thread_Wait_Close(&t->thread); } Event_Close(&t->startEvent); @@ -342,7 +341,7 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) for (;;) { if (Event_Wait(&t->startEvent) != 0) - return SZ_ERROR_THREAD; + return (THREAD_FUNC_RET_TYPE)SZ_ERROR_THREAD; if (t->stop) return 0; { @@ -358,7 +357,7 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); if (numFinished == mtc->numStartedThreads) if (Event_Set(&mtc->finishedEvent) != 0) - return SZ_ERROR_THREAD; + return (THREAD_FUNC_RET_TYPE)SZ_ERROR_THREAD; } #endif } @@ -1,16 +1,21 @@ /* MtDec.c -- Multi-thread Decoder -2019-02-02 : Igor Pavlov : Public domain */ +2021-02-27 : Igor Pavlov : Public domain */ #include "Precomp.h" // #define SHOW_DEBUG_INFO // #include <stdio.h> +#include <string.h> #ifdef SHOW_DEBUG_INFO #include <stdio.h> #endif +#include "MtDec.h" + +#ifndef _7ZIP_ST + #ifdef SHOW_DEBUG_INFO #define PRF(x) x #else @@ -19,10 +24,6 @@ #define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) -#include "MtDec.h" - -#ifndef _7ZIP_ST - void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) { p->progress = progress; @@ -77,7 +78,7 @@ void MtProgress_SetError(CMtProgress *p, SRes res) } -#define RINOK_THREAD(x) RINOK(x) +#define RINOK_THREAD(x) RINOK_WRes(x) static WRes ArEvent_OptCreate_And_Reset(CEvent *p) @@ -156,8 +157,7 @@ static void MtDecThread_CloseThread(CMtDecThread *t) { Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ Event_Set(&t->canRead); - Thread_Wait(&t->thread); - Thread_Close(&t->thread); + Thread_Wait_Close(&t->thread); } Event_Close(&t->canRead); @@ -289,12 +289,13 @@ static WRes ThreadFunc2(CMtDecThread *t) Byte *afterEndData = NULL; size_t afterEndData_Size = 0; + BoolInt afterEndData_IsCross = False; BoolInt canCreateNewThread = False; // CMtDecCallbackInfo parse; CMtDecThread *nextThread; - PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + PRF_STR_INT("=============== Event_Wait(&t->canRead)", t->index); RINOK_THREAD(Event_Wait(&t->canRead)); if (p->exitThread) @@ -418,10 +419,12 @@ static WRes ThreadFunc2(CMtDecThread *t) parse.srcFinished = finish; parse.canCreateNewThread = True; - // PRF(printf("\nParse size = %d\n", (unsigned)size)) + PRF(printf("\nParse size = %d\n", (unsigned)size)); p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + PRF(printf(" Parse processed = %d, state = %d \n", (unsigned)parse.srcSize, (unsigned)parse.state)); + needWrite = True; canCreateNewThread = parse.canCreateNewThread; @@ -478,16 +481,12 @@ static WRes ThreadFunc2(CMtDecThread *t) if (parse.state == MTDEC_PARSE_END) { - p->crossStart = 0; - p->crossEnd = 0; - - if (crossSize != 0) - memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data - afterEndData_Size = size - parse.srcSize; afterEndData = parseData + parse.srcSize; - + afterEndData_Size = size - parse.srcSize; + if (crossSize != 0) + afterEndData_IsCross = True; // we reduce data size to required bytes (parsed only) - inDataSize -= (size - parse.srcSize); + inDataSize -= afterEndData_Size; if (!prev) inDataSize_Start = parse.srcSize; break; @@ -752,13 +751,15 @@ static WRes ThreadFunc2(CMtDecThread *t) { // p->inProcessed += inCodePos; + PRF(printf("\n--Write afterSize = %d\n", (unsigned)afterEndData_Size)); + res = p->mtCallback->Write(p->mtCallbackObject, t->index, res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite - afterEndData, afterEndData_Size, + afterEndData, afterEndData_Size, afterEndData_IsCross, &needContinue, &canRecode); - - // res= E_INVALIDARG; // for test + + // res = SZ_ERROR_FAIL; // for test PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); @@ -847,7 +848,7 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) res = ThreadFunc2(t); p = t->mtDec; if (res == 0) - return p->exitThreadWRes; + return (THREAD_FUNC_RET_TYPE)(UINT_PTR)p->exitThreadWRes; { // it's unexpected situation for some threading function error if (p->exitThreadWRes == 0) @@ -858,15 +859,14 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) Event_Set(&p->threads[0].canWrite); MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); } - return res; + return (THREAD_FUNC_RET_TYPE)(UINT_PTR)res; } static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) { + #ifdef USE_ALLOCA CMtDecThread *t = (CMtDecThread *)pp; - // fprintf(stderr, "\n%d = %p - before", t->index, &t); - #ifdef USE_ALLOCA t->allocaPtr = alloca(t->index * 128); #endif return ThreadFunc1(pp); @@ -1092,13 +1092,14 @@ SRes MtDec_Code(CMtDec *p) { WRes wres; - WRes sres; + SRes sres; CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; // wres = MtDecThread_CreateAndStart(nextThread); wres = MtDecThread_CreateEvents(nextThread); if (wres == 0) { wres = Event_Set(&nextThread->canWrite); if (wres == 0) { wres = Event_Set(&nextThread->canRead); - if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres == 0) { THREAD_FUNC_RET_TYPE res = ThreadFunc(nextThread); + wres = (WRes)(UINT_PTR)res; if (wres != 0) { p->needContinue = False; @@ -1130,8 +1131,8 @@ SRes MtDec_Code(CMtDec *p) return SZ_OK; // if (sres != SZ_OK) - return sres; - // return E_FAIL; + return sres; + // return SZ_ERROR_FAIL; } } @@ -1,5 +1,5 @@ /* MtDec.h -- Multi-thread Decoder -2018-07-04 : Igor Pavlov : Public domain */ +2020-03-05 : Igor Pavlov : Public domain */ #ifndef __MT_DEC_H #define __MT_DEC_H @@ -108,11 +108,12 @@ typedef struct */ SRes (*Write)(void *p, unsigned coderIndex, BoolInt needWriteToStream, - const Byte *src, size_t srcSize, + const Byte *src, size_t srcSize, BoolInt isCross, // int srcFinished, BoolInt *needContinue, BoolInt *canRecode); -} IMtDecCallback; + +} IMtDecCallback2; @@ -132,7 +133,7 @@ typedef struct _CMtDec ICompressProgress *progress; ISzAllocPtr alloc; - IMtDecCallback *mtCallback; + IMtDecCallback2 *mtCallback; void *mtCallbackObject; @@ -1,5 +1,5 @@ /* Ppmd.h -- PPMD codec common code -2017-04-03 : Igor Pavlov : Public domain +2021-04-13 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #ifndef __PPMD_H @@ -9,7 +9,16 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ EXTERN_C_BEGIN -#ifdef MY_CPU_32BIT +#if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) +/* + PPMD code always uses 32-bit internal fields in PPMD structures to store internal references in main block. + if (PPMD_32BIT is defined), the PPMD code stores internal pointers to 32-bit reference fields. + if (PPMD_32BIT is NOT defined), the PPMD code stores internal UInt32 offsets to reference fields. + if (pointer size is 64-bit), then (PPMD_32BIT) mode is not allowed, + if (pointer size is 32-bit), then (PPMD_32BIT) mode is optional, + and it's allowed to disable PPMD_32BIT mode even if pointer is 32-bit. + PPMD code works slightly faster in (PPMD_32BIT) mode. +*/ #define PPMD_32BIT #endif @@ -28,7 +37,7 @@ EXTERN_C_BEGIN #define PPMD_N4 ((128 + 3 - 1 * PPMD_N1 - 2 * PPMD_N2 - 3 * PPMD_N3) / 4) #define PPMD_NUM_INDEXES (PPMD_N1 + PPMD_N2 + PPMD_N3 + PPMD_N4) -#pragma pack(push, 1) +MY_CPU_pragma_pack_push_1 /* Most compilers works OK here even without #pragma pack(push, 1), but some GCC compilers need it. */ /* SEE-contexts for PPM-contexts with masked symbols */ @@ -40,41 +49,114 @@ typedef struct } CPpmd_See; #define Ppmd_See_Update(p) if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ - { (p)->Summ <<= 1; (p)->Count = (Byte)(3 << (p)->Shift++); } + { (p)->Summ = (UInt16)((p)->Summ << 1); (p)->Count = (Byte)(3 << (p)->Shift++); } + typedef struct { Byte Symbol; Byte Freq; - UInt16 SuccessorLow; - UInt16 SuccessorHigh; + UInt16 Successor_0; + UInt16 Successor_1; } CPpmd_State; -#pragma pack(pop) - -typedef - #ifdef PPMD_32BIT - CPpmd_State * - #else - UInt32 - #endif - CPpmd_State_Ref; - -typedef - #ifdef PPMD_32BIT - void * - #else - UInt32 - #endif - CPpmd_Void_Ref; - -typedef - #ifdef PPMD_32BIT - Byte * - #else - UInt32 - #endif - CPpmd_Byte_Ref; +typedef struct CPpmd_State2_ +{ + Byte Symbol; + Byte Freq; +} CPpmd_State2; + +typedef struct CPpmd_State4_ +{ + UInt16 Successor_0; + UInt16 Successor_1; +} CPpmd_State4; + +MY_CPU_pragma_pop + +/* + PPMD code can write full CPpmd_State structure data to CPpmd*_Context + at (byte offset = 2) instead of some fields of original CPpmd*_Context structure. + + If we use pointers to different types, but that point to shared + memory space, we can have aliasing problem (strict aliasing). + + XLC compiler in -O2 mode can change the order of memory write instructions + in relation to read instructions, if we have use pointers to different types. + + To solve that aliasing problem we use combined CPpmd*_Context structure + with unions that contain the fields from both structures: + the original CPpmd*_Context and CPpmd_State. + So we can access the fields from both structures via one pointer, + and the compiler doesn't change the order of write instructions + in relation to read instructions. + + If we don't use memory write instructions to shared memory in + some local code, and we use only reading instructions (read only), + then probably it's safe to use pointers to different types for reading. +*/ + + + +#ifdef PPMD_32BIT + + #define Ppmd_Ref_Type(type) type * + #define Ppmd_GetRef(p, ptr) (ptr) + #define Ppmd_GetPtr(p, ptr) (ptr) + #define Ppmd_GetPtr_Type(p, ptr, note_type) (ptr) + +#else + + #define Ppmd_Ref_Type(type) UInt32 + #define Ppmd_GetRef(p, ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) + #define Ppmd_GetPtr(p, offs) ((void *)((p)->Base + (offs))) + #define Ppmd_GetPtr_Type(p, offs, type) ((type *)Ppmd_GetPtr(p, offs)) + +#endif // PPMD_32BIT + + +typedef Ppmd_Ref_Type(CPpmd_State) CPpmd_State_Ref; +typedef Ppmd_Ref_Type(void) CPpmd_Void_Ref; +typedef Ppmd_Ref_Type(Byte) CPpmd_Byte_Ref; + + +/* +#ifdef MY_CPU_LE_UNALIGN +// the unaligned 32-bit access latency can be too large, if the data is not in L1 cache. +#define Ppmd_GET_SUCCESSOR(p) ((CPpmd_Void_Ref)*(const UInt32 *)(const void *)&(p)->Successor_0) +#define Ppmd_SET_SUCCESSOR(p, v) *(UInt32 *)(void *)(void *)&(p)->Successor_0 = (UInt32)(v) + +#else +*/ + +/* + We can write 16-bit halves to 32-bit (Successor) field in any selected order. + But the native order is more consistent way. + So we use the native order, if LE/BE order can be detected here at compile time. +*/ + +#ifdef MY_CPU_BE + + #define Ppmd_GET_SUCCESSOR(p) \ + ( (CPpmd_Void_Ref) (((UInt32)(p)->Successor_0 << 16) | (p)->Successor_1) ) + + #define Ppmd_SET_SUCCESSOR(p, v) { \ + (p)->Successor_0 = (UInt16)(((UInt32)(v) >> 16) /* & 0xFFFF */); \ + (p)->Successor_1 = (UInt16)((UInt32)(v) /* & 0xFFFF */); } + +#else + + #define Ppmd_GET_SUCCESSOR(p) \ + ( (CPpmd_Void_Ref) ((p)->Successor_0 | ((UInt32)(p)->Successor_1 << 16)) ) + + #define Ppmd_SET_SUCCESSOR(p, v) { \ + (p)->Successor_0 = (UInt16)((UInt32)(v) /* & 0xFFFF */); \ + (p)->Successor_1 = (UInt16)(((UInt32)(v) >> 16) /* & 0xFFFF */); } + +#endif + +// #endif + #define PPMD_SetAllBitsIn256Bytes(p) \ { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2018-07-04 : Igor Pavlov : Public domain +2021-04-13 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,7 +8,12 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Ppmd7.h" -const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +/* define PPMD7_ORDER_0_SUPPPORT to suport order-0 mode, unsupported by orignal PPMd var.H. code */ +// #define PPMD7_ORDER_0_SUPPPORT + +MY_ALIGN(16) +static const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +MY_ALIGN(16) static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; #define MAX_FREQ 124 @@ -16,13 +21,10 @@ static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) #define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) -#define I2U(indx) (p->Indx2Units[indx]) +#define I2U(indx) ((unsigned)p->Indx2Units[indx]) +#define I2U_UInt16(indx) ((UInt16)p->Indx2Units[indx]) -#ifdef PPMD_32BIT - #define REF(ptr) (ptr) -#else - #define REF(ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) -#endif +#define REF(ptr) Ppmd_GetRef(p, ptr) #define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) @@ -35,13 +37,7 @@ typedef CPpmd7_Context * CTX_PTR; struct CPpmd7_Node_; -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Node_ * - #else - UInt32 - #endif - CPpmd7_Node_Ref; +typedef Ppmd_Ref_Type(struct CPpmd7_Node_) CPpmd7_Node_Ref; typedef struct CPpmd7_Node_ { @@ -51,17 +47,13 @@ typedef struct CPpmd7_Node_ CPpmd7_Node_Ref Prev; } CPpmd7_Node; -#ifdef PPMD_32BIT - #define NODE(ptr) (ptr) -#else - #define NODE(offs) ((CPpmd7_Node *)(p->Base + (offs))) -#endif +#define NODE(r) Ppmd_GetPtr_Type(p, r, CPpmd7_Node) void Ppmd7_Construct(CPpmd7 *p) { unsigned i, k, m; - p->Base = 0; + p->Base = NULL; for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) { @@ -77,6 +69,7 @@ void Ppmd7_Construct(CPpmd7 *p) for (i = 0; i < 3; i++) p->NS2Indx[i] = (Byte)i; + for (m = i, k = 1; i < 256; i++) { p->NS2Indx[i] = (Byte)m; @@ -84,54 +77,63 @@ void Ppmd7_Construct(CPpmd7 *p) k = (++m) - 2; } - memset(p->HB2Flag, 0, 0x40); - memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); + memcpy(p->ExpEscape, PPMD7_kExpEscape, 16); } + void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->Base); p->Size = 0; - p->Base = 0; + p->Base = NULL; } + BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { - size_t size2; Ppmd7_Free(p, alloc); - size2 = 0 - #ifndef PPMD_32BIT - + UNIT_SIZE - #endif - ; - p->AlignOffset = - #ifdef PPMD_32BIT - (4 - size) & 3; - #else - 4 - (size & 3); - #endif - if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size + size2)) == 0) + p->AlignOffset = (4 - size) & 3; + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL) return False; p->Size = size; } return True; } + + +// ---------- Internal Memory Allocator ---------- + +/* We can use CPpmd7_Node in list of free units (as in Ppmd8) + But we still need one additional list walk pass in GlueFreeBlocks(). + So we use simple CPpmd_Void_Ref instead of CPpmd7_Node in InsertNode() / RemoveNode() +*/ + +#define EMPTY_NODE 0 + + static void InsertNode(CPpmd7 *p, void *node, unsigned indx) { *((CPpmd_Void_Ref *)node) = p->FreeList[indx]; + // ((CPpmd7_Node *)node)->Next = (CPpmd7_Node_Ref)p->FreeList[indx]; + p->FreeList[indx] = REF(node); + } + static void *RemoveNode(CPpmd7 *p, unsigned indx) { CPpmd_Void_Ref *node = (CPpmd_Void_Ref *)Ppmd7_GetPtr(p, p->FreeList[indx]); p->FreeList[indx] = *node; + // CPpmd7_Node *node = NODE((CPpmd7_Node_Ref)p->FreeList[indx]); + // p->FreeList[indx] = node->Next; return node; } + static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) { unsigned i, nu = I2U(oldIndx) - I2U(newIndx); @@ -144,123 +146,167 @@ static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) InsertNode(p, ptr, i); } -static void GlueFreeBlocks(CPpmd7 *p) + +/* we use CPpmd7_Node_Union union to solve XLC -O2 strict pointer aliasing problem */ + +typedef union _CPpmd7_Node_Union { - #ifdef PPMD_32BIT - CPpmd7_Node headItem; - CPpmd7_Node_Ref head = &headItem; - #else - CPpmd7_Node_Ref head = p->AlignOffset + p->Size; - #endif - - CPpmd7_Node_Ref n = head; - unsigned i; + CPpmd7_Node Node; + CPpmd7_Node_Ref NextRef; +} CPpmd7_Node_Union; + +/* Original PPmdH (Ppmd7) code uses doubly linked list in GlueFreeBlocks() + we use single linked list similar to Ppmd8 code */ + +static void GlueFreeBlocks(CPpmd7 *p) +{ + /* + we use first UInt16 field of 12-bytes UNITs as record type stamp + CPpmd_State { Byte Symbol; Byte Freq; : Freq != 0 + CPpmd7_Context { UInt16 NumStats; : NumStats != 0 + CPpmd7_Node { UInt16 Stamp : Stamp == 0 for free record + : Stamp == 1 for head record and guard + Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd7_Context record. + */ + CPpmd7_Node_Ref head, n = 0; + p->GlueCount = 255; - /* create doubly-linked list of free blocks */ - for (i = 0; i < PPMD_NUM_INDEXES; i++) + + /* we set guard NODE at LoUnit */ + if (p->LoUnit != p->HiUnit) + ((CPpmd7_Node *)(void *)p->LoUnit)->Stamp = 1; + { - UInt16 nu = I2U(i); - CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; - p->FreeList[i] = 0; - while (next != 0) + /* Create list of free blocks. + We still need one additional list walk pass before Glue. */ + unsigned i; + for (i = 0; i < PPMD_NUM_INDEXES; i++) { - CPpmd7_Node *node = NODE(next); - node->Next = n; - n = NODE(n)->Prev = next; - next = *(const CPpmd7_Node_Ref *)node; - node->Stamp = 0; - node->NU = (UInt16)nu; + const UInt16 nu = I2U_UInt16(i); + CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; + p->FreeList[i] = 0; + while (next != 0) + { + /* Don't change the order of the following commands: */ + CPpmd7_Node_Union *un = (CPpmd7_Node_Union *)NODE(next); + const CPpmd7_Node_Ref tmp = next; + next = un->NextRef; + un->Node.Stamp = EMPTY_NODE; + un->Node.NU = nu; + un->Node.Next = n; + n = tmp; + } } } - NODE(head)->Stamp = 1; - NODE(head)->Next = n; - NODE(n)->Prev = head; - if (p->LoUnit != p->HiUnit) - ((CPpmd7_Node *)p->LoUnit)->Stamp = 1; - - /* Glue free blocks */ - while (n != head) + + head = n; + /* Glue and Fill must walk the list in same direction */ { - CPpmd7_Node *node = NODE(n); - UInt32 nu = (UInt32)node->NU; - for (;;) + /* Glue free blocks */ + CPpmd7_Node_Ref *prev = &head; + while (n) { - CPpmd7_Node *node2 = NODE(n) + nu; - nu += node2->NU; - if (node2->Stamp != 0 || nu >= 0x10000) - break; - NODE(node2->Prev)->Next = node2->Next; - NODE(node2->Next)->Prev = node2->Prev; - node->NU = (UInt16)nu; + CPpmd7_Node *node = NODE(n); + UInt32 nu = node->NU; + n = node->Next; + if (nu == 0) + { + *prev = n; + continue; + } + prev = &node->Next; + for (;;) + { + CPpmd7_Node *node2 = node + nu; + nu += node2->NU; + if (node2->Stamp != EMPTY_NODE || nu >= 0x10000) + break; + node->NU = (UInt16)nu; + node2->NU = 0; + } } - n = node->Next; } - + /* Fill lists of free blocks */ - for (n = NODE(head)->Next; n != head;) + for (n = head; n != 0;) { CPpmd7_Node *node = NODE(n); - unsigned nu; - CPpmd7_Node_Ref next = node->Next; - for (nu = node->NU; nu > 128; nu -= 128, node += 128) + UInt32 nu = node->NU; + unsigned i; + n = node->Next; + if (nu == 0) + continue; + for (; nu > 128; nu -= 128, node += 128) InsertNode(p, node, PPMD_NUM_INDEXES - 1); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, node + k, nu - k - 1); + InsertNode(p, node + k, (unsigned)nu - k - 1); } InsertNode(p, node, i); - n = next; } } + +MY_NO_INLINE static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) { unsigned i; - void *retVal; + if (p->GlueCount == 0) { GlueFreeBlocks(p); if (p->FreeList[indx] != 0) return RemoveNode(p, indx); } + i = indx; + do { if (++i == PPMD_NUM_INDEXES) { UInt32 numBytes = U2B(I2U(indx)); + Byte *us = p->UnitsStart; p->GlueCount--; - return ((UInt32)(p->UnitsStart - p->Text) > numBytes) ? (p->UnitsStart -= numBytes) : (NULL); + return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : NULL; } } while (p->FreeList[i] == 0); - retVal = RemoveNode(p, i); - SplitBlock(p, retVal, i, indx); - return retVal; + + { + void *block = RemoveNode(p, i); + SplitBlock(p, block, i, indx); + return block; + } } + static void *AllocUnits(CPpmd7 *p, unsigned indx) { - UInt32 numBytes; if (p->FreeList[indx] != 0) return RemoveNode(p, indx); - numBytes = U2B(I2U(indx)); - if (numBytes <= (UInt32)(p->HiUnit - p->LoUnit)) { - void *retVal = p->LoUnit; - p->LoUnit += numBytes; - return retVal; + UInt32 numBytes = U2B(I2U(indx)); + Byte *lo = p->LoUnit; + if ((UInt32)(p->HiUnit - lo) >= numBytes) + { + p->LoUnit = lo + numBytes; + return lo; + } } return AllocUnitsRare(p, indx); } + #define MyMem12Cpy(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *s = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; s += 3; d += 3; } while (--n); } + { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ + do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } + +/* static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU) { unsigned i0 = U2I(oldNU); @@ -277,20 +323,25 @@ static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU SplitBlock(p, oldPtr, i0, i1); return oldPtr; } +*/ -#define SUCCESSOR(p) ((CPpmd_Void_Ref)((p)->SuccessorLow | ((UInt32)(p)->SuccessorHigh << 16))) +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) { - (p)->SuccessorLow = (UInt16)((UInt32)(v) & 0xFFFF); - (p)->SuccessorHigh = (UInt16)(((UInt32)(v) >> 16) & 0xFFFF); + Ppmd_SET_SUCCESSOR(p, v); } -static void RestartModel(CPpmd7 *p) + + +MY_NO_INLINE +static +void RestartModel(CPpmd7 *p) { - unsigned i, k, m; + unsigned i, k; memset(p->FreeList, 0, sizeof(p->FreeList)); + p->Text = p->Base + p->AlignOffset; p->HiUnit = p->Text + p->Size; p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; @@ -300,57 +351,110 @@ static void RestartModel(CPpmd7 *p) p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; p->PrevSuccess = 0; - p->MinContext = p->MaxContext = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - p->MinContext->Suffix = 0; - p->MinContext->NumStats = 256; - p->MinContext->SummFreq = 256 + 1; - p->FoundState = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ - p->LoUnit += U2B(256 / 2); - p->MinContext->Stats = REF(p->FoundState); - for (i = 0; i < 256; i++) { - CPpmd_State *s = &p->FoundState[i]; - s->Symbol = (Byte)i; - s->Freq = 1; - SetSuccessor(s, 0); + CPpmd7_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + + p->LoUnit += U2B(256 / 2); + p->MaxContext = p->MinContext = mc; + p->FoundState = s; + + mc->NumStats = 256; + mc->Union2.SummFreq = 256 + 1; + mc->Union4.Stats = REF(s); + mc->Suffix = 0; + + for (i = 0; i < 256; i++, s++) + { + s->Symbol = (Byte)i; + s->Freq = 1; + SetSuccessor(s, 0); + } + + #ifdef PPMD7_ORDER_0_SUPPPORT + if (p->MaxOrder == 0) + { + CPpmd_Void_Ref r = REF(mc); + s = p->FoundState; + for (i = 0; i < 256; i++, s++) + SetSuccessor(s, r); + return; + } + #endif } for (i = 0; i < 128; i++) + + + for (k = 0; k < 8; k++) { + unsigned m; UInt16 *dest = p->BinSumm[i] + k; UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 2)); for (m = 0; m < 64; m += 8) dest[m] = val; } - + + for (i = 0; i < 25; i++) - for (k = 0; k < 16; k++) + { + + CPpmd_See *s = p->See[i]; + + + + unsigned summ = ((5 * i + 10) << (PPMD_PERIOD_BITS - 4)); + for (k = 0; k < 16; k++, s++) { - CPpmd_See *s = &p->See[i][k]; - s->Summ = (UInt16)((5 * i + 10) << (s->Shift = PPMD_PERIOD_BITS - 4)); + s->Summ = (UInt16)summ; + s->Shift = (PPMD_PERIOD_BITS - 4); s->Count = 4; } + } + + p->DummySee.Summ = 0; /* unused */ + p->DummySee.Shift = PPMD_PERIOD_BITS; + p->DummySee.Count = 64; /* unused */ } + void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) { p->MaxOrder = maxOrder; + RestartModel(p); - p->DummySee.Shift = PPMD_PERIOD_BITS; - p->DummySee.Summ = 0; /* unused */ - p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) + + +/* + CreateSuccessors() + It's called when (FoundState->Successor) is RAW-Successor, + that is the link to position in Raw text. + So we create Context records and write the links to + FoundState->Successor and to identical RAW-Successors in suffix + contexts of MinContex. + + The function returns: + if (OrderFall == 0) then MinContext is already at MAX order, + { return pointer to new or existing context of same MAX order } + else + { return pointer to new real context that will be (Order+1) in comparison with MinContext + + also it can return pointer to real context of same order, +*/ + +MY_NO_INLINE +static CTX_PTR CreateSuccessors(CPpmd7 *p) { - CPpmd_State upState; CTX_PTR c = p->MinContext; CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); - CPpmd_State *ps[PPMD7_MAX_ORDER]; + Byte newSym, newFreq; unsigned numPs = 0; - - if (!skip) + CPpmd_State *ps[PPMD7_MAX_ORDER]; + + if (p->OrderFall != 0) ps[numPs++] = p->FoundState; while (c->Suffix) @@ -358,44 +462,70 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) CPpmd_Void_Ref successor; CPpmd_State *s; c = SUFFIX(c); + + if (c->NumStats != 1) { - for (s = STATS(c); s->Symbol != p->FoundState->Symbol; s++); + Byte sym = p->FoundState->Symbol; + for (s = STATS(c); s->Symbol != sym; s++); + } else + { s = ONE_STATE(c); + + } successor = SUCCESSOR(s); if (successor != upBranch) { + // (c) is real record Context here, c = CTX(successor); if (numPs == 0) + { + // (c) is real record MAX Order Context here, + // So we don't need to create any new contexts. return c; + } break; } ps[numPs++] = s; } - upState.Symbol = *(const Byte *)Ppmd7_GetPtr(p, upBranch); - SetSuccessor(&upState, upBranch + 1); + // All created contexts will have single-symbol with new RAW-Successor + // All new RAW-Successors will point to next position in RAW text + // after FoundState->Successor + + newSym = *(const Byte *)Ppmd7_GetPtr(p, upBranch); + upBranch++; + if (c->NumStats == 1) - upState.Freq = ONE_STATE(c)->Freq; + newFreq = ONE_STATE(c)->Freq; else { UInt32 cf, s0; CPpmd_State *s; - for (s = STATS(c); s->Symbol != upState.Symbol; s++); - cf = s->Freq - 1; - s0 = c->SummFreq - c->NumStats - cf; - upState.Freq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((2 * cf + 3 * s0 - 1) / (2 * s0)))); + for (s = STATS(c); s->Symbol != newSym; s++); + cf = (UInt32)s->Freq - 1; + s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf; + /* + cf - is frequency of symbol that will be Successor in new context records. + s0 - is commulative frequency sum of another symbols from parent context. + max(newFreq)= (s->Freq + 1), when (s0 == 1) + we have requirement (Ppmd7Context_OneState()->Freq <= 128) in BinSumm[] + so (s->Freq < 128) - is requirement for multi-symbol contexts + */ + newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : (2 * cf + s0 - 1) / (2 * s0) + 1)); } + // Create new single-symbol contexts from low order to high order in loop + do { - /* Create Child */ - CTX_PTR c1; /* = AllocContext(p); */ + CTX_PTR c1; + /* = AllocContext(p); */ if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); + c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); else if (p->FreeList[0] != 0) c1 = (CTX_PTR)RemoveNode(p, 0); else @@ -404,8 +534,11 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) if (!c1) return NULL; } + c1->NumStats = 1; - *ONE_STATE(c1) = upState; + ONE_STATE(c1)->Symbol = newSym; + ONE_STATE(c1)->Freq = newFreq; + SetSuccessor(ONE_STATE(c1), upBranch); c1->Suffix = REF(c); SetSuccessor(ps[--numPs], REF(c1)); c = c1; @@ -415,21 +548,26 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) return c; } -static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) -{ - CPpmd_State tmp = *t1; - *t1 = *t2; - *t2 = tmp; -} -static void UpdateModel(CPpmd7 *p) + +#define SwapStates(s) \ + { CPpmd_State tmp = s[0]; s[0] = s[-1]; s[-1] = tmp; } + + +void Ppmd7_UpdateModel(CPpmd7 *p); +MY_NO_INLINE +void Ppmd7_UpdateModel(CPpmd7 *p) { - CPpmd_Void_Ref successor, fSuccessor = SUCCESSOR(p->FoundState); - CTX_PTR c; + CPpmd_Void_Ref maxSuccessor, minSuccessor; + CTX_PTR c, mc; unsigned s0, ns; - + + + if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) { + /* Update Freqs in Suffix Context */ + c = SUFFIX(p->MinContext); if (c->NumStats == 1) @@ -441,27 +579,39 @@ static void UpdateModel(CPpmd7 *p) else { CPpmd_State *s = STATS(c); - if (s->Symbol != p->FoundState->Symbol) + Byte sym = p->FoundState->Symbol; + + if (s->Symbol != sym) { - do { s++; } while (s->Symbol != p->FoundState->Symbol); + do + { + // s++; if (s->Symbol == sym) break; + s++; + } + while (s->Symbol != sym); + if (s[0].Freq >= s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SwapStates(s); s--; } } + if (s->Freq < MAX_FREQ - 9) { - s->Freq += 2; - c->SummFreq += 2; + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); } } } + if (p->OrderFall == 0) { - p->MinContext = p->MaxContext = CreateSuccessors(p, True); - if (p->MinContext == 0) + /* MAX ORDER context */ + /* (FoundState->Successor) is RAW-Successor. */ + p->MaxContext = p->MinContext = CreateSuccessors(p); + if (!p->MinContext) { RestartModel(p); return; @@ -469,45 +619,93 @@ static void UpdateModel(CPpmd7 *p) SetSuccessor(p->FoundState, REF(p->MinContext)); return; } + + + /* NON-MAX ORDER context */ - *p->Text++ = p->FoundState->Symbol; - successor = REF(p->Text); - if (p->Text >= p->UnitsStart) { - RestartModel(p); - return; + Byte *text = p->Text; + *text++ = p->FoundState->Symbol; + p->Text = text; + if (text >= p->UnitsStart) + { + RestartModel(p); + return; + } + maxSuccessor = REF(text); } - if (fSuccessor) + minSuccessor = SUCCESSOR(p->FoundState); + + if (minSuccessor) { - if (fSuccessor <= successor) + // there is Successor for FoundState in MinContext. + // So the next context will be one order higher than MinContext. + + if (minSuccessor <= maxSuccessor) { - CTX_PTR cs = CreateSuccessors(p, False); - if (cs == NULL) + // minSuccessor is RAW-Successor. So we will create real contexts records: + CTX_PTR cs = CreateSuccessors(p); + if (!cs) { RestartModel(p); return; } - fSuccessor = REF(cs); + minSuccessor = REF(cs); } + + // minSuccessor now is real Context pointer that points to existing (Order+1) context + if (--p->OrderFall == 0) { - successor = fSuccessor; + /* + if we move to MaxOrder context, then minSuccessor will be common Succesor for both: + MinContext that is (MaxOrder - 1) + MaxContext that is (MaxOrder) + so we don't need new RAW-Successor, and we can use real minSuccessor + as succssors for both MinContext and MaxContext. + */ + maxSuccessor = minSuccessor; + + /* + if (MaxContext != MinContext) + { + there was order fall from MaxOrder and we don't need current symbol + to transfer some RAW-Succesors to real contexts. + So we roll back pointer in raw data for one position. + } + */ p->Text -= (p->MaxContext != p->MinContext); } } else { - SetSuccessor(p->FoundState, successor); - fSuccessor = REF(p->MinContext); + /* + FoundState has NULL-Successor here. + And only root 0-order context can contain NULL-Successors. + We change Successor in FoundState to RAW-Successor, + And next context will be same 0-order root Context. + */ + SetSuccessor(p->FoundState, maxSuccessor); + minSuccessor = REF(p->MinContext); } - - s0 = p->MinContext->SummFreq - (ns = p->MinContext->NumStats) - (p->FoundState->Freq - 1); - - for (c = p->MaxContext; c != p->MinContext; c = SUFFIX(c)) + + mc = p->MinContext; + c = p->MaxContext; + + p->MaxContext = p->MinContext = CTX(minSuccessor); + + if (c == mc) + return; + + // s0 : is pure Escape Freq + s0 = mc->Union2.SummFreq - (ns = mc->NumStats) - ((unsigned)p->FoundState->Freq - 1); + + do { unsigned ns1; - UInt32 cf, sf; + UInt32 sum; + if ((ns1 = c->NumStats) != 1) { if ((ns1 & 1) == 0) @@ -527,80 +725,127 @@ static void UpdateModel(CPpmd7 *p) oldPtr = STATS(c); MyMem12Cpy(ptr, oldPtr, oldNU); InsertNode(p, oldPtr, i); - c->Stats = STATS_REF(ptr); + c->Union4.Stats = STATS_REF(ptr); } } - c->SummFreq = (UInt16)(c->SummFreq + (2 * ns1 < ns) + 2 * ((4 * ns1 <= ns) & (c->SummFreq <= 8 * ns1))); + sum = c->Union2.SummFreq; + /* max increase of Escape_Freq is 3 here. + total increase of Union2.SummFreq for all symbols is less than 256 here */ + sum += (UInt32)(2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1)); + /* original PPMdH uses 16-bit variable for (sum) here. + But (sum < 0x9000). So we don't truncate (sum) to 16-bit */ + // sum = (UInt16)sum; } else { + // instead of One-symbol context we create 2-symbol context CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); if (!s) { RestartModel(p); return; } - *s = *ONE_STATE(c); - c->Stats = REF(s); - if (s->Freq < MAX_FREQ / 4 - 1) - s->Freq <<= 1; - else - s->Freq = MAX_FREQ - 4; - c->SummFreq = (UInt16)(s->Freq + p->InitEsc + (ns > 3)); - } - cf = 2 * (UInt32)p->FoundState->Freq * (c->SummFreq + 6); - sf = (UInt32)s0 + c->SummFreq; - if (cf < 6 * sf) - { - cf = 1 + (cf > sf) + (cf >= 4 * sf); - c->SummFreq += 3; - } - else - { - cf = 4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); - c->SummFreq = (UInt16)(c->SummFreq + cf); + { + unsigned freq = c->Union2.State2.Freq; + // s = *ONE_STATE(c); + s->Symbol = c->Union2.State2.Symbol; + s->Successor_0 = c->Union4.State4.Successor_0; + s->Successor_1 = c->Union4.State4.Successor_1; + // SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of + // (Successor_0 and Successor_1) in LE/BE. + c->Union4.Stats = REF(s); + if (freq < MAX_FREQ / 4 - 1) + freq <<= 1; + else + freq = MAX_FREQ - 4; + // (max(s->freq) == 120), when we convert from 1-symbol into 2-symbol context + s->Freq = (Byte)freq; + // max(InitEsc = PPMD7_kExpEscape[*]) is 25. So the max(escapeFreq) is 26 here + sum = freq + p->InitEsc + (ns > 3); + } } + { CPpmd_State *s = STATS(c) + ns1; - SetSuccessor(s, successor); + UInt32 cf = 2 * (sum + 6) * (UInt32)p->FoundState->Freq; + UInt32 sf = (UInt32)s0 + sum; s->Symbol = p->FoundState->Symbol; - s->Freq = (Byte)cf; c->NumStats = (UInt16)(ns1 + 1); + SetSuccessor(s, maxSuccessor); + + if (cf < 6 * sf) + { + cf = (UInt32)1 + (cf > sf) + (cf >= 4 * sf); + sum += 3; + /* It can add (0, 1, 2) to Escape_Freq */ + } + else + { + cf = (UInt32)4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); + sum += cf; + } + + c->Union2.SummFreq = (UInt16)sum; + s->Freq = (Byte)cf; } + c = SUFFIX(c); } - p->MaxContext = p->MinContext = CTX(fSuccessor); + while (c != mc); } + + +MY_NO_INLINE static void Rescale(CPpmd7 *p) { unsigned i, adder, sumFreq, escFreq; CPpmd_State *stats = STATS(p->MinContext); CPpmd_State *s = p->FoundState; + + /* Sort the list by Freq */ + if (s != stats) { CPpmd_State tmp = *s; - for (; s != stats; s--) + do s[0] = s[-1]; + while (--s != stats); *s = tmp; } - escFreq = p->MinContext->SummFreq - s->Freq; - s->Freq += 4; - adder = (p->OrderFall != 0); - s->Freq = (Byte)((s->Freq + adder) >> 1); + sumFreq = s->Freq; + escFreq = p->MinContext->Union2.SummFreq - sumFreq; + + /* + if (p->OrderFall == 0), adder = 0 : it's allowed to remove symbol from MAX Order context + if (p->OrderFall != 0), adder = 1 : it's NOT allowed to remove symbol from NON-MAX Order context + */ + + adder = (p->OrderFall != 0); + + #ifdef PPMD7_ORDER_0_SUPPPORT + adder |= (p->MaxOrder == 0); // we don't remove symbols from order-0 context + #endif + + sumFreq = (sumFreq + 4 + adder) >> 1; + i = (unsigned)p->MinContext->NumStats - 1; + s->Freq = (Byte)sumFreq; - i = p->MinContext->NumStats - 1; do { - escFreq -= (++s)->Freq; - s->Freq = (Byte)((s->Freq + adder) >> 1); - sumFreq += s->Freq; - if (s[0].Freq > s[-1].Freq) + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + adder) >> 1; + sumFreq += freq; + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { + CPpmd_State tmp = *s; CPpmd_State *s1 = s; - CPpmd_State tmp = *s1; do + { s1[0] = s1[-1]; - while (--s1 != stats && tmp.Freq > s1[-1].Freq); + } + while (--s1 != stats && freq > s1[-1].Freq); *s1 = tmp; } } @@ -608,47 +853,89 @@ static void Rescale(CPpmd7 *p) if (s->Freq == 0) { - unsigned numStats = p->MinContext->NumStats; - unsigned n0, n1; - do { i++; } while ((--s)->Freq == 0); + /* Remove all items with Freq == 0 */ + CPpmd7_Context *mc; + unsigned numStats, numStatsNew, n0, n1; + + i = 0; do { i++; } while ((--s)->Freq == 0); + + /* We increase (escFreq) for the number of removed symbols. + So we will have (0.5) increase for Escape_Freq in avarage per + removed symbol after Escape_Freq halving */ escFreq += i; - p->MinContext->NumStats = (UInt16)(p->MinContext->NumStats - i); - if (p->MinContext->NumStats == 1) + mc = p->MinContext; + numStats = mc->NumStats; + numStatsNew = numStats - i; + mc->NumStats = (UInt16)(numStatsNew); + n0 = (numStats + 1) >> 1; + + if (numStatsNew == 1) { - CPpmd_State tmp = *stats; + /* Create Single-Symbol context */ + unsigned freq = stats->Freq; + do { - tmp.Freq = (Byte)(tmp.Freq - (tmp.Freq >> 1)); escFreq >>= 1; + freq = (freq + 1) >> 1; } while (escFreq > 1); - InsertNode(p, stats, U2I(((numStats + 1) >> 1))); - *(p->FoundState = ONE_STATE(p->MinContext)) = tmp; + + s = ONE_STATE(mc); + *s = *stats; + s->Freq = (Byte)freq; // (freq <= 260 / 4) + p->FoundState = s; + InsertNode(p, stats, U2I(n0)); return; } - n0 = (numStats + 1) >> 1; - n1 = (p->MinContext->NumStats + 1) >> 1; + + n1 = (numStatsNew + 1) >> 1; if (n0 != n1) - p->MinContext->Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + { + // p->MinContext->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + unsigned i0 = U2I(n0); + unsigned i1 = U2I(n1); + if (i0 != i1) + { + if (p->FreeList[i1] != 0) + { + void *ptr = RemoveNode(p, i1); + p->MinContext->Union4.Stats = STATS_REF(ptr); + MyMem12Cpy(ptr, (const void *)stats, n1); + InsertNode(p, stats, i0); + } + else + SplitBlock(p, stats, i0, i1); + } + } + } + { + CPpmd7_Context *mc = p->MinContext; + mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); + // Escape_Freq halving here + p->FoundState = STATS(mc); } - p->MinContext->SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); - p->FoundState = STATS(p->MinContext); } + CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) { CPpmd_See *see; - unsigned nonMasked = p->MinContext->NumStats - numMasked; - if (p->MinContext->NumStats != 256) + const CPpmd7_Context *mc = p->MinContext; + unsigned numStats = mc->NumStats; + if (numStats != 256) { - see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + - (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + - 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + - 4 * (unsigned)(numMasked > nonMasked) + + unsigned nonMasked = numStats - numMasked; + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + + (nonMasked < (unsigned)SUFFIX(mc)->NumStats - numStats) + + 2 * (unsigned)(mc->Union2.SummFreq < 11 * numStats) + + 4 * (unsigned)(numMasked > nonMasked) + p->HiBitsFlag; { - unsigned r = (see->Summ >> see->Shift); - see->Summ = (UInt16)(see->Summ - r); + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + unsigned summ = (UInt16)see->Summ; // & 0xFFFF + unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); *escFreq = r + (r == 0); } } @@ -660,53 +947,158 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) return see; } + static void NextContext(CPpmd7 *p) { CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); - if (p->OrderFall == 0 && (Byte *)c > p->Text) - p->MinContext = p->MaxContext = c; + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; else - UpdateModel(p); + Ppmd7_UpdateModel(p); } + void Ppmd7_Update1(CPpmd7 *p) { CPpmd_State *s = p->FoundState; - s->Freq += 4; - p->MinContext->SummFreq += 4; - if (s[0].Freq > s[-1].Freq) + unsigned freq = s->Freq; + freq += 4; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SwapStates(s); p->FoundState = --s; - if (s->Freq > MAX_FREQ) + if (freq > MAX_FREQ) Rescale(p); } NextContext(p); } + void Ppmd7_Update1_0(CPpmd7 *p) { - p->PrevSuccess = (2 * p->FoundState->Freq > p->MinContext->SummFreq); - p->RunLength += p->PrevSuccess; - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) + CPpmd_State *s = p->FoundState; + CPpmd7_Context *mc = p->MinContext; + unsigned freq = s->Freq; + unsigned summFreq = mc->Union2.SummFreq; + p->PrevSuccess = (2 * freq > summFreq); + p->RunLength += (int)p->PrevSuccess; + mc->Union2.SummFreq = (UInt16)(summFreq + 4); + freq += 4; + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) Rescale(p); NextContext(p); } + +/* void Ppmd7_UpdateBin(CPpmd7 *p) { - p->FoundState->Freq = (Byte)(p->FoundState->Freq + (p->FoundState->Freq < 128 ? 1: 0)); + unsigned freq = p->FoundState->Freq; + p->FoundState->Freq = (Byte)(freq + (freq < 128)); p->PrevSuccess = 1; p->RunLength++; NextContext(p); } +*/ void Ppmd7_Update2(CPpmd7 *p) { - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) - Rescale(p); + CPpmd_State *s = p->FoundState; + unsigned freq = s->Freq; + freq += 4; p->RunLength = p->InitRL; - UpdateModel(p); + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) + Rescale(p); + Ppmd7_UpdateModel(p); +} + + + +/* +PPMd Memory Map: +{ + [ 0 ] contains subset of original raw text, that is required to create context + records, Some symbols are not written, when max order context was reached + [ Text ] free area + [ UnitsStart ] CPpmd_State vectors and CPpmd7_Context records + [ LoUnit ] free area for CPpmd_State and CPpmd7_Context items +[ HiUnit ] CPpmd7_Context records + [ Size ] end of array } + +These addresses don't cross at any time. +And the following condtions is true for addresses: + (0 <= Text < UnitsStart <= LoUnit <= HiUnit <= Size) + +Raw text is BYTE--aligned. +the data in block [ UnitsStart ... Size ] contains 12-bytes aligned UNITs. + +Last UNIT of array at offset (Size - 12) is root order-0 CPpmd7_Context record. +The code can free UNITs memory blocks that were allocated to store CPpmd_State vectors. +The code doesn't free UNITs allocated for CPpmd7_Context records. + +The code calls RestartModel(), when there is no free memory for allocation. +And RestartModel() changes the state to orignal start state, with full free block. + + +The code allocates UNITs with the following order: + +Allocation of 1 UNIT for Context record + - from free space (HiUnit) down to (LoUnit) + - from FreeList[0] + - AllocUnitsRare() + +AllocUnits() for CPpmd_State vectors: + - from FreeList[i] + - from free space (LoUnit) up to (HiUnit) + - AllocUnitsRare() + +AllocUnitsRare() + - if (GlueCount == 0) + { Glue lists, GlueCount = 255, allocate from FreeList[i]] } + - loop for all higher sized FreeList[...] lists + - from (UnitsStart - Text), GlueCount-- + - ERROR + + +Each Record with Context contains the CPpmd_State vector, where each +CPpmd_State contains the link to Successor. +There are 3 types of Successor: + 1) NULL-Successor - NULL pointer. NULL-Successor links can be stored + only in 0-order Root Context Record. + We use 0 value as NULL-Successor + 2) RAW-Successor - the link to position in raw text, + that "RAW-Successor" is being created after first + occurrence of new symbol for some existing context record. + (RAW-Successor > 0). + 3) RECORD-Successor - the link to CPpmd7_Context record of (Order+1), + that record is being created when we go via RAW-Successor again. + +For any successors at any time: the following condtions are true for Successor links: +(NULL-Successor < RAW-Successor < UnitsStart <= RECORD-Successor) + + +---------- Symbol Frequency, SummFreq and Range in Range_Coder ---------- + +CPpmd7_Context::SummFreq = Sum(Stats[].Freq) + Escape_Freq + +The PPMd code tries to fulfill the condition: + (SummFreq <= (256 * 128 = RC::kBot)) + +We have (Sum(Stats[].Freq) <= 256 * 124), because of (MAX_FREQ = 124) +So (4 = 128 - 124) is average reserve for Escape_Freq for each symbol. +If (CPpmd_State::Freq) is not aligned for 4, the reserve can be 5, 6 or 7. +SummFreq and Escape_Freq can be changed in Rescale() and *Update*() functions. +Rescale() can remove symbols only from max-order contexts. So Escape_Freq can increase after multiple calls of Rescale() for +max-order context. + +When the PPMd code still break (Total <= RC::Range) condition in range coder, +we have two ways to resolve that problem: + 1) we can report error, if we want to keep compatibility with original PPMd code that has no fix for such cases. + 2) we can reduce (Total) value to (RC::Range) by reducing (Escape_Freq) part of (Total) value. +*/ @@ -1,10 +1,8 @@ -/* Ppmd7.h -- PPMdH compression codec -2018-07-04 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -/* This code supports virtual RangeDecoder and includes the implementation -of RangeCoder from 7z, instead of RangeCoder from original PPMd var.H. -If you need the compatibility with original PPMd var.H, you can use external RangeDecoder */ +/* Ppmd7.h -- Ppmd7 (PPMdH) compression codec +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #ifndef __PPMD7_H #define __PPMD7_H @@ -21,23 +19,56 @@ EXTERN_C_BEGIN struct CPpmd7_Context_; -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Context_ * - #else - UInt32 - #endif - CPpmd7_Context_Ref; +typedef Ppmd_Ref_Type(struct CPpmd7_Context_) CPpmd7_Context_Ref; + +// MY_CPU_pragma_pack_push_1 typedef struct CPpmd7_Context_ { UInt16 NumStats; - UInt16 SummFreq; - CPpmd_State_Ref Stats; + + + union + { + UInt16 SummFreq; + CPpmd_State2 State2; + } Union2; + + union + { + CPpmd_State_Ref Stats; + CPpmd_State4 State4; + } Union4; + CPpmd7_Context_Ref Suffix; } CPpmd7_Context; -#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->SummFreq) +// MY_CPU_pragma_pop + +#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->Union2) + + + + +typedef struct +{ + UInt32 Range; + UInt32 Code; + UInt32 Low; + IByteIn *Stream; +} CPpmd7_RangeDec; + + +typedef struct +{ + UInt32 Range; + Byte Cache; + // Byte _dummy_[3]; + UInt64 Low; + UInt64 CacheSize; + IByteOut *Stream; +} CPpmd7z_RangeEnc; + typedef struct { @@ -48,17 +79,30 @@ typedef struct UInt32 Size; UInt32 GlueCount; - Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; UInt32 AlignOffset; + Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; - Byte Indx2Units[PPMD_NUM_INDEXES]; + + + + union + { + CPpmd7_RangeDec dec; + CPpmd7z_RangeEnc enc; + } rc; + + Byte Indx2Units[PPMD_NUM_INDEXES + 2]; // +2 for alignment Byte Units2Indx[128]; CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; - Byte NS2Indx[256], NS2BSIndx[256], HB2Flag[256]; + + Byte NS2BSIndx[256], NS2Indx[256]; + Byte ExpEscape[16]; CPpmd_See DummySee, See[25][16]; UInt16 BinSumm[128][64]; + // int LastSymbol; } CPpmd7; + void Ppmd7_Construct(CPpmd7 *p); BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); @@ -68,74 +112,69 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); /* ---------- Internal Functions ---------- */ -extern const Byte PPMD7_kExpEscape[16]; - -#ifdef PPMD_32BIT - #define Ppmd7_GetPtr(p, ptr) (ptr) - #define Ppmd7_GetContext(p, ptr) (ptr) - #define Ppmd7_GetStats(p, ctx) ((ctx)->Stats) -#else - #define Ppmd7_GetPtr(p, offs) ((void *)((p)->Base + (offs))) - #define Ppmd7_GetContext(p, offs) ((CPpmd7_Context *)Ppmd7_GetPtr((p), (offs))) - #define Ppmd7_GetStats(p, ctx) ((CPpmd_State *)Ppmd7_GetPtr((p), ((ctx)->Stats))) -#endif +#define Ppmd7_GetPtr(p, ptr) Ppmd_GetPtr(p, ptr) +#define Ppmd7_GetContext(p, ptr) Ppmd_GetPtr_Type(p, ptr, CPpmd7_Context) +#define Ppmd7_GetStats(p, ctx) Ppmd_GetPtr_Type(p, (ctx)->Union4.Stats, CPpmd_State) void Ppmd7_Update1(CPpmd7 *p); void Ppmd7_Update1_0(CPpmd7 *p); void Ppmd7_Update2(CPpmd7 *p); -void Ppmd7_UpdateBin(CPpmd7 *p); + +#define PPMD7_HiBitsFlag_3(sym) ((((unsigned)sym + 0xC0) >> (8 - 3)) & (1 << 3)) +#define PPMD7_HiBitsFlag_4(sym) ((((unsigned)sym + 0xC0) >> (8 - 4)) & (1 << 4)) +// #define PPMD7_HiBitsFlag_3(sym) ((sym) < 0x40 ? 0 : (1 << 3)) +// #define PPMD7_HiBitsFlag_4(sym) ((sym) < 0x40 ? 0 : (1 << 4)) #define Ppmd7_GetBinSumm(p) \ - &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ - p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ - (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ - 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ - ((p->RunLength >> 26) & 0x20)] + &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1] \ + [ p->PrevSuccess + ((p->RunLength >> 26) & 0x20) \ + + p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] \ + + PPMD7_HiBitsFlag_4(Ppmd7Context_OneState(p->MinContext)->Symbol) \ + + (p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol)) ] CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); +/* +We support two versions of Ppmd7 (PPMdH) methods that use same CPpmd7 structure: + 1) Ppmd7a_*: original PPMdH + 2) Ppmd7z_*: modified PPMdH with 7z Range Coder +Ppmd7_*: the structures and functions that are common for both versions of PPMd7 (PPMdH) +*/ + /* ---------- Decode ---------- */ -typedef struct IPpmd7_RangeDec IPpmd7_RangeDec; +#define PPMD7_SYM_END (-1) +#define PPMD7_SYM_ERROR (-2) -struct IPpmd7_RangeDec -{ - UInt32 (*GetThreshold)(const IPpmd7_RangeDec *p, UInt32 total); - void (*Decode)(const IPpmd7_RangeDec *p, UInt32 start, UInt32 size); - UInt32 (*DecodeBit)(const IPpmd7_RangeDec *p, UInt32 size0); -}; +/* +You must set (CPpmd7::rc.dec.Stream) before Ppmd7*_RangeDec_Init() -typedef struct -{ - IPpmd7_RangeDec vt; - UInt32 Range; - UInt32 Code; - IByteIn *Stream; -} CPpmd7z_RangeDec; +Ppmd7*_DecodeSymbol() +out: + >= 0 : decoded byte + -1 : PPMD7_SYM_END : End of payload marker + -2 : PPMD7_SYM_ERROR : Data error +*/ -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); -#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +/* Ppmd7a_* : original PPMdH */ +BoolInt Ppmd7a_RangeDec_Init(CPpmd7_RangeDec *p); +#define Ppmd7a_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +int Ppmd7a_DecodeSymbol(CPpmd7 *p); -int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); +/* Ppmd7z_* : modified PPMdH with 7z Range Coder */ +BoolInt Ppmd7z_RangeDec_Init(CPpmd7_RangeDec *p); +#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +int Ppmd7z_DecodeSymbol(CPpmd7 *p); +// Byte *Ppmd7z_DecodeSymbols(CPpmd7 *p, Byte *buf, const Byte *lim); /* ---------- Encode ---------- */ -typedef struct -{ - UInt64 Low; - UInt32 Range; - Byte Cache; - UInt64 CacheSize; - IByteOut *Stream; -} CPpmd7z_RangeEnc; - -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p); -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p); - -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol); +void Ppmd7z_Init_RangeEnc(CPpmd7 *p); +void Ppmd7z_Flush_RangeEnc(CPpmd7 *p); +// void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol); +void Ppmd7z_EncodeSymbols(CPpmd7 *p, const Byte *buf, const Byte *lim); EXTERN_C_END diff --git a/C/Ppmd7Dec.c b/C/Ppmd7Dec.c index 311e9f9d..55d74ff9 100644 --- a/C/Ppmd7Dec.c +++ b/C/Ppmd7Dec.c @@ -1,6 +1,8 @@ -/* Ppmd7Dec.c -- PPMdH Decoder -2018-07-04 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ +/* Ppmd7Dec.c -- Ppmd7z (PPMdH with 7z Range Coder) Decoder +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #include "Precomp.h" @@ -8,184 +10,288 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) + +#define READ_BYTE(p) IByteIn_Read((p)->Stream) + +BoolInt Ppmd7z_RangeDec_Init(CPpmd7_RangeDec *p) { unsigned i; p->Code = 0; p->Range = 0xFFFFFFFF; - if (IByteIn_Read(p->Stream) != 0) + if (READ_BYTE(p) != 0) return False; for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); + p->Code = (p->Code << 8) | READ_BYTE(p); return (p->Code < 0xFFFFFFFF); } -#define GET_Ppmd7z_RangeDec CPpmd7z_RangeDec *p = CONTAINER_FROM_VTBL(pp, CPpmd7z_RangeDec, vt); - -static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) -{ - GET_Ppmd7z_RangeDec - return p->Code / (p->Range /= total); -} +#define RC_NORM_BASE(p) if ((p)->Range < kTopValue) \ + { (p)->Code = ((p)->Code << 8) | READ_BYTE(p); (p)->Range <<= 8; -static void Range_Normalize(CPpmd7z_RangeDec *p) -{ - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); - p->Range <<= 8; - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); - p->Range <<= 8; - } - } -} +#define RC_NORM_1(p) RC_NORM_BASE(p) } +#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} -static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) -{ - GET_Ppmd7z_RangeDec - p->Code -= start * p->Range; - p->Range *= size; - Range_Normalize(p); -} +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) -static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) -{ - GET_Ppmd7z_RangeDec - UInt32 newBound = (p->Range >> 14) * size0; - UInt32 symbol; - if (p->Code < newBound) - { - symbol = 0; - p->Range = newBound; - } - else - { - symbol = 1; - p->Code -= newBound; - p->Range -= newBound; - } - Range_Normalize(p); - return symbol; -} +#define R (&p->rc.dec) -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) +MY_FORCE_INLINE +// MY_NO_INLINE +static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) { - p->vt.GetThreshold = Range_GetThreshold; - p->vt.Decode = Range_Decode; - p->vt.DecodeBit = Range_DecodeBit; + + + R->Code -= start * R->Range; + R->Range *= size; + RC_NORM_LOCAL(R) } +#define RC_Decode(start, size) RangeDec_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) + -#define MASK(sym) ((signed char *)charMask)[sym] +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +typedef CPpmd7_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +void Ppmd7_UpdateModel(CPpmd7 *p); -int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc) +#define MASK(sym) ((unsigned char *)charMask)[sym] +// MY_FORCE_INLINE +// static +int Ppmd7z_DecodeSymbol(CPpmd7 *p) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) { CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); unsigned i; UInt32 count, hiCnt; - if ((count = rc->GetThreshold(rc, p->MinContext->SummFreq)) < (hiCnt = s->Freq)) + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + + + + count = RC_GetThreshold(summFreq); + hiCnt = count; + + if ((Int32)(count -= s->Freq) < 0) { - Byte symbol; - rc->Decode(rc, 0, s->Freq); + Byte sym; + RC_DecodeFinal(0, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update1_0(p); - return symbol; + return sym; } + p->PrevSuccess = 0; - i = p->MinContext->NumStats - 1; + i = (unsigned)p->MinContext->NumStats - 1; + do { - if ((hiCnt += (++s)->Freq) > count) + if ((Int32)(count -= (++s)->Freq) < 0) { - Byte symbol; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); + Byte sym; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update1(p); - return symbol; + return sym; } } while (--i); - if (count >= p->MinContext->SummFreq) - return -2; - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; - rc->Decode(rc, hiCnt, p->MinContext->SummFreq - hiCnt); + + if (hiCnt >= summFreq) + return PPMD7_SYM_ERROR; + + hiCnt -= count; + RC_Decode(hiCnt, summFreq - hiCnt); + + p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { + CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); UInt16 *prob = Ppmd7_GetBinSumm(p); - if (rc->DecodeBit(rc, *prob) == 0) + UInt32 pr = *prob; + UInt32 size0 = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + + if (R->Code < size0) { - Byte symbol; - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - symbol = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; - Ppmd7_UpdateBin(p); - return symbol; + Byte sym; + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + + // RangeDec_DecodeBit0(size0); + R->Range = size0; + RC_NORM_1(R) + /* we can use single byte normalization here because of + (min(BinSumm[][]) = 95) > (1 << (14 - 8)) */ + + // sym = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; + // Ppmd7_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + sym = s->Symbol; + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 128)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; + else + Ppmd7_UpdateModel(p); + } + return sym; } - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + + // RangeDec_DecodeBit1(size0); + + R->Code -= size0; + R->Range -= size0; + RC_NORM_LOCAL(R) + PPMD_SetAllBitsIn256Bytes(charMask); MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } + for (;;) { - CPpmd_State *ps[256], *s; + CPpmd_State *s, *s2; UInt32 freqSum, count, hiCnt; + CPpmd_See *see; - unsigned i, num, numMasked = p->MinContext->NumStats; + CPpmd7_Context *mc; + unsigned numMasked; + RC_NORM_REMOTE(R) + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) - return -1; - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + if (!mc->Suffix) + return PPMD7_SYM_END; + mc = Ppmd7_GetContext(p, mc->Suffix); } - while (p->MinContext->NumStats == numMasked); - hiCnt = 0; - s = Ppmd7_GetStats(p, p->MinContext); - i = 0; - num = p->MinContext->NumStats - numMasked; - do + while (mc->NumStats == numMasked); + + s = Ppmd7_GetStats(p, mc); + { - int k = (int)(MASK(s->Symbol)); - hiCnt += (s->Freq & k); - ps[i] = s++; - i -= k; + unsigned num = mc->NumStats; + unsigned num2 = num / 2; + + num &= 1; + hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + s += num; + p->MinContext = mc; + + do + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); + hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + } + while (--num2); } - while (i != num); - + see = Ppmd7_MakeEscFreq(p, numMasked, &freqSum); freqSum += hiCnt; - count = rc->GetThreshold(rc, freqSum); + + + + + count = RC_GetThreshold(freqSum); if (count < hiCnt) { - Byte symbol; - CPpmd_State **pps = ps; - for (hiCnt = 0; (hiCnt += (*pps)->Freq) <= count; pps++); - s = *pps; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); + Byte sym; + + s = Ppmd7_GetStats(p, p->MinContext); + hiCnt = count; + // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // if ((Int32)count >= 0) + { + for (;;) + { + count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + }; + } + s--; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + + // new (see->Summ) value can overflow over 16-bits in some rare cases Ppmd_See_Update(see); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update2(p); - return symbol; + return sym; } + if (count >= freqSum) - return -2; - rc->Decode(rc, hiCnt, freqSum - hiCnt); + return PPMD7_SYM_ERROR; + + RC_Decode(hiCnt, freqSum - hiCnt); + + // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. + // new (see->Summ) value can overflow over 16-bits in some rare cases see->Summ = (UInt16)(see->Summ + freqSum); - do { MASK(ps[--i]->Symbol) = 0; } while (i != 0); + + s = Ppmd7_GetStats(p, p->MinContext); + s2 = s + p->MinContext->NumStats; + do + { + MASK(s->Symbol) = 0; + s++; + } + while (s != s2); + } +} + +/* +Byte *Ppmd7z_DecodeSymbols(CPpmd7 *p, Byte *buf, const Byte *lim) +{ + int sym = 0; + if (buf != lim) + do + { + sym = Ppmd7z_DecodeSymbol(p); + if (sym < 0) + break; + *buf = (Byte)sym; } + while (++buf < lim); + p->LastSymbol = sym; + return buf; } +*/ diff --git a/C/Ppmd7Enc.c b/C/Ppmd7Enc.c index 286b8712..62139c5b 100644 --- a/C/Ppmd7Enc.c +++ b/C/Ppmd7Enc.c @@ -1,6 +1,8 @@ -/* Ppmd7Enc.c -- PPMdH Encoder -2017-04-03 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ +/* Ppmd7Enc.c -- Ppmd7z (PPMdH with 7z Range Coder) Encoder +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #include "Precomp.h" @@ -8,65 +10,60 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p) +#define R (&p->rc.enc) + +void Ppmd7z_Init_RangeEnc(CPpmd7 *p) { - p->Low = 0; - p->Range = 0xFFFFFFFF; - p->Cache = 0; - p->CacheSize = 1; + R->Low = 0; + R->Range = 0xFFFFFFFF; + R->Cache = 0; + R->CacheSize = 1; } -static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) +MY_NO_INLINE +static void RangeEnc_ShiftLow(CPpmd7 *p) { - if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0) + if ((UInt32)R->Low < (UInt32)0xFF000000 || (unsigned)(R->Low >> 32) != 0) { - Byte temp = p->Cache; + Byte temp = R->Cache; do { - IByteOut_Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + IByteOut_Write(R->Stream, (Byte)(temp + (Byte)(R->Low >> 32))); temp = 0xFF; } - while (--p->CacheSize != 0); - p->Cache = (Byte)((UInt32)p->Low >> 24); + while (--R->CacheSize != 0); + R->Cache = (Byte)((UInt32)R->Low >> 24); } - p->CacheSize++; - p->Low = (UInt32)p->Low << 8; + R->CacheSize++; + R->Low = (UInt32)((UInt32)R->Low << 8); } -static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total) -{ - p->Low += start * (p->Range /= total); - p->Range *= size; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} +#define RC_NORM_BASE(p) if (R->Range < kTopValue) { R->Range <<= 8; RangeEnc_ShiftLow(p); +#define RC_NORM_1(p) RC_NORM_BASE(p) } +#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} -static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0) -{ - p->Range = (p->Range >> 14) * size0; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +/* +#define RangeEnc_Encode(p, start, _size_) \ + { UInt32 size = _size_; \ + R->Low += start * R->Range; \ + R->Range *= size; \ + RC_NORM_LOCAL(p); } +*/ -static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0) +MY_FORCE_INLINE +// MY_NO_INLINE +static void RangeEnc_Encode(CPpmd7 *p, UInt32 start, UInt32 size) { - UInt32 newBound = (p->Range >> 14) * size0; - p->Low += newBound; - p->Range -= newBound; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } + R->Low += start * R->Range; + R->Range *= size; + RC_NORM_LOCAL(p); } -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) +void Ppmd7z_Flush_RangeEnc(CPpmd7 *p) { unsigned i; for (i = 0; i < 5; i++) @@ -74,31 +71,53 @@ void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) } -#define MASK(sym) ((signed char *)charMask)[sym] -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) +#define RC_Encode(start, size) RangeEnc_Encode(p, start, size); +#define RC_EncodeFinal(start, size) RC_Encode(start, size); RC_NORM_REMOTE(p); + +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +#define SUFFIX(ctx) CTX((ctx)->Suffix) +typedef CPpmd7_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) + +void Ppmd7_UpdateModel(CPpmd7 *p); + +#define MASK(sym) ((unsigned char *)charMask)[sym] + +MY_FORCE_INLINE +static +void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) { CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); UInt32 sum; unsigned i; + + + + + R->Range /= p->MinContext->Union2.SummFreq; + if (s->Symbol == symbol) { - RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq); + // R->Range /= p->MinContext->Union2.SummFreq; + RC_EncodeFinal(0, s->Freq); p->FoundState = s; Ppmd7_Update1_0(p); return; } p->PrevSuccess = 0; sum = s->Freq; - i = p->MinContext->NumStats - 1; + i = (unsigned)p->MinContext->NumStats - 1; do { if ((++s)->Symbol == symbol) { - RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq); + // R->Range /= p->MinContext->Union2.SummFreq; + RC_EncodeFinal(sum, s->Freq); p->FoundState = s; Ppmd7_Update1(p); return; @@ -106,82 +125,199 @@ void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) sum += s->Freq; } while (--i); + + // R->Range /= p->MinContext->Union2.SummFreq; + RC_Encode(sum, p->MinContext->Union2.SummFreq - sum); - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; + p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); - RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); + // MASK(s->Symbol) = 0; + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { UInt16 *prob = Ppmd7_GetBinSumm(p); CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); + UInt32 pr = *prob; + UInt32 bound = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); if (s->Symbol == symbol) { - RangeEnc_EncodeBit_0(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - p->FoundState = s; - Ppmd7_UpdateBin(p); + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + // RangeEnc_EncodeBit_0(p, bound); + R->Range = bound; + RC_NORM_1(p); + + // p->FoundState = s; + // Ppmd7_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 128)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; + else + Ppmd7_UpdateModel(p); + } return; } - else - { - RangeEnc_EncodeBit_1(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - p->PrevSuccess = 0; - } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + // RangeEnc_EncodeBit_1(p, bound); + R->Low += bound; + R->Range -= bound; + RC_NORM_LOCAL(p) + + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; } + for (;;) { - UInt32 escFreq; CPpmd_See *see; CPpmd_State *s; - UInt32 sum; - unsigned i, numMasked = p->MinContext->NumStats; + UInt32 sum, escFreq; + CPpmd7_Context *mc; + unsigned i, numMasked; + + RC_NORM_REMOTE(p) + + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) + if (!mc->Suffix) return; /* EndMarker (symbol = -1) */ - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + mc = Ppmd7_GetContext(p, mc->Suffix); + i = mc->NumStats; } - while (p->MinContext->NumStats == numMasked); + while (i == numMasked); + + p->MinContext = mc; - see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); - s = Ppmd7_GetStats(p, p->MinContext); + // see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); + { + if (i != 256) + { + unsigned nonMasked = i - numMasked; + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + + p->HiBitsFlag + + (nonMasked < (unsigned)SUFFIX(mc)->NumStats - i) + + 2 * (unsigned)(mc->Union2.SummFreq < 11 * i) + + 4 * (unsigned)(numMasked > nonMasked); + { + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + unsigned summ = (UInt16)see->Summ; // & 0xFFFF + unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); + escFreq = r + (r == 0); + } + } + else + { + see = &p->DummySee; + escFreq = 1; + } + } + + s = Ppmd7_GetStats(p, mc); sum = 0; - i = p->MinContext->NumStats; + // i = mc->NumStats; + do { - int cur = s->Symbol; - if (cur == symbol) + unsigned cur = s->Symbol; + if ((int)cur == symbol) { UInt32 low = sum; - CPpmd_State *s1 = s; - do + UInt32 freq = s->Freq; + unsigned num2; + + Ppmd_See_Update(see); + p->FoundState = s; + sum += escFreq; + + num2 = i / 2; + i &= 1; + sum += freq & (0 - (UInt32)i); + if (num2 != 0) { - sum += (s->Freq & (int)(MASK(s->Symbol))); - s++; + s += i; + for (;;) + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + sum += (s[-2].Freq & (unsigned)(MASK(sym0))); + sum += (s[-1].Freq & (unsigned)(MASK(sym1))); + if (--num2 == 0) + break; + } } - while (--i); - RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq); - Ppmd_See_Update(see); - p->FoundState = s1; + + + R->Range /= sum; + RC_EncodeFinal(low, freq); Ppmd7_Update2(p); return; } - sum += (s->Freq & (int)(MASK(cur))); - MASK(cur) = 0; + sum += (s->Freq & (unsigned)(MASK(cur))); s++; } while (--i); - RangeEnc_Encode(rc, sum, escFreq, sum + escFreq); - see->Summ = (UInt16)(see->Summ + sum + escFreq); + { + UInt32 total = sum + escFreq; + see->Summ = (UInt16)(see->Summ + total); + + R->Range /= total; + RC_Encode(sum, escFreq); + } + + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + s--; + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } +} + + +void Ppmd7z_EncodeSymbols(CPpmd7 *p, const Byte *buf, const Byte *lim) +{ + for (; buf < lim; buf++) + { + Ppmd7z_EncodeSymbol(p, *buf); } } diff --git a/C/Ppmd7aDec.c b/C/Ppmd7aDec.c new file mode 100644 index 00000000..c4245784 --- /dev/null +++ b/C/Ppmd7aDec.c @@ -0,0 +1,279 @@ +/* Ppmd7aDec.c -- PPMd7a (PPMdH) Decoder +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain + Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ + +#include "Precomp.h" + +#include "Ppmd7.h" + +#define kTop (1 << 24) +#define kBot (1 << 15) + +#define READ_BYTE(p) IByteIn_Read((p)->Stream) + +BoolInt Ppmd7a_RangeDec_Init(CPpmd7_RangeDec *p) +{ + unsigned i; + p->Code = 0; + p->Range = 0xFFFFFFFF; + p->Low = 0; + + for (i = 0; i < 4; i++) + p->Code = (p->Code << 8) | READ_BYTE(p); + return (p->Code < 0xFFFFFFFF); +} + +#define RC_NORM(p) \ + while ((p->Low ^ (p->Low + p->Range)) < kTop \ + || (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) { \ + p->Code = (p->Code << 8) | READ_BYTE(p); \ + p->Range <<= 8; p->Low <<= 8; } + +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +#define R (&p->rc.dec) + +MY_FORCE_INLINE +// MY_NO_INLINE +static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) +{ + start *= R->Range; + R->Low += start; + R->Code -= start; + R->Range *= size; + RC_NORM_LOCAL(R) +} + +#define RC_Decode(start, size) RangeDec_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) + + +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +typedef CPpmd7_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +void Ppmd7_UpdateModel(CPpmd7 *p); + +#define MASK(sym) ((unsigned char *)charMask)[sym] + + +int Ppmd7a_DecodeSymbol(CPpmd7 *p) +{ + size_t charMask[256 / sizeof(size_t)]; + + if (p->MinContext->NumStats != 1) + { + CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); + unsigned i; + UInt32 count, hiCnt; + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + if (summFreq > R->Range) + return PPMD7_SYM_ERROR; + + count = RC_GetThreshold(summFreq); + hiCnt = count; + + if ((Int32)(count -= s->Freq) < 0) + { + Byte sym; + RC_DecodeFinal(0, s->Freq); + p->FoundState = s; + sym = s->Symbol; + Ppmd7_Update1_0(p); + return sym; + } + + p->PrevSuccess = 0; + i = (unsigned)p->MinContext->NumStats - 1; + + do + { + if ((Int32)(count -= (++s)->Freq) < 0) + { + Byte sym; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + p->FoundState = s; + sym = s->Symbol; + Ppmd7_Update1(p); + return sym; + } + } + while (--i); + + if (hiCnt >= summFreq) + return PPMD7_SYM_ERROR; + + hiCnt -= count; + RC_Decode(hiCnt, summFreq - hiCnt); + + p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); + PPMD_SetAllBitsIn256Bytes(charMask); + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } + else + { + CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); + UInt16 *prob = Ppmd7_GetBinSumm(p); + UInt32 pr = *prob; + UInt32 size0 = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + + if (R->Code < size0) + { + Byte sym; + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + + // RangeDec_DecodeBit0(size0); + R->Range = size0; + RC_NORM(R) + + + + // sym = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; + // Ppmd7_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + sym = s->Symbol; + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 128)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; + else + Ppmd7_UpdateModel(p); + } + return sym; + } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + + // RangeDec_DecodeBit1(size0); + R->Low += size0; + R->Code -= size0; + R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - size0; + RC_NORM_LOCAL(R) + + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; + p->PrevSuccess = 0; + } + + for (;;) + { + CPpmd_State *s, *s2; + UInt32 freqSum, count, hiCnt; + + CPpmd_See *see; + CPpmd7_Context *mc; + unsigned numMasked; + RC_NORM_REMOTE(R) + mc = p->MinContext; + numMasked = mc->NumStats; + + do + { + p->OrderFall++; + if (!mc->Suffix) + return PPMD7_SYM_END; + mc = Ppmd7_GetContext(p, mc->Suffix); + } + while (mc->NumStats == numMasked); + + s = Ppmd7_GetStats(p, mc); + + { + unsigned num = mc->NumStats; + unsigned num2 = num / 2; + + num &= 1; + hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + s += num; + p->MinContext = mc; + + do + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); + hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + } + while (--num2); + } + + see = Ppmd7_MakeEscFreq(p, numMasked, &freqSum); + freqSum += hiCnt; + + if (freqSum > R->Range) + return PPMD7_SYM_ERROR; + + count = RC_GetThreshold(freqSum); + + if (count < hiCnt) + { + Byte sym; + + s = Ppmd7_GetStats(p, p->MinContext); + hiCnt = count; + // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // if ((Int32)count >= 0) + { + for (;;) + { + count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + }; + } + s--; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + + // new (see->Summ) value can overflow over 16-bits in some rare cases + Ppmd_See_Update(see); + p->FoundState = s; + sym = s->Symbol; + Ppmd7_Update2(p); + return sym; + } + + if (count >= freqSum) + return PPMD7_SYM_ERROR; + + RC_Decode(hiCnt, freqSum - hiCnt); + + // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. + // new (see->Summ) value can overflow over 16-bits in some rare cases + see->Summ = (UInt16)(see->Summ + freqSum); + + s = Ppmd7_GetStats(p, p->MinContext); + s2 = s + p->MinContext->NumStats; + do + { + MASK(s->Symbol) = 0; + s++; + } + while (s != s2); + } +} @@ -1,5 +1,5 @@ /* Ppmd8.c -- PPMdI codec -2018-07-04 : Igor Pavlov : Public domain +2021-04-13 : Igor Pavlov : Public domain This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,7 +8,12 @@ This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Ppmd8.h" -const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; + + + +MY_ALIGN(16) +static const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +MY_ALIGN(16) static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; #define MAX_FREQ 124 @@ -16,13 +21,10 @@ static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) #define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) -#define I2U(indx) (p->Indx2Units[indx]) +#define I2U(indx) ((unsigned)p->Indx2Units[indx]) -#ifdef PPMD_32BIT - #define REF(ptr) (ptr) -#else - #define REF(ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) -#endif + +#define REF(ptr) Ppmd_GetRef(p, ptr) #define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) @@ -35,34 +37,23 @@ typedef CPpmd8_Context * CTX_PTR; struct CPpmd8_Node_; -typedef - #ifdef PPMD_32BIT - struct CPpmd8_Node_ * - #else - UInt32 - #endif - CPpmd8_Node_Ref; +typedef Ppmd_Ref_Type(struct CPpmd8_Node_) CPpmd8_Node_Ref; typedef struct CPpmd8_Node_ { UInt32 Stamp; + CPpmd8_Node_Ref Next; UInt32 NU; } CPpmd8_Node; -#ifdef PPMD_32BIT - #define NODE(ptr) (ptr) -#else - #define NODE(offs) ((CPpmd8_Node *)(p->Base + (offs))) -#endif - -#define EMPTY_NODE 0xFFFFFFFF +#define NODE(r) Ppmd_GetPtr_Type(p, r, CPpmd8_Node) void Ppmd8_Construct(CPpmd8 *p) { unsigned i, k, m; - p->Base = 0; + p->Base = NULL; for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) { @@ -78,39 +69,51 @@ void Ppmd8_Construct(CPpmd8 *p) for (i = 0; i < 5; i++) p->NS2Indx[i] = (Byte)i; + for (m = i, k = 1; i < 260; i++) { p->NS2Indx[i] = (Byte)m; if (--k == 0) k = (++m) - 4; } + + memcpy(p->ExpEscape, PPMD8_kExpEscape, 16); } + void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->Base); p->Size = 0; - p->Base = 0; + p->Base = NULL; } + BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { Ppmd8_Free(p, alloc); - p->AlignOffset = - #ifdef PPMD_32BIT - (4 - size) & 3; - #else - 4 - (size & 3); - #endif - if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == 0) + p->AlignOffset = (4 - size) & 3; + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL) return False; p->Size = size; } return True; } + + +// ---------- Internal Memory Allocator ---------- + + + + + + +#define EMPTY_NODE 0xFFFFFFFF + + static void InsertNode(CPpmd8 *p, void *node, unsigned indx) { ((CPpmd8_Node *)node)->Stamp = EMPTY_NODE; @@ -120,14 +123,17 @@ static void InsertNode(CPpmd8 *p, void *node, unsigned indx) p->Stamps[indx]++; } + static void *RemoveNode(CPpmd8 *p, unsigned indx) { CPpmd8_Node *node = NODE((CPpmd8_Node_Ref)p->FreeList[indx]); p->FreeList[indx] = node->Next; p->Stamps[indx]--; + return node; } + static void SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) { unsigned i, nu = I2U(oldIndx) - I2U(newIndx); @@ -140,51 +146,96 @@ static void SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) InsertNode(p, ptr, i); } + + + + + + + + + + + + + static void GlueFreeBlocks(CPpmd8 *p) { - CPpmd8_Node_Ref head = 0; - CPpmd8_Node_Ref *prev = &head; - unsigned i; + /* + we use first UInt32 field of 12-bytes UNITs as record type stamp + CPpmd_State { Byte Symbol; Byte Freq; : Freq != 0xFF + CPpmd8_Context { Byte NumStats; Byte Flags; UInt16 SummFreq; : Flags != 0xFF ??? + CPpmd8_Node { UInt32 Stamp : Stamp == 0xFFFFFFFF for free record + : Stamp == 0 for guard + Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd8_Context record + */ + CPpmd8_Node_Ref n; p->GlueCount = 1 << 13; memset(p->Stamps, 0, sizeof(p->Stamps)); - /* Order-0 context is always at top UNIT, so we don't need guard NODE at the end. - All blocks up to p->LoUnit can be free, so we need guard NODE at LoUnit. */ + /* we set guard NODE at LoUnit */ if (p->LoUnit != p->HiUnit) - ((CPpmd8_Node *)p->LoUnit)->Stamp = 0; + ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; - /* Glue free blocks */ - for (i = 0; i < PPMD_NUM_INDEXES; i++) { - CPpmd8_Node_Ref next = (CPpmd8_Node_Ref)p->FreeList[i]; - p->FreeList[i] = 0; - while (next != 0) + /* Glue free blocks */ + CPpmd8_Node_Ref *prev = &n; + unsigned i; + for (i = 0; i < PPMD_NUM_INDEXES; i++) { - CPpmd8_Node *node = NODE(next); - if (node->NU != 0) + + CPpmd8_Node_Ref next = (CPpmd8_Node_Ref)p->FreeList[i]; + p->FreeList[i] = 0; + while (next != 0) { - CPpmd8_Node *node2; + CPpmd8_Node *node = NODE(next); + UInt32 nu = node->NU; *prev = next; - prev = &(node->Next); - while ((node2 = node + node->NU)->Stamp == EMPTY_NODE) + next = node->Next; + if (nu != 0) { - node->NU += node2->NU; - node2->NU = 0; + CPpmd8_Node *node2; + prev = &(node->Next); + while ((node2 = node + nu)->Stamp == EMPTY_NODE) + { + nu += node2->NU; + node2->NU = 0; + node->NU = nu; + } } } - next = node->Next; } + + *prev = 0; } - *prev = 0; + + + + + + + + + + + + + + + + + + + /* Fill lists of free blocks */ - while (head != 0) + while (n != 0) { - CPpmd8_Node *node = NODE(head); - unsigned nu; - head = node->Next; - nu = node->NU; + CPpmd8_Node *node = NODE(n); + UInt32 nu = node->NU; + unsigned i; + n = node->Next; if (nu == 0) continue; for (; nu > 128; nu -= 128, node += 128) @@ -192,57 +243,70 @@ static void GlueFreeBlocks(CPpmd8 *p) if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, node + k, nu - k - 1); + InsertNode(p, node + k, (unsigned)nu - k - 1); } InsertNode(p, node, i); } } + +MY_NO_INLINE static void *AllocUnitsRare(CPpmd8 *p, unsigned indx) { unsigned i; - void *retVal; + if (p->GlueCount == 0) { GlueFreeBlocks(p); if (p->FreeList[indx] != 0) return RemoveNode(p, indx); } + i = indx; + do { if (++i == PPMD_NUM_INDEXES) { UInt32 numBytes = U2B(I2U(indx)); + Byte *us = p->UnitsStart; p->GlueCount--; - return ((UInt32)(p->UnitsStart - p->Text) > numBytes) ? (p->UnitsStart -= numBytes) : (NULL); + return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : (NULL); } } while (p->FreeList[i] == 0); - retVal = RemoveNode(p, i); - SplitBlock(p, retVal, i, indx); - return retVal; + + { + void *block = RemoveNode(p, i); + SplitBlock(p, block, i, indx); + return block; + } } + static void *AllocUnits(CPpmd8 *p, unsigned indx) { - UInt32 numBytes; if (p->FreeList[indx] != 0) return RemoveNode(p, indx); - numBytes = U2B(I2U(indx)); - if (numBytes <= (UInt32)(p->HiUnit - p->LoUnit)) { - void *retVal = p->LoUnit; - p->LoUnit += numBytes; - return retVal; + UInt32 numBytes = U2B(I2U(indx)); + Byte *lo = p->LoUnit; + if ((UInt32)(p->HiUnit - lo) >= numBytes) + { + p->LoUnit = lo + numBytes; + return lo; + } } return AllocUnitsRare(p, indx); } + #define MyMem12Cpy(dest, src, num) \ { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } + + static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU) { unsigned i0 = U2I(oldNU); @@ -260,11 +324,13 @@ static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU return oldPtr; } + static void FreeUnits(CPpmd8 *p, void *ptr, unsigned nu) { InsertNode(p, ptr, U2I(nu)); } + static void SpecialFreeUnit(CPpmd8 *p, void *ptr) { if ((Byte *)ptr != p->UnitsStart) @@ -272,77 +338,91 @@ static void SpecialFreeUnit(CPpmd8 *p, void *ptr) else { #ifdef PPMD8_FREEZE_SUPPORT - *(UInt32 *)ptr = EMPTY_NODE; /* it's used for (Flags == 0xFF) check in RemoveBinContexts */ + *(UInt32 *)ptr = EMPTY_NODE; /* it's used for (Flags == 0xFF) check in RemoveBinContexts() */ #endif p->UnitsStart += UNIT_SIZE; } } + +/* static void *MoveUnitsUp(CPpmd8 *p, void *oldPtr, unsigned nu) { unsigned indx = U2I(nu); void *ptr; - if ((Byte *)oldPtr > p->UnitsStart + 16 * 1024 || REF(oldPtr) > p->FreeList[indx]) + if ((Byte *)oldPtr > p->UnitsStart + (1 << 14) || REF(oldPtr) > p->FreeList[indx]) return oldPtr; ptr = RemoveNode(p, indx); MyMem12Cpy(ptr, oldPtr, nu); - if ((Byte*)oldPtr != p->UnitsStart) + if ((Byte *)oldPtr != p->UnitsStart) InsertNode(p, oldPtr, indx); else p->UnitsStart += U2B(I2U(indx)); return ptr; } +*/ static void ExpandTextArea(CPpmd8 *p) { UInt32 count[PPMD_NUM_INDEXES]; unsigned i; + memset(count, 0, sizeof(count)); if (p->LoUnit != p->HiUnit) - ((CPpmd8_Node *)p->LoUnit)->Stamp = 0; + ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; { - CPpmd8_Node *node = (CPpmd8_Node *)p->UnitsStart; - for (; node->Stamp == EMPTY_NODE; node += node->NU) + CPpmd8_Node *node = (CPpmd8_Node *)(void *)p->UnitsStart; + while (node->Stamp == EMPTY_NODE) { + UInt32 nu = node->NU; node->Stamp = 0; - count[U2I(node->NU)]++; + count[U2I(nu)]++; + node += nu; } p->UnitsStart = (Byte *)node; } for (i = 0; i < PPMD_NUM_INDEXES; i++) { - CPpmd8_Node_Ref *next = (CPpmd8_Node_Ref *)&p->FreeList[i]; - while (count[i] != 0) + UInt32 cnt = count[i]; + if (cnt == 0) + continue; { - CPpmd8_Node *node = NODE(*next); - while (node->Stamp == 0) + CPpmd8_Node_Ref *prev = (CPpmd8_Node_Ref *)&p->FreeList[i]; + CPpmd8_Node_Ref n = *prev; + p->Stamps[i] -= cnt; + for (;;) { - *next = node->Next; - node = NODE(*next); - p->Stamps[i]--; - if (--count[i] == 0) + CPpmd8_Node *node = NODE(n); + n = node->Next; + if (node->Stamp != 0) + { + prev = &node->Next; + continue; + } + *prev = n; + if (--cnt == 0) break; } - next = &node->Next; } } } -#define SUCCESSOR(p) ((CPpmd_Void_Ref)((p)->SuccessorLow | ((UInt32)(p)->SuccessorHigh << 16))) +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) { - (p)->SuccessorLow = (UInt16)((UInt32)(v) & 0xFFFF); - (p)->SuccessorHigh = (UInt16)(((UInt32)(v) >> 16) & 0xFFFF); + Ppmd_SET_SUCCESSOR(p, v); } #define RESET_TEXT(offs) { p->Text = p->Base + p->AlignOffset + (offs); } -static void RestartModel(CPpmd8 *p) +MY_NO_INLINE +static +void RestartModel(CPpmd8 *p) { - unsigned i, k, m, r; + unsigned i, k, m; memset(p->FreeList, 0, sizeof(p->FreeList)); memset(p->Stamps, 0, sizeof(p->Stamps)); @@ -355,30 +435,47 @@ static void RestartModel(CPpmd8 *p) p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; p->PrevSuccess = 0; - p->MinContext = p->MaxContext = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - p->MinContext->Suffix = 0; - p->MinContext->NumStats = 255; - p->MinContext->Flags = 0; - p->MinContext->SummFreq = 256 + 1; - p->FoundState = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ - p->LoUnit += U2B(256 / 2); - p->MinContext->Stats = REF(p->FoundState); - for (i = 0; i < 256; i++) { - CPpmd_State *s = &p->FoundState[i]; - s->Symbol = (Byte)i; - s->Freq = 1; - SetSuccessor(s, 0); + CPpmd8_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + + p->LoUnit += U2B(256 / 2); + p->MaxContext = p->MinContext = mc; + p->FoundState = s; + mc->Flags = 0; + mc->NumStats = 256 - 1; + mc->Union2.SummFreq = 256 + 1; + mc->Union4.Stats = REF(s); + mc->Suffix = 0; + + for (i = 0; i < 256; i++, s++) + { + s->Symbol = (Byte)i; + s->Freq = 1; + SetSuccessor(s, 0); + } } + + + + + + + + + + + for (i = m = 0; m < 25; m++) { while (p->NS2Indx[i] == m) i++; for (k = 0; k < 8; k++) { - UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 1)); + unsigned r; UInt16 *dest = p->BinSumm[m] + k; + UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 1)); for (r = 0; r < 64; r += 8) dest[r] = val; } @@ -386,50 +483,104 @@ static void RestartModel(CPpmd8 *p) for (i = m = 0; m < 24; m++) { + unsigned summ; + CPpmd_See *s; while (p->NS2Indx[(size_t)i + 3] == m + 3) i++; - for (k = 0; k < 32; k++) + s = p->See[m]; + summ = ((2 * i + 5) << (PPMD_PERIOD_BITS - 4)); + for (k = 0; k < 32; k++, s++) { - CPpmd_See *s = &p->See[m][k]; - s->Summ = (UInt16)((2 * i + 5) << (s->Shift = PPMD_PERIOD_BITS - 4)); + s->Summ = (UInt16)summ; + s->Shift = (PPMD_PERIOD_BITS - 4); s->Count = 7; } } + + p->DummySee.Summ = 0; /* unused */ + p->DummySee.Shift = PPMD_PERIOD_BITS; + p->DummySee.Count = 64; /* unused */ } + void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod) { p->MaxOrder = maxOrder; p->RestoreMethod = restoreMethod; RestartModel(p); - p->DummySee.Shift = PPMD_PERIOD_BITS; - p->DummySee.Summ = 0; /* unused */ - p->DummySee.Count = 64; /* unused */ } + +#define FLAG_RESCALED (1 << 2) +// #define FLAG_SYM_HIGH (1 << 3) +#define FLAG_PREV_HIGH (1 << 4) + +#define HiBits_Prepare(sym) ((unsigned)(sym) + 0xC0) + +#define HiBits_Convert_3(flags) (((flags) >> (8 - 3)) & (1 << 3)) +#define HiBits_Convert_4(flags) (((flags) >> (8 - 4)) & (1 << 4)) + +#define PPMD8_HiBitsFlag_3(sym) HiBits_Convert_3(HiBits_Prepare(sym)) +#define PPMD8_HiBitsFlag_4(sym) HiBits_Convert_4(HiBits_Prepare(sym)) + +// #define PPMD8_HiBitsFlag_3(sym) (0x08 * ((sym) >= 0x40)) +// #define PPMD8_HiBitsFlag_4(sym) (0x10 * ((sym) >= 0x40)) + +/* +Refresh() is called when we remove some symbols (successors) in context. +It increases Escape_Freq for sum of all removed symbols. +*/ + static void Refresh(CPpmd8 *p, CTX_PTR ctx, unsigned oldNU, unsigned scale) { unsigned i = ctx->NumStats, escFreq, sumFreq, flags; CPpmd_State *s = (CPpmd_State *)ShrinkUnits(p, STATS(ctx), oldNU, (i + 2) >> 1); - ctx->Stats = REF(s); - #ifdef PPMD8_FREEZE_SUPPORT - /* fixed over Shkarin's code. Fixed code is not compatible with original code for some files in FREEZE mode. */ - scale |= (ctx->SummFreq >= ((UInt32)1 << 15)); - #endif - flags = (ctx->Flags & (0x10 + 0x04 * scale)) + 0x08 * (s->Symbol >= 0x40); - escFreq = ctx->SummFreq - s->Freq; - sumFreq = (s->Freq = (Byte)((s->Freq + scale) >> scale)); + ctx->Union4.Stats = REF(s); + + // #ifdef PPMD8_FREEZE_SUPPORT + /* + (ctx->Union2.SummFreq >= ((UInt32)1 << 15)) can be in FREEZE mode for some files. + It's not good for range coder. So new versions of support fix: + - original PPMdI code rev.1 + + original PPMdI code rev.2 + - 7-Zip default ((PPMD8_FREEZE_SUPPORT is not defined) + + 7-Zip (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) + if we use that fixed line, we can lose compatibility with some files created before fix + if we don't use that fixed line, the program can work incorrectly in FREEZE mode in rare case. + */ + // if (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) + { + scale |= (ctx->Union2.SummFreq >= ((UInt32)1 << 15)); + } + // #endif + + + + flags = HiBits_Prepare(s->Symbol); + { + unsigned freq = s->Freq; + escFreq = ctx->Union2.SummFreq - freq; + freq = (freq + scale) >> scale; + sumFreq = freq; + s->Freq = (Byte)freq; + } + do { - escFreq -= (++s)->Freq; - sumFreq += (s->Freq = (Byte)((s->Freq + scale) >> scale)); - flags |= 0x08 * (s->Symbol >= 0x40); + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + scale) >> scale; + sumFreq += freq; + s->Freq = (Byte)freq; + flags |= HiBits_Prepare(s->Symbol); } while (--i); - ctx->SummFreq = (UInt16)(sumFreq + ((escFreq + scale) >> scale)); - ctx->Flags = (Byte)flags; + + ctx->Union2.SummFreq = (UInt16)(sumFreq + ((escFreq + scale) >> scale)); + ctx->Flags = (Byte)((ctx->Flags & (FLAG_PREV_HIGH + FLAG_RESCALED * scale)) + HiBits_Convert_3(flags)); } + static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) { CPpmd_State tmp = *t1; @@ -437,98 +588,169 @@ static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) *t2 = tmp; } + +/* +CutOff() reduces contexts: + It conversts Successors at MaxOrder to another Contexts to NULL-Successors + It removes RAW-Successors and NULL-Successors that are not Order-0 + and it removes contexts when it has no Successors. + if the (Union4.Stats) is close to (UnitsStart), it moves it up. +*/ + static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) { - int i; - unsigned tmp; - CPpmd_State *s; + int ns = ctx->NumStats; + unsigned nu; + CPpmd_State *stats; - if (!ctx->NumStats) + if (ns == 0) { - s = ONE_STATE(ctx); - if ((Byte *)Ppmd8_GetPtr(p, SUCCESSOR(s)) >= p->UnitsStart) + CPpmd_State *s = ONE_STATE(ctx); + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart) { if (order < p->MaxOrder) - SetSuccessor(s, CutOff(p, CTX(SUCCESSOR(s)), order + 1)); + successor = CutOff(p, CTX(successor), order + 1); else - SetSuccessor(s, 0); - if (SUCCESSOR(s) || order <= 9) /* O_BOUND */ + successor = 0; + SetSuccessor(s, successor); + if (successor || order <= 9) /* O_BOUND */ return REF(ctx); } SpecialFreeUnit(p, ctx); return 0; } - ctx->Stats = STATS_REF(MoveUnitsUp(p, STATS(ctx), tmp = ((unsigned)ctx->NumStats + 2) >> 1)); + nu = ((unsigned)ns + 2) >> 1; + // ctx->Union4.Stats = STATS_REF(MoveUnitsUp(p, STATS(ctx), nu)); + { + unsigned indx = U2I(nu); + stats = STATS(ctx); - for (s = STATS(ctx) + (i = ctx->NumStats); s >= STATS(ctx); s--) - if ((Byte *)Ppmd8_GetPtr(p, SUCCESSOR(s)) < p->UnitsStart) + if ((UInt32)((Byte *)stats - p->UnitsStart) <= (1 << 14) + && (CPpmd_Void_Ref)ctx->Union4.Stats <= p->FreeList[indx]) { - CPpmd_State *s2 = STATS(ctx) + (i--); - SetSuccessor(s, 0); - SwapStates(s, s2); + void *ptr = RemoveNode(p, indx); + ctx->Union4.Stats = STATS_REF(ptr); + MyMem12Cpy(ptr, (const void *)stats, nu); + if ((Byte *)stats != p->UnitsStart) + InsertNode(p, stats, indx); + else + p->UnitsStart += U2B(I2U(indx)); + stats = ptr; } - else if (order < p->MaxOrder) - SetSuccessor(s, CutOff(p, CTX(SUCCESSOR(s)), order + 1)); - else - SetSuccessor(s, 0); - - if (i != ctx->NumStats && order) + } + + { + CPpmd_State *s = stats + (unsigned)ns; + do + { + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) < p->UnitsStart) + { + CPpmd_State *s2 = stats + (unsigned)(ns--); + if (order) + { + if (s != s2) + *s = *s2; + } + else + { + SwapStates(s, s2); + SetSuccessor(s2, 0); + } + } + else + { + if (order < p->MaxOrder) + SetSuccessor(s, CutOff(p, CTX(successor), order + 1)); + else + SetSuccessor(s, 0); + } + } + while (--s >= stats); + } + + if (ns != ctx->NumStats && order) { - ctx->NumStats = (Byte)i; - s = STATS(ctx); - if (i < 0) + if (ns < 0) { - FreeUnits(p, s, tmp); + FreeUnits(p, stats, nu); SpecialFreeUnit(p, ctx); return 0; } - if (i == 0) + ctx->NumStats = (Byte)ns; + if (ns == 0) { - ctx->Flags = (Byte)((ctx->Flags & 0x10) + 0x08 * (s->Symbol >= 0x40)); - *ONE_STATE(ctx) = *s; - FreeUnits(p, s, tmp); - /* 9.31: the code was fixed. It's was not BUG, if Freq <= MAX_FREQ = 124 */ - ONE_STATE(ctx)->Freq = (Byte)(((unsigned)ONE_STATE(ctx)->Freq + 11) >> 3); + const Byte sym = stats->Symbol; + ctx->Flags = (Byte)((ctx->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(sym)); + // *ONE_STATE(ctx) = *stats; + ctx->Union2.State2.Symbol = sym; + ctx->Union2.State2.Freq = (Byte)(((unsigned)stats->Freq + 11) >> 3); + ctx->Union4.State4.Successor_0 = stats->Successor_0; + ctx->Union4.State4.Successor_1 = stats->Successor_1; + FreeUnits(p, stats, nu); } else - Refresh(p, ctx, tmp, ctx->SummFreq > 16 * i); + { + Refresh(p, ctx, nu, ctx->Union2.SummFreq > 16 * (unsigned)ns); + } } + return REF(ctx); } + + #ifdef PPMD8_FREEZE_SUPPORT + +/* +RemoveBinContexts() + It conversts Successors at MaxOrder to another Contexts to NULL-Successors + It changes RAW-Successors to NULL-Successors + removes Bin Context without Successor, if suffix of that context is also binary. +*/ + static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, CTX_PTR ctx, unsigned order) { - CPpmd_State *s; if (!ctx->NumStats) { - s = ONE_STATE(ctx); - if ((Byte *)Ppmd8_GetPtr(p, SUCCESSOR(s)) >= p->UnitsStart && order < p->MaxOrder) - SetSuccessor(s, RemoveBinContexts(p, CTX(SUCCESSOR(s)), order + 1)); + CPpmd_State *s = ONE_STATE(ctx); + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) + successor = RemoveBinContexts(p, CTX(successor), order + 1); else - SetSuccessor(s, 0); + successor = 0; + SetSuccessor(s, successor); /* Suffix context can be removed already, since different (high-order) Successors may refer to same context. So we check Flags == 0xFF (Stamp == EMPTY_NODE) */ - if (!SUCCESSOR(s) && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF)) + if (!successor && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF)) { FreeUnits(p, ctx, 1); return 0; } - else - return REF(ctx); } - - for (s = STATS(ctx) + ctx->NumStats; s >= STATS(ctx); s--) - if ((Byte *)Ppmd8_GetPtr(p, SUCCESSOR(s)) >= p->UnitsStart && order < p->MaxOrder) - SetSuccessor(s, RemoveBinContexts(p, CTX(SUCCESSOR(s)), order + 1)); - else - SetSuccessor(s, 0); + else + { + CPpmd_State *s = STATS(ctx) + ctx->NumStats; + do + { + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) + SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1)); + else + SetSuccessor(s, 0); + } + while (--s >= STATS(ctx)); + } return REF(ctx); } + #endif + + static UInt32 GetUsedMemory(const CPpmd8 *p) { UInt32 v = 0; @@ -544,7 +766,8 @@ static UInt32 GetUsedMemory(const CPpmd8 *p) #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1) #endif -static void RestoreModel(CPpmd8 *p, CTX_PTR c1 + +static void RestoreModel(CPpmd8 *p, CTX_PTR ctxError #ifdef PPMD8_FREEZE_SUPPORT , CTX_PTR fSuccessor #endif @@ -553,36 +776,55 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR c1 CTX_PTR c; CPpmd_State *s; RESET_TEXT(0); - for (c = p->MaxContext; c != c1; c = SUFFIX(c)) + + // we go here in cases of error of allocation for context (c1) + // Order(MinContext) < Order(ctxError) <= Order(MaxContext) + + // We remove last symbol from each of contexts [p->MaxContext ... ctxError) contexts + // So we rollback all created (symbols) before error. + for (c = p->MaxContext; c != ctxError; c = SUFFIX(c)) if (--(c->NumStats) == 0) { s = STATS(c); - c->Flags = (Byte)((c->Flags & 0x10) + 0x08 * (s->Symbol >= 0x40)); - *ONE_STATE(c) = *s; + c->Flags = (Byte)((c->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(s->Symbol)); + // *ONE_STATE(c) = *s; + c->Union2.State2.Symbol = s->Symbol; + c->Union2.State2.Freq = (Byte)(((unsigned)s->Freq + 11) >> 3); + c->Union4.State4.Successor_0 = s->Successor_0; + c->Union4.State4.Successor_1 = s->Successor_1; + SpecialFreeUnit(p, s); - ONE_STATE(c)->Freq = (Byte)(((unsigned)ONE_STATE(c)->Freq + 11) >> 3); } else - Refresh(p, c, (c->NumStats+3) >> 1, 0); + { + /* Refresh() can increase Escape_Freq on value of Freq of last symbol, that was added before error. + so the largest possible increase for Escape_Freq is (8) from value before ModelUpoadet() */ + Refresh(p, c, ((unsigned)c->NumStats + 3) >> 1, 0); + } + // increase Escape Freq for context [ctxError ... p->MinContext) for (; c != p->MinContext; c = SUFFIX(c)) - if (!c->NumStats) - ONE_STATE(c)->Freq = (Byte)(ONE_STATE(c)->Freq - (ONE_STATE(c)->Freq >> 1)); - else if ((c->SummFreq += 4) > 128 + 4 * c->NumStats) - Refresh(p, c, (c->NumStats + 2) >> 1, 1); + if (c->NumStats == 0) + { + // ONE_STATE(c) + c->Union2.State2.Freq = (Byte)(((unsigned)c->Union2.State2.Freq + 1) >> 1); + } + else if ((c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 4)) > 128 + 4 * c->NumStats) + Refresh(p, c, ((unsigned)c->NumStats + 2) >> 1, 1); #ifdef PPMD8_FREEZE_SUPPORT if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) { p->MaxContext = fSuccessor; - p->GlueCount += !(p->Stamps[1] & 1); + p->GlueCount += !(p->Stamps[1] & 1); // why? } else if (p->RestoreMethod == PPMD8_RESTORE_METHOD_FREEZE) { while (p->MaxContext->Suffix) p->MaxContext = SUFFIX(p->MaxContext); RemoveBinContexts(p, p->MaxContext, 0); - p->RestoreMethod++; + // we change the current mode to (PPMD8_RESTORE_METHOD_FREEZE + 1) + p->RestoreMethod = PPMD8_RESTORE_METHOD_FREEZE + 1; p->GlueCount = 0; p->OrderFall = p->MaxOrder; } @@ -603,16 +845,19 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR c1 p->GlueCount = 0; p->OrderFall = p->MaxOrder; } + p->MinContext = p->MaxContext; } + + +MY_NO_INLINE static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c) { - CPpmd_State upState; - Byte flags; + CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); - /* fixed over Shkarin's code. Maybe it could work without + 1 too. */ - CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; + Byte newSym, newFreq, flags; unsigned numPs = 0; + CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; /* fixed over Shkarin's code. Maybe it could work without + 1 too. */ if (!skip) ps[numPs++] = p->FoundState; @@ -622,19 +867,13 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT CPpmd_Void_Ref successor; CPpmd_State *s; c = SUFFIX(c); - if (s1) - { - s = s1; - s1 = NULL; - } + + if (s1) { s = s1; s1 = NULL; } else if (c->NumStats != 0) { - for (s = STATS(c); s->Symbol != p->FoundState->Symbol; s++); - if (s->Freq < MAX_FREQ - 9) - { - s->Freq++; - c->SummFreq++; - } + Byte sym = p->FoundState->Symbol; + for (s = STATS(c); s->Symbol != sym; s++); + if (s->Freq < MAX_FREQ - 9) { s->Freq++; c->Union2.SummFreq++; } } else { @@ -644,36 +883,54 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT successor = SUCCESSOR(s); if (successor != upBranch) { + c = CTX(successor); if (numPs == 0) + { + + return c; + } break; } ps[numPs++] = s; } - upState.Symbol = *(const Byte *)Ppmd8_GetPtr(p, upBranch); - SetSuccessor(&upState, upBranch + 1); - flags = (Byte)(0x10 * (p->FoundState->Symbol >= 0x40) + 0x08 * (upState.Symbol >= 0x40)); - + + + + + newSym = *(const Byte *)Ppmd8_GetPtr(p, upBranch); + upBranch++; + flags = (Byte)(PPMD8_HiBitsFlag_4(p->FoundState->Symbol) + PPMD8_HiBitsFlag_3(newSym)); + if (c->NumStats == 0) - upState.Freq = ONE_STATE(c)->Freq; + newFreq = c->Union2.State2.Freq; else { UInt32 cf, s0; CPpmd_State *s; - for (s = STATS(c); s->Symbol != upState.Symbol; s++); - cf = s->Freq - 1; - s0 = c->SummFreq - c->NumStats - cf; - upState.Freq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((cf + 2 * s0 - 3) / s0))); + for (s = STATS(c); s->Symbol != newSym; s++); + cf = (UInt32)s->Freq - 1; + s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf; + /* + + + max(newFreq)= (s->Freq - 1), when (s0 == 1) + + + */ + newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((cf + 2 * s0 - 3) / s0))); } + + do { - /* Create Child */ - CTX_PTR c1; /* = AllocContext(p); */ + CTX_PTR c1; + /* = AllocContext(p); */ if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); + c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); else if (p->FreeList[0] != 0) c1 = (CTX_PTR)RemoveNode(p, 0); else @@ -682,9 +939,11 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT if (!c1) return NULL; } - c1->NumStats = 0; c1->Flags = flags; - *ONE_STATE(c1) = upState; + c1->NumStats = 0; + c1->Union2.State2.Symbol = newSym; + c1->Union2.State2.Freq = newFreq; + SetSuccessor(ONE_STATE(c1), upBranch); c1->Suffix = REF(c); SetSuccessor(ps[--numPs], REF(c1)); c = c1; @@ -694,6 +953,7 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT return c; } + static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) { CPpmd_State *s = NULL; @@ -739,8 +999,8 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) do { s++; } while (s->Symbol != p->FoundState->Symbol); if (s->Freq < MAX_FREQ - 9) { - s->Freq += 2; - c->SummFreq += 2; + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); } } else @@ -776,33 +1036,42 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) p->FoundState = s; successor = CreateSuccessors(p, False, NULL, c); - if (successor == NULL) + if (!successor) SetSuccessor(s, 0); else SetSuccessor(s, REF(successor)); p->FoundState = s2; } - if (p->OrderFall == 1 && c1 == p->MaxContext) { - SetSuccessor(p->FoundState, SUCCESSOR(s)); - p->Text--; + CPpmd_Void_Ref successor = SUCCESSOR(s); + if (p->OrderFall == 1 && c1 == p->MaxContext) + { + SetSuccessor(p->FoundState, successor); + p->Text--; + } + if (successor == 0) + return NULL; + return CTX(successor); } - if (SUCCESSOR(s) == 0) - return NULL; - return CTX(SUCCESSOR(s)); } -static void UpdateModel(CPpmd8 *p) + + +void Ppmd8_UpdateModel(CPpmd8 *p); +MY_NO_INLINE +void Ppmd8_UpdateModel(CPpmd8 *p) { - CPpmd_Void_Ref successor, fSuccessor = SUCCESSOR(p->FoundState); + CPpmd_Void_Ref maxSuccessor, minSuccessor = SUCCESSOR(p->FoundState); CTX_PTR c; unsigned s0, ns, fFreq = p->FoundState->Freq; Byte flag, fSymbol = p->FoundState->Symbol; + { CPpmd_State *s = NULL; - if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) { + /* Update Freqs in Suffix Context */ + c = SUFFIX(p->MinContext); if (c->NumStats == 0) @@ -813,91 +1082,134 @@ static void UpdateModel(CPpmd8 *p) } else { + Byte sym = p->FoundState->Symbol; s = STATS(c); - if (s->Symbol != p->FoundState->Symbol) + + if (s->Symbol != sym) { - do { s++; } while (s->Symbol != p->FoundState->Symbol); + do + { + + s++; + } + while (s->Symbol != sym); + if (s[0].Freq >= s[-1].Freq) { SwapStates(&s[0], &s[-1]); s--; } } + if (s->Freq < MAX_FREQ - 9) { - s->Freq += 2; - c->SummFreq += 2; + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); } } } c = p->MaxContext; - if (p->OrderFall == 0 && fSuccessor) + if (p->OrderFall == 0 && minSuccessor) { CTX_PTR cs = CreateSuccessors(p, True, s, p->MinContext); - if (cs == 0) + if (!cs) { SetSuccessor(p->FoundState, 0); - RESTORE_MODEL(c, CTX(fSuccessor)); - } - else - { - SetSuccessor(p->FoundState, REF(cs)); - p->MaxContext = cs; + RESTORE_MODEL(c, CTX(minSuccessor)); + return; } + SetSuccessor(p->FoundState, REF(cs)); + p->MinContext = p->MaxContext = cs; return; } - *p->Text++ = p->FoundState->Symbol; - successor = REF(p->Text); - if (p->Text >= p->UnitsStart) + + + { - RESTORE_MODEL(c, CTX(fSuccessor)); /* check it */ - return; + Byte *text = p->Text; + *text++ = p->FoundState->Symbol; + p->Text = text; + if (text >= p->UnitsStart) + { + RESTORE_MODEL(c, CTX(minSuccessor)); /* check it */ + return; + } + maxSuccessor = REF(text); } - - if (!fSuccessor) + + if (!minSuccessor) { CTX_PTR cs = ReduceOrder(p, s, p->MinContext); - if (cs == NULL) + if (!cs) { - RESTORE_MODEL(c, 0); + RESTORE_MODEL(c, NULL); return; } - fSuccessor = REF(cs); + minSuccessor = REF(cs); } - else if ((Byte *)Ppmd8_GetPtr(p, fSuccessor) < p->UnitsStart) + else if ((Byte *)Ppmd8_GetPtr(p, minSuccessor) < p->UnitsStart) { CTX_PTR cs = CreateSuccessors(p, False, s, p->MinContext); - if (cs == NULL) + if (!cs) { - RESTORE_MODEL(c, 0); + RESTORE_MODEL(c, NULL); return; } - fSuccessor = REF(cs); + minSuccessor = REF(cs); } if (--p->OrderFall == 0) { - successor = fSuccessor; + maxSuccessor = minSuccessor; p->Text -= (p->MaxContext != p->MinContext); } #ifdef PPMD8_FREEZE_SUPPORT else if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) { - successor = fSuccessor; + maxSuccessor = minSuccessor; RESET_TEXT(0); p->OrderFall = 0; } #endif + } + + + + + + + + + + - s0 = p->MinContext->SummFreq - (ns = p->MinContext->NumStats) - fFreq; - flag = (Byte)(0x08 * (fSymbol >= 0x40)); + + + + + + + + + + + + + + + + + + flag = (Byte)(PPMD8_HiBitsFlag_3(fSymbol)); + s0 = p->MinContext->Union2.SummFreq - (ns = p->MinContext->NumStats) - fFreq; for (; c != p->MinContext; c = SUFFIX(c)) { unsigned ns1; - UInt32 cf, sf; + UInt32 sum; + if ((ns1 = c->NumStats) != 0) { if ((ns1 & 1) != 0) @@ -911,91 +1223,133 @@ static void UpdateModel(CPpmd8 *p) void *oldPtr; if (!ptr) { - RESTORE_MODEL(c, CTX(fSuccessor)); + RESTORE_MODEL(c, CTX(minSuccessor)); return; } oldPtr = STATS(c); MyMem12Cpy(ptr, oldPtr, oldNU); InsertNode(p, oldPtr, i); - c->Stats = STATS_REF(ptr); + c->Union4.Stats = STATS_REF(ptr); } } - c->SummFreq = (UInt16)(c->SummFreq + (3 * ns1 + 1 < ns)); + sum = c->Union2.SummFreq; + /* max increase of Escape_Freq is 1 here. + an average increase is 1/3 per symbol */ + sum += (3 * ns1 + 1 < ns); + /* original PPMdH uses 16-bit variable for (sum) here. + But (sum < ???). Do we need to truncate (sum) to 16-bit */ + // sum = (UInt16)sum; } else { - CPpmd_State *s2 = (CPpmd_State*)AllocUnits(p, 0); - if (!s2) + + CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); + if (!s) { - RESTORE_MODEL(c, CTX(fSuccessor)); + RESTORE_MODEL(c, CTX(minSuccessor)); return; } - *s2 = *ONE_STATE(c); - c->Stats = REF(s2); - if (s2->Freq < MAX_FREQ / 4 - 1) - s2->Freq <<= 1; - else - s2->Freq = MAX_FREQ - 4; - c->SummFreq = (UInt16)(s2->Freq + p->InitEsc + (ns > 2)); - } - cf = 2 * fFreq * (c->SummFreq + 6); - sf = (UInt32)s0 + c->SummFreq; - if (cf < 6 * sf) - { - cf = 1 + (cf > sf) + (cf >= 4 * sf); - c->SummFreq += 4; - } - else - { - cf = 4 + (cf > 9 * sf) + (cf > 12 * sf) + (cf > 15 * sf); - c->SummFreq = (UInt16)(c->SummFreq + cf); + { + unsigned freq = c->Union2.State2.Freq; + // s = *ONE_STATE(c); + s->Symbol = c->Union2.State2.Symbol; + s->Successor_0 = c->Union4.State4.Successor_0; + s->Successor_1 = c->Union4.State4.Successor_1; + // SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of + // (Successor_0 and Successor_1) in LE/BE. + c->Union4.Stats = REF(s); + if (freq < MAX_FREQ / 4 - 1) + freq <<= 1; + else + freq = MAX_FREQ - 4; + + s->Freq = (Byte)freq; + + sum = freq + p->InitEsc + (ns > 2); // Ppmd8 (> 2) + } } + { - CPpmd_State *s2 = STATS(c) + ns1 + 1; - SetSuccessor(s2, successor); - s2->Symbol = fSymbol; - s2->Freq = (Byte)cf; - c->Flags |= flag; + CPpmd_State *s = STATS(c) + ns1 + 1; + UInt32 cf = 2 * (sum + 6) * (UInt32)fFreq; + UInt32 sf = (UInt32)s0 + sum; + s->Symbol = fSymbol; c->NumStats = (Byte)(ns1 + 1); + SetSuccessor(s, maxSuccessor); + c->Flags |= flag; + if (cf < 6 * sf) + { + cf = (unsigned)1 + (cf > sf) + (cf >= 4 * sf); + sum += 4; + /* It can add (1, 2, 3) to Escape_Freq */ + } + else + { + cf = (unsigned)4 + (cf > 9 * sf) + (cf > 12 * sf) + (cf > 15 * sf); + sum += cf; + } + + c->Union2.SummFreq = (UInt16)sum; + s->Freq = (Byte)cf; } + } - p->MaxContext = p->MinContext = CTX(fSuccessor); + p->MaxContext = p->MinContext = CTX(minSuccessor); } + + +MY_NO_INLINE static void Rescale(CPpmd8 *p) { unsigned i, adder, sumFreq, escFreq; CPpmd_State *stats = STATS(p->MinContext); CPpmd_State *s = p->FoundState; + + /* Sort the list by Freq */ + if (s != stats) { CPpmd_State tmp = *s; - for (; s != stats; s--) + do s[0] = s[-1]; + while (--s != stats); *s = tmp; } - escFreq = p->MinContext->SummFreq - s->Freq; - s->Freq += 4; - adder = (p->OrderFall != 0 - #ifdef PPMD8_FREEZE_SUPPORT - || p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE - #endif - ); - s->Freq = (Byte)((s->Freq + adder) >> 1); + sumFreq = s->Freq; + escFreq = p->MinContext->Union2.SummFreq - sumFreq; + + + + + + adder = (p->OrderFall != 0); + + #ifdef PPMD8_FREEZE_SUPPORT + adder |= (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE); + #endif + + sumFreq = (sumFreq + 4 + adder) >> 1; i = p->MinContext->NumStats; + s->Freq = (Byte)sumFreq; + do { - escFreq -= (++s)->Freq; - s->Freq = (Byte)((s->Freq + adder) >> 1); - sumFreq += s->Freq; - if (s[0].Freq > s[-1].Freq) + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + adder) >> 1; + sumFreq += freq; + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { + CPpmd_State tmp = *s; CPpmd_State *s1 = s; - CPpmd_State tmp = *s1; do + { s1[0] = s1[-1]; - while (--s1 != stats && tmp.Freq > s1[-1].Freq); + } + while (--s1 != stats && freq > s1[-1].Freq); *s1 = tmp; } } @@ -1003,49 +1357,89 @@ static void Rescale(CPpmd8 *p) if (s->Freq == 0) { - unsigned numStats = p->MinContext->NumStats; - unsigned n0, n1; - do { i++; } while ((--s)->Freq == 0); + /* Remove all items with Freq == 0 */ + CPpmd8_Context *mc; + unsigned numStats, numStatsNew, n0, n1; + + i = 0; do { i++; } while ((--s)->Freq == 0); + + + + escFreq += i; - p->MinContext->NumStats = (Byte)(p->MinContext->NumStats - i); - if (p->MinContext->NumStats == 0) + mc = p->MinContext; + numStats = mc->NumStats; + numStatsNew = numStats - i; + mc->NumStats = (Byte)(numStatsNew); + n0 = (numStats + 2) >> 1; + + if (numStatsNew == 0) { - CPpmd_State tmp = *stats; - tmp.Freq = (Byte)((2 * tmp.Freq + escFreq - 1) / escFreq); - if (tmp.Freq > MAX_FREQ / 3) - tmp.Freq = MAX_FREQ / 3; - InsertNode(p, stats, U2I((numStats + 2) >> 1)); - p->MinContext->Flags = (Byte)((p->MinContext->Flags & 0x10) + 0x08 * (tmp.Symbol >= 0x40)); - *(p->FoundState = ONE_STATE(p->MinContext)) = tmp; + + unsigned freq = (2 * (unsigned)stats->Freq + escFreq - 1) / escFreq; + if (freq > MAX_FREQ / 3) + freq = MAX_FREQ / 3; + mc->Flags = (Byte)((mc->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(stats->Symbol)); + + + + + + s = ONE_STATE(mc); + *s = *stats; + s->Freq = (Byte)freq; + p->FoundState = s; + InsertNode(p, stats, U2I(n0)); return; } - n0 = (numStats + 2) >> 1; - n1 = (p->MinContext->NumStats + 2) >> 1; + + n1 = (numStatsNew + 2) >> 1; if (n0 != n1) - p->MinContext->Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); - p->MinContext->Flags &= ~0x08; - p->MinContext->Flags |= 0x08 * ((s = STATS(p->MinContext))->Symbol >= 0x40); - i = p->MinContext->NumStats; - do { p->MinContext->Flags |= 0x08*((++s)->Symbol >= 0x40); } while (--i); + mc->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + { + // here we are for max order only. So Ppmd8_MakeEscFreq() doesn't use mc->Flags + // but we still need current (Flags & FLAG_PREV_HIGH), if we will convert context to 1-symbol context later. + /* + unsigned flags = HiBits_Prepare((s = STATS(mc))->Symbol); + i = mc->NumStats; + do { flags |= HiBits_Prepare((++s)->Symbol); } while (--i); + mc->Flags = (Byte)((mc->Flags & ~FLAG_SYM_HIGH) + HiBits_Convert_3(flags)); + */ + } + } + + + + + + + { + CPpmd8_Context *mc = p->MinContext; + mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); + mc->Flags |= FLAG_RESCALED; + p->FoundState = STATS(mc); } - p->MinContext->SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); - p->MinContext->Flags |= 0x4; - p->FoundState = STATS(p->MinContext); } + CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) { CPpmd_See *see; - if (p->MinContext->NumStats != 0xFF) + const CPpmd8_Context *mc = p->MinContext; + unsigned numStats = mc->NumStats; + if (numStats != 0xFF) { - see = p->See[(size_t)(unsigned)p->NS2Indx[(size_t)(unsigned)p->MinContext->NumStats + 2] - 3] + - (p->MinContext->SummFreq > 11 * ((unsigned)p->MinContext->NumStats + 1)) + - 2 * (unsigned)(2 * (unsigned)p->MinContext->NumStats < - ((unsigned)SUFFIX(p->MinContext)->NumStats + numMasked1)) + - p->MinContext->Flags; + // (3 <= numStats + 2 <= 256) (3 <= NS2Indx[3] and NS2Indx[256] === 26) + see = p->See[(size_t)(unsigned)p->NS2Indx[(size_t)numStats + 2] - 3] + + (mc->Union2.SummFreq > 11 * (numStats + 1)) + + 2 * (unsigned)(2 * numStats < ((unsigned)SUFFIX(mc)->NumStats + numMasked1)) + + mc->Flags; + { - unsigned r = (see->Summ >> see->Shift); - see->Summ = (UInt16)(see->Summ - r); + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + unsigned summ = (UInt16)see->Summ; // & 0xFFFF + unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); *escFreq = r + (r == 0); } } @@ -1057,67 +1451,87 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) return see; } + static void NextContext(CPpmd8 *p) { CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); - if (p->OrderFall == 0 && (Byte *)c >= p->UnitsStart) - p->MinContext = p->MaxContext = c; + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; else - { - UpdateModel(p); - p->MinContext = p->MaxContext; - } + Ppmd8_UpdateModel(p); } + void Ppmd8_Update1(CPpmd8 *p) { CPpmd_State *s = p->FoundState; - s->Freq += 4; - p->MinContext->SummFreq += 4; - if (s[0].Freq > s[-1].Freq) + unsigned freq = s->Freq; + freq += 4; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SwapStates(s, &s[-1]); p->FoundState = --s; - if (s->Freq > MAX_FREQ) + if (freq > MAX_FREQ) Rescale(p); } NextContext(p); } + void Ppmd8_Update1_0(CPpmd8 *p) { - p->PrevSuccess = (2 * p->FoundState->Freq >= p->MinContext->SummFreq); - p->RunLength += p->PrevSuccess; - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) + CPpmd_State *s = p->FoundState; + CPpmd8_Context *mc = p->MinContext; + unsigned freq = s->Freq; + unsigned summFreq = mc->Union2.SummFreq; + p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=) + p->RunLength += (int)p->PrevSuccess; + mc->Union2.SummFreq = (UInt16)(summFreq + 4); + freq += 4; + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) Rescale(p); NextContext(p); } + +/* void Ppmd8_UpdateBin(CPpmd8 *p) { - p->FoundState->Freq = (Byte)(p->FoundState->Freq + (p->FoundState->Freq < 196)); + unsigned freq = p->FoundState->Freq; + p->FoundState->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) p->PrevSuccess = 1; p->RunLength++; NextContext(p); } +*/ void Ppmd8_Update2(CPpmd8 *p) { - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) - Rescale(p); + CPpmd_State *s = p->FoundState; + unsigned freq = s->Freq; + freq += 4; p->RunLength = p->InitRL; - UpdateModel(p); - p->MinContext = p->MaxContext; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) + Rescale(p); + Ppmd8_UpdateModel(p); } /* H->I changes: NS2Indx - GlewCount, and Glue method + GlueCount, and Glue method BinSum See / EscFreq CreateSuccessors updates more suffix contexts - UpdateModel consts. + Ppmd8_UpdateModel consts. PrevSuccess Update + +Flags: + (1 << 2) - the Context was Rescaled + (1 << 3) - there is symbol in Stats with (sym >= 0x40) in + (1 << 4) - main symbol of context is (sym >= 0x40) */ @@ -1,5 +1,5 @@ -/* Ppmd8.h -- PPMdI codec -2018-07-04 : Igor Pavlov : Public domain +/* Ppmd8.h -- Ppmd8 (PPMdI) compression codec +2021-04-13 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -14,35 +14,45 @@ EXTERN_C_BEGIN #define PPMD8_MIN_ORDER 2 #define PPMD8_MAX_ORDER 16 + + + struct CPpmd8_Context_; -typedef - #ifdef PPMD_32BIT - struct CPpmd8_Context_ * - #else - UInt32 - #endif - CPpmd8_Context_Ref; +typedef Ppmd_Ref_Type(struct CPpmd8_Context_) CPpmd8_Context_Ref; -#pragma pack(push, 1) +// MY_CPU_pragma_pack_push_1 typedef struct CPpmd8_Context_ { Byte NumStats; Byte Flags; - UInt16 SummFreq; - CPpmd_State_Ref Stats; + + union + { + UInt16 SummFreq; + CPpmd_State2 State2; + } Union2; + + union + { + CPpmd_State_Ref Stats; + CPpmd_State4 State4; + } Union4; + CPpmd8_Context_Ref Suffix; } CPpmd8_Context; -#pragma pack(pop) +// MY_CPU_pragma_pop -#define Ppmd8Context_OneState(p) ((CPpmd_State *)&(p)->SummFreq) +#define Ppmd8Context_OneState(p) ((CPpmd_State *)&(p)->Union2) -/* The BUG in Shkarin's code for FREEZE mode was fixed, but that fixed - code is not compatible with original code for some files compressed +/* PPMdI code rev.2 contains the fix over PPMdI code rev.1. + But the code PPMdI.2 is not compatible with PPMdI.1 for some files compressed in FREEZE mode. So we disable FREEZE mode support. */ +// #define PPMD8_FREEZE_SUPPORT + enum { PPMD8_RESTORE_METHOD_RESTART, @@ -50,22 +60,28 @@ enum #ifdef PPMD8_FREEZE_SUPPORT , PPMD8_RESTORE_METHOD_FREEZE #endif + , PPMD8_RESTORE_METHOD_UNSUPPPORTED }; + + + + + + + typedef struct { CPpmd8_Context *MinContext, *MaxContext; CPpmd_State *FoundState; - unsigned OrderFall, InitEsc, PrevSuccess, MaxOrder; + unsigned OrderFall, InitEsc, PrevSuccess, MaxOrder, RestoreMethod; Int32 RunLength, InitRL; /* must be 32-bit at least */ UInt32 Size; UInt32 GlueCount; - Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; UInt32 AlignOffset; - unsigned RestoreMethod; + Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; - /* Range Coder */ UInt32 Range; UInt32 Code; UInt32 Low; @@ -75,16 +91,18 @@ typedef struct IByteOut *Out; } Stream; - Byte Indx2Units[PPMD_NUM_INDEXES]; + Byte Indx2Units[PPMD_NUM_INDEXES + 2]; // +2 for alignment Byte Units2Indx[128]; CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; UInt32 Stamps[PPMD_NUM_INDEXES]; - Byte NS2BSIndx[256], NS2Indx[260]; + Byte ExpEscape[16]; CPpmd_See DummySee, See[24][32]; UInt16 BinSumm[25][64]; + } CPpmd8; + void Ppmd8_Construct(CPpmd8 *p); BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc); @@ -94,43 +112,69 @@ void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod); /* ---------- Internal Functions ---------- */ -extern const Byte PPMD8_kExpEscape[16]; - -#ifdef PPMD_32BIT - #define Ppmd8_GetPtr(p, ptr) (ptr) - #define Ppmd8_GetContext(p, ptr) (ptr) - #define Ppmd8_GetStats(p, ctx) ((ctx)->Stats) -#else - #define Ppmd8_GetPtr(p, offs) ((void *)((p)->Base + (offs))) - #define Ppmd8_GetContext(p, offs) ((CPpmd8_Context *)Ppmd8_GetPtr((p), (offs))) - #define Ppmd8_GetStats(p, ctx) ((CPpmd_State *)Ppmd8_GetPtr((p), ((ctx)->Stats))) -#endif +#define Ppmd8_GetPtr(p, ptr) Ppmd_GetPtr(p, ptr) +#define Ppmd8_GetContext(p, ptr) Ppmd_GetPtr_Type(p, ptr, CPpmd8_Context) +#define Ppmd8_GetStats(p, ctx) Ppmd_GetPtr_Type(p, (ctx)->Union4.Stats, CPpmd_State) void Ppmd8_Update1(CPpmd8 *p); void Ppmd8_Update1_0(CPpmd8 *p); void Ppmd8_Update2(CPpmd8 *p); -void Ppmd8_UpdateBin(CPpmd8 *p); + + + + + #define Ppmd8_GetBinSumm(p) \ - &p->BinSumm[p->NS2Indx[(size_t)Ppmd8Context_OneState(p->MinContext)->Freq - 1]][ \ - p->NS2BSIndx[Ppmd8_GetContext(p, p->MinContext->Suffix)->NumStats] + \ - p->PrevSuccess + p->MinContext->Flags + ((p->RunLength >> 26) & 0x20)] + &p->BinSumm[p->NS2Indx[(size_t)Ppmd8Context_OneState(p->MinContext)->Freq - 1]] \ + [ p->PrevSuccess + ((p->RunLength >> 26) & 0x20) \ + + p->NS2BSIndx[Ppmd8_GetContext(p, p->MinContext->Suffix)->NumStats] + \ + + p->MinContext->Flags ] + CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked, UInt32 *scale); +/* 20.01: the original PPMdI encoder and decoder probably could work incorrectly in some rare cases, + where the original PPMdI code can give "Divide by Zero" operation. + We use the following fix to allow correct working of encoder and decoder in any cases. + We correct (Escape_Freq) and (_sum_), if (_sum_) is larger than p->Range) */ +#define PPMD8_CORRECT_SUM_RANGE(p, _sum_) if (_sum_ > p->Range /* /1 */) _sum_ = p->Range; + + /* ---------- Decode ---------- */ -BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p); +#define PPMD8_SYM_END (-1) +#define PPMD8_SYM_ERROR (-2) + +/* +You must set (CPpmd8::Stream.In) before Ppmd8_RangeDec_Init() + +Ppmd8_DecodeSymbol() +out: + >= 0 : decoded byte + -1 : PPMD8_SYM_END : End of payload marker + -2 : PPMD8_SYM_ERROR : Data error +*/ + + +BoolInt Ppmd8_Init_RangeDec(CPpmd8 *p); #define Ppmd8_RangeDec_IsFinishedOK(p) ((p)->Code == 0) -int Ppmd8_DecodeSymbol(CPpmd8 *p); /* returns: -1 as EndMarker, -2 as DataError */ +int Ppmd8_DecodeSymbol(CPpmd8 *p); + + + + + + /* ---------- Encode ---------- */ -#define Ppmd8_RangeEnc_Init(p) { (p)->Low = 0; (p)->Range = 0xFFFFFFFF; } -void Ppmd8_RangeEnc_FlushData(CPpmd8 *p); -void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol); /* symbol = -1 means EndMarker */ +#define Ppmd8_Init_RangeEnc(p) { (p)->Low = 0; (p)->Range = 0xFFFFFFFF; } +void Ppmd8_Flush_RangeEnc(CPpmd8 *p); +void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol); + EXTERN_C_END diff --git a/C/Ppmd8Dec.c b/C/Ppmd8Dec.c index a18ec677..d205de28 100644 --- a/C/Ppmd8Dec.c +++ b/C/Ppmd8Dec.c @@ -1,5 +1,5 @@ -/* Ppmd8Dec.c -- PPMdI Decoder -2018-07-04 : Igor Pavlov : Public domain +/* Ppmd8Dec.c -- Ppmd8 (PPMdI) Decoder +2021-04-13 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -11,147 +11,269 @@ This code is based on: #define kTop (1 << 24) #define kBot (1 << 15) -BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p) +#define READ_BYTE(p) IByteIn_Read((p)->Stream.In) + +BoolInt Ppmd8_Init_RangeDec(CPpmd8 *p) { unsigned i; - p->Low = 0; - p->Range = 0xFFFFFFFF; p->Code = 0; + p->Range = 0xFFFFFFFF; + p->Low = 0; + for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream.In); + p->Code = (p->Code << 8) | READ_BYTE(p); return (p->Code < 0xFFFFFFFF); } -static UInt32 RangeDec_GetThreshold(CPpmd8 *p, UInt32 total) -{ - return p->Code / (p->Range /= total); -} +#define RC_NORM(p) \ + while ((p->Low ^ (p->Low + p->Range)) < kTop \ + || (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) { \ + p->Code = (p->Code << 8) | READ_BYTE(p); \ + p->Range <<= 8; p->Low <<= 8; } + +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) +#define R p + +MY_FORCE_INLINE +// MY_NO_INLINE static void RangeDec_Decode(CPpmd8 *p, UInt32 start, UInt32 size) { - start *= p->Range; - p->Low += start; - p->Code -= start; - p->Range *= size; - - while ((p->Low ^ (p->Low + p->Range)) < kTop || - (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) - { - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream.In); - p->Range <<= 8; - p->Low <<= 8; - } + start *= R->Range; + R->Low += start; + R->Code -= start; + R->Range *= size; + RC_NORM_LOCAL(R) } -#define MASK(sym) ((signed char *)charMask)[sym] +#define RC_Decode(start, size) RangeDec_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) + + +#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) +typedef CPpmd8_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +void Ppmd8_UpdateModel(CPpmd8 *p); + +#define MASK(sym) ((unsigned char *)charMask)[sym] + int Ppmd8_DecodeSymbol(CPpmd8 *p) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 0) { CPpmd_State *s = Ppmd8_GetStats(p, p->MinContext); unsigned i; UInt32 count, hiCnt; - if ((count = RangeDec_GetThreshold(p, p->MinContext->SummFreq)) < (hiCnt = s->Freq)) + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + PPMD8_CORRECT_SUM_RANGE(p, summFreq) + + + count = RC_GetThreshold(summFreq); + hiCnt = count; + + if ((Int32)(count -= s->Freq) < 0) { - Byte symbol; - RangeDec_Decode(p, 0, s->Freq); + Byte sym; + RC_DecodeFinal(0, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd8_Update1_0(p); - return symbol; + return sym; } + p->PrevSuccess = 0; i = p->MinContext->NumStats; + do { - if ((hiCnt += (++s)->Freq) > count) + if ((Int32)(count -= (++s)->Freq) < 0) { - Byte symbol; - RangeDec_Decode(p, hiCnt - s->Freq, s->Freq); + Byte sym; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd8_Update1(p); - return symbol; + return sym; } } while (--i); - if (count >= p->MinContext->SummFreq) - return -2; - RangeDec_Decode(p, hiCnt, p->MinContext->SummFreq - hiCnt); + + if (hiCnt >= summFreq) + return PPMD8_SYM_ERROR; + + hiCnt -= count; + RC_Decode(hiCnt, summFreq - hiCnt); + + PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats; - do { MASK((--s)->Symbol) = 0; } while (--i); + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { + CPpmd_State *s = Ppmd8Context_OneState(p->MinContext); UInt16 *prob = Ppmd8_GetBinSumm(p); - if (((p->Code / (p->Range >>= 14)) < *prob)) + UInt32 pr = *prob; + UInt32 size0 = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + + if (R->Code < size0) { - Byte symbol; - RangeDec_Decode(p, 0, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - symbol = (p->FoundState = Ppmd8Context_OneState(p->MinContext))->Symbol; - Ppmd8_UpdateBin(p); - return symbol; + Byte sym; + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + + // RangeDec_DecodeBit0(size0); + R->Range = size0; + RC_NORM(R) + + + + // sym = (p->FoundState = Ppmd8Context_OneState(p->MinContext))->Symbol; + // Ppmd8_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + sym = s->Symbol; + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 196)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; + else + Ppmd8_UpdateModel(p); + } + return sym; } - RangeDec_Decode(p, *prob, (1 << 14) - *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD8_kExpEscape[*prob >> 10]; + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + + // RangeDec_DecodeBit1(rc2, size0); + R->Low += size0; + R->Code -= size0; + R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - size0; + RC_NORM_LOCAL(R) + PPMD_SetAllBitsIn256Bytes(charMask); MASK(Ppmd8Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } + for (;;) { - CPpmd_State *ps[256], *s; + CPpmd_State *s, *s2; UInt32 freqSum, count, hiCnt; + UInt32 freqSum2; CPpmd_See *see; - unsigned i, num, numMasked = p->MinContext->NumStats; + CPpmd8_Context *mc; + unsigned numMasked; + RC_NORM_REMOTE(R) + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) - return -1; - p->MinContext = Ppmd8_GetContext(p, p->MinContext->Suffix); + if (!mc->Suffix) + return PPMD8_SYM_END; + mc = Ppmd8_GetContext(p, mc->Suffix); } - while (p->MinContext->NumStats == numMasked); - hiCnt = 0; - s = Ppmd8_GetStats(p, p->MinContext); - i = 0; - num = p->MinContext->NumStats - numMasked; - do + while (mc->NumStats == numMasked); + + s = Ppmd8_GetStats(p, mc); + { - int k = (int)(MASK(s->Symbol)); - hiCnt += (s->Freq & k); - ps[i] = s++; - i -= k; + unsigned num = (unsigned)mc->NumStats + 1; + unsigned num2 = num / 2; + + num &= 1; + hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + s += num; + p->MinContext = mc; + + do + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); + hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + } + while (--num2); } - while (i != num); see = Ppmd8_MakeEscFreq(p, numMasked, &freqSum); freqSum += hiCnt; - count = RangeDec_GetThreshold(p, freqSum); + freqSum2 = freqSum; + PPMD8_CORRECT_SUM_RANGE(R, freqSum2); + + + count = RC_GetThreshold(freqSum2); if (count < hiCnt) { - Byte symbol; - CPpmd_State **pps = ps; - for (hiCnt = 0; (hiCnt += (*pps)->Freq) <= count; pps++); - s = *pps; - RangeDec_Decode(p, hiCnt - s->Freq, s->Freq); + Byte sym; + // Ppmd_See_Update(see); // new (see->Summ) value can overflow over 16-bits in some rare cases + s = Ppmd8_GetStats(p, p->MinContext); + hiCnt = count; + + + { + for (;;) + { + count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + } + } + s--; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + + // new (see->Summ) value can overflow over 16-bits in some rare cases Ppmd_See_Update(see); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd8_Update2(p); - return symbol; + return sym; } - if (count >= freqSum) - return -2; - RangeDec_Decode(p, hiCnt, freqSum - hiCnt); + + if (count >= freqSum2) + return PPMD8_SYM_ERROR; + + RC_Decode(hiCnt, freqSum2 - hiCnt); + + // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. + // new (see->Summ) value can overflow over 16-bits in some rare cases see->Summ = (UInt16)(see->Summ + freqSum); - do { MASK(ps[--i]->Symbol) = 0; } while (i != 0); + + s = Ppmd8_GetStats(p, p->MinContext); + s2 = s + p->MinContext->NumStats + 1; + do + { + MASK(s->Symbol) = 0; + s++; + } + while (s != s2); } } diff --git a/C/Ppmd8Enc.c b/C/Ppmd8Enc.c index 1cbc17f1..32ff8052 100644 --- a/C/Ppmd8Enc.c +++ b/C/Ppmd8Enc.c @@ -1,5 +1,5 @@ -/* Ppmd8Enc.c -- PPMdI Encoder -2017-04-03 : Igor Pavlov : Public domain +/* Ppmd8Enc.c -- Ppmd8 (PPMdI) Encoder +2021-04-13 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -11,59 +11,100 @@ This code is based on: #define kTop (1 << 24) #define kBot (1 << 15) -void Ppmd8_RangeEnc_FlushData(CPpmd8 *p) +#define WRITE_BYTE(p) IByteOut_Write(p->Stream.Out, (Byte)(p->Low >> 24)) + +void Ppmd8_Flush_RangeEnc(CPpmd8 *p) { unsigned i; for (i = 0; i < 4; i++, p->Low <<= 8 ) - IByteOut_Write(p->Stream.Out, (Byte)(p->Low >> 24)); + WRITE_BYTE(p); } -static void RangeEnc_Normalize(CPpmd8 *p) -{ - while ((p->Low ^ (p->Low + p->Range)) < kTop || - (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) - { - IByteOut_Write(p->Stream.Out, (Byte)(p->Low >> 24)); - p->Range <<= 8; - p->Low <<= 8; - } -} + + + + +#define RC_NORM(p) \ + while ((p->Low ^ (p->Low + p->Range)) < kTop \ + || (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) \ + { WRITE_BYTE(p); p->Range <<= 8; p->Low <<= 8; } + + + + + + + + + + + + + +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +// #define RC_PRE(total) p->Range /= total; +// #define RC_PRE(total) + +#define R p + + + + +MY_FORCE_INLINE +// MY_NO_INLINE static void RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 total) { - p->Low += start * (p->Range /= total); - p->Range *= size; - RangeEnc_Normalize(p); + R->Low += start * (R->Range /= total); + R->Range *= size; + RC_NORM_LOCAL(R); } -static void RangeEnc_EncodeBit_0(CPpmd8 *p, UInt32 size0) -{ - p->Range >>= 14; - p->Range *= size0; - RangeEnc_Normalize(p); -} -static void RangeEnc_EncodeBit_1(CPpmd8 *p, UInt32 size0) -{ - p->Low += size0 * (p->Range >>= 14); - p->Range *= ((1 << 14) - size0); - RangeEnc_Normalize(p); -} -#define MASK(sym) ((signed char *)charMask)[sym] + + + + + +#define RC_Encode(start, size, total) RangeEnc_Encode(p, start, size, total); +#define RC_EncodeFinal(start, size, total) RC_Encode(start, size, total); RC_NORM_REMOTE(p); + +#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) + +typedef CPpmd8_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) + +void Ppmd8_UpdateModel(CPpmd8 *p); + +#define MASK(sym) ((unsigned char *)charMask)[sym] + +// MY_FORCE_INLINE +// static void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 0) { CPpmd_State *s = Ppmd8_GetStats(p, p->MinContext); UInt32 sum; unsigned i; + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + PPMD8_CORRECT_SUM_RANGE(p, summFreq) + + // RC_PRE(summFreq); + if (s->Symbol == symbol) { - RangeEnc_Encode(p, 0, s->Freq, p->MinContext->SummFreq); + + RC_EncodeFinal(0, s->Freq, summFreq); p->FoundState = s; Ppmd8_Update1_0(p); return; @@ -75,7 +116,8 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) { if ((++s)->Symbol == symbol) { - RangeEnc_Encode(p, sum, s->Freq, p->MinContext->SummFreq); + + RC_EncodeFinal(sum, s->Freq, summFreq); p->FoundState = s; Ppmd8_Update1(p); return; @@ -84,80 +126,189 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) } while (--i); + + RC_Encode(sum, summFreq - sum, summFreq); + + PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats; - do { MASK((--s)->Symbol) = 0; } while (--i); - RangeEnc_Encode(p, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); + // MASK(s->Symbol) = 0; + // i = p->MinContext->NumStats; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { UInt16 *prob = Ppmd8_GetBinSumm(p); CPpmd_State *s = Ppmd8Context_OneState(p->MinContext); + UInt32 pr = *prob; + UInt32 bound = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); if (s->Symbol == symbol) { - RangeEnc_EncodeBit_0(p, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - p->FoundState = s; - Ppmd8_UpdateBin(p); + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + // RangeEnc_EncodeBit_0(p, bound); + R->Range = bound; + RC_NORM(R); + + // p->FoundState = s; + // Ppmd8_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; + else + Ppmd8_UpdateModel(p); + } return; } - else - { - RangeEnc_EncodeBit_1(p, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD8_kExpEscape[*prob >> 10]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - p->PrevSuccess = 0; - } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + // RangeEnc_EncodeBit_1(p, bound); + R->Low += bound; + R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - bound; + RC_NORM_LOCAL(R) + + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; } + for (;;) { - UInt32 escFreq; CPpmd_See *see; CPpmd_State *s; - UInt32 sum; - unsigned i, numMasked = p->MinContext->NumStats; + UInt32 sum, escFreq; + CPpmd8_Context *mc; + unsigned i, numMasked; + + RC_NORM_REMOTE(p) + + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) + if (!mc->Suffix) return; /* EndMarker (symbol = -1) */ - p->MinContext = Ppmd8_GetContext(p, p->MinContext->Suffix); + mc = Ppmd8_GetContext(p, mc->Suffix); + } - while (p->MinContext->NumStats == numMasked); + while (mc->NumStats == numMasked); + p->MinContext = mc; + see = Ppmd8_MakeEscFreq(p, numMasked, &escFreq); + + + + + + + + + + + + + + + + + + + + + + + + s = Ppmd8_GetStats(p, p->MinContext); sum = 0; - i = p->MinContext->NumStats + 1; + i = (unsigned)p->MinContext->NumStats + 1; + do { - int cur = s->Symbol; - if (cur == symbol) + unsigned cur = s->Symbol; + if ((int)cur == symbol) { UInt32 low = sum; - CPpmd_State *s1 = s; - do + UInt32 freq = s->Freq; + unsigned num2; + + Ppmd_See_Update(see); + p->FoundState = s; + sum += escFreq; + + num2 = i / 2; + i &= 1; + sum += freq & (0 - (UInt32)i); + if (num2 != 0) { - sum += (s->Freq & (int)(MASK(s->Symbol))); - s++; + s += i; + for (;;) + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + sum += (s[-2].Freq & (unsigned)(MASK(sym0))); + sum += (s[-1].Freq & (unsigned)(MASK(sym1))); + if (--num2 == 0) + break; + } } - while (--i); - RangeEnc_Encode(p, low, s1->Freq, sum + escFreq); - Ppmd_See_Update(see); - p->FoundState = s1; + + PPMD8_CORRECT_SUM_RANGE(p, sum); + + RC_EncodeFinal(low, freq, sum); Ppmd8_Update2(p); return; } - sum += (s->Freq & (int)(MASK(cur))); - MASK(cur) = 0; + sum += (s->Freq & (unsigned)(MASK(cur))); s++; } while (--i); - RangeEnc_Encode(p, sum, escFreq, sum + escFreq); - see->Summ = (UInt16)(see->Summ + sum + escFreq); + { + UInt32 total = sum + escFreq; + see->Summ = (UInt16)(see->Summ + total); + PPMD8_CORRECT_SUM_RANGE(p, total); + + RC_Encode(sum, total - sum, total); + } + + { + CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + s--; + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } } @@ -1,5 +1,5 @@ /* Sha1.c -- SHA-1 Hash -2017-04-03 : Igor Pavlov : Public domain +2021-04-01 : Igor Pavlov : Public domain This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -10,331 +10,434 @@ This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ l #include "RotateDefs.h" #include "Sha1.h" -// define it for speed optimization +#if defined(_MSC_VER) && (_MSC_VER < 1900) +// #define USE_MY_MM +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + #ifdef _MSC_VER + #if _MSC_VER >= 1200 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#elif defined(MY_CPU_ARM_OR_ARM64) + #ifdef _MSC_VER + #if _MSC_VER >= 1910 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#endif + +void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); + +#ifdef _SHA_SUPPORTED + void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); + + static SHA1_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS = Sha1_UpdateBlocks; + static SHA1_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS_HW; + + #define UPDATE_BLOCKS(p) p->func_UpdateBlocks +#else + #define UPDATE_BLOCKS(p) Sha1_UpdateBlocks +#endif + + +BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) +{ + SHA1_FUNC_UPDATE_BLOCKS func = Sha1_UpdateBlocks; + + #ifdef _SHA_SUPPORTED + if (algo != SHA1_ALGO_SW) + { + if (algo == SHA1_ALGO_DEFAULT) + func = g_FUNC_UPDATE_BLOCKS; + else + { + if (algo != SHA1_ALGO_HW) + return False; + func = g_FUNC_UPDATE_BLOCKS_HW; + if (!func) + return False; + } + } + #else + if (algo > 1) + return False; + #endif + + p->func_UpdateBlocks = func; + return True; +} + + +/* define it for speed optimization */ // #define _SHA1_UNROLL +// allowed unroll steps: (1, 2, 4, 5, 20) + #ifdef _SHA1_UNROLL - #define kNumW 16 - #define WW(i) W[(i)&15] + #define STEP_PRE 20 + #define STEP_MAIN 20 #else + #define _SHA1_BIG_W + #define STEP_PRE 5 + #define STEP_MAIN 5 +#endif + + +#ifdef _SHA1_BIG_W #define kNumW 80 - #define WW(i) W[i] + #define w(i) W[i] +#else + #define kNumW 16 + #define w(i) W[(i)&15] #endif -#define w0(i) (W[i] = data[i]) +#define w0(i) (W[i] = GetBe32(data + (size_t)(i) * 4)) +#define w1(i) (w(i) = rotlFixed(w((size_t)(i)-3) ^ w((size_t)(i)-8) ^ w((size_t)(i)-14) ^ w((size_t)(i)-16), 1)) -#define w1(i) (WW(i) = rotlFixed(WW((i)-3) ^ WW((i)-8) ^ WW((i)-14) ^ WW((i)-16), 1)) +#define f0(x,y,z) ( 0x5a827999 + (z^(x&(y^z))) ) +#define f1(x,y,z) ( 0x6ed9eba1 + (x^y^z) ) +#define f2(x,y,z) ( 0x8f1bbcdc + ((x&y)|(z&(x|y))) ) +#define f3(x,y,z) ( 0xca62c1d6 + (x^y^z) ) -#define f1(x,y,z) (z^(x&(y^z))) -#define f2(x,y,z) (x^y^z) -#define f3(x,y,z) ((x&y)|(z&(x|y))) -#define f4(x,y,z) (x^y^z) +/* +#define T1(fx, ww) \ + tmp = e + fx(b,c,d) + ww + rotlFixed(a, 5); \ + e = d; \ + d = c; \ + c = rotlFixed(b, 30); \ + b = a; \ + a = tmp; \ +*/ -#define RK(a,b,c,d,e, fx, w, k) e += fx(b,c,d) + w + k + rotlFixed(a,5); b = rotlFixed(b,30); +#define T5(a,b,c,d,e, fx, ww) \ + e += fx(b,c,d) + ww + rotlFixed(a, 5); \ + b = rotlFixed(b, 30); \ -#define R0(a,b,c,d,e, i) RK(a,b,c,d,e, f1, w0(i), 0x5A827999) -#define R1(a,b,c,d,e, i) RK(a,b,c,d,e, f1, w1(i), 0x5A827999) -#define R2(a,b,c,d,e, i) RK(a,b,c,d,e, f2, w1(i), 0x6ED9EBA1) -#define R3(a,b,c,d,e, i) RK(a,b,c,d,e, f3, w1(i), 0x8F1BBCDC) -#define R4(a,b,c,d,e, i) RK(a,b,c,d,e, f4, w1(i), 0xCA62C1D6) -#define RX_1_4(rx1, rx4, i) \ - rx1(a,b,c,d,e, i); \ - rx4(e,a,b,c,d, i+1); \ - rx4(d,e,a,b,c, i+2); \ - rx4(c,d,e,a,b, i+3); \ - rx4(b,c,d,e,a, i+4); \ +/* +#define R1(i, fx, wx) \ + T1 ( fx, wx(i)); \ -#define RX_5(rx, i) RX_1_4(rx, rx, i); +#define R2(i, fx, wx) \ + R1 ( (i) , fx, wx); \ + R1 ( (i) + 1, fx, wx); \ + +#define R4(i, fx, wx) \ + R2 ( (i) , fx, wx); \ + R2 ( (i) + 2, fx, wx); \ +*/ + +#define M5(i, fx, wx0, wx1) \ + T5 ( a,b,c,d,e, fx, wx0((i) ) ); \ + T5 ( e,a,b,c,d, fx, wx1((i)+1) ); \ + T5 ( d,e,a,b,c, fx, wx1((i)+2) ); \ + T5 ( c,d,e,a,b, fx, wx1((i)+3) ); \ + T5 ( b,c,d,e,a, fx, wx1((i)+4) ); \ + +#define R5(i, fx, wx) \ + M5 ( i, fx, wx, wx) \ -#ifdef _SHA1_UNROLL - #define RX_15 \ - RX_5(R0, 0); \ - RX_5(R0, 5); \ - RX_5(R0, 10); +#if STEP_PRE > 5 + + #define R20_START \ + R5 ( 0, f0, w0); \ + R5 ( 5, f0, w0); \ + R5 ( 10, f0, w0); \ + M5 ( 15, f0, w0, w1); \ + + #elif STEP_PRE == 5 - #define RX_20(rx, i) \ - RX_5(rx, i); \ - RX_5(rx, i + 5); \ - RX_5(rx, i + 10); \ - RX_5(rx, i + 15); + #define R20_START \ + { size_t i; for (i = 0; i < 15; i += STEP_PRE) \ + { R5(i, f0, w0); } } \ + M5 ( 15, f0, w0, w1); \ #else - -#define RX_15 { size_t i; for (i = 0; i < 15; i += 5) { RX_5(R0, i); } } -#define RX_20(rx, ii) { size_t i; i = ii; for (; i < ii + 20; i += 5) { RX_5(rx, i); } } + + #if STEP_PRE == 1 + #define R_PRE R1 + #elif STEP_PRE == 2 + #define R_PRE R2 + #elif STEP_PRE == 4 + #define R_PRE R4 + #endif + + #define R20_START \ + { size_t i; for (i = 0; i < 16; i += STEP_PRE) \ + { R_PRE(i, f0, w0); } } \ + R4 ( 16, f0, w1); \ #endif -void Sha1_Init(CSha1 *p) + +#if STEP_MAIN > 5 + + #define R20(ii, fx) \ + R5 ( (ii) , fx, w1); \ + R5 ( (ii) + 5 , fx, w1); \ + R5 ( (ii) + 10, fx, w1); \ + R5 ( (ii) + 15, fx, w1); \ + +#else + + #if STEP_MAIN == 1 + #define R_MAIN R1 + #elif STEP_MAIN == 2 + #define R_MAIN R2 + #elif STEP_MAIN == 4 + #define R_MAIN R4 + #elif STEP_MAIN == 5 + #define R_MAIN R5 + #endif + + #define R20(ii, fx) \ + { size_t i; for (i = (ii); i < (ii) + 20; i += STEP_MAIN) \ + { R_MAIN(i, fx, w1); } } \ + +#endif + + + +void Sha1_InitState(CSha1 *p) { + p->count = 0; p->state[0] = 0x67452301; p->state[1] = 0xEFCDAB89; p->state[2] = 0x98BADCFE; p->state[3] = 0x10325476; p->state[4] = 0xC3D2E1F0; - p->count = 0; } -void Sha1_GetBlockDigest(CSha1 *p, const UInt32 *data, UInt32 *destDigest) +void Sha1_Init(CSha1 *p) { - UInt32 a, b, c, d, e; - UInt32 W[kNumW]; - - a = p->state[0]; - b = p->state[1]; - c = p->state[2]; - d = p->state[3]; - e = p->state[4]; - - RX_15 - - RX_1_4(R0, R1, 15); - - RX_20(R2, 20); - RX_20(R3, 40); - RX_20(R4, 60); - - destDigest[0] = p->state[0] + a; - destDigest[1] = p->state[1] + b; - destDigest[2] = p->state[2] + c; - destDigest[3] = p->state[3] + d; - destDigest[4] = p->state[4] + e; + p->func_UpdateBlocks = + #ifdef _SHA_SUPPORTED + g_FUNC_UPDATE_BLOCKS; + #else + NULL; + #endif + Sha1_InitState(p); } -void Sha1_UpdateBlock_Rar(CSha1 *p, UInt32 *data, int returnRes) + +MY_NO_INLINE +void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks) { UInt32 a, b, c, d, e; UInt32 W[kNumW]; + // if (numBlocks != 0x1264378347) return; + if (numBlocks == 0) + return; - a = p->state[0]; - b = p->state[1]; - c = p->state[2]; - d = p->state[3]; - e = p->state[4]; - - RX_15 - - RX_1_4(R0, R1, 15); - - RX_20(R2, 20); - RX_20(R3, 40); - RX_20(R4, 60); - - p->state[0] += a; - p->state[1] += b; - p->state[2] += c; - p->state[3] += d; - p->state[4] += e; + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; - if (returnRes) + do { - size_t i; - for (i = 0 ; i < SHA1_NUM_BLOCK_WORDS; i++) - data[i] = W[kNumW - SHA1_NUM_BLOCK_WORDS + i]; + #if STEP_PRE < 5 || STEP_MAIN < 5 + UInt32 tmp; + #endif + + R20_START + R20(20, f1); + R20(40, f2); + R20(60, f3); + + a += state[0]; + b += state[1]; + c += state[2]; + d += state[3]; + e += state[4]; + + state[0] = a; + state[1] = b; + state[2] = c; + state[3] = d; + state[4] = e; + + data += 64; } + while (--numBlocks); } -#define Sha1_UpdateBlock(p) Sha1_GetBlockDigest(p, p->buffer, p->state) + +#define Sha1_UpdateBlock(p) UPDATE_BLOCKS(p)(p->state, p->buffer, 1) void Sha1_Update(CSha1 *p, const Byte *data, size_t size) { - unsigned pos, pos2; if (size == 0) return; - pos = (unsigned)p->count & 0x3F; - p->count += size; - pos2 = pos & 3; - pos >>= 2; - - if (pos2 != 0) - { - UInt32 w; - pos2 = (3 - pos2) * 8; - w = ((UInt32)*data++) << pos2; - if (--size && pos2) - { - pos2 -= 8; - w |= ((UInt32)*data++) << pos2; - if (--size && pos2) - { - pos2 -= 8; - w |= ((UInt32)*data++) << pos2; - size--; - } - } - p->buffer[pos] |= w; - if (pos2 == 0) - pos++; - } - for (;;) { - if (pos == SHA1_NUM_BLOCK_WORDS) + unsigned pos = (unsigned)p->count & 0x3F; + unsigned num; + + p->count += size; + + num = 64 - pos; + if (num > size) { - for (;;) - { - size_t i; - Sha1_UpdateBlock(p); - if (size < SHA1_BLOCK_SIZE) - break; - size -= SHA1_BLOCK_SIZE; - for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i += 2) - { - p->buffer[i ] = GetBe32(data); - p->buffer[i + 1] = GetBe32(data + 4); - data += 8; - } - } - pos = 0; + memcpy(p->buffer + pos, data, size); + return; } - if (size < 4) - break; - - p->buffer[pos] = GetBe32(data); - data += 4; - size -= 4; - pos++; - } - - if (size != 0) - { - UInt32 w = ((UInt32)data[0]) << 24; - if (size > 1) + + if (pos != 0) { - w |= ((UInt32)data[1]) << 16; - if (size > 2) - w |= ((UInt32)data[2]) << 8; + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + Sha1_UpdateBlock(p); } - p->buffer[pos] = w; } -} - -void Sha1_Update_Rar(CSha1 *p, Byte *data, size_t size /* , int rar350Mode */) -{ - int returnRes = False; - - unsigned pos = (unsigned)p->count & 0x3F; - p->count += size; - - while (size--) { - unsigned pos2 = (pos & 3); - UInt32 v = ((UInt32)*data++) << (8 * (3 - pos2)); - UInt32 *ref = &(p->buffer[pos >> 2]); - pos++; - if (pos2 == 0) - { - *ref = v; - continue; - } - *ref |= v; - - if (pos == SHA1_BLOCK_SIZE) - { - pos = 0; - Sha1_UpdateBlock_Rar(p, p->buffer, returnRes); - if (returnRes) - { - size_t i; - for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) - { - UInt32 d = p->buffer[i]; - Byte *prev = data + i * 4 - SHA1_BLOCK_SIZE; - SetUi32(prev, d); - } - } - // returnRes = rar350Mode; - returnRes = True; - } + size_t numBlocks = size >> 6; + UPDATE_BLOCKS(p)(p->state, data, numBlocks); + size &= 0x3F; + if (size == 0) + return; + data += (numBlocks << 6); + memcpy(p->buffer, data, size); } } + void Sha1_Final(CSha1 *p, Byte *digest) { unsigned pos = (unsigned)p->count & 0x3F; - unsigned pos2 = (pos & 3); - UInt64 numBits; - UInt32 w; - unsigned i; - pos >>= 2; + + p->buffer[pos++] = 0x80; - w = 0; - if (pos2 != 0) - w = p->buffer[pos]; - p->buffer[pos++] = w | (((UInt32)0x80000000) >> (8 * pos2)); + if (pos > (64 - 8)) + { + while (pos != 64) { p->buffer[pos++] = 0; } + // memset(&p->buf.buffer[pos], 0, 64 - pos); + Sha1_UpdateBlock(p); + pos = 0; + } - while (pos != (SHA1_NUM_BLOCK_WORDS - 2)) + /* + if (pos & 3) { - pos &= 0xF; - if (pos == 0) - Sha1_UpdateBlock(p); - p->buffer[pos++] = 0; + p->buffer[pos] = 0; + p->buffer[pos + 1] = 0; + p->buffer[pos + 2] = 0; + pos += 3; + pos &= ~3; } - - numBits = (p->count << 3); - p->buffer[SHA1_NUM_BLOCK_WORDS - 2] = (UInt32)(numBits >> 32); - p->buffer[SHA1_NUM_BLOCK_WORDS - 1] = (UInt32)(numBits); - Sha1_UpdateBlock(p); + { + for (; pos < 64 - 8; pos += 4) + *(UInt32 *)(&p->buffer[pos]) = 0; + } + */ + + memset(&p->buffer[pos], 0, (64 - 8) - pos); - for (i = 0; i < SHA1_NUM_DIGEST_WORDS; i++) { - UInt32 v = p->state[i]; - SetBe32(digest, v); - digest += 4; + UInt64 numBits = (p->count << 3); + SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); + SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); } + + Sha1_UpdateBlock(p); - Sha1_Init(p); -} + SetBe32(digest, p->state[0]); + SetBe32(digest + 4, p->state[1]); + SetBe32(digest + 8, p->state[2]); + SetBe32(digest + 12, p->state[3]); + SetBe32(digest + 16, p->state[4]); + -void Sha1_32_PrepareBlock(const CSha1 *p, UInt32 *block, unsigned size) -{ - const UInt64 numBits = (p->count + size) << 5; - block[SHA1_NUM_BLOCK_WORDS - 2] = (UInt32)(numBits >> 32); - block[SHA1_NUM_BLOCK_WORDS - 1] = (UInt32)(numBits); - block[size++] = 0x80000000; - while (size != (SHA1_NUM_BLOCK_WORDS - 2)) - block[size++] = 0; + + Sha1_InitState(p); } -void Sha1_32_Update(CSha1 *p, const UInt32 *data, size_t size) + +void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size) { - unsigned pos = (unsigned)p->count & 0xF; - p->count += size; - while (size--) + const UInt64 numBits = (p->count + size) << 3; + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 2], (UInt32)(numBits >> 32)); + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 1], (UInt32)(numBits)); + // SetBe32((UInt32 *)(block + size), 0x80000000); + SetUi32((UInt32 *)(void *)(block + size), 0x80); + size += 4; + while (size != (SHA1_NUM_BLOCK_WORDS - 2) * 4) { - p->buffer[pos++] = *data++; - if (pos == SHA1_NUM_BLOCK_WORDS) - { - pos = 0; - Sha1_UpdateBlock(p); - } + *((UInt32 *)(void *)(block + size)) = 0; + size += 4; } } -void Sha1_32_Final(CSha1 *p, UInt32 *digest) +void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest) { - UInt64 numBits; - unsigned pos = (unsigned)p->count & 0xF; - p->buffer[pos++] = 0x80000000; + MY_ALIGN (16) + UInt32 st[SHA1_NUM_DIGEST_WORDS]; + + st[0] = p->state[0]; + st[1] = p->state[1]; + st[2] = p->state[2]; + st[3] = p->state[3]; + st[4] = p->state[4]; + + UPDATE_BLOCKS(p)(st, data, 1); - while (pos != (SHA1_NUM_BLOCK_WORDS - 2)) + SetBe32(destDigest + 0 , st[0]); + SetBe32(destDigest + 1 * 4, st[1]); + SetBe32(destDigest + 2 * 4, st[2]); + SetBe32(destDigest + 3 * 4, st[3]); + SetBe32(destDigest + 4 * 4, st[4]); +} + + +void Sha1Prepare() +{ + #ifdef _SHA_SUPPORTED + SHA1_FUNC_UPDATE_BLOCKS f, f_hw; + f = Sha1_UpdateBlocks; + f_hw = NULL; + #ifdef MY_CPU_X86_OR_AMD64 + #ifndef USE_MY_MM + if (CPU_IsSupported_SHA() + && CPU_IsSupported_SSSE3() + // && CPU_IsSupported_SSE41() + ) + #endif + #else + if (CPU_IsSupported_SHA1()) + #endif { - pos &= 0xF; - if (pos == 0) - Sha1_UpdateBlock(p); - p->buffer[pos++] = 0; + // printf("\n========== HW SHA1 ======== \n"); + f = f_hw = Sha1_UpdateBlocks_HW; } - - numBits = (p->count << 5); - p->buffer[SHA1_NUM_BLOCK_WORDS - 2] = (UInt32)(numBits >> 32); - p->buffer[SHA1_NUM_BLOCK_WORDS - 1] = (UInt32)(numBits); - - Sha1_GetBlockDigest(p, p->buffer, digest); - - Sha1_Init(p); + g_FUNC_UPDATE_BLOCKS = f; + g_FUNC_UPDATE_BLOCKS_HW = f_hw; + #endif } @@ -1,5 +1,5 @@ /* Sha1.h -- SHA-1 Hash -2016-05-20 : Igor Pavlov : Public domain */ +2021-02-08 : Igor Pavlov : Public domain */ #ifndef __7Z_SHA1_H #define __7Z_SHA1_H @@ -14,24 +14,62 @@ EXTERN_C_BEGIN #define SHA1_BLOCK_SIZE (SHA1_NUM_BLOCK_WORDS * 4) #define SHA1_DIGEST_SIZE (SHA1_NUM_DIGEST_WORDS * 4) +typedef void (MY_FAST_CALL *SHA1_FUNC_UPDATE_BLOCKS)(UInt32 state[5], const Byte *data, size_t numBlocks); + +/* + if (the system supports different SHA1 code implementations) + { + (CSha1::func_UpdateBlocks) will be used + (CSha1::func_UpdateBlocks) can be set by + Sha1_Init() - to default (fastest) + Sha1_SetFunction() - to any algo + } + else + { + (CSha1::func_UpdateBlocks) is ignored. + } +*/ + typedef struct { - UInt32 state[SHA1_NUM_DIGEST_WORDS]; + SHA1_FUNC_UPDATE_BLOCKS func_UpdateBlocks; UInt64 count; - UInt32 buffer[SHA1_NUM_BLOCK_WORDS]; + UInt64 __pad_2[2]; + UInt32 state[SHA1_NUM_DIGEST_WORDS]; + UInt32 __pad_3[3]; + Byte buffer[SHA1_BLOCK_SIZE]; } CSha1; -void Sha1_Init(CSha1 *p); -void Sha1_GetBlockDigest(CSha1 *p, const UInt32 *data, UInt32 *destDigest); +#define SHA1_ALGO_DEFAULT 0 +#define SHA1_ALGO_SW 1 +#define SHA1_ALGO_HW 2 + +/* +Sha1_SetFunction() +return: + 0 - (algo) value is not supported, and func_UpdateBlocks was not changed + 1 - func_UpdateBlocks was set according (algo) value. +*/ + +BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo); + +void Sha1_InitState(CSha1 *p); +void Sha1_Init(CSha1 *p); void Sha1_Update(CSha1 *p, const Byte *data, size_t size); void Sha1_Final(CSha1 *p, Byte *digest); -void Sha1_Update_Rar(CSha1 *p, Byte *data, size_t size /* , int rar350Mode */); +void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size); +void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest); + +// void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); + +/* +call Sha1Prepare() once at program start. +It prepares all supported implementations, and detects the fastest implementation. +*/ -void Sha1_32_PrepareBlock(const CSha1 *p, UInt32 *block, unsigned size); -void Sha1_32_Update(CSha1 *p, const UInt32 *data, size_t size); -void Sha1_32_Final(CSha1 *p, UInt32 *digest); +void Sha1Prepare(void); EXTERN_C_END diff --git a/C/Sha1Opt.c b/C/Sha1Opt.c new file mode 100644 index 00000000..63132da3 --- /dev/null +++ b/C/Sha1Opt.c @@ -0,0 +1,373 @@ +/* Sha1Opt.c -- SHA-1 optimized code for SHA-1 hardware instructions +2021-04-01 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#if defined(_MSC_VER) +#if (_MSC_VER < 1900) && (_MSC_VER >= 1200) +// #define USE_MY_MM +#endif +#endif + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + #if defined(_MSC_VER) + // SSSE3: for clang-cl: + #include <tmmintrin.h> + #define __SHA__ + #endif + #endif + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + // #pragma GCC target("sha,ssse3") + #endif + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #ifdef USE_MY_MM + #define USE_VER_MIN 1300 + #else + #define USE_VER_MIN 1910 + #endif + #if _MSC_VER >= USE_VER_MIN + #define USE_HW_SHA + #endif + #endif +// #endif // MY_CPU_X86_OR_AMD64 + +#ifdef USE_HW_SHA + +// #pragma message("Sha1 HW") +// #include <wmmintrin.h> + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +#include <immintrin.h> +#else +#include <emmintrin.h> + +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +// #include <intrin.h> +#endif + +#ifdef USE_MY_MM +#include "My_mm.h" +#endif + +#endif + +/* +SHA1 uses: +SSE2: + _mm_loadu_si128 + _mm_storeu_si128 + _mm_set_epi32 + _mm_add_epi32 + _mm_shuffle_epi32 / pshufd + _mm_xor_si128 + _mm_cvtsi128_si32 + _mm_cvtsi32_si128 +SSSE3: + _mm_shuffle_epi8 / pshufb + +SHA: + _mm_sha1* +*/ + +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define XOR_SI128(dest, src) dest = _mm_xor_si128(dest, src); +#define SHUFFLE_EPI8(dest, mask) dest = _mm_shuffle_epi8(dest, mask); +#define SHUFFLE_EPI32(dest, mask) dest = _mm_shuffle_epi32(dest, mask); + +#define SHA1_RND4(abcd, e0, f) abcd = _mm_sha1rnds4_epu32(abcd, e0, f); +#define SHA1_NEXTE(e, m) e = _mm_sha1nexte_epu32(e, m); + + + + + +#define SHA1_MSG1(dest, src) dest = _mm_sha1msg1_epu32(dest, src); +#define SHA1_MSG2(dest, src) dest = _mm_sha1msg2_epu32(dest, src); + + +#define LOAD_SHUFFLE(m, k) \ + m = _mm_loadu_si128((const __m128i *)(const void *)(data + (k) * 16)); \ + SHUFFLE_EPI8(m, mask); \ + +#define SM1(m0, m1, m2, m3) \ + SHA1_MSG1(m0, m1); \ + +#define SM2(m0, m1, m2, m3) \ + XOR_SI128(m3, m1); \ + SHA1_MSG2(m3, m2); \ + +#define SM3(m0, m1, m2, m3) \ + XOR_SI128(m3, m1); \ + SM1(m0, m1, m2, m3) \ + SHA1_MSG2(m3, m2); \ + +#define NNN(m0, m1, m2, m3) + + + + + + + + + + + + + + + + + +#define R4(k, e0, e1, m0, m1, m2, m3, OP) \ + e1 = abcd; \ + SHA1_RND4(abcd, e0, (k) / 5); \ + SHA1_NEXTE(e1, m1); \ + OP(m0, m1, m2, m3); \ + +#define R16(k, mx, OP0, OP1, OP2, OP3) \ + R4 ( (k)*4+0, e0,e1, m0,m1,m2,m3, OP0 ) \ + R4 ( (k)*4+1, e1,e0, m1,m2,m3,m0, OP1 ) \ + R4 ( (k)*4+2, e0,e1, m2,m3,m0,m1, OP2 ) \ + R4 ( (k)*4+3, e1,e0, m3,mx,m1,m2, OP3 ) \ + +#define PREPARE_STATE \ + SHUFFLE_EPI32 (abcd, 0x1B); \ + SHUFFLE_EPI32 (e0, 0x1B); \ + + + + + +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +{ + const __m128i mask = _mm_set_epi32(0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f); + + __m128i abcd, e0; + + if (numBlocks == 0) + return; + + abcd = _mm_loadu_si128((const __m128i *) (const void *) &state[0]); // dbca + e0 = _mm_cvtsi32_si128((int)state[4]); // 000e + + PREPARE_STATE + + do + { + __m128i abcd_save, e2; + __m128i m0, m1, m2, m3; + __m128i e1; + + + abcd_save = abcd; + e2 = e0; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + ADD_EPI32(e0, m0); + + R16 ( 0, m0, SM1, SM3, SM3, SM3 ); + R16 ( 1, m0, SM3, SM3, SM3, SM3 ); + R16 ( 2, m0, SM3, SM3, SM3, SM3 ); + R16 ( 3, m0, SM3, SM3, SM3, SM3 ); + R16 ( 4, e2, SM2, NNN, NNN, NNN ); + + ADD_EPI32(abcd, abcd_save); + + data += 64; + } + while (--numBlocks); + + PREPARE_STATE + + _mm_storeu_si128((__m128i *) (void *) state, abcd); + *(state+4) = (UInt32)_mm_cvtsi128_si32(e0); +} + +#endif // USE_HW_SHA + +#elif defined(MY_CPU_ARM_OR_ARM64) + + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_SHA + #endif + #endif + +#ifdef USE_HW_SHA + +// #pragma message("=== Sha1 HW === ") + +#if defined(__clang__) || defined(__GNUC__) + #ifdef MY_CPU_ARM64 + #define ATTRIB_SHA __attribute__((__target__("+crypto"))) + #else + #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#include <arm64_neon.h> +#else +#include <arm_neon.h> +#endif + +typedef uint32x4_t v128; +// typedef __n128 v128; // MSVC + +#ifdef MY_CPU_BE + #define MY_rev32_for_LE(x) +#else + #define MY_rev32_for_LE(x) x = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x))) +#endif + +#define LOAD_128(_p) (*(const v128 *)(const void *)(_p)) +#define STORE_128(_p, _v) *(v128 *)(void *)(_p) = (_v) + +#define LOAD_SHUFFLE(m, k) \ + m = LOAD_128((data + (k) * 16)); \ + MY_rev32_for_LE(m); \ + +#define SU0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3); +#define SU1(dest, src) dest = vsha1su1q_u32(dest, src); +#define C(e) abcd = vsha1cq_u32(abcd, e, t); +#define P(e) abcd = vsha1pq_u32(abcd, e, t); +#define M(e) abcd = vsha1mq_u32(abcd, e, t); +#define H(e) e = vsha1h_u32(vgetq_lane_u32(abcd, 0)) +#define T(m, c) t = vaddq_u32(m, c) + +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + v128 abcd; + v128 c0, c1, c2, c3; + uint32_t e0; + + if (numBlocks == 0) + return; + + c0 = vdupq_n_u32(0x5a827999); + c1 = vdupq_n_u32(0x6ed9eba1); + c2 = vdupq_n_u32(0x8f1bbcdc); + c3 = vdupq_n_u32(0xca62c1d6); + + abcd = LOAD_128(&state[0]); + e0 = state[4]; + + do + { + v128 abcd_save; + v128 m0, m1, m2, m3; + v128 t; + uint32_t e0_save, e1; + + abcd_save = abcd; + e0_save = e0; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + T(m0, c0); H(e1); C(e0); + T(m1, c0); SU0(m0, m1, m2); H(e0); C(e1); + T(m2, c0); SU0(m1, m2, m3); SU1(m0, m3); H(e1); C(e0); + T(m3, c0); SU0(m2, m3, m0); SU1(m1, m0); H(e0); C(e1); + T(m0, c0); SU0(m3, m0, m1); SU1(m2, m1); H(e1); C(e0); + T(m1, c1); SU0(m0, m1, m2); SU1(m3, m2); H(e0); P(e1); + T(m2, c1); SU0(m1, m2, m3); SU1(m0, m3); H(e1); P(e0); + T(m3, c1); SU0(m2, m3, m0); SU1(m1, m0); H(e0); P(e1); + T(m0, c1); SU0(m3, m0, m1); SU1(m2, m1); H(e1); P(e0); + T(m1, c1); SU0(m0, m1, m2); SU1(m3, m2); H(e0); P(e1); + T(m2, c2); SU0(m1, m2, m3); SU1(m0, m3); H(e1); M(e0); + T(m3, c2); SU0(m2, m3, m0); SU1(m1, m0); H(e0); M(e1); + T(m0, c2); SU0(m3, m0, m1); SU1(m2, m1); H(e1); M(e0); + T(m1, c2); SU0(m0, m1, m2); SU1(m3, m2); H(e0); M(e1); + T(m2, c2); SU0(m1, m2, m3); SU1(m0, m3); H(e1); M(e0); + T(m3, c3); SU0(m2, m3, m0); SU1(m1, m0); H(e0); P(e1); + T(m0, c3); SU0(m3, m0, m1); SU1(m2, m1); H(e1); P(e0); + T(m1, c3); SU1(m3, m2); H(e0); P(e1); + T(m2, c3); H(e1); P(e0); + T(m3, c3); H(e0); P(e1); + + abcd = vaddq_u32(abcd, abcd_save); + e0 += e0_save; + + data += 64; + } + while (--numBlocks); + + STORE_128(&state[0], abcd); + state[4] = e0; +} + +#endif // USE_HW_SHA + +#endif // MY_CPU_ARM_OR_ARM64 + + +#ifndef USE_HW_SHA + +// #error Stop_Compiling_UNSUPPORTED_SHA +// #include <stdlib.h> + +// #include "Sha1.h" +void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); + +#pragma message("Sha1 HW-SW stub was used") + +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +{ + Sha1_UpdateBlocks(state, data, numBlocks); + /* + UNUSED_VAR(state); + UNUSED_VAR(data); + UNUSED_VAR(numBlocks); + exit(1); + return; + */ +} + +#endif @@ -1,5 +1,5 @@ -/* Crypto/Sha256.c -- SHA-256 Hash -2017-04-03 : Igor Pavlov : Public domain +/* Sha256.c -- SHA-256 Hash +2021-04-01 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -10,16 +10,107 @@ This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "RotateDefs.h" #include "Sha256.h" +#if defined(_MSC_VER) && (_MSC_VER < 1900) +// #define USE_MY_MM +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + #ifdef _MSC_VER + #if _MSC_VER >= 1200 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#elif defined(MY_CPU_ARM_OR_ARM64) + #ifdef _MSC_VER + #if _MSC_VER >= 1910 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#endif + +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +#ifdef _SHA_SUPPORTED + void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); + + static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS = Sha256_UpdateBlocks; + static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS_HW; + + #define UPDATE_BLOCKS(p) p->func_UpdateBlocks +#else + #define UPDATE_BLOCKS(p) Sha256_UpdateBlocks +#endif + + +BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo) +{ + SHA256_FUNC_UPDATE_BLOCKS func = Sha256_UpdateBlocks; + + #ifdef _SHA_SUPPORTED + if (algo != SHA256_ALGO_SW) + { + if (algo == SHA256_ALGO_DEFAULT) + func = g_FUNC_UPDATE_BLOCKS; + else + { + if (algo != SHA256_ALGO_HW) + return False; + func = g_FUNC_UPDATE_BLOCKS_HW; + if (!func) + return False; + } + } + #else + if (algo > 1) + return False; + #endif + + p->func_UpdateBlocks = func; + return True; +} + + /* define it for speed optimization */ -#ifndef _SFX -#define _SHA256_UNROLL -#define _SHA256_UNROLL2 + +#ifdef _SFX + #define STEP_PRE 1 + #define STEP_MAIN 1 +#else + #define STEP_PRE 2 + #define STEP_MAIN 4 + // #define _SHA256_UNROLL #endif -/* #define _SHA256_UNROLL2 */ +#if STEP_MAIN != 16 + #define _SHA256_BIG_W +#endif -void Sha256_Init(CSha256 *p) + + + +void Sha256_InitState(CSha256 *p) { + p->count = 0; p->state[0] = 0x6a09e667; p->state[1] = 0xbb67ae85; p->state[2] = 0x3c6ef372; @@ -28,7 +119,17 @@ void Sha256_Init(CSha256 *p) p->state[5] = 0x9b05688c; p->state[6] = 0x1f83d9ab; p->state[7] = 0x5be0cd19; - p->count = 0; +} + +void Sha256_Init(CSha256 *p) +{ + p->func_UpdateBlocks = + #ifdef _SHA_SUPPORTED + g_FUNC_UPDATE_BLOCKS; + #else + NULL; + #endif + Sha256_InitState(p); } #define S0(x) (rotrFixed(x, 2) ^ rotrFixed(x,13) ^ rotrFixed(x, 22)) @@ -36,61 +137,100 @@ void Sha256_Init(CSha256 *p) #define s0(x) (rotrFixed(x, 7) ^ rotrFixed(x,18) ^ (x >> 3)) #define s1(x) (rotrFixed(x,17) ^ rotrFixed(x,19) ^ (x >> 10)) -#define blk0(i) (W[i]) -#define blk2(i) (W[i] += s1(W[((i)-2)&15]) + W[((i)-7)&15] + s0(W[((i)-15)&15])) - #define Ch(x,y,z) (z^(x&(y^z))) #define Maj(x,y,z) ((x&y)|(z&(x|y))) -#ifdef _SHA256_UNROLL2 - -#define R(a,b,c,d,e,f,g,h, i) \ - h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ - d += h; \ - h += S0(a) + Maj(a, b, c) -#define RX_8(i) \ - R(a,b,c,d,e,f,g,h, i); \ - R(h,a,b,c,d,e,f,g, i+1); \ - R(g,h,a,b,c,d,e,f, i+2); \ - R(f,g,h,a,b,c,d,e, i+3); \ - R(e,f,g,h,a,b,c,d, i+4); \ - R(d,e,f,g,h,a,b,c, i+5); \ - R(c,d,e,f,g,h,a,b, i+6); \ - R(b,c,d,e,f,g,h,a, i+7) +#define W_PRE(i) (W[(i) + (size_t)(j)] = GetBe32(data + ((size_t)(j) + i) * 4)) -#define RX_16 RX_8(0); RX_8(8); +#define blk2_main(j, i) s1(w(j, (i)-2)) + w(j, (i)-7) + s0(w(j, (i)-15)) +#ifdef _SHA256_BIG_W + // we use +i instead of +(i) to change the order to solve CLANG compiler warning for signed/unsigned. + #define w(j, i) W[(size_t)(j) + i] + #define blk2(j, i) (w(j, i) = w(j, (i)-16) + blk2_main(j, i)) #else + #if STEP_MAIN == 16 + #define w(j, i) W[(i) & 15] + #else + #define w(j, i) W[((size_t)(j) + (i)) & 15] + #endif + #define blk2(j, i) (w(j, i) += blk2_main(j, i)) +#endif -#define a(i) T[(0-(i))&7] -#define b(i) T[(1-(i))&7] -#define c(i) T[(2-(i))&7] -#define d(i) T[(3-(i))&7] -#define e(i) T[(4-(i))&7] -#define f(i) T[(5-(i))&7] -#define g(i) T[(6-(i))&7] -#define h(i) T[(7-(i))&7] - -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ - d(i) += h(i); \ - h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \ +#define W_MAIN(i) blk2(j, i) -#ifdef _SHA256_UNROLL -#define RX_8(i) R(i+0); R(i+1); R(i+2); R(i+3); R(i+4); R(i+5); R(i+6); R(i+7); -#define RX_16 RX_8(0); RX_8(8); +#define T1(wx, i) \ + tmp = h + S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + h = g; \ + g = f; \ + f = e; \ + e = d + tmp; \ + tmp += S0(a) + Maj(a, b, c); \ + d = c; \ + c = b; \ + b = a; \ + a = tmp; \ -#else +#define R1_PRE(i) T1( W_PRE, i) +#define R1_MAIN(i) T1( W_MAIN, i) -#define RX_16 unsigned i; for (i = 0; i < 16; i++) { R(i); } +#if (!defined(_SHA256_UNROLL) || STEP_MAIN < 8) && (STEP_MAIN >= 4) +#define R2_MAIN(i) \ + R1_MAIN(i) \ + R1_MAIN(i + 1) \ #endif + + +#if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 + +#define T4( a,b,c,d,e,f,g,h, wx, i) \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + tmp = h; \ + h += d; \ + d = tmp + S0(a) + Maj(a, b, c); \ + +#define R4( wx, i) \ + T4 ( a,b,c,d,e,f,g,h, wx, (i )); \ + T4 ( d,a,b,c,h,e,f,g, wx, (i+1)); \ + T4 ( c,d,a,b,g,h,e,f, wx, (i+2)); \ + T4 ( b,c,d,a,f,g,h,e, wx, (i+3)); \ + +#define R4_PRE(i) R4( W_PRE, i) +#define R4_MAIN(i) R4( W_MAIN, i) + + +#define T8( a,b,c,d,e,f,g,h, wx, i) \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + d += h; \ + h += S0(a) + Maj(a, b, c); \ + +#define R8( wx, i) \ + T8 ( a,b,c,d,e,f,g,h, wx, i ); \ + T8 ( h,a,b,c,d,e,f,g, wx, i+1); \ + T8 ( g,h,a,b,c,d,e,f, wx, i+2); \ + T8 ( f,g,h,a,b,c,d,e, wx, i+3); \ + T8 ( e,f,g,h,a,b,c,d, wx, i+4); \ + T8 ( d,e,f,g,h,a,b,c, wx, i+5); \ + T8 ( c,d,e,f,g,h,a,b, wx, i+6); \ + T8 ( b,c,d,e,f,g,h,a, wx, i+7); \ + +#define R8_PRE(i) R8( W_PRE, i) +#define R8_MAIN(i) R8( W_MAIN, i) + #endif -static const UInt32 K[64] = { +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); + +// static +extern MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, @@ -109,30 +249,27 @@ static const UInt32 K[64] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 }; -static void Sha256_WriteByteBlock(CSha256 *p) -{ - UInt32 W[16]; - unsigned j; - UInt32 *state; +#define K SHA256_K_ARRAY - #ifdef _SHA256_UNROLL2 - UInt32 a,b,c,d,e,f,g,h; + +MY_NO_INLINE +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + UInt32 W + #ifdef _SHA256_BIG_W + [64]; #else - UInt32 T[8]; + [16]; #endif - for (j = 0; j < 16; j += 4) - { - const Byte *ccc = p->buffer + j * 4; - W[j ] = GetBe32(ccc); - W[j + 1] = GetBe32(ccc + 4); - W[j + 2] = GetBe32(ccc + 8); - W[j + 3] = GetBe32(ccc + 12); - } + unsigned j; - state = p->state; + UInt32 a,b,c,d,e,f,g,h; - #ifdef _SHA256_UNROLL2 + #if !defined(_SHA256_UNROLL) || (STEP_MAIN <= 4) || (STEP_PRE <= 4) + UInt32 tmp; + #endif + a = state[0]; b = state[1]; c = state[2]; @@ -141,39 +278,96 @@ static void Sha256_WriteByteBlock(CSha256 *p) f = state[5]; g = state[6]; h = state[7]; - #else - for (j = 0; j < 8; j++) - T[j] = state[j]; - #endif - for (j = 0; j < 64; j += 16) + while (numBlocks) { - RX_16 + + for (j = 0; j < 16; j += STEP_PRE) + { + #if STEP_PRE > 4 + + #if STEP_PRE < 8 + R4_PRE(0); + #else + R8_PRE(0); + #if STEP_PRE == 16 + R8_PRE(8); + #endif + #endif + + #else + + R1_PRE(0); + #if STEP_PRE >= 2 + R1_PRE(1); + #if STEP_PRE >= 4 + R1_PRE(2); + R1_PRE(3); + #endif + #endif + + #endif + } + + for (j = 16; j < 64; j += STEP_MAIN) + { + #if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 + + #if STEP_MAIN < 8 + R4_MAIN(0); + #else + R8_MAIN(0); + #if STEP_MAIN == 16 + R8_MAIN(8); + #endif + #endif + + #else + + R1_MAIN(0); + #if STEP_MAIN >= 2 + R1_MAIN(1); + #if STEP_MAIN >= 4 + R2_MAIN(2); + #if STEP_MAIN >= 8 + R2_MAIN(4); + R2_MAIN(6); + #if STEP_MAIN >= 16 + R2_MAIN(8); + R2_MAIN(10); + R2_MAIN(12); + R2_MAIN(14); + #endif + #endif + #endif + #endif + #endif + } + + a += state[0]; state[0] = a; + b += state[1]; state[1] = b; + c += state[2]; state[2] = c; + d += state[3]; state[3] = d; + e += state[4]; state[4] = e; + f += state[5]; state[5] = f; + g += state[6]; state[6] = g; + h += state[7]; state[7] = h; + + data += 64; + numBlocks--; } - #ifdef _SHA256_UNROLL2 - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; - state[5] += f; - state[6] += g; - state[7] += h; - #else - for (j = 0; j < 8; j++) - state[j] += T[j]; - #endif - /* Wipe variables */ /* memset(W, 0, sizeof(W)); */ - /* memset(T, 0, sizeof(T)); */ } #undef S0 #undef S1 #undef s0 #undef s1 +#undef K + +#define Sha256_UpdateBlock(p) UPDATE_BLOCKS(p)(p->state, p->buffer, 1) void Sha256_Update(CSha256 *p, const Byte *data, size_t size) { @@ -193,25 +387,26 @@ void Sha256_Update(CSha256 *p, const Byte *data, size_t size) return; } - size -= num; - memcpy(p->buffer + pos, data, num); - data += num; + if (pos != 0) + { + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + Sha256_UpdateBlock(p); + } } - - for (;;) { - Sha256_WriteByteBlock(p); - if (size < 64) - break; - size -= 64; - memcpy(p->buffer, data, 64); - data += 64; - } - - if (size != 0) + size_t numBlocks = size >> 6; + UPDATE_BLOCKS(p)(p->state, data, numBlocks); + size &= 0x3F; + if (size == 0) + return; + data += (numBlocks << 6); memcpy(p->buffer, data, size); + } } + void Sha256_Final(CSha256 *p, Byte *digest) { unsigned pos = (unsigned)p->count & 0x3F; @@ -219,13 +414,30 @@ void Sha256_Final(CSha256 *p, Byte *digest) p->buffer[pos++] = 0x80; - while (pos != (64 - 8)) + if (pos > (64 - 8)) + { + while (pos != 64) { p->buffer[pos++] = 0; } + // memset(&p->buf.buffer[pos], 0, 64 - pos); + Sha256_UpdateBlock(p); + pos = 0; + } + + /* + if (pos & 3) { - pos &= 0x3F; - if (pos == 0) - Sha256_WriteByteBlock(p); - p->buffer[pos++] = 0; + p->buffer[pos] = 0; + p->buffer[pos + 1] = 0; + p->buffer[pos + 2] = 0; + pos += 3; + pos &= ~3; } + { + for (; pos < 64 - 8; pos += 4) + *(UInt32 *)(&p->buffer[pos]) = 0; + } + */ + + memset(&p->buffer[pos], 0, (64 - 8) - pos); { UInt64 numBits = (p->count << 3); @@ -233,16 +445,42 @@ void Sha256_Final(CSha256 *p, Byte *digest) SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); } - Sha256_WriteByteBlock(p); + Sha256_UpdateBlock(p); for (i = 0; i < 8; i += 2) { UInt32 v0 = p->state[i]; - UInt32 v1 = p->state[i + 1]; + UInt32 v1 = p->state[(size_t)i + 1]; SetBe32(digest , v0); SetBe32(digest + 4, v1); digest += 8; } - Sha256_Init(p); + Sha256_InitState(p); +} + + +void Sha256Prepare() +{ + #ifdef _SHA_SUPPORTED + SHA256_FUNC_UPDATE_BLOCKS f, f_hw; + f = Sha256_UpdateBlocks; + f_hw = NULL; + #ifdef MY_CPU_X86_OR_AMD64 + #ifndef USE_MY_MM + if (CPU_IsSupported_SHA() + && CPU_IsSupported_SSSE3() + // && CPU_IsSupported_SSE41() + ) + #endif + #else + if (CPU_IsSupported_SHA2()) + #endif + { + // printf("\n========== HW SHA256 ======== \n"); + f = f_hw = Sha256_UpdateBlocks_HW; + } + g_FUNC_UPDATE_BLOCKS = f; + g_FUNC_UPDATE_BLOCKS_HW = f_hw; + #endif } @@ -1,26 +1,76 @@ /* Sha256.h -- SHA-256 Hash -2013-01-18 : Igor Pavlov : Public domain */ +2021-01-01 : Igor Pavlov : Public domain */ -#ifndef __CRYPTO_SHA256_H -#define __CRYPTO_SHA256_H +#ifndef __7Z_SHA256_H +#define __7Z_SHA256_H #include "7zTypes.h" EXTERN_C_BEGIN -#define SHA256_DIGEST_SIZE 32 +#define SHA256_NUM_BLOCK_WORDS 16 +#define SHA256_NUM_DIGEST_WORDS 8 + +#define SHA256_BLOCK_SIZE (SHA256_NUM_BLOCK_WORDS * 4) +#define SHA256_DIGEST_SIZE (SHA256_NUM_DIGEST_WORDS * 4) + +typedef void (MY_FAST_CALL *SHA256_FUNC_UPDATE_BLOCKS)(UInt32 state[8], const Byte *data, size_t numBlocks); + +/* + if (the system supports different SHA256 code implementations) + { + (CSha256::func_UpdateBlocks) will be used + (CSha256::func_UpdateBlocks) can be set by + Sha256_Init() - to default (fastest) + Sha256_SetFunction() - to any algo + } + else + { + (CSha256::func_UpdateBlocks) is ignored. + } +*/ typedef struct { - UInt32 state[8]; + SHA256_FUNC_UPDATE_BLOCKS func_UpdateBlocks; UInt64 count; - Byte buffer[64]; + UInt64 __pad_2[2]; + UInt32 state[SHA256_NUM_DIGEST_WORDS]; + + Byte buffer[SHA256_BLOCK_SIZE]; } CSha256; + +#define SHA256_ALGO_DEFAULT 0 +#define SHA256_ALGO_SW 1 +#define SHA256_ALGO_HW 2 + +/* +Sha256_SetFunction() +return: + 0 - (algo) value is not supported, and func_UpdateBlocks was not changed + 1 - func_UpdateBlocks was set according (algo) value. +*/ + +BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo); + +void Sha256_InitState(CSha256 *p); void Sha256_Init(CSha256 *p); void Sha256_Update(CSha256 *p, const Byte *data, size_t size); void Sha256_Final(CSha256 *p, Byte *digest); + + + +// void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +/* +call Sha256Prepare() once at program start. +It prepares all supported implementations, and detects the fastest implementation. +*/ + +void Sha256Prepare(void); + EXTERN_C_END #endif diff --git a/C/Sha256Opt.c b/C/Sha256Opt.c new file mode 100644 index 00000000..decc1382 --- /dev/null +++ b/C/Sha256Opt.c @@ -0,0 +1,373 @@ +/* Sha256Opt.c -- SHA-256 optimized code for SHA-256 hardware instructions +2021-04-01 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#if defined(_MSC_VER) +#if (_MSC_VER < 1900) && (_MSC_VER >= 1200) +// #define USE_MY_MM +#endif +#endif + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + #if defined(_MSC_VER) + // SSSE3: for clang-cl: + #include <tmmintrin.h> + #define __SHA__ + #endif + #endif + + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + // #pragma GCC target("sha,ssse3") + #endif + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #ifdef USE_MY_MM + #define USE_VER_MIN 1300 + #else + #define USE_VER_MIN 1910 + #endif + #if _MSC_VER >= USE_VER_MIN + #define USE_HW_SHA + #endif + #endif +// #endif // MY_CPU_X86_OR_AMD64 + +#ifdef USE_HW_SHA + +// #pragma message("Sha256 HW") +// #include <wmmintrin.h> + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +#include <immintrin.h> +#else +#include <emmintrin.h> + +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +// #include <intrin.h> +#endif + +#ifdef USE_MY_MM +#include "My_mm.h" +#endif + +#endif + +/* +SHA256 uses: +SSE2: + _mm_loadu_si128 + _mm_storeu_si128 + _mm_set_epi32 + _mm_add_epi32 + _mm_shuffle_epi32 / pshufd + + + +SSSE3: + _mm_shuffle_epi8 / pshufb + _mm_alignr_epi8 +SHA: + _mm_sha256* +*/ + +// K array must be aligned for 16-bytes at least. +// The compiler can look align attribute and selects +// movdqu - for code without align attribute +// movdqa - for code with align attribute +extern +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +#define K SHA256_K_ARRAY + + +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define SHA256_MSG1(dest, src) dest = _mm_sha256msg1_epu32(dest, src); +#define SHA25G_MSG2(dest, src) dest = _mm_sha256msg2_epu32(dest, src); + + +#define LOAD_SHUFFLE(m, k) \ + m = _mm_loadu_si128((const __m128i *)(const void *)(data + (k) * 16)); \ + m = _mm_shuffle_epi8(m, mask); \ + +#define SM1(g0, g1, g2, g3) \ + SHA256_MSG1(g3, g0); \ + +#define SM2(g0, g1, g2, g3) \ + tmp = _mm_alignr_epi8(g1, g0, 4); \ + ADD_EPI32(g2, tmp); \ + SHA25G_MSG2(g2, g1); \ + +// #define LS0(k, g0, g1, g2, g3) LOAD_SHUFFLE(g0, k) +// #define LS1(k, g0, g1, g2, g3) LOAD_SHUFFLE(g1, k+1) + + +#define NNN(g0, g1, g2, g3) + + +#define RND2(t0, t1) \ + t0 = _mm_sha256rnds2_epu32(t0, t1, msg); + +#define RND2_0(m, k) \ + msg = _mm_add_epi32(m, *(const __m128i *) (const void *) &K[(k) * 4]); \ + RND2(state0, state1); \ + msg = _mm_shuffle_epi32(msg, 0x0E); \ + + +#define RND2_1 \ + RND2(state1, state0); \ + + +// We use scheme with 3 rounds ahead for SHA256_MSG1 / 2 rounds ahead for SHA256_MSG2 + +#define R4(k, g0, g1, g2, g3, OP0, OP1) \ + RND2_0(g0, k); \ + OP0(g0, g1, g2, g3); \ + RND2_1; \ + OP1(g0, g1, g2, g3); \ + +#define R16(k, OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7) \ + R4 ( (k)*4+0, m0, m1, m2, m3, OP0, OP1 ) \ + R4 ( (k)*4+1, m1, m2, m3, m0, OP2, OP3 ) \ + R4 ( (k)*4+2, m2, m3, m0, m1, OP4, OP5 ) \ + R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ + +#define PREPARE_STATE \ + tmp = _mm_shuffle_epi32(state0, 0x1B); /* abcd */ \ + state0 = _mm_shuffle_epi32(state1, 0x1B); /* efgh */ \ + state1 = state0; \ + state0 = _mm_unpacklo_epi64(state0, tmp); /* cdgh */ \ + state1 = _mm_unpackhi_epi64(state1, tmp); /* abef */ \ + + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + const __m128i mask = _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203); + __m128i tmp; + __m128i state0, state1; + + if (numBlocks == 0) + return; + + state0 = _mm_loadu_si128((const __m128i *) (const void *) &state[0]); + state1 = _mm_loadu_si128((const __m128i *) (const void *) &state[4]); + + PREPARE_STATE + + do + { + __m128i state0_save, state1_save; + __m128i m0, m1, m2, m3; + __m128i msg; + // #define msg tmp + + state0_save = state0; + state1_save = state1; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + + + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + + ADD_EPI32(state0, state0_save); + ADD_EPI32(state1, state1_save); + + data += 64; + } + while (--numBlocks); + + PREPARE_STATE + + _mm_storeu_si128((__m128i *) (void *) &state[0], state0); + _mm_storeu_si128((__m128i *) (void *) &state[4], state1); +} + +#endif // USE_HW_SHA + +#elif defined(MY_CPU_ARM_OR_ARM64) + + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_SHA + #endif + #endif + +#ifdef USE_HW_SHA + +// #pragma message("=== Sha256 HW === ") + +#if defined(__clang__) || defined(__GNUC__) + #ifdef MY_CPU_ARM64 + #define ATTRIB_SHA __attribute__((__target__("+crypto"))) + #else + #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#include <arm64_neon.h> +#else +#include <arm_neon.h> +#endif + +typedef uint32x4_t v128; +// typedef __n128 v128; // MSVC + +#ifdef MY_CPU_BE + #define MY_rev32_for_LE(x) +#else + #define MY_rev32_for_LE(x) x = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x))) +#endif + +#define LOAD_128(_p) (*(const v128 *)(const void *)(_p)) +#define STORE_128(_p, _v) *(v128 *)(void *)(_p) = (_v) + +#define LOAD_SHUFFLE(m, k) \ + m = LOAD_128((data + (k) * 16)); \ + MY_rev32_for_LE(m); \ + +// K array must be aligned for 16-bytes at least. +extern +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +#define K SHA256_K_ARRAY + + +#define SHA256_SU0(dest, src) dest = vsha256su0q_u32(dest, src); +#define SHA25G_SU1(dest, src2, src3) dest = vsha256su1q_u32(dest, src2, src3); + +#define SM1(g0, g1, g2, g3) SHA256_SU0(g3, g0) +#define SM2(g0, g1, g2, g3) SHA25G_SU1(g2, g0, g1) +#define NNN(g0, g1, g2, g3) + + +#define R4(k, g0, g1, g2, g3, OP0, OP1) \ + msg = vaddq_u32(g0, *(const v128 *) (const void *) &K[(k) * 4]); \ + tmp = state0; \ + state0 = vsha256hq_u32( state0, state1, msg ); \ + state1 = vsha256h2q_u32( state1, tmp, msg ); \ + OP0(g0, g1, g2, g3); \ + OP1(g0, g1, g2, g3); \ + + +#define R16(k, OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7) \ + R4 ( (k)*4+0, m0, m1, m2, m3, OP0, OP1 ) \ + R4 ( (k)*4+1, m1, m2, m3, m0, OP2, OP3 ) \ + R4 ( (k)*4+2, m2, m3, m0, m1, OP4, OP5 ) \ + R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ + + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + v128 state0, state1; + + if (numBlocks == 0) + return; + + state0 = LOAD_128(&state[0]); + state1 = LOAD_128(&state[4]); + + do + { + v128 state0_save, state1_save; + v128 m0, m1, m2, m3; + v128 msg, tmp; + + state0_save = state0; + state1_save = state1; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + + state0 = vaddq_u32(state0, state0_save); + state1 = vaddq_u32(state1, state1_save); + + data += 64; + } + while (--numBlocks); + + STORE_128(&state[0], state0); + STORE_128(&state[4], state1); +} + +#endif // USE_HW_SHA + +#endif // MY_CPU_ARM_OR_ARM64 + + +#ifndef USE_HW_SHA + +// #error Stop_Compiling_UNSUPPORTED_SHA +// #include <stdlib.h> + +// #include "Sha256.h" +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +#pragma message("Sha256 HW-SW stub was used") + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + Sha256_UpdateBlocks(state, data, numBlocks); + /* + UNUSED_VAR(state); + UNUSED_VAR(data); + UNUSED_VAR(numBlocks); + exit(1); + return; + */ +} + +#endif diff --git a/C/Threads.c b/C/Threads.c index 930ad271..bd9553dc 100644 --- a/C/Threads.c +++ b/C/Threads.c @@ -1,8 +1,10 @@ /* Threads.c -- multithreading library -2017-06-26 : Igor Pavlov : Public domain */ +2021-04-25 : Igor Pavlov : Public domain */ #include "Precomp.h" +#ifdef _WIN32 + #ifndef UNDER_CE #include <process.h> #endif @@ -29,28 +31,103 @@ WRes HandlePtr_Close(HANDLE *p) return 0; } -WRes Handle_WaitObject(HANDLE h) { return (WRes)WaitForSingleObject(h, INFINITE); } +WRes Handle_WaitObject(HANDLE h) +{ + DWORD dw = WaitForSingleObject(h, INFINITE); + /* + (dw) result: + WAIT_OBJECT_0 // 0 + WAIT_ABANDONED // 0x00000080 : is not compatible with Win32 Error space + WAIT_TIMEOUT // 0x00000102 : is compatible with Win32 Error space + WAIT_FAILED // 0xFFFFFFFF + */ + if (dw == WAIT_FAILED) + { + dw = GetLastError(); + if (dw == 0) + return WAIT_FAILED; + } + return (WRes)dw; +} + +#define Thread_Wait(p) Handle_WaitObject(*(p)) + +WRes Thread_Wait_Close(CThread *p) +{ + WRes res = Thread_Wait(p); + WRes res2 = Thread_Close(p); + return (res != 0 ? res : res2); +} WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) { /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */ - + #ifdef UNDER_CE - + DWORD threadId; *p = CreateThread(0, 0, func, param, 0, &threadId); - + #else - + unsigned threadId; - *p = (HANDLE)_beginthreadex(NULL, 0, func, param, 0, &threadId); - + *p = (HANDLE)(_beginthreadex(NULL, 0, func, param, 0, &threadId)); + #endif /* maybe we must use errno here, but probably GetLastError() is also OK. */ return HandleToWRes(*p); } + +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity) +{ + #ifdef UNDER_CE + + UNUSED_VAR(affinity) + return Thread_Create(p, func, param); + + #else + + /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */ + HANDLE h; + WRes wres; + unsigned threadId; + h = (HANDLE)(_beginthreadex(NULL, 0, func, param, CREATE_SUSPENDED, &threadId)); + *p = h; + wres = HandleToWRes(h); + if (h) + { + { + // DWORD_PTR prevMask = + SetThreadAffinityMask(h, (DWORD_PTR)affinity); + /* + if (prevMask == 0) + { + // affinity change is non-critical error, so we can ignore it + // wres = GetError(); + } + */ + } + { + DWORD prevSuspendCount = ResumeThread(h); + /* ResumeThread() returns: + 0 : was_not_suspended + 1 : was_resumed + -1 : error + */ + if (prevSuspendCount == (DWORD)-1) + wres = GetError(); + } + } + + /* maybe we must use errno here, but probably GetLastError() is also OK. */ + return wres; + + #endif +} + + static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) { *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL); @@ -68,6 +145,7 @@ WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) { return AutoResetEven WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) { + // negative ((LONG)maxCount) is not supported in WIN32::CreateSemaphore() *p = CreateSemaphore(NULL, (LONG)initCount, (LONG)maxCount, NULL); return HandleToWRes(*p); } @@ -93,3 +171,336 @@ WRes CriticalSection_Init(CCriticalSection *p) #endif return 0; } + + + + +#else // _WIN32 + +// ---------- POSIX ---------- + +#ifndef __APPLE__ +#ifndef _7ZIP_AFFINITY_DISABLE +// _GNU_SOURCE can be required for pthread_setaffinity_np() / CPU_ZERO / CPU_SET +#define _GNU_SOURCE +#endif +#endif + +#include "Threads.h" + +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#ifdef _7ZIP_AFFINITY_SUPPORTED +// #include <sched.h> +#endif + + +// #include <stdio.h> +// #define PRF(p) p +#define PRF(p) + +#define Print(s) PRF(printf("\n%s\n", s)) + +// #include <stdio.h> + +WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet) +{ + // new thread in Posix probably inherits affinity from parrent thread + Print("Thread_Create_With_CpuSet"); + + pthread_attr_t attr; + int ret; + // int ret2; + + p->_created = 0; + + RINOK(pthread_attr_init(&attr)); + + ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + if (!ret) + { + if (cpuSet) + { + #ifdef _7ZIP_AFFINITY_SUPPORTED + + /* + printf("\n affinity :"); + unsigned i; + for (i = 0; i < sizeof(*cpuSet) && i < 8; i++) + { + Byte b = *((const Byte *)cpuSet + i); + char temp[32]; + #define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + temp[0] = GET_HEX_CHAR((b & 0xF)); + temp[1] = GET_HEX_CHAR((b >> 4)); + // temp[0] = GET_HEX_CHAR((b >> 4)); // big-endian + // temp[1] = GET_HEX_CHAR((b & 0xF)); // big-endian + temp[2] = 0; + printf("%s", temp); + } + printf("\n"); + */ + + // ret2 = + pthread_attr_setaffinity_np(&attr, sizeof(*cpuSet), cpuSet); + // if (ret2) ret = ret2; + #endif + } + + ret = pthread_create(&p->_tid, &attr, func, param); + + if (!ret) + { + p->_created = 1; + /* + if (cpuSet) + { + // ret2 = + pthread_setaffinity_np(p->_tid, sizeof(*cpuSet), cpuSet); + // if (ret2) ret = ret2; + } + */ + } + } + // ret2 = + pthread_attr_destroy(&attr); + // if (ret2 != 0) ret = ret2; + return ret; +} + + +WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) +{ + return Thread_Create_With_CpuSet(p, func, param, NULL); +} + + +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity) +{ + Print("Thread_Create_WithAffinity"); + CCpuSet cs; + unsigned i; + CpuSet_Zero(&cs); + for (i = 0; i < sizeof(affinity) * 8; i++) + { + if (affinity == 0) + break; + if (affinity & 1) + { + CpuSet_Set(&cs, i); + } + affinity >>= 1; + } + return Thread_Create_With_CpuSet(p, func, param, &cs); +} + + +WRes Thread_Close(CThread *p) +{ + // Print("Thread_Close"); + int ret; + if (!p->_created) + return 0; + + ret = pthread_detach(p->_tid); + p->_tid = 0; + p->_created = 0; + return ret; +} + + +WRes Thread_Wait_Close(CThread *p) +{ + // Print("Thread_Wait_Close"); + void *thread_return; + int ret; + if (!p->_created) + return EINVAL; + + ret = pthread_join(p->_tid, &thread_return); + // probably we can't use that (_tid) after pthread_join(), so we close thread here + p->_created = 0; + p->_tid = 0; + return ret; +} + + + +static WRes Event_Create(CEvent *p, int manualReset, int signaled) +{ + RINOK(pthread_mutex_init(&p->_mutex, NULL)); + RINOK(pthread_cond_init(&p->_cond, NULL)); + p->_manual_reset = manualReset; + p->_state = (signaled ? True : False); + p->_created = 1; + return 0; +} + +WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled) + { return Event_Create(p, True, signaled); } +WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p) + { return ManualResetEvent_Create(p, 0); } +WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled) + { return Event_Create(p, False, signaled); } +WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) + { return AutoResetEvent_Create(p, 0); } + + +WRes Event_Set(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + p->_state = True; + int res1 = pthread_cond_broadcast(&p->_cond); + int res2 = pthread_mutex_unlock(&p->_mutex); + return (res2 ? res2 : res1); +} + +WRes Event_Reset(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + p->_state = False; + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Event_Wait(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + while (p->_state == False) + { + // ETIMEDOUT + // ret = + pthread_cond_wait(&p->_cond, &p->_mutex); + // if (ret != 0) break; + } + if (p->_manual_reset == False) + { + p->_state = False; + } + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Event_Close(CEvent *p) +{ + if (!p->_created) + return 0; + p->_created = 0; + { + int res1 = pthread_mutex_destroy(&p->_mutex); + int res2 = pthread_cond_destroy(&p->_cond); + return (res1 ? res1 : res2); + } +} + + +WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + if (initCount > maxCount || maxCount < 1) + return EINVAL; + RINOK(pthread_mutex_init(&p->_mutex, NULL)); + RINOK(pthread_cond_init(&p->_cond, NULL)); + p->_count = initCount; + p->_maxCount = maxCount; + p->_created = 1; + return 0; +} + +WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) +{ + UInt32 newCount; + int ret; + + if (releaseCount < 1) + return EINVAL; + + RINOK(pthread_mutex_lock(&p->_mutex)); + + newCount = p->_count + releaseCount; + if (newCount > p->_maxCount) + ret = ERROR_TOO_MANY_POSTS; // EINVAL; + else + { + p->_count = newCount; + ret = pthread_cond_broadcast(&p->_cond); + } + RINOK(pthread_mutex_unlock(&p->_mutex)); + return ret; +} + +WRes Semaphore_Wait(CSemaphore *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + while (p->_count < 1) + { + pthread_cond_wait(&p->_cond, &p->_mutex); + } + p->_count--; + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Semaphore_Close(CSemaphore *p) +{ + if (!p->_created) + return 0; + p->_created = 0; + { + int res1 = pthread_mutex_destroy(&p->_mutex); + int res2 = pthread_cond_destroy(&p->_cond); + return (res1 ? res1 : res2); + } +} + + + +WRes CriticalSection_Init(CCriticalSection *p) +{ + // Print("CriticalSection_Init"); + if (!p) + return EINTR; + return pthread_mutex_init(&p->_mutex, NULL); +} + +void CriticalSection_Enter(CCriticalSection *p) +{ + // Print("CriticalSection_Enter"); + if (p) + { + // int ret = + pthread_mutex_lock(&p->_mutex); + } +} + +void CriticalSection_Leave(CCriticalSection *p) +{ + // Print("CriticalSection_Leave"); + if (p) + { + // int ret = + pthread_mutex_unlock(&p->_mutex); + } +} + +void CriticalSection_Delete(CCriticalSection *p) +{ + // Print("CriticalSection_Delete"); + if (p) + { + // int ret = + pthread_mutex_destroy(&p->_mutex); + } +} + +LONG InterlockedIncrement(LONG volatile *addend) +{ + // Print("InterlockedIncrement"); + #ifdef USE_HACK_UNSAFE_ATOMIC + LONG val = *addend + 1; + *addend = val; + return val; + #else + return __sync_add_and_fetch(addend, 1); + #endif +} + +#endif // _WIN32 diff --git a/C/Threads.h b/C/Threads.h index e53ace43..6cb4aa4b 100644 --- a/C/Threads.h +++ b/C/Threads.h @@ -1,38 +1,106 @@ /* Threads.h -- multithreading library -2017-06-18 : Igor Pavlov : Public domain */ +2021-04-25 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H #ifdef _WIN32 -#include <windows.h> +#include <Windows.h> +#else + +#if !defined(__APPLE__) && !defined(_AIX) +#ifndef _7ZIP_AFFINITY_DISABLE +#define _7ZIP_AFFINITY_SUPPORTED +// #define _GNU_SOURCE +#endif +#endif + +#include <pthread.h> #endif #include "7zTypes.h" EXTERN_C_BEGIN +#ifdef _WIN32 + WRes HandlePtr_Close(HANDLE *h); WRes Handle_WaitObject(HANDLE h); typedef HANDLE CThread; -#define Thread_Construct(p) *(p) = NULL + +#define Thread_Construct(p) { *(p) = NULL; } #define Thread_WasCreated(p) (*(p) != NULL) #define Thread_Close(p) HandlePtr_Close(p) -#define Thread_Wait(p) Handle_WaitObject(*(p)) +// #define Thread_Wait(p) Handle_WaitObject(*(p)) typedef -#ifdef UNDER_CE - DWORD + #ifdef UNDER_CE + DWORD + #else + unsigned + #endif + THREAD_FUNC_RET_TYPE; + +typedef DWORD_PTR CAffinityMask; +typedef DWORD_PTR CCpuSet; + +#define CpuSet_Zero(p) { *(p) = 0; } +#define CpuSet_Set(p, cpu) { *(p) |= ((DWORD_PTR)1 << (cpu)); } + +#else // _WIN32 + +typedef struct _CThread +{ + pthread_t _tid; + int _created; +} CThread; + +#define Thread_Construct(p) { (p)->_tid = 0; (p)->_created = 0; } +#define Thread_WasCreated(p) ((p)->_created != 0) +WRes Thread_Close(CThread *p); +// #define Thread_Wait Thread_Wait_Close + +typedef void * THREAD_FUNC_RET_TYPE; + +typedef UInt64 CAffinityMask; + +#ifdef _7ZIP_AFFINITY_SUPPORTED + +typedef cpu_set_t CCpuSet; +#define CpuSet_Zero(p) CPU_ZERO(p) +#define CpuSet_Set(p, cpu) CPU_SET(cpu, p) +#define CpuSet_IsSet(p, cpu) CPU_ISSET(cpu, p) + #else - unsigned + +typedef UInt64 CCpuSet; +#define CpuSet_Zero(p) { *(p) = 0; } +#define CpuSet_Set(p, cpu) { *(p) |= ((UInt64)1 << (cpu)); } +#define CpuSet_IsSet(p, cpu) ((*(p) & ((UInt64)1 << (cpu))) != 0) + #endif - THREAD_FUNC_RET_TYPE; + + +#endif // _WIN32 + #define THREAD_FUNC_CALL_TYPE MY_STD_CALL #define THREAD_FUNC_DECL THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE typedef THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE * THREAD_FUNC_TYPE)(void *); WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param); +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity); +WRes Thread_Wait_Close(CThread *p); + +#ifdef _WIN32 +#define Thread_Create_With_CpuSet(p, func, param, cs) \ + Thread_Create_With_Affinity(p, func, param, *cs) +#else +WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet); +#endif + + +#ifdef _WIN32 typedef HANDLE CEvent; typedef CEvent CAutoResetEvent; @@ -63,6 +131,67 @@ WRes CriticalSection_Init(CCriticalSection *p); #define CriticalSection_Enter(p) EnterCriticalSection(p) #define CriticalSection_Leave(p) LeaveCriticalSection(p) + +#else // _WIN32 + +typedef struct _CEvent +{ + int _created; + int _manual_reset; + int _state; + pthread_mutex_t _mutex; + pthread_cond_t _cond; +} CEvent; + +typedef CEvent CAutoResetEvent; +typedef CEvent CManualResetEvent; + +#define Event_Construct(p) (p)->_created = 0 +#define Event_IsCreated(p) ((p)->_created) + +WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled); +WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p); +WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); +WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); +WRes Event_Set(CEvent *p); +WRes Event_Reset(CEvent *p); +WRes Event_Wait(CEvent *p); +WRes Event_Close(CEvent *p); + + +typedef struct _CSemaphore +{ + int _created; + UInt32 _count; + UInt32 _maxCount; + pthread_mutex_t _mutex; + pthread_cond_t _cond; +} CSemaphore; + +#define Semaphore_Construct(p) (p)->_created = 0 +#define Semaphore_IsCreated(p) ((p)->_created) + +WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); +#define Semaphore_Release1(p) Semaphore_ReleaseN(p, 1) +WRes Semaphore_Wait(CSemaphore *p); +WRes Semaphore_Close(CSemaphore *p); + + +typedef struct _CCriticalSection +{ + pthread_mutex_t _mutex; +} CCriticalSection; + +WRes CriticalSection_Init(CCriticalSection *p); +void CriticalSection_Delete(CCriticalSection *cs); +void CriticalSection_Enter(CCriticalSection *cs); +void CriticalSection_Leave(CCriticalSection *cs); + +LONG InterlockedIncrement(LONG volatile *addend); + +#endif // _WIN32 + EXTERN_C_END #endif diff --git a/C/Util/7z/7zMain.c b/C/Util/7z/7zMain.c index 6ccc8305..9d555098 100644 --- a/C/Util/7z/7zMain.c +++ b/C/Util/7z/7zMain.c @@ -1,5 +1,5 @@ /* 7zMain.c - Test application for 7z Decoder -2019-02-02 : Igor Pavlov : Public domain */ +2021-04-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -20,6 +20,13 @@ #ifdef _WIN32 #include <direct.h> #else +#include <stdlib.h> +#include <time.h> +#ifdef __GNUC__ +#include <sys/time.h> +#endif +#include <fcntl.h> +// #include <utime.h> #include <sys/stat.h> #include <errno.h> #endif @@ -108,7 +115,7 @@ static Byte *Utf16_To_Utf8(Byte *dest, const UInt16 *src, const UInt16 *srcLim) if (val < 0x80) { - *dest++ = (char)val; + *dest++ = (Byte)val; continue; } @@ -162,21 +169,21 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s ) { unsigned len = 0; - for (len = 0; s[len] != 0; len++); + for (len = 0; s[len] != 0; len++) {} #ifndef _USE_UTF8 { - unsigned size = len * 3 + 100; + const unsigned size = len * 3 + 100; if (!Buf_EnsureSize(buf, size)) return SZ_ERROR_MEM; { buf->data[0] = 0; if (len != 0) { - char defaultChar = '_'; + const char defaultChar = '_'; BOOL defUsed; - unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); + const unsigned numChars = (unsigned)WideCharToMultiByte( + codePage, 0, (LPCWSTR)s, (int)len, (char *)buf->data, (int)size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0; @@ -192,8 +199,8 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s #ifdef _WIN32 #ifndef USE_WINDOWS_FILE static UINT g_FileCodePage = CP_ACP; + #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage #endif - #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage #else #define MY_FILE_CODE_PAGE_PARAM #endif @@ -300,17 +307,142 @@ static void UIntToStr_2(char *s, unsigned value) s[1] = (char)('0' + (value % 10)); } + #define PERIOD_4 (4 * 365 + 1) #define PERIOD_100 (PERIOD_4 * 25 - 1) #define PERIOD_400 (PERIOD_100 * 4 + 1) -static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) + + +#ifndef _WIN32 + +// MS uses long for BOOL, but long is 32-bit in MS. So we use int. +// typedef long BOOL; +typedef int BOOL; + +typedef struct _FILETIME +{ + DWORD dwLowDateTime; + DWORD dwHighDateTime; +} FILETIME; + +static LONG TIME_GetBias() +{ + time_t utc = time(NULL); + struct tm *ptm = localtime(&utc); + int localdaylight = ptm->tm_isdst; /* daylight for local timezone */ + ptm = gmtime(&utc); + ptm->tm_isdst = localdaylight; /* use local daylight, not that of Greenwich */ + LONG bias = (int)(mktime(ptm)-utc); + return bias; +} + +#define TICKS_PER_SEC 10000000 + +#define GET_TIME_64(pft) ((pft)->dwLowDateTime | ((UInt64)(pft)->dwHighDateTime << 32)) + +#define SET_FILETIME(ft, v64) \ + (ft)->dwLowDateTime = (DWORD)v64; \ + (ft)->dwHighDateTime = (DWORD)(v64 >> 32); + +#define WINAPI +#define TRUE 1 + +static BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime) +{ + UInt64 v = GET_TIME_64(fileTime); + v = (UInt64)((Int64)v - (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(localFileTime, v); + return TRUE; +} + +static const UInt32 kNumTimeQuantumsInSecond = 10000000; +static const UInt32 kFileTimeStartYear = 1601; +static const UInt32 kUnixTimeStartYear = 1970; +static const UInt64 kUnixTimeOffset = + (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); + +static Int64 Time_FileTimeToUnixTime64(const FILETIME *ft) +{ + UInt64 winTime = GET_TIME_64(ft); + return (Int64)(winTime / kNumTimeQuantumsInSecond) - (Int64)kUnixTimeOffset; +} + +#if defined(_AIX) + #define MY_ST_TIMESPEC st_timespec +#else + #define MY_ST_TIMESPEC timespec +#endif + +static void FILETIME_To_timespec(const FILETIME *ft, struct MY_ST_TIMESPEC *ts) +{ + if (ft) + { + const Int64 sec = Time_FileTimeToUnixTime64(ft); + // time_t is long + const time_t sec2 = (time_t)sec; + if (sec2 == sec) + { + ts->tv_sec = sec2; + UInt64 winTime = GET_TIME_64(ft); + ts->tv_nsec = (long)((winTime % 10000000) * 100);; + return; + } + } + // else + { + ts->tv_sec = 0; + // ts.tv_nsec = UTIME_NOW; // set to the current time + ts->tv_nsec = UTIME_OMIT; // keep old timesptamp + } +} + +static WRes Set_File_FILETIME(const UInt16 *name, const FILETIME *mTime) +{ + struct timespec times[2]; + + const int flags = 0; // follow link + // = AT_SYMLINK_NOFOLLOW; // don't follow link + + CBuf buf; + int res; + Buf_Init(&buf); + RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); + FILETIME_To_timespec(NULL, ×[0]); + FILETIME_To_timespec(mTime, ×[1]); + res = utimensat(AT_FDCWD, (const char *)buf.data, times, flags); + Buf_Free(&buf, &g_Alloc); + if (res == 0) + return 0; + return errno; +} + +#endif + +static void NtfsFileTime_to_FILETIME(const CNtfsFileTime *t, FILETIME *ft) +{ + ft->dwLowDateTime = (DWORD)(t->Low); + ft->dwHighDateTime = (DWORD)(t->High); +} + +static void ConvertFileTimeToString(const CNtfsFileTime *nTime, char *s) { unsigned year, mon, hour, min, sec; Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; unsigned t; UInt32 v; - UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); + // UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); + UInt64 v64; + { + FILETIME fileTime, locTime; + NtfsFileTime_to_FILETIME(nTime, &fileTime); + if (!FileTimeToLocalFileTime(&fileTime, &locTime)) + { + locTime.dwHighDateTime = + locTime.dwLowDateTime = 0; + } + v64 = locTime.dwLowDateTime | ((UInt64)locTime.dwHighDateTime << 32); + } v64 /= 10000000; sec = (unsigned)(v64 % 60); v64 /= 60; min = (unsigned)(v64 % 60); v64 /= 60; @@ -354,6 +486,43 @@ static void PrintError(char *s) PrintLF(); } +static void PrintError_WRes(const char *message, WRes wres) +{ + Print("\nERROR: "); + Print(message); + PrintLF(); + { + char s[32]; + UIntToStr(s, (unsigned)wres, 1); + Print("System error code: "); + Print(s); + } + // sprintf(buffer + strlen(buffer), "\nSystem error code: %d", (unsigned)wres); + #ifdef _WIN32 + { + char *s = NULL; + if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, wres, 0, (LPSTR) &s, 0, NULL) != 0 && s) + { + Print(" : "); + Print(s); + LocalFree(s); + } + } + #else + { + const char *s = strerror(wres); + if (s) + { + Print(" : "); + Print(s); + } + } + #endif + PrintLF(); +} + static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE @@ -413,17 +582,22 @@ int MY_CDECL main(int numargs, char *args[]) allocImp = g_Alloc; allocTempImp = g_Alloc; - #ifdef UNDER_CE - if (InFile_OpenW(&archiveStream.file, L"\test.7z")) - #else - if (InFile_Open(&archiveStream.file, args[2])) - #endif { - PrintError("can not open input file"); - return 1; + WRes wres = + #ifdef UNDER_CE + InFile_OpenW(&archiveStream.file, L"\test.7z"); // change it + #else + InFile_Open(&archiveStream.file, args[2]); + #endif + if (wres != 0) + { + PrintError_WRes("cannot open input file", wres); + return 1; + } } FileInStream_CreateVTable(&archiveStream); + archiveStream.wres = 0; LookToRead2_CreateVTable(&lookStream, False); lookStream.buf = NULL; @@ -483,7 +657,7 @@ int MY_CDECL main(int numargs, char *args[]) size_t outSizeProcessed = 0; // const CSzFileItem *f = db.Files + i; size_t len; - unsigned isDir = SzArEx_IsDir(&db, i); + const BoolInt isDir = SzArEx_IsDir(&db, i); if (listCommand == 0 && isDir && !fullPaths) continue; len = SzArEx_GetFileNameUtf16(&db, i, NULL); @@ -546,8 +720,8 @@ int MY_CDECL main(int numargs, char *args[]) } Print(testCommand ? - "Testing ": - "Extracting "); + "T ": + "- "); res = PrintString(temp); if (res != SZ_OK) break; @@ -591,27 +765,37 @@ int MY_CDECL main(int numargs, char *args[]) PrintLF(); continue; } - else if (OutFile_OpenUtf16(&outFile, destPath)) + else { - PrintError("can not open output file"); - res = SZ_ERROR_FAIL; - break; + WRes wres = OutFile_OpenUtf16(&outFile, destPath); + if (wres != 0) + { + PrintError_WRes("cannot open output file", wres); + res = SZ_ERROR_FAIL; + break; + } } processedSize = outSizeProcessed; - if (File_Write(&outFile, outBuffer + offset, &processedSize) != 0 || processedSize != outSizeProcessed) { - PrintError("can not write output file"); - res = SZ_ERROR_FAIL; - break; + WRes wres = File_Write(&outFile, outBuffer + offset, &processedSize); + if (wres != 0 || processedSize != outSizeProcessed) + { + PrintError_WRes("cannot write output file", wres); + res = SZ_ERROR_FAIL; + break; + } } - #ifdef USE_WINDOWS_FILE { - FILETIME mtime, ctime; + FILETIME mtime; FILETIME *mtimePtr = NULL; + + #ifdef USE_WINDOWS_FILE + FILETIME ctime; FILETIME *ctimePtr = NULL; + #endif if (SzBitWithVals_Check(&db.MTime, i)) { @@ -620,6 +804,8 @@ int MY_CDECL main(int numargs, char *args[]) mtime.dwHighDateTime = (DWORD)(t->High); mtimePtr = &mtime; } + + #ifdef USE_WINDOWS_FILE if (SzBitWithVals_Check(&db.CTime, i)) { const CNtfsFileTime *t = &db.CTime.Vals[i]; @@ -627,16 +813,29 @@ int MY_CDECL main(int numargs, char *args[]) ctime.dwHighDateTime = (DWORD)(t->High); ctimePtr = &ctime; } + if (mtimePtr || ctimePtr) SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); - } - #endif + #endif - if (File_Close(&outFile)) - { - PrintError("can not close output file"); - res = SZ_ERROR_FAIL; - break; + { + WRes wres = File_Close(&outFile); + if (wres != 0) + { + PrintError_WRes("cannot close output file", wres); + res = SZ_ERROR_FAIL; + break; + } + } + + #ifndef USE_WINDOWS_FILE + #ifdef _WIN32 + mtimePtr = mtimePtr; + #else + if (mtimePtr) + Set_File_FILETIME(destPath, mtimePtr); + #endif + #endif } #ifdef USE_WINDOWS_FILE @@ -672,13 +871,15 @@ int MY_CDECL main(int numargs, char *args[]) if (res == SZ_ERROR_UNSUPPORTED) PrintError("decoder doesn't support this archive"); else if (res == SZ_ERROR_MEM) - PrintError("can not allocate memory"); + PrintError("cannot allocate memory"); else if (res == SZ_ERROR_CRC) PrintError("CRC error"); + else if (res == SZ_ERROR_READ /* || archiveStream.Res != 0 */) + PrintError_WRes("Read Error", archiveStream.wres); else { char s[32]; - UInt64ToStr(res, s, 0); + UInt64ToStr((unsigned)res, s, 0); PrintError(s); } diff --git a/C/Util/7z/makefile.gcc b/C/Util/7z/makefile.gcc index 51053bad..4263d675 100644 --- a/C/Util/7z/makefile.gcc +++ b/C/Util/7z/makefile.gcc @@ -1,75 +1,34 @@ -PROG = 7zDec -CXX = gcc -LIB = -RM = rm -f -CFLAGS = -c -O2 -Wall - -OBJS = 7zMain.o 7zAlloc.o 7zArcIn.o 7zBuf.o 7zBuf2.o 7zCrc.o 7zCrcOpt.o 7zDec.o CpuArch.o Delta.o LzmaDec.o Lzma2Dec.o Bra.o Bra86.o BraIA64.o Bcj2.o Ppmd7.o Ppmd7Dec.o 7zFile.o 7zStream.o - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) - -7zMain.o: 7zMain.c - $(CXX) $(CFLAGS) 7zMain.c - -7zAlloc.o: ../../7zAlloc.c - $(CXX) $(CFLAGS) ../../7zAlloc.c - -7zArcIn.o: ../../7zArcIn.c - $(CXX) $(CFLAGS) ../../7zArcIn.c - -7zBuf.o: ../../7zBuf.c - $(CXX) $(CFLAGS) ../../7zBuf.c - -7zBuf2.o: ../../7zBuf2.c - $(CXX) $(CFLAGS) ../../7zBuf2.c - -7zCrc.o: ../../7zCrc.c - $(CXX) $(CFLAGS) ../../7zCrc.c - -7zCrcOpt.o: ../../7zCrc.c - $(CXX) $(CFLAGS) ../../7zCrcOpt.c - -7zDec.o: ../../7zDec.c - $(CXX) $(CFLAGS) -D_7ZIP_PPMD_SUPPPORT ../../7zDec.c - -CpuArch.o: ../../CpuArch.c - $(CXX) $(CFLAGS) ../../CpuArch.c - -Delta.o: ../../Delta.c - $(CXX) $(CFLAGS) ../../Delta.c - -LzmaDec.o: ../../LzmaDec.c - $(CXX) $(CFLAGS) ../../LzmaDec.c - -Lzma2Dec.o: ../../Lzma2Dec.c - $(CXX) $(CFLAGS) ../../Lzma2Dec.c - -Bra.o: ../../Bra.c - $(CXX) $(CFLAGS) ../../Bra.c - -Bra86.o: ../../Bra86.c - $(CXX) $(CFLAGS) ../../Bra86.c - -BraIA64.o: ../../BraIA64.c - $(CXX) $(CFLAGS) ../../BraIA64.c - -Bcj2.o: ../../Bcj2.c - $(CXX) $(CFLAGS) ../../Bcj2.c - -Ppmd7.o: ../../Ppmd7.c - $(CXX) $(CFLAGS) ../../Ppmd7.c - -Ppmd7Dec.o: ../../Ppmd7Dec.c - $(CXX) $(CFLAGS) ../../Ppmd7Dec.c - -7zFile.o: ../../7zFile.c - $(CXX) $(CFLAGS) ../../7zFile.c - -7zStream.o: ../../7zStream.c - $(CXX) $(CFLAGS) ../../7zStream.c - -clean: - -$(RM) $(PROG) $(OBJS) +PROG = 7zdec + +LOCAL_FLAGS = -D_7ZIP_PPMD_SUPPPORT + +include ../../../CPP/7zip/LzmaDec_gcc.mak + + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $O/Bcj2.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/Lzma2Dec.o \ + $O/LzmaDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/7zAlloc.o \ + $O/7zArcIn.o \ + $O/7zBuf.o \ + $O/7zBuf2.o \ + $O/7zDec.o \ + $O/7zMain.o \ + $O/7zFile.o \ + $O/7zStream.o \ + + +include ../../7zip_gcc_c.mak diff --git a/C/Util/7zipInstall/7zipInstall.c b/C/Util/7zipInstall/7zipInstall.c index 2022e2f3..7aa11de7 100644 --- a/C/Util/7zipInstall/7zipInstall.c +++ b/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2019-02-19 : Igor Pavlov : Public domain */ +2021-02-23 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,9 +12,6 @@ #include <windows.h> #include <ShlObj.h> -#define LLL_(quote) L##quote -#define LLL(quote) LLL_(quote) - #include "../../7z.h" #include "../../7zAlloc.h" #include "../../7zCrc.h" @@ -25,11 +22,15 @@ #include "resource.h" +#define LLL_(quote) L##quote +#define LLL(quote) LLL_(quote) + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) #define wcscat lstrcatW #define wcslen lstrlenW #define wcscpy lstrcpyW -#define wcsncpy lstrcpynW +// wcsncpy() and lstrcpynW() work differently. We don't use them. #define kInputBufSize ((size_t)1 << 18) @@ -38,7 +39,7 @@ #define _7ZIP_CUR_VER ((MY_VER_MAJOR << 16) | MY_VER_MINOR) #define _7ZIP_DLL_VER_COMPAT ((16 << 16) | 3) -static LPCWSTR const k_7zip = L"7-Zip"; +static LPCSTR const k_7zip = "7-Zip"; static LPCWSTR const k_Reg_Software_7zip = L"Software\\7-Zip"; @@ -51,11 +52,27 @@ static LPCWSTR const k_Reg_Software_7zip = L"Software\\7-Zip"; #define k_7zip_with_Ver_base L"7-Zip " LLL(MY_VERSION) #ifdef _64BIT_INSTALLER - #define k_7zip_with_Ver k_7zip_with_Ver_base L" (x64)" + + // #define USE_7ZIP_32_DLL + + #if defined(_M_ARM64) || defined(_M_ARM) + #define k_Postfix L" (arm64)" + #else + #define k_Postfix L" (x64)" + #define USE_7ZIP_32_DLL + #endif #else - #define k_7zip_with_Ver k_7zip_with_Ver_base + #if defined(_M_ARM64) || defined(_M_ARM) + #define k_Postfix L" (arm)" + #else + // #define k_Postfix L" (x86)" + #define k_Postfix + #endif #endif +#define k_7zip_with_Ver k_7zip_with_Ver_base k_Postfix + + static LPCWSTR const k_7zip_with_Ver_str = k_7zip_with_Ver; static LPCWSTR const k_7zip_Setup = k_7zip_with_Ver L" Setup"; @@ -100,24 +117,47 @@ static HWND g_Progress_HWND; static DWORD g_TotalSize; +static WCHAR cmd[MAX_PATH + 4]; +static WCHAR cmdError[MAX_PATH + 4]; static WCHAR path[MAX_PATH * 2 + 40]; -#define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c))) +// #define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c))) -static void PrintErrorMessage(const char *s) + +static void CpyAscii(wchar_t *dest, const char *s) { - WCHAR s2[256 + 4]; - unsigned i; - for (i = 0; i < 256; i++) + for (;;) { - Byte b = s[i]; + Byte b = (Byte)*s++; + *dest++ = b; if (b == 0) - break; - s2[i] = b; + return; + } +} + +static void CatAscii(wchar_t *dest, const char *s) +{ + dest += wcslen(dest); + CpyAscii(dest, s); +} + +static void PrintErrorMessage(const char *s1, const wchar_t *s2) +{ + WCHAR m[MAX_PATH + 512]; + m[0] = 0; + CatAscii(m, "ERROR:"); + if (s1) + { + CatAscii(m, "\n"); + CatAscii(m, s1); } - s2[i] = 0; - MessageBoxW(g_HWND, s2, k_7zip_with_Ver_str, MB_ICONERROR); + if (s2) + { + CatAscii(m, "\n"); + wcscat(m, s2); + } + MessageBoxW(g_HWND, m, k_7zip_with_Ver_str, MB_ICONERROR | MB_OK); } @@ -347,7 +387,7 @@ static LONG MyRegistry_CreateKeyAndVal(HKEY parentKey, LPCWSTR keyName, LPCWSTR } -#ifdef _64BIT_INSTALLER +#ifdef USE_7ZIP_32_DLL static LONG MyRegistry_CreateKey_32(HKEY parentKey, LPCWSTR name, HKEY *destKey) { @@ -441,7 +481,7 @@ static void HexToString(UInt32 val, WCHAR *s) #ifndef UNDER_CE -int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM lp, LPARAM data) +static int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM lp, LPARAM data) { UNUSED_VAR(lp) UNUSED_VAR(data) @@ -560,11 +600,11 @@ static void Set7zipPostfix(WCHAR *s) NormalizePrefix(s); if (FindSubString(s, "7-Zip")) return; - wcscat(s, L"7-Zip\\"); + CatAscii(s, "7-Zip\\"); } -static int Install(); +static int Install(void); static void OnClose() { @@ -612,7 +652,7 @@ static INT_PTR CALLBACK MyDlgProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM } if (!g_Install_was_Pressed) { - SendMessage(hwnd, WM_NEXTDLGCTL, (WPARAM)GetDlgItem(hwnd, IDCANCEL), TRUE); + SendMessage(hwnd, WM_NEXTDLGCTL, (WPARAM)(void *)GetDlgItem(hwnd, IDCANCEL), TRUE); EnableWindow(g_Path_HWND, FALSE); EnableWindow(GetDlgItem(hwnd, IDB_EXTRACT_SET_PATH), FALSE); @@ -723,7 +763,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) { #ifdef UNDER_CE - // wcscpy(link, L"\\Program Files\\"); + // CpyAscii(link, "\\Program Files\\"); UNUSED_VAR(hwndOwner) #else @@ -744,8 +784,8 @@ static void SetShellProgramsGroup(HWND hwndOwner) continue; NormalizePrefix(link); - wcscat(link, k_7zip); - // wcscat(link, L"2"); + CatAscii(link, k_7zip); + // CatAscii(link, "2"); if (i != 0) MyCreateDir(link); @@ -758,14 +798,14 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (k = 0; k < 2; k++) { - wcscpy(link + baseLen, k == 0 ? - L"7-Zip File Manager.lnk" : - L"7-Zip Help.lnk" + CpyAscii(link + baseLen, k == 0 ? + "7-Zip File Manager.lnk" : + "7-Zip Help.lnk" ); wcscpy(destPath, path); - wcscat(destPath, k == 0 ? - L"7zFM.exe" : - L"7-zip.chm"); + CatAscii(destPath, k == 0 ? + "7zFM.exe" : + "7-zip.chm"); if (i == 0) DeleteFileW(link); @@ -789,7 +829,7 @@ static void WriteCLSID() HKEY destKey; LONG res; - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL MyRegistry_CreateKeyAndVal_32(HKEY_CLASSES_ROOT, k_Reg_CLSID_7zip, NULL, k_7zip_ShellExtension); @@ -797,9 +837,9 @@ static void WriteCLSID() if (res == ERROR_SUCCESS) { - WCHAR destPath[MAX_PATH + 10]; + WCHAR destPath[MAX_PATH + 40]; wcscpy(destPath, path); - wcscat(destPath, L"7-zip32.dll"); + CatAscii(destPath, "7-zip32.dll"); /* res = */ MyRegistry_SetString(destKey, NULL, destPath); /* res = */ MyRegistry_SetString(destKey, L"ThreadingModel", L"Apartment"); // DeleteRegValue(destKey, L"InprocServer32"); @@ -816,9 +856,9 @@ static void WriteCLSID() if (res == ERROR_SUCCESS) { - WCHAR destPath[MAX_PATH + 10]; + WCHAR destPath[MAX_PATH + 40]; wcscpy(destPath, path); - wcscat(destPath, L"7-zip.dll"); + CatAscii(destPath, "7-zip.dll"); /* res = */ MyRegistry_SetString(destKey, NULL, destPath); /* res = */ MyRegistry_SetString(destKey, L"ThreadingModel", L"Apartment"); // DeleteRegValue(destKey, L"InprocServer32"); @@ -826,13 +866,13 @@ static void WriteCLSID() } } -static LPCWSTR const k_ShellEx_Items[] = +static LPCSTR const k_ShellEx_Items[] = { - L"*\\shellex\\ContextMenuHandlers" - , L"Directory\\shellex\\ContextMenuHandlers" - , L"Folder\\shellex\\ContextMenuHandlers" - , L"Directory\\shellex\\DragDropHandlers" - , L"Drive\\shellex\\DragDropHandlers" + "*\\shellex\\ContextMenuHandlers" + , "Directory\\shellex\\ContextMenuHandlers" + , "Folder\\shellex\\ContextMenuHandlers" + , "Directory\\shellex\\DragDropHandlers" + , "Drive\\shellex\\DragDropHandlers" }; static void WriteShellEx() @@ -840,31 +880,31 @@ static void WriteShellEx() unsigned i; WCHAR destPath[MAX_PATH + 40]; - for (i = 0; i < sizeof(k_ShellEx_Items) / sizeof(k_ShellEx_Items[0]); i++) + for (i = 0; i < ARRAY_SIZE(k_ShellEx_Items); i++) { - wcscpy(destPath, k_ShellEx_Items[i]); - wcscat(destPath, L"\\7-Zip"); + CpyAscii(destPath, k_ShellEx_Items[i]); + CatAscii(destPath, "\\7-Zip"); - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL MyRegistry_CreateKeyAndVal_32(HKEY_CLASSES_ROOT, destPath, NULL, k_7zip_CLSID); #endif MyRegistry_CreateKeyAndVal (HKEY_CLASSES_ROOT, destPath, NULL, k_7zip_CLSID); } - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL MyRegistry_CreateKeyAndVal_32(HKEY_LOCAL_MACHINE, k_Shell_Approved, k_7zip_CLSID, k_7zip_ShellExtension); #endif MyRegistry_CreateKeyAndVal (HKEY_LOCAL_MACHINE, k_Shell_Approved, k_7zip_CLSID, k_7zip_ShellExtension); + wcscpy(destPath, path); + CatAscii(destPath, "7zFM.exe"); + { HKEY destKey = 0; LONG res = MyRegistry_CreateKey(HKEY_LOCAL_MACHINE, L"Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\7zFM.exe", &destKey); if (res == ERROR_SUCCESS) { - wcscpy(destPath, path); - wcscat(destPath, L"7zFM.exe"); - MyRegistry_SetString(destKey, NULL, destPath); MyRegistry_SetString(destKey, L"Path", path); RegCloseKey(destKey); @@ -877,17 +917,14 @@ static void WriteShellEx() LONG res = MyRegistry_CreateKey(HKEY_LOCAL_MACHINE, L"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\7-Zip", &destKey); if (res == ERROR_SUCCESS) { - // wcscpy(destPath, path); - // wcscat(destPath, L"7zFM.exe"); MyRegistry_SetString(destKey, L"DisplayName", k_7zip_with_Ver_str); MyRegistry_SetString(destKey, L"DisplayVersion", LLL(MY_VERSION_NUMBERS)); - MyRegistry_SetString(destKey, L"DisplayIcon", destPath); + MyRegistry_SetString(destKey, L"InstallLocation", path); - wcscpy(destPath, path); - MyRegistry_SetString(destKey, L"InstallLocation", destPath); - wcscat(destPath, L"Uninstall.exe"); - // wcscat(destPath, L"\""); + destPath[0] = '\"'; + wcscpy(destPath + 1, path); + CatAscii(destPath, "Uninstall.exe\""); MyRegistry_SetString(destKey, L"UninstallString", destPath); MyRegistry_SetDWORD(destKey, L"NoModify", 1); @@ -912,17 +949,27 @@ static void WriteShellEx() static const wchar_t *GetCmdParam(const wchar_t *s) { + unsigned pos = 0; BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; + if (c == 0 || (c == L' ' && !quoteMode)) + break; if (c == L'\"') + { quoteMode = !quoteMode; - else if (c == 0 || (c == L' ' && !quoteMode)) - return s; + continue; + } + if (pos >= ARRAY_SIZE(cmd) - 1) + exit(1); + cmd[pos++] = c; } + cmd[pos] = 0; + return s; } + static void RemoveQuotes(wchar_t *s) { const wchar_t *src = s; @@ -937,7 +984,7 @@ static void RemoveQuotes(wchar_t *s) } } -#define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') +// #define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') typedef BOOL (WINAPI *Func_IsWow64Process)(HANDLE, PBOOL); @@ -984,24 +1031,36 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { const wchar_t *s2 = GetCmdParam(s); - if (s[0] == '/') + BoolInt error = True; + if (cmd[0] == '/') { - if (s[1] == 'S' && IS_LIMIT_CHAR(s[2])) - g_SilentMode = True; - else if (s[1] == 'D' && s[2] == '=') + if (cmd[1] == 'S') { - size_t num; - s += 3; - num = s2 - s; - if (num > MAX_PATH) - num = MAX_PATH; - wcsncpy(path, s, (unsigned)num); - RemoveQuotes(path); + if (cmd[2] == 0) + { + g_SilentMode = True; + error = False; + } + } + else if (cmd[1] == 'D' && cmd[2] == '=') + { + wcscpy(path, cmd + 3); + // RemoveQuotes(path); + error = False; } } s = s2; + if (error && cmdError[0] == 0) + wcscpy(cmdError, cmd); } } + + if (cmdError[0] != 0) + { + if (!g_SilentMode) + PrintErrorMessage("Unsupported command:", cmdError); + return 1; + } } #if defined(_64BIT_INSTALLER) && !defined(_WIN64) @@ -1016,7 +1075,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (!isWow64) { if (!g_SilentMode) - PrintErrorMessage("This installation requires Windows x64"); + PrintErrorMessage("This installation requires Windows " MY_CPU_NAME, NULL); return 1; } } @@ -1040,27 +1099,27 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { /* #ifdef UNDER_CE - wcscpy(path, L"\\Program Files\\"); + CpyAscii(path, "\\Program Files\\"); #else #ifdef _64BIT_INSTALLER { DWORD ttt = GetEnvironmentVariableW(L"ProgramW6432", path, MAX_PATH); if (ttt == 0 || ttt > MAX_PATH) - wcscpy(path, L"C:\\"); + CpyAscii(path, "C:\\"); } #else if (!SHGetSpecialFolderPathW(0, path, CSIDL_PROGRAM_FILES, FALSE)) - wcscpy(path, L"C:\\"); + CpyAscii(path, "C:\\"); #endif #endif */ if (!MyRegistry_QueryString2(HKEY_LOCAL_MACHINE, L"Software\\Microsoft\\Windows\\CurrentVersion", L"ProgramFilesDir", path)) - wcscpy(path, + CpyAscii(path, #ifdef UNDER_CE - L"\\Program Files\\" + "\\Program Files\\" #else - L"C:\\" + "C:\\" #endif ); @@ -1122,7 +1181,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, SetDlgItemTextW(g_HWND, IDOK, L"Close"); EnableWindow(GetDlgItem(g_HWND, IDOK), TRUE); EnableWindow(GetDlgItem(g_HWND, IDCANCEL), FALSE); - SendMessage(g_HWND, WM_NEXTDLGCTL, (WPARAM)GetDlgItem(g_HWND, IDOK), TRUE); + SendMessage(g_HWND, WM_NEXTDLGCTL, (WPARAM)(void *)GetDlgItem(g_HWND, IDOK), TRUE); } } } @@ -1155,7 +1214,7 @@ static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message) -static int Install() +static int Install(void) { CFileInStream archiveStream; CLookToRead2 lookStream; @@ -1212,6 +1271,7 @@ if (res == SZ_OK) LookToRead2_CreateVTable(&lookStream, False); lookStream.buf = NULL; + RemoveQuotes(path); { // Remove post spaces unsigned endPos = 0; @@ -1227,6 +1287,11 @@ if (res == SZ_OK) } path[endPos] = 0; + if (path[0] == 0) + { + PrintErrorMessage("Incorrect path", NULL); + return 1; + } } NormalizePrefix(path); @@ -1367,7 +1432,7 @@ if (res == SZ_OK) break; } wcscpy(path, origPath); - wcscat(path, L".tmp"); + CatAscii(path, ".tmp"); if (tempIndex > 1) HexToString(tempIndex, path + wcslen(path)); if (GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES) @@ -1391,7 +1456,7 @@ if (res == SZ_OK) } if (FindSubString(temp, "7-zip.dll") - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL || FindSubString(temp, "7-zip32.dll") #endif ) @@ -1411,9 +1476,9 @@ if (res == SZ_OK) WCHAR message[MAX_PATH * 3 + 100]; int mbRes; - wcscpy(message, L"Can't open file\n"); + CpyAscii(message, "Can't open file\n"); wcscat(message, path); - wcscat(message, L"\n"); + CatAscii(message, "\n"); GetErrorMessage(openRes, message + wcslen(message)); @@ -1572,7 +1637,7 @@ if (res == SZ_OK) WCHAR m[MAX_PATH + 100]; m[0] = 0; GetErrorMessage(winRes, m); - MessageBoxW(g_HWND, m, k_7zip_with_Ver_str, MB_ICONERROR); + PrintErrorMessage(NULL, m); } else { @@ -1590,7 +1655,7 @@ if (res == SZ_OK) if (!errorMessage) errorMessage = "ERROR"; - PrintErrorMessage(errorMessage); + PrintErrorMessage(errorMessage, NULL); } } diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index 16761170..b02fe5a1 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,15 +1,15 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2019-02-02 : Igor Pavlov : Public domain */ +2021-02-23 : Igor Pavlov : Public domain */ #include "Precomp.h" -#define SZ_ERROR_ABORT 100 - #ifdef _MSC_VER #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union #pragma warning(disable : 4011) // vs2010: identifier truncated to _CRT_SECURE_CPP_OVERLOAD_SECURE #endif +// #define SZ_ERROR_ABORT 100 + #include <windows.h> #include <ShlObj.h> @@ -20,7 +20,9 @@ #define LLL_(quote) L##quote #define LLL(quote) LLL_(quote) -// static const WCHAR * const k_7zip = L"7-Zip"; +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + +// static LPCWSTR const k_7zip = L"7-Zip"; // #define _64BIT_INSTALLER 1 @@ -31,18 +33,33 @@ #define k_7zip_with_Ver_base L"7-Zip " LLL(MY_VERSION) #ifdef _64BIT_INSTALLER - #define k_7zip_with_Ver k_7zip_with_Ver_base L" (x64)" + + // #define USE_7ZIP_32_DLL + + #if defined(_M_ARM64) || defined(_M_ARM) + #define k_Postfix L" (arm64)" + #else + #define k_Postfix L" (x64)" + #define USE_7ZIP_32_DLL + #endif #else - #define k_7zip_with_Ver k_7zip_with_Ver_base + #if defined(_M_ARM64) || defined(_M_ARM) + #define k_Postfix L" (arm)" + #else + // #define k_Postfix L" (x86)" + #define k_Postfix + #endif #endif -// static const WCHAR * const k_7zip_with_Ver_str = k_7zip_with_Ver; +#define k_7zip_with_Ver k_7zip_with_Ver_base k_Postfix -static const WCHAR * const k_Reg_Software_7zip = L"Software\\7-Zip"; +static LPCWSTR const k_7zip_with_Ver_Uninstall = k_7zip_with_Ver L" Uninstall"; -static const WCHAR * const k_Reg_Path = L"Path"; +static LPCWSTR const k_Reg_Software_7zip = L"Software\\7-Zip"; + +static LPCWSTR const k_Reg_Path = L"Path"; -static const WCHAR * const k_Reg_Path32 = L"Path" +static LPCWSTR const k_Reg_Path32 = L"Path" #ifdef _64BIT_INSTALLER L"64" #else @@ -64,8 +81,8 @@ static const WCHAR * const k_Reg_Path32 = L"Path" #define k_7zip_CLSID L"{23170F69-40C1-278A-1000-000100020000}" -static const WCHAR * const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID; -static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32"; +static LPCWSTR const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID; +static LPCWSTR const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32"; #define g_AllUsers True @@ -79,9 +96,12 @@ static HWND g_Path_HWND; static HWND g_InfoLine_HWND; static HWND g_Progress_HWND; -typedef WINADVAPI LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); +// WINADVAPI +typedef LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); static Func_RegDeleteKeyExW func_RegDeleteKeyExW; +static WCHAR cmd[MAX_PATH + 4]; +static WCHAR cmdError[MAX_PATH + 4]; static WCHAR path[MAX_PATH * 2 + 40]; static WCHAR workDir[MAX_PATH + 10]; static WCHAR modulePath[MAX_PATH + 10]; @@ -90,10 +110,47 @@ static WCHAR tempPath[MAX_PATH * 2 + 40]; static WCHAR cmdLine[MAX_PATH * 3 + 40]; static WCHAR copyPath[MAX_PATH * 2 + 40]; -static const WCHAR * const kUninstallExe = L"Uninstall.exe"; +static LPCWSTR const kUninstallExe = L"Uninstall.exe"; #define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c))) + +static void CpyAscii(wchar_t *dest, const char *s) +{ + for (;;) + { + Byte b = (Byte)*s++; + *dest++ = b; + if (b == 0) + return; + } +} + +static void CatAscii(wchar_t *dest, const char *s) +{ + dest += wcslen(dest); + CpyAscii(dest, s); +} + +static void PrintErrorMessage(const char *s1, const wchar_t *s2) +{ + WCHAR m[MAX_PATH + 512]; + m[0] = 0; + CatAscii(m, "ERROR:"); + if (s1) + { + CatAscii(m, "\n"); + CatAscii(m, s1); + } + if (s2) + { + CatAscii(m, "\n"); + wcscat(m, s2); + } + MessageBoxW(g_HWND, m, k_7zip_with_Ver_Uninstall, MB_ICONERROR | MB_OK); +} + + static BoolInt AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) { for (;;) @@ -171,7 +228,7 @@ static LONG MyRegistry_DeleteKey(HKEY parentKey, LPCWSTR name) #endif } -#ifdef _64BIT_INSTALLER +#ifdef USE_7ZIP_32_DLL static int MyRegistry_QueryString2_32(HKEY hKey, LPCWSTR keyName, LPCWSTR valName, LPWSTR dest) { @@ -295,7 +352,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) continue; NormalizePrefix(link); - wcscat(link, L"7-Zip\\"); + CatAscii(link, "7-Zip\\"); { const size_t baseLen = wcslen(link); @@ -304,13 +361,13 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (k = 0; k < 2; k++) { - wcscpy(link + baseLen, k == 0 ? - L"7-Zip File Manager.lnk" : - L"7-Zip Help.lnk"); + CpyAscii(link + baseLen, k == 0 ? + "7-Zip File Manager.lnk" : + "7-Zip Help.lnk"); wcscpy(destPath, path); - wcscat(destPath, k == 0 ? - L"7zFM.exe" : - L"7-zip.chm"); + CatAscii(destPath, k == 0 ? + "7zFM.exe" : + "7-zip.chm"); if (CreateShellLink(link, destPath) == S_OK) { @@ -331,20 +388,20 @@ static void SetShellProgramsGroup(HWND hwndOwner) } -static const WCHAR * const k_ShellEx_Items[] = +static LPCSTR const k_ShellEx_Items[] = { - L"*\\shellex\\ContextMenuHandlers" - , L"Directory\\shellex\\ContextMenuHandlers" - , L"Folder\\shellex\\ContextMenuHandlers" - , L"Directory\\shellex\\DragDropHandlers" - , L"Drive\\shellex\\DragDropHandlers" + "*\\shellex\\ContextMenuHandlers" + , "Directory\\shellex\\ContextMenuHandlers" + , "Folder\\shellex\\ContextMenuHandlers" + , "Directory\\shellex\\DragDropHandlers" + , "Drive\\shellex\\DragDropHandlers" }; -static const WCHAR * const k_Shell_Approved = L"Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved"; +static LPCWSTR const k_Shell_Approved = L"Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved"; -static const WCHAR * const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\7zFM.exe"; +static LPCWSTR const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\7zFM.exe"; #define k_REG_Uninstall L"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\" -static const WCHAR * const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip"; +static LPCWSTR const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip"; static BoolInt AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) @@ -370,11 +427,11 @@ static void WriteCLSID() { unsigned i; - for (i = 0; i < sizeof(k_ShellEx_Items) / sizeof(k_ShellEx_Items[0]); i++) + for (i = 0; i < ARRAY_SIZE(k_ShellEx_Items); i++) { WCHAR destPath[MAX_PATH]; - wcscpy(destPath, k_ShellEx_Items[i]); - wcscat(destPath, L"\\7-Zip"); + CpyAscii(destPath, k_ShellEx_Items[i]); + CatAscii(destPath, "\\7-Zip"); MyRegistry_DeleteKey(HKEY_CLASSES_ROOT, destPath); } @@ -393,7 +450,7 @@ static void WriteCLSID() } - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL if (MyRegistry_QueryString2_32(HKEY_CLASSES_ROOT, k_Reg_CLSID_7zip_Inproc, NULL, s)) { @@ -407,11 +464,11 @@ static void WriteCLSID() { unsigned i; - for (i = 0; i < sizeof(k_ShellEx_Items) / sizeof(k_ShellEx_Items[0]); i++) + for (i = 0; i < ARRAY_SIZE(k_ShellEx_Items); i++) { WCHAR destPath[MAX_PATH]; - wcscpy(destPath, k_ShellEx_Items[i]); - wcscat(destPath, L"\\7-Zip"); + CpyAscii(destPath, k_ShellEx_Items[i]); + CatAscii(destPath, "\\7-Zip"); MyRegistry_DeleteKey_32(HKEY_CLASSES_ROOT, destPath); } @@ -444,17 +501,27 @@ static void WriteCLSID() static const wchar_t *GetCmdParam(const wchar_t *s) { + unsigned pos = 0; BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; + if (c == 0 || (c == L' ' && !quoteMode)) + break; if (c == L'\"') + { quoteMode = !quoteMode; - else if (c == 0 || (c == L' ' && !quoteMode)) - return s; + continue; + } + if (pos >= ARRAY_SIZE(cmd) - 1) + exit(1); + cmd[pos++] = c; } + cmd[pos] = 0; + return s; } +/* static void RemoveQuotes(wchar_t *s) { const wchar_t *src = s; @@ -468,6 +535,7 @@ static void RemoveQuotes(wchar_t *s) return; } } +*/ static BoolInt DoesFileOrDirExist() { @@ -489,7 +557,7 @@ static BOOL RemoveFileAfterReboot() return RemoveFileAfterReboot2(path); } -#define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') +// #define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') static BoolInt IsThereSpace(const wchar_t *s) { @@ -507,10 +575,10 @@ static void AddPathParam(wchar_t *dest, const wchar_t *src) { BoolInt needQuote = IsThereSpace(src); if (needQuote) - wcscat(dest, L"\""); + CatAscii(dest, "\""); wcscat(dest, src); if (needQuote) - wcscat(dest, L"\""); + CatAscii(dest, "\""); } @@ -543,12 +611,12 @@ static BOOL RemoveDir() -#define k_Lang L"Lang" +#define k_Lang "Lang" // NUM_LANG_TXT_FILES files are placed before en.ttt -#define NUM_LANG_TXT_FILES 88 +#define NUM_LANG_TXT_FILES 92 -#ifdef _64BIT_INSTALLER +#ifdef USE_7ZIP_32_DLL #define NUM_EXTRA_FILES_64BIT 1 #else #define NUM_EXTRA_FILES_64BIT 0 @@ -560,7 +628,7 @@ static const char * const k_Names = "af an ar ast az ba be bg bn br ca co cs cy da de el eo es et eu ext" " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kab kk ko ku ku-ckb ky" " lij lt lv mk mn mng mng2 mr ms nb ne nl nn pa-in pl ps pt pt-br ro ru" - " sa si sk sl sq sr-spc sr-spl sv ta th tr tt ug uk uz va vi yo zh-cn zh-tw" + " sa si sk sl sq sr-spc sr-spl sv sw ta tg th tk tr tt ug uk uz uz-cyrl va vi yo zh-cn zh-tw" " en.ttt" " descript.ion" " History.txt" @@ -573,7 +641,7 @@ static const char * const k_Names = " 7zG.exe" " 7z.dll" " 7zFM.exe" - #ifdef _64BIT_INSTALLER + #ifdef USE_7ZIP_32_DLL " 7-zip32.dll" #endif " 7-zip.dll" @@ -628,7 +696,7 @@ static int Install() temp = path + pathLen; if (i <= NUM_LANG_TXT_FILES) - wcscpy(temp, k_Lang L"\\"); + CpyAscii(temp, k_Lang "\\"); { WCHAR *dest = temp + wcslen(temp); @@ -648,7 +716,7 @@ static int Install() } if (i < NUM_LANG_TXT_FILES) - wcscat(temp, L".txt"); + CatAscii(temp, ".txt"); if (!g_SilentMode) SetWindowTextW(g_InfoLine_HWND, temp); @@ -673,7 +741,7 @@ static int Install() } } - wcscpy(path + pathLen, k_Lang); + CpyAscii(path + pathLen, k_Lang); RemoveDir(); path[pathLen] = 0; @@ -706,8 +774,8 @@ static int Install() WCHAR m[MAX_PATH + 100]; m[0] = 0; if (winRes == 0 || !GetErrorMessage(winRes, m)) - wcscpy(m, L"ERROR"); - MessageBoxW(g_HWND, m, L"Error", MB_ICONERROR | MB_OK); + CpyAscii(m, "ERROR"); + PrintErrorMessage("System ERROR:", m); } return 1; @@ -720,7 +788,7 @@ static void OnClose() { if (MessageBoxW(g_HWND, L"Do you want to cancel uninstallation?", - k_7zip_with_Ver, + k_7zip_with_Ver_Uninstall, MB_ICONQUESTION | MB_YESNO | MB_DEFBUTTON2) != IDYES) return; } @@ -739,7 +807,7 @@ static INT_PTR CALLBACK MyDlgProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM g_InfoLine_HWND = GetDlgItem(hwnd, IDT_CUR_FILE); g_Progress_HWND = GetDlgItem(hwnd, IDC_PROGRESS); - SetWindowTextW(hwnd, k_7zip_with_Ver L" Uninstall"); + SetWindowTextW(hwnd, k_7zip_with_Ver_Uninstall); SetDlgItemTextW(hwnd, IDE_EXTRACT_PATH, path); ShowWindow(g_Progress_HWND, SW_HIDE); @@ -759,7 +827,7 @@ static INT_PTR CALLBACK MyDlgProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM } if (!g_Install_was_Pressed) { - SendMessage(hwnd, WM_NEXTDLGCTL, (WPARAM)GetDlgItem(hwnd, IDCANCEL), TRUE); + SendMessage(hwnd, WM_NEXTDLGCTL, (WPARAM)(void *)GetDlgItem(hwnd, IDCANCEL), TRUE); EnableWindow(g_Path_HWND, FALSE); EnableWindow(GetDlgItem(hwnd, IDOK), FALSE); @@ -822,7 +890,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { const wchar_t *s = GetCommandLineW(); - + #ifndef UNDER_CE s = GetCmdParam(s); #endif @@ -844,32 +912,47 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { const wchar_t *s2 = GetCmdParam(s); - if (s[0] == '/') + BoolInt error = True; + if (cmd[0] == '/') { - if (s[1] == 'S' && IS_LIMIT_CHAR(s[2])) - g_SilentMode = True; - else if (s[1] == 'N' && IS_LIMIT_CHAR(s[2])) - useTemp = False; - else if (s[1] == 'D' && s[2] == '=') + if (cmd[1] == 'S') { - size_t num; - s += 3; - num = s2 - s; - if (num <= MAX_PATH) + if (cmd[2] == 0) + { + g_SilentMode = True; + error = False; + } + } + else if (cmd[1] == 'N') + { + if (cmd[2] == 0) { - wcsncpy(workDir, s, num); - workDir[num] = 0; - RemoveQuotes(workDir); useTemp = False; + error = False; } } + else if (cmd[1] == 'D' && cmd[2] == '=') + { + wcscpy(workDir, cmd + 3); + // RemoveQuotes(workDir); + useTemp = False; + error = False; + } } s = s2; + if (error && cmdError[0] == 0) + wcscpy(cmdError, cmd); } } - } - + if (cmdError[0] != 0) + { + if (!g_SilentMode) + PrintErrorMessage("Unsupported command:", cmdError); + return 1; + } + } + { wchar_t *name; DWORD len = GetModuleFileNameW(NULL, modulePath, MAX_PATH); @@ -925,7 +1008,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, for (i = 0; i < 100; i++, d += GetTickCount()) { - wcscpy(path + pathLen, L"7z"); + CpyAscii(path + pathLen, "7z"); { wchar_t *s = path + wcslen(path); @@ -944,7 +1027,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, continue; if (CreateDirectoryW(path, NULL)) { - wcscat(path, WSTRING_PATH_SEPARATOR); + CatAscii(path, STRING_PATH_SEPARATOR); wcscpy(tempPath, path); break; } @@ -955,7 +1038,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (tempPath[0] != 0) { wcscpy(copyPath, tempPath); - wcscat(copyPath, L"Uninst.exe"); // we need not "Uninstall.exe" here + CatAscii(copyPath, "Uninst.exe"); // we need not "Uninstall.exe" here if (CopyFileW(modulePath, copyPath, TRUE)) { @@ -969,7 +1052,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, // maybe CreateProcess supports path with spaces even without quotes. AddPathParam(cmdLine, copyPath); - wcscat(cmdLine, L" /N /D="); + CatAscii(cmdLine, " /N /D="); AddPathParam(cmdLine, modulePrefix); if (cmdParams[0] != 0 && wcslen(cmdParams) < MAX_PATH * 2 + 10) @@ -1066,7 +1149,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, SetDlgItemTextW(g_HWND, IDOK, L"Close"); EnableWindow(GetDlgItem(g_HWND, IDOK), TRUE); EnableWindow(GetDlgItem(g_HWND, IDCANCEL), FALSE); - SendMessage(g_HWND, WM_NEXTDLGCTL, (WPARAM)GetDlgItem(g_HWND, IDOK), TRUE); + SendMessage(g_HWND, WM_NEXTDLGCTL, (WPARAM)(void *)GetDlgItem(g_HWND, IDOK), TRUE); } } } diff --git a/C/Util/Lzma/LzmaUtil.c b/C/Util/Lzma/LzmaUtil.c index 739bc0fd..e613b9c2 100644 --- a/C/Util/Lzma/LzmaUtil.c +++ b/C/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2018-07-04 : Igor Pavlov : Public domain */ +2021-02-15 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -15,9 +15,9 @@ #include "../../LzmaDec.h" #include "../../LzmaEnc.h" -static const char * const kCantReadMessage = "Can not read input file"; -static const char * const kCantWriteMessage = "Can not write output file"; -static const char * const kCantAllocateMessage = "Can not allocate memory"; +static const char * const kCantReadMessage = "Cannot read input file"; +static const char * const kCantWriteMessage = "Cannot write output file"; +static const char * const kCantAllocateMessage = "Cannot allocate memory"; static const char * const kDataErrorMessage = "Data error"; static void PrintHelp(char *buffer) @@ -37,9 +37,25 @@ static int PrintError(char *buffer, const char *message) return 1; } +static int PrintError_WRes(char *buffer, const char *message, WRes wres) +{ + strcat(buffer, "\nError: "); + strcat(buffer, message); + sprintf(buffer + strlen(buffer), "\nSystem error code: %d", (unsigned)wres); + #ifndef _WIN32 + { + const char *s = strerror(wres); + if (s) + sprintf(buffer + strlen(buffer), " : %s", s); + } + #endif + strcat(buffer, "\n"); + return 1; +} + static int PrintErrorNumber(char *buffer, SRes val) { - sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); + sprintf(buffer + strlen(buffer), "\n7-Zip error code: %d\n", (unsigned)val); return 1; } @@ -181,9 +197,11 @@ static int main2(int numArgs, const char *args[], char *rs) FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); + inStream.wres = 0; FileOutStream_CreateVTable(&outStream); File_Construct(&outStream.file); + outStream.wres = 0; if (numArgs == 1) { @@ -206,14 +224,19 @@ static int main2(int numArgs, const char *args[], char *rs) return PrintError(rs, "Incorrect UInt32 or UInt64"); } - if (InFile_Open(&inStream.file, args[2]) != 0) - return PrintError(rs, "Can not open input file"); + { + WRes wres = InFile_Open(&inStream.file, args[2]); + if (wres != 0) + return PrintError_WRes(rs, "Cannot open input file", wres); + } if (numArgs > 3) { + WRes wres; useOutFile = True; - if (OutFile_Open(&outStream.file, args[3]) != 0) - return PrintError(rs, "Can not open output file"); + wres = OutFile_Open(&outStream.file, args[3]); + if (wres != 0) + return PrintError_WRes(rs, "Cannot open output file", wres); } else if (encodeMode) PrintUserError(rs); @@ -221,7 +244,9 @@ static int main2(int numArgs, const char *args[], char *rs) if (encodeMode) { UInt64 fileSize; - File_GetLength(&inStream.file, &fileSize); + WRes wres = File_GetLength(&inStream.file, &fileSize); + if (wres != 0) + return PrintError_WRes(rs, "Cannot get file length", wres); res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); } else @@ -240,9 +265,9 @@ static int main2(int numArgs, const char *args[], char *rs) else if (res == SZ_ERROR_DATA) return PrintError(rs, kDataErrorMessage); else if (res == SZ_ERROR_WRITE) - return PrintError(rs, kCantWriteMessage); + return PrintError_WRes(rs, kCantWriteMessage, outStream.wres); else if (res == SZ_ERROR_READ) - return PrintError(rs, kCantReadMessage); + return PrintError_WRes(rs, kCantReadMessage, inStream.wres); return PrintErrorNumber(rs, res); } return 0; diff --git a/C/Util/Lzma/makefile.gcc b/C/Util/Lzma/makefile.gcc index 67aa8b17..f9d59e3e 100644 --- a/C/Util/Lzma/makefile.gcc +++ b/C/Util/Lzma/makefile.gcc @@ -1,44 +1,19 @@ -PROG = lzma -CXX = g++ -LIB = -RM = rm -f -CFLAGS = -c -O2 -Wall -D_7ZIP_ST +PROG = 7lzma -OBJS = \ - LzmaUtil.o \ - Alloc.o \ - LzFind.o \ - LzmaDec.o \ - LzmaEnc.o \ - 7zFile.o \ - 7zStream.o \ - - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) $(LIB2) - -LzmaUtil.o: LzmaUtil.c - $(CXX) $(CFLAGS) LzmaUtil.c - -Alloc.o: ../../Alloc.c - $(CXX) $(CFLAGS) ../../Alloc.c +include ../../../CPP/7zip/LzmaDec_gcc.mak -LzFind.o: ../../LzFind.c - $(CXX) $(CFLAGS) ../../LzFind.c -LzmaDec.o: ../../LzmaDec.c - $(CXX) $(CFLAGS) ../../LzmaDec.c - -LzmaEnc.o: ../../LzmaEnc.c - $(CXX) $(CFLAGS) ../../LzmaEnc.c - -7zFile.o: ../../7zFile.c - $(CXX) $(CFLAGS) ../../7zFile.c - -7zStream.o: ../../7zStream.c - $(CXX) $(CFLAGS) ../../7zStream.c - -clean: - -$(RM) $(PROG) $(OBJS) +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $O/7zFile.o \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/LzFind.o \ + $O/LzFindMt.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/LzmaUtil.o \ + $O/Threads.o \ + + +include ../../7zip_gcc_c.mak @@ -1,5 +1,5 @@ /* Xz.c - Xz -2017-05-12 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -41,7 +41,7 @@ void Xz_Free(CXzStream *p, ISzAllocPtr alloc) unsigned XzFlags_GetCheckSize(CXzStreamFlags f) { unsigned t = XzFlags_GetCheckType(f); - return (t == 0) ? 0 : (4 << ((t - 1) / 3)); + return (t == 0) ? 0 : ((unsigned)4 << ((t - 1) / 3)); } void XzCheck_Init(CXzCheck *p, unsigned mode) @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2018-07-04 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -47,7 +47,7 @@ typedef struct CXzFilter filters[XZ_NUM_FILTERS_MAX]; } CXzBlock; -#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) +#define XzBlock_GetNumFilters(p) (((unsigned)(p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) #define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) #define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) @@ -277,7 +277,10 @@ void XzUnpacker_Free(CXzUnpacker *p); { XzUnpacker_Init() for() + { XzUnpacker_Code(); + } + XzUnpacker_IsStreamWasFinished() } Interface-2 : Direct output buffer: @@ -288,7 +291,10 @@ void XzUnpacker_Free(CXzUnpacker *p); XzUnpacker_Init() XzUnpacker_SetOutBufMode(); // to set output buffer and size for() + { XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + XzUnpacker_IsStreamWasFinished() } Interface-3 : Direct output buffer : One call full decoding @@ -296,6 +302,7 @@ void XzUnpacker_Free(CXzUnpacker *p); It uses Interface-2 internally. { XzUnpacker_CodeFull() + XzUnpacker_IsStreamWasFinished() } */ @@ -309,8 +316,12 @@ Returns: SZ_OK status: CODER_STATUS_NOT_FINISHED, - CODER_STATUS_NEEDS_MORE_INPUT - maybe there are more xz streams, - call XzUnpacker_IsStreamWasFinished to check that current stream was finished + CODER_STATUS_NEEDS_MORE_INPUT - the decoder can return it in two cases: + 1) it needs more input data to finish current xz stream + 2) xz stream was finished successfully. But the decoder supports multiple + concatented xz streams. So it expects more input data for new xz streams. + Call XzUnpacker_IsStreamWasFinished() to check that latest xz stream was finished successfully. + SZ_ERROR_MEM - Memory allocation error SZ_ERROR_DATA - Data error SZ_ERROR_UNSUPPORTED - Unsupported method or method properties @@ -335,12 +346,17 @@ SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status); +/* +If you decode full xz stream(s), then you can call XzUnpacker_IsStreamWasFinished() +after successful XzUnpacker_CodeFull() or after last call of XzUnpacker_Code(). +*/ + BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, +XzUnpacker_GetExtraSize() returns then number of unconfirmed bytes, if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. -These bytes can be some bytes after xz archive, or +These bytes can be some data after xz archive, or it can be start of new xz stream. Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of @@ -371,19 +387,46 @@ BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); -/* ---------- Multi Threading Decoding ---------- */ + + + +/* ---- Single-Thread and Multi-Thread xz Decoding with Input/Output Streams ---- */ + +/* + if (CXzDecMtProps::numThreads > 1), the decoder can try to use + Multi-Threading. The decoder analyses xz block header, and if + there are pack size and unpack size values stored in xz block header, + the decoder reads compressed data of block to internal buffers, + and then it can start parallel decoding, if there are another blocks. + The decoder can switch back to Single-Thread decoding after some conditions. + + The sequence of calls for xz decoding with in/out Streams: + { + XzDecMt_Create() + XzDecMtProps_Init(XzDecMtProps) to set default values of properties + // then you can change some XzDecMtProps parameters with required values + // here you can set the number of threads and (memUseMax) - the maximum + Memory usage for multithreading decoding. + for() + { + XzDecMt_Decode() // one call per one file + } + XzDecMt_Destroy() + } +*/ typedef struct { - size_t inBufSize_ST; - size_t outStep_ST; - BoolInt ignoreErrors; + size_t inBufSize_ST; // size of input buffer for Single-Thread decoding + size_t outStep_ST; // size of output buffer for Single-Thread decoding + BoolInt ignoreErrors; // if set to 1, the decoder can ignore some errors and it skips broken parts of data. #ifndef _7ZIP_ST - unsigned numThreads; - size_t inBufSize_MT; - size_t memUseMax; + unsigned numThreads; // the number of threads for Multi-Thread decoding. if (umThreads == 1) it will use Single-thread decoding + size_t inBufSize_MT; // size of small input data buffers for Multi-Thread decoding. Big number of such small buffers can be created + size_t memUseMax; // the limit of total memory usage for Multi-Thread decoding. + // it's recommended to set (memUseMax) manually to value that is smaller of total size of RAM in computer. #endif } CXzDecMtProps; @@ -393,7 +436,7 @@ void XzDecMtProps_Init(CXzDecMtProps *p); typedef void * CXzDecMtHandle; /* - alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + alloc : XzDecMt uses CAlignOffsetAlloc internally for addresses allocated by (alloc). allocMid : for big allocations, aligned allocation is better */ @@ -407,33 +450,46 @@ typedef struct Byte NumStreams_Defined; Byte NumBlocks_Defined; - Byte DataAfterEnd; + Byte DataAfterEnd; // there are some additional data after good xz streams, and that data is not new xz stream. Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data - UInt64 InSize; // pack size processed + UInt64 InSize; // pack size processed. That value doesn't include the data after + // end of xz stream, if that data was not correct UInt64 OutSize; UInt64 NumStreams; UInt64 NumBlocks; - SRes DecodeRes; - SRes ReadRes; - SRes ProgressRes; - SRes CombinedRes; - SRes CombinedRes_Type; + SRes DecodeRes; // the error code of xz streams data decoding + SRes ReadRes; // error code from ISeqInStream:Read() + SRes ProgressRes; // error code from ICompressProgress:Progress() + SRes CombinedRes; // Combined result error code that shows main rusult + // = S_OK, if there is no error. + // but check also (DataAfterEnd) that can show additional minor errors. + + SRes CombinedRes_Type; // = SZ_ERROR_READ, if error from ISeqInStream + // = SZ_ERROR_PROGRESS, if error from ICompressProgress + // = SZ_ERROR_WRITE, if error from ISeqOutStream + // = SZ_ERROR_* codes for decoding } CXzStatInfo; void XzStatInfo_Clear(CXzStatInfo *p); /* + XzDecMt_Decode() -SRes: - SZ_OK - OK +SRes: it's combined decoding result. It also is equal to stat->CombinedRes. + + SZ_OK - no error + check also output value in (stat->DataAfterEnd) + that can show additional possible error + SZ_ERROR_MEM - Memory allocation error SZ_ERROR_NO_ARCHIVE - is not xz archive SZ_ERROR_ARCHIVE - Headers error SZ_ERROR_DATA - Data Error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties SZ_ERROR_CRC - CRC Error SZ_ERROR_INPUT_EOF - it needs more input data SZ_ERROR_WRITE - ISeqOutStream error @@ -451,8 +507,9 @@ SRes XzDecMt_Decode(CXzDecMtHandle p, // Byte *outBuf, size_t *outBufSize, ISeqInStream *inStream, // const Byte *inData, size_t inDataSize, - CXzStatInfo *stat, - int *isMT, // 0 means that ST (Single-Thread) version was used + CXzStatInfo *stat, // out: decoding results and statistics + int *isMT, // out: 0 means that ST (Single-Thread) version was used + // 1 means that MT (Multi-Thread) version was used ICompressProgress *progress); EXTERN_C_END diff --git a/C/XzCrc64Opt.c b/C/XzCrc64Opt.c index b2852de4..93a9ffff 100644 --- a/C/XzCrc64Opt.c +++ b/C/XzCrc64Opt.c @@ -1,5 +1,5 @@ /* XzCrc64Opt.c -- CRC64 calculation -2017-06-30 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,7 @@ #define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; @@ -16,7 +17,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con v = CRC64_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { - UInt32 d = (UInt32)v ^ *(const UInt32 *)p; + UInt32 d = (UInt32)v ^ *(const UInt32 *)(const void *)p; v = (v >> 32) ^ (table + 0x300)[((d ) & 0xFF)] ^ (table + 0x200)[((d >> 8) & 0xFF)] @@ -45,6 +46,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con #define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; @@ -54,7 +56,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size v = CRC64_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { - UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; + UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)(const void *)p; v = (v << 32) ^ (table + 0x000)[((d ) & 0xFF)] ^ (table + 0x100)[((d >> 8) & 0xFF)] @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2019-02-02 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -240,6 +240,7 @@ static SRes BraState_Code2(void *pp, } +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; @@ -1038,7 +1039,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (p->outBuf ? NULL : dest), &destLen2, destFinish, src, &srcLen2, srcFinished2, finishMode2); - + *status = p->decoder.status; XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); if (!p->outBuf) @@ -1275,9 +1276,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { + const Byte *ptr = p->buf; p->state = XZ_STATE_STREAM_FOOTER; p->pos = 0; - if (CRC_GET_DIGEST(p->crc) != GetUi32(p->buf)) + if (CRC_GET_DIGEST(p->crc) != GetUi32(ptr)) return SZ_ERROR_CRC; } break; @@ -1456,7 +1458,6 @@ typedef struct ISeqInStream *inStream; ISeqOutStream *outStream; ICompressProgress *progress; - // CXzStatInfo *stat; BoolInt finishMode; BoolInt outSize_Defined; @@ -1492,8 +1493,9 @@ typedef struct UInt64 numBlocks; // UInt64 numBadBlocks; - SRes mainErrorCode; - + SRes mainErrorCode; // it's set to error code, if the size Code() output doesn't patch the size from Parsing stage + // it can be = SZ_ERROR_INPUT_EOF + // it can be = SZ_ERROR_DATA, in some another cases BoolInt isBlockHeaderState_Parse; BoolInt isBlockHeaderState_Write; UInt64 outProcessed_Parse; @@ -1877,7 +1879,7 @@ static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) { // if (res == SZ_ERROR_MEM) return res; if (me->props.ignoreErrors && res != SZ_ERROR_MEM) - return S_OK; + return SZ_OK; return res; } } @@ -1898,15 +1900,18 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, *outCodePos = coder->outCodeSize; *stop = True; + if (srcSize > coder->inPreSize - coder->inCodeSize) + return SZ_ERROR_FAIL; + if (coder->inCodeSize < coder->inPreHeaderSize) { - UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; - size_t step = srcSize; - if (step > rem) - step = (size_t)rem; + size_t step = coder->inPreHeaderSize - coder->inCodeSize; + if (step > srcSize) + step = srcSize; src += step; srcSize -= step; coder->inCodeSize += step; + *inCodePos = coder->inCodeSize; if (coder->inCodeSize < coder->inPreHeaderSize) { *stop = False; @@ -1956,7 +1961,7 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, { *inCodePos = coder->inPreSize; *outCodePos = coder->outPreSize; - return S_OK; + return SZ_OK; } return coder->codeRes; } @@ -1966,7 +1971,7 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, BoolInt needWriteToStream, - const Byte *src, size_t srcSize, + const Byte *src, size_t srcSize, BoolInt isCross, // int srcFinished, BoolInt *needContinue, BoolInt *canRecode) @@ -1985,7 +1990,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, if (!coder->dec.headerParsedOk || !coder->outBuf) { if (me->finishedDecoderIndex < 0) - me->finishedDecoderIndex = coderIndex; + me->finishedDecoderIndex = (int)coderIndex; return SZ_OK; } @@ -2077,7 +2082,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, if (coder->codeRes != SZ_OK) if (!me->props.ignoreErrors) { - me->finishedDecoderIndex = coderIndex; + me->finishedDecoderIndex = (int)coderIndex; return res; } @@ -2086,7 +2091,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, if (coder->inPreSize != coder->inCodeSize || coder->blockPackTotal != coder->inCodeSize) { - me->finishedDecoderIndex = coderIndex; + me->finishedDecoderIndex = (int)coderIndex; return SZ_OK; } @@ -2125,22 +2130,41 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, return SZ_OK; } + /* + We have processed all xz-blocks of stream, + And xz unpacker is at XZ_STATE_BLOCK_HEADER state, where + (src) is a pointer to xz-Index structure. + We finish reading of current xz-Stream, including Zero padding after xz-Stream. + We exit, if we reach extra byte (first byte of new-Stream or another data). + But we don't update input stream pointer for that new extra byte. + If extra byte is not correct first byte of xz-signature, + we have SZ_ERROR_NO_ARCHIVE error here. + */ + res = XzUnpacker_Code(dec, NULL, &outSizeCur, src, &srcProcessed, me->mtc.readWasFinished, // srcFinished CODER_FINISH_END, // CODER_FINISH_ANY, &status); + + // res = SZ_ERROR_ARCHIVE; // for failure test me->status = status; me->codeRes = res; + if (isCross) + me->mtc.crossStart += srcProcessed; + me->mtc.inProcessed += srcProcessed; me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + srcSize -= srcProcessed; + src += srcProcessed; + if (res != SZ_OK) { - return S_OK; + return SZ_OK; // return res; } @@ -2149,20 +2173,26 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, *needContinue = True; me->isBlockHeaderState_Parse = False; me->isBlockHeaderState_Write = False; + + if (!isCross) { Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); if (!crossBuf) return SZ_ERROR_MEM; - memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + if (srcSize != 0) + memcpy(crossBuf, src, srcSize); + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize; } - me->mtc.crossStart = 0; - me->mtc.crossEnd = srcSize - srcProcessed; + + PRF_STR_INT("XZ_STATE_STREAM_HEADER crossEnd = ", (unsigned)me->mtc.crossEnd); + return SZ_OK; } - if (status != CODER_STATUS_NEEDS_MORE_INPUT) + if (status != CODER_STATUS_NEEDS_MORE_INPUT || srcSize != 0) { - return E_FAIL; + return SZ_ERROR_FAIL; } if (me->mtc.readWasFinished) @@ -2174,7 +2204,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, { size_t inPos; size_t inLim; - const Byte *inData; + // const Byte *inData; UInt64 inProgressPrev = me->mtc.inProcessed; // XzDecMt_Prepare_InBuf_ST(p); @@ -2184,9 +2214,8 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, inPos = 0; inLim = 0; - // outProcessed = 0; - inData = crossBuf; + // inData = crossBuf; for (;;) { @@ -2201,7 +2230,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, { inPos = 0; inLim = me->mtc.inBufSize; - me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)crossBuf, &inLim); me->mtc.readProcessed += inLim; if (inLim == 0 || me->mtc.readRes != SZ_OK) me->mtc.readWasFinished = True; @@ -2213,7 +2242,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, res = XzUnpacker_Code(dec, NULL, &outProcessed, - inData + inPos, &inProcessed, + crossBuf + inPos, &inProcessed, (inProcessed == 0), // srcFinished CODER_FINISH_END, &status); @@ -2225,7 +2254,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, if (res != SZ_OK) { - return S_OK; + return SZ_OK; // return res; } @@ -2240,7 +2269,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, } if (status != CODER_STATUS_NEEDS_MORE_INPUT) - return E_FAIL; + return SZ_ERROR_FAIL; if (me->mtc.progress) { @@ -2276,13 +2305,6 @@ void XzStatInfo_Clear(CXzStatInfo *p) p->NumStreams_Defined = False; p->NumBlocks_Defined = False; - // p->IsArc = False; - // p->UnexpectedEnd = False; - // p->Unsupported = False; - // p->HeadersError = False; - // p->DataError = False; - // p->CrcError = False; - p->DataAfterEnd = False; p->DecodingTruncated = False; @@ -2296,6 +2318,16 @@ void XzStatInfo_Clear(CXzStatInfo *p) +/* + XzDecMt_Decode_ST() can return SZ_OK or the following errors + - SZ_ERROR_MEM for memory allocation error + - error from XzUnpacker_Code() function + - SZ_ERROR_WRITE for ISeqOutStream::Write(). stat->CombinedRes_Type = SZ_ERROR_WRITE in that case + - ICompressProgress::Progress() error, stat->CombinedRes_Type = SZ_ERROR_PROGRESS. + But XzDecMt_Decode_ST() doesn't return ISeqInStream::Read() errors. + ISeqInStream::Read() result is set to p->readRes. + also it can set stat->CombinedRes_Type to SZ_ERROR_WRITE or SZ_ERROR_PROGRESS. +*/ static SRes XzDecMt_Decode_ST(CXzDecMt *p #ifndef _7ZIP_ST @@ -2384,7 +2416,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p inPos = 0; inLim = p->inBufSize; inData = p->inBuf; - p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readRes = ISeqInStream_Read(p->inStream, (void *)p->inBuf, &inLim); p->readProcessed += inLim; if (inLim == 0 || p->readRes != SZ_OK) p->readWasFinished = True; @@ -2426,8 +2458,8 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p if (finished || outProcessed >= outSize) if (outPos != 0) { - size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); - p->outProcessed += written; + const size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + // p->outProcessed += written; // 21.01: BUG fixed if (written != outPos) { stat->CombinedRes_Type = SZ_ERROR_WRITE; @@ -2438,9 +2470,8 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p if (p->progress && res == SZ_OK) { - UInt64 inDelta = p->inProcessed - inPrev; - UInt64 outDelta = p->outProcessed - outPrev; - if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + if (p->inProcessed - inPrev >= (1 << 22) || + p->outProcessed - outPrev >= (1 << 22)) { res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); if (res != SZ_OK) @@ -2455,14 +2486,31 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p } if (finished) - return res; + { + // p->codeRes is preliminary error from XzUnpacker_Code. + // and it can be corrected later as final result + // so we return SZ_OK here instead of (res); + return SZ_OK; + // return res; + } } } -static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + + +/* +XzStatInfo_SetStat() transforms + CXzUnpacker return code and status to combined CXzStatInfo results. + it can convert SZ_OK to SZ_ERROR_INPUT_EOF + it can convert SZ_ERROR_NO_ARCHIVE to SZ_OK and (DataAfterEnd = 1) +*/ + +static void XzStatInfo_SetStat(const CXzUnpacker *dec, int finishMode, - UInt64 readProcessed, UInt64 inProcessed, - SRes res, ECoderStatus status, + // UInt64 readProcessed, + UInt64 inProcessed, + SRes res, // it's result from CXzUnpacker unpacker + ECoderStatus status, BoolInt decodingTruncated, CXzStatInfo *stat) { @@ -2484,12 +2532,20 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, if (status == CODER_STATUS_NEEDS_MORE_INPUT) { // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + // any extra data is part of correct data extraSize = 0; + // if xz stream was not finished, then we need more data if (!XzUnpacker_IsStreamWasFinished(dec)) res = SZ_ERROR_INPUT_EOF; } - else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) - res = SZ_ERROR_DATA; + else + { + // CODER_STATUS_FINISHED_WITH_MARK is not possible for multi stream xz decoding + // so he we have (status == CODER_STATUS_NOT_FINISHED) + // if (status != CODER_STATUS_FINISHED_WITH_MARK) + if (!decodingTruncated || finishMode) + res = SZ_ERROR_DATA; + } } else if (res == SZ_ERROR_NO_ARCHIVE) { @@ -2497,24 +2553,29 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, SZ_ERROR_NO_ARCHIVE is possible for 2 states: XZ_STATE_STREAM_HEADER - if bad signature or bad CRC XZ_STATE_STREAM_PADDING - if non-zero padding data - extraSize / inProcessed don't include "bad" byte + extraSize and inProcessed don't include "bad" byte */ - if (inProcessed != extraSize) // if good streams before error - if (extraSize != 0 || readProcessed != inProcessed) + // if (inProcessed == extraSize), there was no any good xz stream header, and we keep error + if (inProcessed != extraSize) // if there were good xz streams before error + { + // if (extraSize != 0 || readProcessed != inProcessed) { + // he we suppose that all xz streams were finsihed OK, and we have + // some extra data after all streams stat->DataAfterEnd = True; - // there is some good xz stream before. So we set SZ_OK res = SZ_OK; } + } } - stat->DecodeRes = res; + if (stat->DecodeRes == SZ_OK) + stat->DecodeRes = res; stat->InSize -= extraSize; - return res; } + SRes XzDecMt_Decode(CXzDecMtHandle pp, const CXzDecMtProps *props, const UInt64 *outDataSize, int finishMode, @@ -2557,8 +2618,9 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, p->inProcessed = 0; p->readProcessed = 0; p->readWasFinished = False; + p->readRes = SZ_OK; - p->codeRes = 0; + p->codeRes = SZ_OK; p->status = CODER_STATUS_NOT_SPECIFIED; XzUnpacker_Init(&p->dec); @@ -2589,8 +2651,9 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, if (p->props.numThreads > 1) { - IMtDecCallback vt; - + IMtDecCallback2 vt; + BoolInt needContinue; + SRes res; // we just free ST buffers here // but we still keep state variables, that was set in XzUnpacker_Init() XzDecMt_FreeSt(p); @@ -2628,45 +2691,45 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, vt.Code = XzDecMt_Callback_Code; vt.Write = XzDecMt_Callback_Write; - { - BoolInt needContinue; - - SRes res = MtDec_Code(&p->mtc); - - stat->InSize = p->mtc.inProcessed; - p->inProcessed = p->mtc.inProcessed; - p->readRes = p->mtc.readRes; - p->readWasFinished = p->mtc.readWasFinished; - p->readProcessed = p->mtc.readProcessed; + res = MtDec_Code(&p->mtc); - tMode = True; - needContinue = False; - if (res == SZ_OK) + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) { - if (p->mtc.mtProgress.res != SZ_OK) - { - res = p->mtc.mtProgress.res; - stat->ProgressRes = res; - stat->CombinedRes_Type = SZ_ERROR_PROGRESS; - } - else - needContinue = p->mtc.needContinue; + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; } - - if (!needContinue) + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { { SRes codeRes; BoolInt truncated = False; ECoderStatus status; - CXzUnpacker *dec; + const CXzUnpacker *dec; stat->OutSize = p->outProcessed; if (p->finishedDecoderIndex >= 0) { - CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + const CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; codeRes = coder->codeRes; dec = &coder->dec; status = coder->status; @@ -2679,41 +2742,46 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, truncated = p->parsing_Truncated; } else - return E_FAIL; + return SZ_ERROR_FAIL; + + if (p->mainErrorCode != SZ_OK) + stat->DecodeRes = p->mainErrorCode; XzStatInfo_SetStat(dec, p->finishMode, - p->mtc.readProcessed, p->mtc.inProcessed, + // p->mtc.readProcessed, + p->mtc.inProcessed, codeRes, status, truncated, stat); + } - if (res == SZ_OK) + if (res == SZ_OK) + { + stat->ReadRes = p->mtc.readRes; + + if (p->writeRes != SZ_OK) { - if (p->writeRes != SZ_OK) - { - res = p->writeRes; - stat->CombinedRes_Type = SZ_ERROR_WRITE; - } - else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) - { - res = p->mtc.readRes; - stat->ReadRes = res; - stat->CombinedRes_Type = SZ_ERROR_READ; - } - else if (p->mainErrorCode != SZ_OK) - { - res = p->mainErrorCode; - } + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; } - - stat->CombinedRes = res; - if (stat->CombinedRes_Type == SZ_OK) - stat->CombinedRes_Type = res; - return res; + else if (p->mtc.readRes != SZ_OK + // && p->mtc.inProcessed == p->mtc.readProcessed + && stat->DecodeRes == SZ_ERROR_INPUT_EOF) + { + res = p->mtc.readRes; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (stat->DecodeRes != SZ_OK) + res = stat->DecodeRes; } - - PRF_STR("----- decoding ST -----"); + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; } + + PRF_STR("----- decoding ST -----"); } #endif @@ -2729,33 +2797,35 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, , stat ); + #ifndef _7ZIP_ST + // we must set error code from MT decoding at first + if (p->mainErrorCode != SZ_OK) + stat->DecodeRes = p->mainErrorCode; + #endif + XzStatInfo_SetStat(&p->dec, p->finishMode, - p->readProcessed, p->inProcessed, + // p->readProcessed, + p->inProcessed, p->codeRes, p->status, False, // truncated stat); + stat->ReadRes = p->readRes; + if (res == SZ_OK) { - /* - if (p->writeRes != SZ_OK) - { - res = p->writeRes; - stat->CombinedRes_Type = SZ_ERROR_WRITE; - } - else - */ - if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + if (p->readRes != SZ_OK + // && p->inProcessed == p->readProcessed + && stat->DecodeRes == SZ_ERROR_INPUT_EOF) { + // we set read error as combined error, only if that error was the reason + // of decoding problem res = p->readRes; - stat->ReadRes = res; stat->CombinedRes_Type = SZ_ERROR_READ; } - #ifndef _7ZIP_ST - else if (p->mainErrorCode != SZ_OK) - res = p->mainErrorCode; - #endif + else if (stat->DecodeRes != SZ_OK) + res = stat->DecodeRes; } stat->CombinedRes = res; @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2019-02-02 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -36,7 +36,7 @@ #define XzBlock_ClearFlags(p) (p)->flags = 0; -#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); +#define XzBlock_SetNumFilters(p, n) (p)->flags = (Byte)((p)->flags | ((n) - 1)); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; @@ -552,7 +552,7 @@ static void XzEncProps_Normalize_Fixed(CXzProps *p) numBlocks++; if (numBlocks < (unsigned)t2) { - t2r = (unsigned)numBlocks; + t2r = (int)numBlocks; if (t2r == 0) t2r = 1; t3 = t1 * t2r; @@ -751,7 +751,8 @@ static SRes Xz_CompressBlock( } else if (fp->ipDefined) { - SetUi32(filter->props, fp->ip); + Byte *ptr = filter->props; + SetUi32(ptr, fp->ip); filter->propsSize = 4; } } @@ -1196,7 +1197,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr p->outBufSize = destBlockSize; } - p->mtCoder.numThreadsMax = props->numBlockThreads_Max; + p->mtCoder.numThreadsMax = (unsigned)props->numBlockThreads_Max; p->mtCoder.expectedDataSize = p->expectedDataSize; RINOK(MtCoder_Code(&p->mtCoder)); @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2018-07-04 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -152,7 +152,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff { UInt64 indexSize; Byte buf[XZ_STREAM_FOOTER_SIZE]; - UInt64 pos = *startOffset; + UInt64 pos = (UInt64)*startOffset; if ((pos & 3) != 0 || pos < XZ_STREAM_FOOTER_SIZE) return SZ_ERROR_NO_ARCHIVE; @@ -202,8 +202,13 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff if (!XzFlags_IsSupported(p->flags)) return SZ_ERROR_UNSUPPORTED; - if (GetUi32(buf) != CrcCalc(buf + 4, 6)) - return SZ_ERROR_ARCHIVE; + { + /* to eliminate GCC 6.3 warning: + dereferencing type-punned pointer will break strict-aliasing rules */ + const Byte *buf_ptr = buf; + if (GetUi32(buf_ptr) != CrcCalc(buf + 4, 6)) + return SZ_ERROR_ARCHIVE; + } indexSize = ((UInt64)GetUi32(buf + 4) + 1) << 2; @@ -222,7 +227,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_ARCHIVE; pos -= (totalSize + XZ_STREAM_HEADER_SIZE); RINOK(LookInStream_SeekTo(stream, pos)); - *startOffset = pos; + *startOffset = (Int64)pos; } { CXzStreamFlags headerFlags; @@ -294,12 +299,12 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr SRes res; Xz_Construct(&st); res = Xz_ReadBackward(&st, stream, startOffset, alloc); - st.startOffset = *startOffset; + st.startOffset = (UInt64)*startOffset; RINOK(res); if (p->num == p->numAllocated) { - size_t newNum = p->num + p->num / 4 + 1; - Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); + const size_t newNum = p->num + p->num / 4 + 1; + void *data = ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); if (!data) return SZ_ERROR_MEM; p->numAllocated = newNum; @@ -311,8 +316,8 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr p->streams[p->num++] = st; if (*startOffset == 0) break; - RINOK(LookInStream_SeekTo(stream, *startOffset)); - if (progress && ICompressProgress_Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) + RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset)); + if (progress && ICompressProgress_Progress(progress, (UInt64)(endOffset - *startOffset), (UInt64)(Int64)-1) != SZ_OK) return SZ_ERROR_PROGRESS; } return SZ_OK; diff --git a/C/var_clang.mak b/C/var_clang.mak new file mode 100644 index 00000000..a6df26e7 --- /dev/null +++ b/C/var_clang.mak @@ -0,0 +1,11 @@ +PLATFORM= +O=b/c +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/C/var_clang_arm64.mak b/C/var_clang_arm64.mak new file mode 100644 index 00000000..4b354096 --- /dev/null +++ b/C/var_clang_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/C/var_clang_x64.mak b/C/var_clang_x64.mak new file mode 100644 index 00000000..fefed51c --- /dev/null +++ b/C/var_clang_x64.mak @@ -0,0 +1,12 @@ +PLATFORM=x64 +O=b/c_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/C/var_clang_x86.mak b/C/var_clang_x86.mak new file mode 100644 index 00000000..5f3c2d9c --- /dev/null +++ b/C/var_clang_x86.mak @@ -0,0 +1,12 @@ +PLATFORM=x86 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/C/var_gcc.mak b/C/var_gcc.mak new file mode 100644 index 00000000..664491cf --- /dev/null +++ b/C/var_gcc.mak @@ -0,0 +1,12 @@ +PLATFORM= +O=b/g +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/C/var_gcc_arm64.mak b/C/var_gcc_arm64.mak new file mode 100644 index 00000000..4bbb687d --- /dev/null +++ b/C/var_gcc_arm64.mak @@ -0,0 +1,12 @@ +PLATFORM=arm64 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-mtune=cortex-a53 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/C/var_gcc_x64.mak b/C/var_gcc_x64.mak new file mode 100644 index 00000000..1acf604f --- /dev/null +++ b/C/var_gcc_x64.mak @@ -0,0 +1,10 @@ +PLATFORM=x64 +O=b/g_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ diff --git a/C/var_gcc_x86.mak b/C/var_gcc_x86.mak new file mode 100644 index 00000000..288bf94b --- /dev/null +++ b/C/var_gcc_x86.mak @@ -0,0 +1,11 @@ +PLATFORM=x86 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + diff --git a/C/var_mac_arm64.mak b/C/var_mac_arm64.mak new file mode 100644 index 00000000..adf5fa1d --- /dev/null +++ b/C/var_mac_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/m_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-arch arm64 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/C/var_mac_x64.mak b/C/var_mac_x64.mak new file mode 100644 index 00000000..13d7aa7f --- /dev/null +++ b/C/var_mac_x64.mak @@ -0,0 +1,11 @@ +PLATFORM=x64 +O=b/m_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-arch x86_64 +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/C/warn_clang.mak b/C/warn_clang.mak new file mode 100644 index 00000000..ed4f908f --- /dev/null +++ b/C/warn_clang.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_1 = \ + -Wno-deprecated-copy-dtor \ + + + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_1) diff --git a/C/warn_clang_mac.mak b/C/warn_clang_mac.mak new file mode 100644 index 00000000..41044a2c --- /dev/null +++ b/C/warn_clang_mac.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_MAC = \ + -Wno-poison-system-directories \ + -Wno-c++11-long-long \ + -Wno-atomic-implicit-seq-cst \ + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_MAC) diff --git a/C/warn_gcc.mak b/C/warn_gcc.mak new file mode 100644 index 00000000..5fb747dc --- /dev/null +++ b/C/warn_gcc.mak @@ -0,0 +1,53 @@ +CFLAGS_WARN_GCC_4_5 = \ + +CFLAGS_WARN_GCC_6 = \ + -Waddress \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wcast-align \ + -Wcomment \ + -Wdiv-by-zero \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Winit-self \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + +# -Wno-strict-aliasing + +CFLAGS_WARN_GCC_9 = \ + -Waddress \ + -Waddress-of-packed-member \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wbool-operation \ + -Wcast-align \ + -Wcast-align=strict \ + -Wcomment \ + -Wdangling-else \ + -Wdiv-by-zero \ + -Wduplicated-branches \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Wimplicit-fallthrough=5 \ + -Winit-self \ + -Wint-in-bool-context \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + -Wconversion \ + +# -Wno-sign-conversion \ + +CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ + -Wno-strict-aliasing \ + + +CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) \ + +# $(CFLAGS_WARN_GCC_PPMD_UNALIGNED) + +
\ No newline at end of file diff --git a/CPP/7zip/7zip_gcc.mak b/CPP/7zip/7zip_gcc.mak new file mode 100644 index 00000000..122686ff --- /dev/null +++ b/CPP/7zip/7zip_gcc.mak @@ -0,0 +1,1183 @@ +# USE_CLANG=1 +# USE_ASM = 1 +# IS_X64 = 1 +# MY_ARCH = + + +MY_ARCH_2 = $(MY_ARCH) + +MY_ASM = jwasm +MY_ASM = asmc + +PROGPATH = $(O)/$(PROG) + + +ifneq ($(CC), xlc) +CFLAGS_WARN_WALL = -Wall -Werror -Wextra +endif + +# for object file +CFLAGS_BASE_LIST = -c +# CFLAGS_BASE_LIST = -S +CFLAGS_BASE = -O2 $(CFLAGS_BASE_LIST) $(CFLAGS_WARN_WALL) $(CFLAGS_WARN) \ + -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ + -fPIC + + +ifdef SystemDrive +IS_MINGW = 1 +endif + +ifdef IS_MINGW +LDFLAGS_STATIC_2 = -static +else +ifndef DEF_FILE +ifndef IS_NOT_STANDALONE +ifndef MY_DYNAMIC_LINK +ifneq ($(CC), clang) +LDFLAGS_STATIC_2 = +# -static +# -static-libstdc++ -static-libgcc +endif +endif +endif +endif +endif + +LDFLAGS_STATIC = -DNDEBUG $(LDFLAGS_STATIC_2) + +ifndef O + ifdef IS_MINGW + O=_o + else + O=_o + endif +endif + + +ifdef DEF_FILE + + +ifdef IS_MINGW +SHARED_EXT=.dll +LDFLAGS = -shared -DEF $(DEF_FILE) $(LDFLAGS_STATIC) +else +SHARED_EXT=.so +LDFLAGS = -shared -fPIC $(LDFLAGS_STATIC) +CC_SHARED=-fPIC +endif + + +else + +LDFLAGS = $(LDFLAGS_STATIC) +# -s is not required for clang, do we need it for GGC ??? +# -s + +#-static -static-libgcc -static-libstdc++ + +ifdef IS_MINGW +SHARED_EXT=.exe +else +SHARED_EXT= +endif + +endif + + +PROGPATH = $(O)/$(PROG)$(SHARED_EXT) + + +ifdef IS_MINGW + +RM = del +MY_MKDIR=mkdir +LIB2 = -loleaut32 -luuid -ladvapi32 -lUser32 + +CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE +# -Wno-delete-non-virtual-dtor + +DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll + +else + +RM = rm -f +MY_MKDIR=mkdir -p +# CFLAGS_BASE := $(CFLAGS_BASE) -D_7ZIP_ST +# CXXFLAGS_EXTRA = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + +# LOCAL_LIBS=-lpthread +# LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl +LIB2 = -lpthread -ldl + + + +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(OBJS) + +endif + + + +CFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CFLAGS_BASE2) $(CFLAGS_BASE) $(CC_SHARED) -o $@ + + +ifdef IS_MINGW +AFLAGS_ABI = -coff -DABI_CDECL +AFLAGS = $(AFLAGS_ABI) -Fo$(O)/$(basename $(<F)).o +else +ifdef IS_X64 +AFLAGS_ABI = -elf64 -DABI_LINUX +else +AFLAGS_ABI = -elf -DABI_LINUX -DABI_CDECL +# -DABI_CDECL +# -DABI_LINUX +# -DABI_CDECL +endif +AFLAGS = $(AFLAGS_ABI) -Fo$(O)/ +endif + +ifdef USE_ASM +CONSOLE_ASM_FLAGS=-D_7ZIP_ASM +else +CONSOLE_ASM_FLAGS= +endif + +CXX_WARN_FLAGS = +#-Wno-invalid-offsetof +#-Wno-reorder + +CXXFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(CXXFLAGS_EXTRA) $(CC_SHARED) -o $@ $(CXX_WARN_FLAGS) + +all: $(O) $(PROGPATH) + +$(O): + $(MY_MKDIR) $(O) + +$(PROGPATH): $(OBJS) + $(CXX) -o $(PROGPATH) -s $(MY_ARCH_2) $(LDFLAGS) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) + +# -s strips debug sections from executable in GCC + + + + +ifndef NO_DEFAULT_RES +$O/resource.o: resource.rc + windres.exe $(RFLAGS) resource.rc $O/resource.o +endif + +$O/LzmaAlone.o: LzmaAlone.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/CommandLineParser.o: ../../../Common/CommandLineParser.cpp + $(CXX) $(CXXFLAGS) $< +$O/CRC.o: ../../../Common/CRC.cpp + $(CXX) $(CXXFLAGS) $< + +$O/CrcReg.o: ../../../Common/CrcReg.cpp + $(CXX) $(CXXFLAGS) $< + +$O/DynLimBuf.o: ../../../Common/DynLimBuf.cpp + $(CXX) $(CXXFLAGS) $< +$O/IntToString.o: ../../../Common/IntToString.cpp + $(CXX) $(CXXFLAGS) $< +$O/Lang.o: ../../../Common/Lang.cpp + $(CXX) $(CXXFLAGS) $< +$O/ListFileUtils.o: ../../../Common/ListFileUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyMap.o: ../../../Common/MyMap.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyString.o: ../../../Common/MyString.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyVector.o: ../../../Common/MyVector.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyWindows.o: ../../../Common/MyWindows.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyWindows2.o: ../../../Common/MyWindows2.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyXml.o: ../../../Common/MyXml.cpp + $(CXX) $(CXXFLAGS) $< +$O/NewHandler.o: ../../../Common/NewHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/Random.o: ../../../Common/Random.cpp + $(CXX) $(CXXFLAGS) $< +$O/Sha1Prepare.o: ../../../Common/Sha1Prepare.cpp + $(CXX) $(CXXFLAGS) $< +$O/Sha1Reg.o: ../../../Common/Sha1Reg.cpp + $(CXX) $(CXXFLAGS) $< +$O/Sha256Prepare.o: ../../../Common/Sha256Prepare.cpp + $(CXX) $(CXXFLAGS) $< +$O/Sha256Reg.o: ../../../Common/Sha256Reg.cpp + $(CXX) $(CXXFLAGS) $< +$O/StdInStream.o: ../../../Common/StdInStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/StdOutStream.o: ../../../Common/StdOutStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/StringConvert.o: ../../../Common/StringConvert.cpp + $(CXX) $(CXXFLAGS) $< +$O/StringToInt.o: ../../../Common/StringToInt.cpp + $(CXX) $(CXXFLAGS) $< +$O/TextConfig.o: ../../../Common/TextConfig.cpp + $(CXX) $(CXXFLAGS) $< +$O/UTFConvert.o: ../../../Common/UTFConvert.cpp + $(CXX) $(CXXFLAGS) $< +$O/Wildcard.o: ../../../Common/Wildcard.cpp + $(CXX) $(CXXFLAGS) $< +$O/XzCrc64Init.o: ../../../Common/XzCrc64Init.cpp + $(CXX) $(CXXFLAGS) $< +$O/XzCrc64Reg.o: ../../../Common/XzCrc64Reg.cpp + $(CXX) $(CXXFLAGS) $< + + + +$O/Clipboard.o: ../../../Windows/Clipboard.cpp + $(CXX) $(CXXFLAGS) $< +$O/COM.o: ../../../Windows/COM.cpp + $(CXX) $(CXXFLAGS) $< +$O/CommonDialog.o: ../../../Windows/CommonDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/Console.o: ../../../Windows/Console.cpp + $(CXX) $(CXXFLAGS) $< +$O/DLL.o: ../../../Windows/DLL.cpp + $(CXX) $(CXXFLAGS) $< +$O/ErrorMsg.o: ../../../Windows/ErrorMsg.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileDir.o: ../../../Windows/FileDir.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileFind.o: ../../../Windows/FileFind.cpp + $(CXX) $(CXXFLAGS) $< + +$O/FileIO.o: ../../../Windows/FileIO.cpp + $(CXX) $(CXXFLAGS) $< + +$O/FileLink.o: ../../../Windows/FileLink.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileMapping.o: ../../../Windows/FileMapping.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileName.o: ../../../Windows/FileName.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileSystem.o: ../../../Windows/FileSystem.cpp + $(CXX) $(CXXFLAGS) $< +$O/MemoryGlobal.o: ../../../Windows/MemoryGlobal.cpp + $(CXX) $(CXXFLAGS) $< +$O/MemoryLock.o: ../../../Windows/MemoryLock.cpp + $(CXX) $(CXXFLAGS) $< +$O/Menu.o: ../../../Windows/Menu.cpp + $(CXX) $(CXXFLAGS) $< +$O/NationalTime.o: ../../../Windows/NationalTime.cpp + $(CXX) $(CXXFLAGS) $< +$O/Net.o: ../../../Windows/Net.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProcessMessages.o: ../../../Windows/ProcessMessages.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProcessUtils.o: ../../../Windows/ProcessUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropVariant.o: ../../../Windows/PropVariant.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropVariantConv.o: ../../../Windows/PropVariantConv.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropVariantUtils.o: ../../../Windows/PropVariantUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/Registry.o: ../../../Windows/Registry.cpp + $(CXX) $(CXXFLAGS) $< +$O/ResourceString.o: ../../../Windows/ResourceString.cpp + $(CXX) $(CXXFLAGS) $< +$O/SecurityUtils.o: ../../../Windows/SecurityUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/Shell.o: ../../../Windows/Shell.cpp + $(CXX) $(CXXFLAGS) $< +$O/Synchronization.o: ../../../Windows/Synchronization.cpp + $(CXX) $(CXXFLAGS) $< +$O/System.o: ../../../Windows/System.cpp + $(CXX) $(CXXFLAGS) $< +$O/SystemInfo.o: ../../../Windows/SystemInfo.cpp + $(CXX) $(CXXFLAGS) $< +$O/TimeUtils.o: ../../../Windows/TimeUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/Window.o: ../../../Windows/Window.cpp + $(CXX) $(CXXFLAGS) $< + + + +$O/ComboBox.o: ../../../Windows/Control/ComboBox.cpp + $(CXX) $(CXXFLAGS) $< +$O/Dialog.o: ../../../Windows/Control/Dialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ImageList.o: ../../../Windows/Control/ImageList.cpp + $(CXX) $(CXXFLAGS) $< +$O/ListView.o: ../../../Windows/Control/ListView.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropertyPage.o: ../../../Windows/Control/PropertyPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/Window2.o: ../../../Windows/Control/Window2.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/CreateCoder.o: ../../Common/CreateCoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/CWrappers.o: ../../Common/CWrappers.cpp + $(CXX) $(CXXFLAGS) $< +$O/FilePathAutoRename.o: ../../Common/FilePathAutoRename.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileStreams.o: ../../Common/FileStreams.cpp + $(CXX) $(CXXFLAGS) $< +$O/FilterCoder.o: ../../Common/FilterCoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/InBuffer.o: ../../Common/InBuffer.cpp + $(CXX) $(CXXFLAGS) $< +$O/InOutTempBuffer.o: ../../Common/InOutTempBuffer.cpp + $(CXX) $(CXXFLAGS) $< +$O/LimitedStreams.o: ../../Common/LimitedStreams.cpp + $(CXX) $(CXXFLAGS) $< +$O/LockedStream.o: ../../Common/LockedStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/MemBlocks.o: ../../Common/MemBlocks.cpp + $(CXX) $(CXXFLAGS) $< +$O/MethodId.o: ../../Common/MethodId.cpp + $(CXX) $(CXXFLAGS) $< +$O/MethodProps.o: ../../Common/MethodProps.cpp + $(CXX) $(CXXFLAGS) $< +$O/OffsetStream.o: ../../Common/OffsetStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/OutBuffer.o: ../../Common/OutBuffer.cpp + $(CXX) $(CXXFLAGS) $< +$O/OutMemStream.o: ../../Common/OutMemStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProgressMt.o: ../../Common/ProgressMt.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProgressUtils.o: ../../Common/ProgressUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropId.o: ../../Common/PropId.cpp + $(CXX) $(CXXFLAGS) $< +$O/StreamBinder.o: ../../Common/StreamBinder.cpp + $(CXX) $(CXXFLAGS) $< +$O/StreamObjects.o: ../../Common/StreamObjects.cpp + $(CXX) $(CXXFLAGS) $< +$O/StreamUtils.o: ../../Common/StreamUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/UniqBlocks.o: ../../Common/UniqBlocks.cpp + $(CXX) $(CXXFLAGS) $< +$O/VirtThread.o: ../../Common/VirtThread.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/ApmHandler.o: ../../Archive/ApmHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveExports.o: ../../Archive/ArchiveExports.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArHandler.o: ../../Archive/ArHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArjHandler.o: ../../Archive/ArjHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/Base64Handler.o: ../../Archive/Base64Handler.cpp + $(CXX) $(CXXFLAGS) $< +$O/Bz2Handler.o: ../../Archive/Bz2Handler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ComHandler.o: ../../Archive/ComHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/CpioHandler.o: ../../Archive/CpioHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/CramfsHandler.o: ../../Archive/CramfsHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/DeflateProps.o: ../../Archive/DeflateProps.cpp + $(CXX) $(CXXFLAGS) $< +$O/DllExports.o: ../../Archive/DllExports.cpp + $(CXX) $(CXXFLAGS) $< +$O/DllExports2.o: ../../Archive/DllExports2.cpp + $(CXX) $(CXXFLAGS) $< +$O/DmgHandler.o: ../../Archive/DmgHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ElfHandler.o: ../../Archive/ElfHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtHandler.o: ../../Archive/ExtHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/FatHandler.o: ../../Archive/FatHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/FlvHandler.o: ../../Archive/FlvHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/GptHandler.o: ../../Archive/GptHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/GzHandler.o: ../../Archive/GzHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/HandlerCont.o: ../../Archive/HandlerCont.cpp + $(CXX) $(CXXFLAGS) $< +$O/HfsHandler.o: ../../Archive/HfsHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/IhexHandler.o: ../../Archive/IhexHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzhHandler.o: ../../Archive/LzhHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzmaHandler.o: ../../Archive/LzmaHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/MachoHandler.o: ../../Archive/MachoHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/MbrHandler.o: ../../Archive/MbrHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/MslzHandler.o: ../../Archive/MslzHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/MubHandler.o: ../../Archive/MubHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/NtfsHandler.o: ../../Archive/NtfsHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/PeHandler.o: ../../Archive/PeHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/PpmdHandler.o: ../../Archive/PpmdHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/QcowHandler.o: ../../Archive/QcowHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/RpmHandler.o: ../../Archive/RpmHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/SplitHandler.o: ../../Archive/SplitHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/SquashfsHandler.o: ../../Archive/SquashfsHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/SwfHandler.o: ../../Archive/SwfHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/UefiHandler.o: ../../Archive/UefiHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/VdiHandler.o: ../../Archive/VdiHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/VhdHandler.o: ../../Archive/VhdHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/VmdkHandler.o: ../../Archive/VmdkHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/XarHandler.o: ../../Archive/XarHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/XzHandler.o: ../../Archive/XzHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZHandler.o: ../../Archive/ZHandler.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/7zCompressionMode.o: ../../Archive/7z/7zCompressionMode.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zDecode.o: ../../Archive/7z/7zDecode.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zEncode.o: ../../Archive/7z/7zEncode.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zExtract.o: ../../Archive/7z/7zExtract.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zFolderInStream.o: ../../Archive/7z/7zFolderInStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zHandler.o: ../../Archive/7z/7zHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zHandlerOut.o: ../../Archive/7z/7zHandlerOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zHeader.o: ../../Archive/7z/7zHeader.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zIn.o: ../../Archive/7z/7zIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zOut.o: ../../Archive/7z/7zOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zProperties.o: ../../Archive/7z/7zProperties.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zRegister.o: ../../Archive/7z/7zRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zSpecStream.o: ../../Archive/7z/7zSpecStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zUpdate.o: ../../Archive/7z/7zUpdate.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/CabBlockInStream.o: ../../Archive/Cab/CabBlockInStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/CabHandler.o: ../../Archive/Cab/CabHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/CabHeader.o: ../../Archive/Cab/CabHeader.cpp + $(CXX) $(CXXFLAGS) $< +$O/CabIn.o: ../../Archive/Cab/CabIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/CabRegister.o: ../../Archive/Cab/CabRegister.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/ChmHandler.o: ../../Archive/Chm/ChmHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ChmIn.o: ../../Archive/Chm/ChmIn.cpp + $(CXX) $(CXXFLAGS) $< + +$O/IsoHandler.o: ../../Archive/Iso/IsoHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/IsoHeader.o: ../../Archive/Iso/IsoHeader.cpp + $(CXX) $(CXXFLAGS) $< +$O/IsoIn.o: ../../Archive/Iso/IsoIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/IsoRegister.o: ../../Archive/Iso/IsoRegister.cpp + $(CXX) $(CXXFLAGS) $< + +$O/NsisDecode.o: ../../Archive/Nsis/NsisDecode.cpp + $(CXX) $(CXXFLAGS) $< +$O/NsisHandler.o: ../../Archive/Nsis/NsisHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/NsisIn.o: ../../Archive/Nsis/NsisIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/NsisRegister.o: ../../Archive/Nsis/NsisRegister.cpp + $(CXX) $(CXXFLAGS) $< + +$O/Rar5Handler.o: ../../Archive/Rar/Rar5Handler.cpp + $(CXX) $(CXXFLAGS) $< +$O/RarHandler.o: ../../Archive/Rar/RarHandler.cpp + $(CXX) $(CXXFLAGS) $< + +$O/TarHandler.o: ../../Archive/Tar/TarHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarHandlerOut.o: ../../Archive/Tar/TarHandlerOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarHeader.o: ../../Archive/Tar/TarHeader.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarIn.o: ../../Archive/Tar/TarIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarOut.o: ../../Archive/Tar/TarOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarRegister.o: ../../Archive/Tar/TarRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/TarUpdate.o: ../../Archive/Tar/TarUpdate.cpp + $(CXX) $(CXXFLAGS) $< + +$O/UdfHandler.o: ../../Archive/Udf/UdfHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/UdfIn.o: ../../Archive/Udf/UdfIn.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/WimHandler.o: ../../Archive/Wim/WimHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/WimHandlerOut.o: ../../Archive/Wim/WimHandlerOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/WimIn.o: ../../Archive/Wim/WimIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/WimRegister.o: ../../Archive/Wim/WimRegister.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/ZipAddCommon.o: ../../Archive/Zip/ZipAddCommon.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipHandler.o: ../../Archive/Zip/ZipHandler.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipHandlerOut.o: ../../Archive/Zip/ZipHandlerOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipIn.o: ../../Archive/Zip/ZipIn.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipItem.o: ../../Archive/Zip/ZipItem.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipOut.o: ../../Archive/Zip/ZipOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipUpdate.o: ../../Archive/Zip/ZipUpdate.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipRegister.o: ../../Archive/Zip/ZipRegister.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/Bcj2Coder.o: ../../Compress/Bcj2Coder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Bcj2Register.o: ../../Compress/Bcj2Register.cpp + $(CXX) $(CXXFLAGS) $< +$O/BcjCoder.o: ../../Compress/BcjCoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/BcjRegister.o: ../../Compress/BcjRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/BitlDecoder.o: ../../Compress/BitlDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/BranchMisc.o: ../../Compress/BranchMisc.cpp + $(CXX) $(CXXFLAGS) $< +$O/BranchRegister.o: ../../Compress/BranchRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/ByteSwap.o: ../../Compress/ByteSwap.cpp + $(CXX) $(CXXFLAGS) $< +$O/BZip2Crc.o: ../../Compress/BZip2Crc.cpp + $(CXX) $(CXXFLAGS) $< +$O/BZip2Decoder.o: ../../Compress/BZip2Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/BZip2Encoder.o: ../../Compress/BZip2Encoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/BZip2Register.o: ../../Compress/BZip2Register.cpp + $(CXX) $(CXXFLAGS) $< +$O/CodecExports.o: ../../Compress/CodecExports.cpp + $(CXX) $(CXXFLAGS) $< +$O/CopyCoder.o: ../../Compress/CopyCoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/CopyRegister.o: ../../Compress/CopyRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/Deflate64Register.o: ../../Compress/Deflate64Register.cpp + $(CXX) $(CXXFLAGS) $< +$O/DeflateDecoder.o: ../../Compress/DeflateDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/DeflateEncoder.o: ../../Compress/DeflateEncoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/DeflateRegister.o: ../../Compress/DeflateRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/DeltaFilter.o: ../../Compress/DeltaFilter.cpp + $(CXX) $(CXXFLAGS) $< +$O/DllExports2Compress.o: ../../Compress/DllExports2Compress.cpp + $(CXX) $(CXXFLAGS) $< +$O/DllExportsCompress.o: ../../Compress/DllExportsCompress.cpp + $(CXX) $(CXXFLAGS) $< +$O/ImplodeDecoder.o: ../../Compress/ImplodeDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ImplodeHuffmanDecoder.o: ../../Compress/ImplodeHuffmanDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzfseDecoder.o: ../../Compress/LzfseDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzhDecoder.o: ../../Compress/LzhDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Lzma2Decoder.o: ../../Compress/Lzma2Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Lzma2Encoder.o: ../../Compress/Lzma2Encoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Lzma2Register.o: ../../Compress/Lzma2Register.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzmaDecoder.o: ../../Compress/LzmaDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzmaEncoder.o: ../../Compress/LzmaEncoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzmaRegister.o: ../../Compress/LzmaRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzmsDecoder.o: ../../Compress/LzmsDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzOutWindow.o: ../../Compress/LzOutWindow.cpp + $(CXX) $(CXXFLAGS) $< +$O/LzxDecoder.o: ../../Compress/LzxDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/PpmdDecoder.o: ../../Compress/PpmdDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/PpmdEncoder.o: ../../Compress/PpmdEncoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/PpmdRegister.o: ../../Compress/PpmdRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/PpmdZip.o: ../../Compress/PpmdZip.cpp + $(CXX) $(CXXFLAGS) $< +$O/QuantumDecoder.o: ../../Compress/QuantumDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar1Decoder.o: ../../Compress/Rar1Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar2Decoder.o: ../../Compress/Rar2Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar3Decoder.o: ../../Compress/Rar3Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar3Vm.o: ../../Compress/Rar3Vm.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar5Decoder.o: ../../Compress/Rar5Decoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/RarCodecsRegister.o: ../../Compress/RarCodecsRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/ShrinkDecoder.o: ../../Compress/ShrinkDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/XpressDecoder.o: ../../Compress/XpressDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/XzDecoder.o: ../../Compress/XzDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/XzEncoder.o: ../../Compress/XzEncoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZDecoder.o: ../../Compress/ZDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZlibDecoder.o: ../../Compress/ZlibDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZlibEncoder.o: ../../Compress/ZlibEncoder.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/7zAes.o: ../../Crypto/7zAes.cpp + $(CXX) $(CXXFLAGS) $< +$O/7zAesRegister.o: ../../Crypto/7zAesRegister.cpp + $(CXX) $(CXXFLAGS) $< +$O/HmacSha1.o: ../../Crypto/HmacSha1.cpp + $(CXX) $(CXXFLAGS) $< +$O/HmacSha256.o: ../../Crypto/HmacSha256.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyAes.o: ../../Crypto/MyAes.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyAesReg.o: ../../Crypto/MyAesReg.cpp + $(CXX) $(CXXFLAGS) $< +$O/Pbkdf2HmacSha1.o: ../../Crypto/Pbkdf2HmacSha1.cpp + $(CXX) $(CXXFLAGS) $< +$O/RandGen.o: ../../Crypto/RandGen.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar20Crypto.o: ../../Crypto/Rar20Crypto.cpp + $(CXX) $(CXXFLAGS) $< +$O/Rar5Aes.o: ../../Crypto/Rar5Aes.cpp + $(CXX) $(CXXFLAGS) $< +$O/RarAes.o: ../../Crypto/RarAes.cpp + $(CXX) $(CXXFLAGS) $< +$O/WzAes.o: ../../Crypto/WzAes.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipCrypto.o: ../../Crypto/ZipCrypto.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipStrong.o: ../../Crypto/ZipStrong.cpp + $(CXX) $(CXXFLAGS) $< + + + +$O/CoderMixer2.o: ../../Archive/Common/CoderMixer2.cpp + $(CXX) $(CXXFLAGS) $< +$O/DummyOutStream.o: ../../Archive/Common/DummyOutStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/FindSignature.o: ../../Archive/Common/FindSignature.cpp + $(CXX) $(CXXFLAGS) $< +$O/HandlerOut.o: ../../Archive/Common/HandlerOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/InStreamWithCRC.o: ../../Archive/Common/InStreamWithCRC.cpp + $(CXX) $(CXXFLAGS) $< +$O/ItemNameUtils.o: ../../Archive/Common/ItemNameUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/MultiStream.o: ../../Archive/Common/MultiStream.cpp + $(CXX) $(CXXFLAGS) $< +$O/OutStreamWithCRC.o: ../../Archive/Common/OutStreamWithCRC.cpp + $(CXX) $(CXXFLAGS) $< +$O/OutStreamWithSha1.o: ../../Archive/Common/OutStreamWithSha1.cpp + $(CXX) $(CXXFLAGS) $< +$O/ParseProperties.o: ../../Archive/Common/ParseProperties.cpp + $(CXX) $(CXXFLAGS) $< + + + + +$O/ArchiveCommandLine.o: ../../UI/Common/ArchiveCommandLine.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveExtractCallback.o: ../../UI/Common/ArchiveExtractCallback.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveName.o: ../../UI/Common/ArchiveName.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveOpenCallback.o: ../../UI/Common/ArchiveOpenCallback.cpp + $(CXX) $(CXXFLAGS) $< +$O/Bench.o: ../../UI/Common/Bench.cpp + $(CXX) $(CXXFLAGS) $< +$O/CompressCall.o: ../../UI/Common/CompressCall.cpp + $(CXX) $(CXXFLAGS) $< +$O/CompressCall2.o: ../../UI/Common/CompressCall2.cpp + $(CXX) $(CXXFLAGS) $< +$O/DefaultName.o: ../../UI/Common/DefaultName.cpp + $(CXX) $(CXXFLAGS) $< +$O/EnumDirItems.o: ../../UI/Common/EnumDirItems.cpp + $(CXX) $(CXXFLAGS) $< +$O/Extract.o: ../../UI/Common/Extract.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtractingFilePath.o: ../../UI/Common/ExtractingFilePath.cpp + $(CXX) $(CXXFLAGS) $< +$O/HashCalc.o: ../../UI/Common/HashCalc.cpp + $(CXX) $(CXXFLAGS) $< +$O/LoadCodecs.o: ../../UI/Common/LoadCodecs.cpp + $(CXX) $(CXXFLAGS) $< +$O/OpenArchive.o: ../../UI/Common/OpenArchive.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropIDUtils.o: ../../UI/Common/PropIDUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/SetProperties.o: ../../UI/Common/SetProperties.cpp + $(CXX) $(CXXFLAGS) $< +$O/SortUtils.o: ../../UI/Common/SortUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/TempFiles.o: ../../UI/Common/TempFiles.cpp + $(CXX) $(CXXFLAGS) $< +$O/Update.o: ../../UI/Common/Update.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateAction.o: ../../UI/Common/UpdateAction.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallback.o: ../../UI/Common/UpdateCallback.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdatePair.o: ../../UI/Common/UpdatePair.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateProduce.o: ../../UI/Common/UpdateProduce.cpp + $(CXX) $(CXXFLAGS) $< +$O/WorkDir.o: ../../UI/Common/WorkDir.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZipRegistry.o: ../../UI/Common/ZipRegistry.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/Agent.o: ../../UI/Agent/Agent.cpp + $(CXX) $(CXXFLAGS) $< +$O/AgentOut.o: ../../UI/Agent/AgentOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/AgentProxy.o: ../../UI/Agent/AgentProxy.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveFolder.o: ../../UI/Agent/ArchiveFolder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveFolderOpen.o: ../../UI/Agent/ArchiveFolderOpen.cpp + $(CXX) $(CXXFLAGS) $< +$O/ArchiveFolderOut.o: ../../UI/Agent/ArchiveFolderOut.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallbackAgent.o: ../../UI/Agent/UpdateCallbackAgent.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/Client7z.o: ../../UI/Client7z/Client7z.cpp + $(CXX) $(CXXFLAGS) $< + + +$O/BenchCon.o: ../../UI/Console/BenchCon.cpp + $(CXX) $(CXXFLAGS) $< +$O/ConsoleClose.o: ../../UI/Console/ConsoleClose.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtractCallbackConsole.o: ../../UI/Console/ExtractCallbackConsole.cpp + $(CXX) $(CXXFLAGS) $< +$O/HashCon.o: ../../UI/Console/HashCon.cpp + $(CXX) $(CXXFLAGS) $< +$O/List.o: ../../UI/Console/List.cpp + $(CXX) $(CXXFLAGS) $< +$O/Main.o: ../../UI/Console/Main.cpp ../../../../C/7zVersion.h + $(CXX) $(CXXFLAGS) $(CONSOLE_VARIANT_FLAGS) $(CONSOLE_ASM_FLAGS) $< +$O/MainAr.o: ../../UI/Console/MainAr.cpp + $(CXX) $(CXXFLAGS) $< +$O/OpenCallbackConsole.o: ../../UI/Console/OpenCallbackConsole.cpp + $(CXX) $(CXXFLAGS) $< +$O/PercentPrinter.o: ../../UI/Console/PercentPrinter.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallbackConsole.o: ../../UI/Console/UpdateCallbackConsole.cpp + $(CXX) $(CXXFLAGS) $< +$O/UserInputUtils.o: ../../UI/Console/UserInputUtils.cpp + $(CXX) $(CXXFLAGS) $< + +$O/BenchmarkDialog.o: ../../UI/GUI/BenchmarkDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/CompressDialog.o: ../../UI/GUI/CompressDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtractDialog.o: ../../UI/GUI/ExtractDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtractGUI.o: ../../UI/GUI/ExtractGUI.cpp + $(CXX) $(CXXFLAGS) $< +$O/GUI.o: ../../UI/GUI/GUI.cpp + $(CXX) $(CXXFLAGS) $< +$O/HashGUI.o: ../../UI/GUI/HashGUI.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallbackGUI.o: ../../UI/GUI/UpdateCallbackGUI.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallbackGUI2.o: ../../UI/GUI/UpdateCallbackGUI2.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateGUI.o: ../../UI/GUI/UpdateGUI.cpp + $(CXX) $(CXXFLAGS) $< + +$O/MyMessages.o: ../../UI/Explorer/MyMessages.cpp + $(CXX) $(CXXFLAGS) $< +$O/ContextMenu.o: ../../UI/Explorer/ContextMenu.cpp + $(CXX) $(CXXFLAGS) $< +$O/DllExportsExplorer.o: ../../UI/Explorer/DllExportsExplorer.cpp + $(CXX) $(CXXFLAGS) $< +$O/RegistryContextMenu.o: ../../UI/Explorer/RegistryContextMenu.cpp + $(CXX) $(CXXFLAGS) $< + + + +$O/AboutDialog.o: ../../UI/FileManager/AboutDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/AltStreamsFolder.o: ../../UI/FileManager/AltStreamsFolder.cpp + $(CXX) $(CXXFLAGS) $< +$O/App.o: ../../UI/FileManager/App.cpp + $(CXX) $(CXXFLAGS) $< +$O/BrowseDialog.o: ../../UI/FileManager/BrowseDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ClassDefs.o: ../../UI/FileManager/ClassDefs.cpp + $(CXX) $(CXXFLAGS) $< +$O/ComboDialog.o: ../../UI/FileManager/ComboDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/CopyDialog.o: ../../UI/FileManager/CopyDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/EditDialog.o: ../../UI/FileManager/EditDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/EditPage.o: ../../UI/FileManager/EditPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/EnumFormatEtc.o: ../../UI/FileManager/EnumFormatEtc.cpp + $(CXX) $(CXXFLAGS) $< +$O/ExtractCallback.o: ../../UI/FileManager/ExtractCallback.cpp + $(CXX) $(CXXFLAGS) $< +$O/FileFolderPluginOpen.o: ../../UI/FileManager/FileFolderPluginOpen.cpp + $(CXX) $(CXXFLAGS) $< +$O/FilePlugins.o: ../../UI/FileManager/FilePlugins.cpp + $(CXX) $(CXXFLAGS) $< +$O/FM.o: ../../UI/FileManager/FM.cpp + $(CXX) $(CXXFLAGS) $< +$O/FoldersPage.o: ../../UI/FileManager/FoldersPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/FormatUtils.o: ../../UI/FileManager/FormatUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/FSDrives.o: ../../UI/FileManager/FSDrives.cpp + $(CXX) $(CXXFLAGS) $< +$O/FSFolder.o: ../../UI/FileManager/FSFolder.cpp + $(CXX) $(CXXFLAGS) $< +$O/FSFolderCopy.o: ../../UI/FileManager/FSFolderCopy.cpp + $(CXX) $(CXXFLAGS) $< +$O/HelpUtils.o: ../../UI/FileManager/HelpUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/LangPage.o: ../../UI/FileManager/LangPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/LangUtils.o: ../../UI/FileManager/LangUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/LinkDialog.o: ../../UI/FileManager/LinkDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ListViewDialog.o: ../../UI/FileManager/ListViewDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/MenuPage.o: ../../UI/FileManager/MenuPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/MessagesDialog.o: ../../UI/FileManager/MessagesDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/MyLoadMenu.o: ../../UI/FileManager/MyLoadMenu.cpp + $(CXX) $(CXXFLAGS) $< +$O/NetFolder.o: ../../UI/FileManager/NetFolder.cpp + $(CXX) $(CXXFLAGS) $< +$O/OpenCallback.o: ../../UI/FileManager/OpenCallback.cpp + $(CXX) $(CXXFLAGS) $< +$O/OptionsDialog.o: ../../UI/FileManager/OptionsDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/OverwriteDialog.o: ../../UI/FileManager/OverwriteDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/Panel.o: ../../UI/FileManager/Panel.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelCopy.o: ../../UI/FileManager/PanelCopy.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelCrc.o: ../../UI/FileManager/PanelCrc.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelDrag.o: ../../UI/FileManager/PanelDrag.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelFolderChange.o: ../../UI/FileManager/PanelFolderChange.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelItemOpen.o: ../../UI/FileManager/PanelItemOpen.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelItems.o: ../../UI/FileManager/PanelItems.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelKey.o: ../../UI/FileManager/PanelKey.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelListNotify.o: ../../UI/FileManager/PanelListNotify.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelMenu.o: ../../UI/FileManager/PanelMenu.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelOperations.o: ../../UI/FileManager/PanelOperations.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelSelect.o: ../../UI/FileManager/PanelSelect.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelSort.o: ../../UI/FileManager/PanelSort.cpp + $(CXX) $(CXXFLAGS) $< +$O/PanelSplitFile.o: ../../UI/FileManager/PanelSplitFile.cpp + $(CXX) $(CXXFLAGS) $< +$O/PasswordDialog.o: ../../UI/FileManager/PasswordDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProgramLocation.o: ../../UI/FileManager/ProgramLocation.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProgressDialog.o: ../../UI/FileManager/ProgressDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/ProgressDialog2.o: ../../UI/FileManager/ProgressDialog2.cpp + $(CXX) $(CXXFLAGS) $< +$O/PropertyName.o: ../../UI/FileManager/PropertyName.cpp + $(CXX) $(CXXFLAGS) $< +$O/RegistryAssociations.o: ../../UI/FileManager/RegistryAssociations.cpp + $(CXX) $(CXXFLAGS) $< +$O/RegistryPlugins.o: ../../UI/FileManager/RegistryPlugins.cpp + $(CXX) $(CXXFLAGS) $< +$O/RegistryUtils.o: ../../UI/FileManager/RegistryUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/RootFolder.o: ../../UI/FileManager/RootFolder.cpp + $(CXX) $(CXXFLAGS) $< +$O/SettingsPage.o: ../../UI/FileManager/SettingsPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/SplitDialog.o: ../../UI/FileManager/SplitDialog.cpp + $(CXX) $(CXXFLAGS) $< +$O/SplitUtils.o: ../../UI/FileManager/SplitUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/StringUtils.o: ../../UI/FileManager/StringUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/SysIconUtils.o: ../../UI/FileManager/SysIconUtils.cpp + $(CXX) $(CXXFLAGS) $< +$O/SystemPage.o: ../../UI/FileManager/SystemPage.cpp + $(CXX) $(CXXFLAGS) $< +$O/TextPairs.o: ../../UI/FileManager/TextPairs.cpp + $(CXX) $(CXXFLAGS) $< +$O/UpdateCallback100.o: ../../UI/FileManager/UpdateCallback100.cpp + $(CXX) $(CXXFLAGS) $< +$O/ViewSettings.o: ../../UI/FileManager/ViewSettings.cpp + $(CXX) $(CXXFLAGS) $< + +$O/SfxCon.o: ../../Bundles/SFXCon/SfxCon.cpp + $(CXX) $(CXXFLAGS) $< + +$O/$(FILE_IO).o: ../../../$(FILE_IO_2).cpp + $(CXX) $(CXXFLAGS) $< + + + + + + +$O/7zAlloc.o: ../../../../C/7zAlloc.c + $(CC) $(CFLAGS) $< +$O/7zArcIn.o: ../../../../C/7zArcIn.c + $(CC) $(CFLAGS) $< +$O/7zBuf.o: ../../../../C/7zBuf.c + $(CC) $(CFLAGS) $< +$O/7zBuf2.o: ../../../../C/7zBuf2.c + $(CC) $(CFLAGS) $< +$O/7zCrc.o: ../../../../C/7zCrc.c + $(CC) $(CFLAGS) $< +$O/7zDec.o: ../../../../C/7zDec.c + $(CC) $(CFLAGS) $< +$O/7zFile.o: ../../../../C/7zFile.c + $(CC) $(CFLAGS) $< +$O/7zStream.o: ../../../../C/7zStream.c + $(CC) $(CFLAGS) $< +$O/Aes.o: ../../../../C/Aes.c + $(CC) $(CFLAGS) $< +$O/Alloc.o: ../../../../C/Alloc.c + $(CC) $(CFLAGS) $< +$O/Bcj2.o: ../../../../C/Bcj2.c + $(CC) $(CFLAGS) $< +$O/Bcj2Enc.o: ../../../../C/Bcj2Enc.c + $(CC) $(CFLAGS) $< +$O/Blake2s.o: ../../../../C/Blake2s.c + $(CC) $(CFLAGS) $< +$O/Bra.o: ../../../../C/Bra.c + $(CC) $(CFLAGS) $< +$O/Bra86.o: ../../../../C/Bra86.c + $(CC) $(CFLAGS) $< +$O/BraIA64.o: ../../../../C/BraIA64.c + $(CC) $(CFLAGS) $< +$O/BwtSort.o: ../../../../C/BwtSort.c + $(CC) $(CFLAGS) $< + +$O/CpuArch.o: ../../../../C/CpuArch.c + $(CC) $(CFLAGS) $< +$O/Delta.o: ../../../../C/Delta.c + $(CC) $(CFLAGS) $< +$O/DllSecur.o: ../../../../C/DllSecur.c + $(CC) $(CFLAGS) $< +$O/HuffEnc.o: ../../../../C/HuffEnc.c + $(CC) $(CFLAGS) $< +$O/LzFind.o: ../../../../C/LzFind.c + $(CC) $(CFLAGS) $< + +# ifdef MT_FILES +$O/LzFindMt.o: ../../../../C/LzFindMt.c + $(CC) $(CFLAGS) $< + +$O/Threads.o: ../../../../C/Threads.c + $(CC) $(CFLAGS) $< +# endif + +$O/LzmaEnc.o: ../../../../C/LzmaEnc.c + $(CC) $(CFLAGS) $< +$O/Lzma86Dec.o: ../../../../C/Lzma86Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma86Enc.o: ../../../../C/Lzma86Enc.c + $(CC) $(CFLAGS) $< +$O/Lzma2Dec.o: ../../../../C/Lzma2Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma2DecMt.o: ../../../../C/Lzma2DecMt.c + $(CC) $(CFLAGS) $< +$O/Lzma2Enc.o: ../../../../C/Lzma2Enc.c + $(CC) $(CFLAGS) $< +$O/LzmaLib.o: ../../../../C/LzmaLib.c + $(CC) $(CFLAGS) $< +$O/MtCoder.o: ../../../../C/MtCoder.c + $(CC) $(CFLAGS) $< +$O/MtDec.o: ../../../../C/MtDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7.o: ../../../../C/Ppmd7.c + $(CC) $(CFLAGS) $< +$O/Ppmd7aDec.o: ../../../../C/Ppmd7aDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Dec.o: ../../../../C/Ppmd7Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Enc.o: ../../../../C/Ppmd7Enc.c + $(CC) $(CFLAGS) $< +$O/Ppmd8.o: ../../../../C/Ppmd8.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Dec.o: ../../../../C/Ppmd8Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Enc.o: ../../../../C/Ppmd8Enc.c + $(CC) $(CFLAGS) $< +$O/Sha1.o: ../../../../C/Sha1.c + $(CC) $(CFLAGS) $< +$O/Sha256.o: ../../../../C/Sha256.c + $(CC) $(CFLAGS) $< +$O/Sort.o: ../../../../C/Sort.c + $(CC) $(CFLAGS) $< +$O/Xz.o: ../../../../C/Xz.c + $(CC) $(CFLAGS) $< +$O/XzCrc64.o: ../../../../C/XzCrc64.c + $(CC) $(CFLAGS) $< + +ifdef USE_ASM +ifdef IS_X64 +USE_X86_ASM=1 +else +ifdef IS_X86 +USE_X86_ASM=1 +endif +endif +endif + +ifdef USE_X86_ASM +$O/7zCrcOpt.o: ../../../../Asm/x86/7zCrcOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/XzCrc64Opt.o: ../../../../Asm/x86/XzCrc64Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/AesOpt.o: ../../../../Asm/x86/AesOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha1Opt.o: ../../../../Asm/x86/Sha1Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha256Opt.o: ../../../../Asm/x86/Sha256Opt.asm + $(MY_ASM) $(AFLAGS) $< +else +$O/7zCrcOpt.o: ../../../../C/7zCrcOpt.c + $(CC) $(CFLAGS) $< +$O/XzCrc64Opt.o: ../../../../C/XzCrc64Opt.c + $(CC) $(CFLAGS) $< +$O/Sha1Opt.o: ../../../../C/Sha1Opt.c + $(CC) $(CFLAGS) $< +$O/Sha256Opt.o: ../../../../C/Sha256Opt.c + $(CC) $(CFLAGS) $< +$O/AesOpt.o: ../../../../C/AesOpt.c + $(CC) $(CFLAGS) $< +endif + + +ifdef USE_LZMA_DEC_ASM + +ifdef IS_X64 +$O/LzmaDecOpt.o: ../../../../Asm/x86/LzmaDecOpt.asm + $(MY_ASM) $(AFLAGS) $< +endif + +ifdef IS_ARM64 +$O/LzmaDecOpt.o: ../../../../Asm/arm64/LzmaDecOpt.S ../../../../Asm/arm64/7zAsm.S + $(CC) $(CFLAGS) $< +endif + +$O/LzmaDec.o: ../../../../C/LzmaDec.c + $(CC) $(CFLAGS) -D_LZMA_DEC_OPT $< + +else + +$O/LzmaDec.o: ../../../../C/LzmaDec.c + $(CC) $(CFLAGS) $< + +endif + + + +$O/XzDec.o: ../../../../C/XzDec.c + $(CC) $(CFLAGS) $< +$O/XzEnc.o: ../../../../C/XzEnc.c + $(CC) $(CFLAGS) $< +$O/XzIn.o: ../../../../C/XzIn.c + $(CC) $(CFLAGS) $< + + +$O/7zMain.o: ../../../../C/Util/7z/7zMain.c + $(CC) $(CFLAGS) $< +$O/LzmaUtil.o: ../../../../C/Util/Lzma/LzmaUtil.c + $(CC) $(CFLAGS) $< + +ifneq ($(CC), xlc) +SHOW_PREDEF=-dM +else +SHOW_PREDEF= -qshowmacros=pre +endif + +predef_cc: + $(CC) $(CFLAGS) -E $(SHOW_PREDEF) ../../../../C/CpuArch.c > predef_cc_log +# $(CC) $(CFLAGS) -E -dM - < /dev/null +predef_cxx: + $(CXX) $(CFLAGS) -E $(SHOW_PREDEF) ../../../Common/CrcReg.cpp > predef_cxx_log + +predef: predef_cc predef_cxx + + +clean: + -$(DEL_OBJ_EXE) diff --git a/CPP/7zip/Aes.mak b/CPP/7zip/Aes.mak index 20f1a72f..7d8da2d8 100644 --- a/CPP/7zip/Aes.mak +++ b/CPP/7zip/Aes.mak @@ -1,7 +1,10 @@ C_OBJS = $(C_OBJS) \ $O\Aes.obj -!IF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" +!IF defined(USE_C_AES) || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +C_OBJS = $(C_OBJS) \ + $O\AesOpt.obj +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" ASM_OBJS = $(ASM_OBJS) \ $O\AesOpt.obj !ENDIF diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index 608293d6..44c90226 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -57,18 +57,20 @@ struct CCompressionMethodMode #endif bool PasswordIsDefined; - UString Password; + UString Password; // _Wipe bool IsEmpty() const { return (Methods.IsEmpty() && !PasswordIsDefined); } CCompressionMethodMode(): - DefaultMethod_was_Inserted(false), - Filter_was_Inserted(false), - PasswordIsDefined(false) + DefaultMethod_was_Inserted(false) + , Filter_was_Inserted(false) #ifndef _7ZIP_ST , NumThreads(1) , MultiThreadMixer(true) #endif + , PasswordIsDefined(false) {} + + ~CCompressionMethodMode() { Password.Wipe_and_Empty(); } }; }} diff --git a/CPP/7zip/Archive/7z/7zDecode.cpp b/CPP/7zip/Archive/7z/7zDecode.cpp index 9df531e0..c27c8fbc 100644 --- a/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/CPP/7zip/Archive/7z/7zDecode.cpp @@ -158,7 +158,7 @@ STDMETHODIMP CLockedSequentialInStreamMT::Read(void *data, UInt32 size, UInt32 * if (_pos != _glob->Pos) { - RINOK(_glob->Stream->Seek(_pos, STREAM_SEEK_SET, NULL)); + RINOK(_glob->Stream->Seek((Int64)_pos, STREAM_SEEK_SET, NULL)); _glob->Pos = _pos; } @@ -200,7 +200,7 @@ STDMETHODIMP CLockedSequentialInStreamST::Read(void *data, UInt32 size, UInt32 * { if (_pos != _glob->Pos) { - RINOK(_glob->Stream->Seek(_pos, STREAM_SEEK_SET, NULL)); + RINOK(_glob->Stream->Seek((Int64)_pos, STREAM_SEEK_SET, NULL)); _glob->Pos = _pos; } @@ -276,6 +276,7 @@ HRESULT CDecoder::Decode( if (!_bindInfoPrev_Defined || !AreBindInfoExEqual(bindInfo, _bindInfoPrev)) { + _bindInfoPrev_Defined = false; _mixerRef.Release(); #ifdef USE_MIXER_MT @@ -348,7 +349,7 @@ HRESULT CDecoder::Decode( _bindInfoPrev_Defined = true; } - _mixer->ReInit(); + RINOK(_mixer->ReInit2()); UInt32 packStreamIndex = 0; UInt32 unpackStreamIndexStart = folders.FoToCoderUnpackSizes[folderIndex]; @@ -396,10 +397,10 @@ HRESULT CDecoder::Decode( if (setDecoderProperties) { const CByteBuffer &props = coderInfo.Props; - size_t size = props.Size(); - if (size > 0xFFFFFFFF) + const UInt32 size32 = (UInt32)props.Size(); + if (props.Size() != size32) return E_NOTIMPL; - HRESULT res = setDecoderProperties->SetDecoderProperties2((const Byte *)props, (UInt32)size); + HRESULT res = setDecoderProperties->SetDecoderProperties2((const Byte *)props, size32); if (res == E_INVALIDARG) res = E_NOTIMPL; RINOK(res); @@ -415,17 +416,17 @@ HRESULT CDecoder::Decode( isEncrypted = true; if (!getTextPassword) return E_NOTIMPL; - CMyComBSTR passwordBSTR; + CMyComBSTR_Wipe passwordBSTR; RINOK(getTextPassword->CryptoGetTextPassword(&passwordBSTR)); passwordIsDefined = true; - password.Empty(); + password.Wipe_and_Empty(); size_t len = 0; if (passwordBSTR) { password = passwordBSTR; len = password.Len(); } - CByteBuffer buffer(len * 2); + CByteBuffer_Wipe buffer(len * 2); for (size_t k = 0; k < len; k++) { wchar_t c = passwordBSTR[k]; @@ -444,7 +445,7 @@ HRESULT CDecoder::Decode( if (setFinishMode) { finishMode = fullUnpack; - RINOK(setFinishMode->SetFinishMode(BoolToInt(finishMode))); + RINOK(setFinishMode->SetFinishMode(BoolToUInt(finishMode))); } } @@ -487,36 +488,49 @@ HRESULT CDecoder::Decode( CLockedInStream *lockedInStreamSpec = new CLockedInStream; CMyComPtr<IUnknown> lockedInStream = lockedInStreamSpec; - bool needMtLock = false; + #ifdef USE_MIXER_MT + #ifdef USE_MIXER_ST + bool needMtLock = _useMixerMT; + #endif + #endif if (folderInfo.PackStreams.Size() > 1) { // lockedInStream.Pos = (UInt64)(Int64)-1; // RINOK(inStream->Seek(0, STREAM_SEEK_CUR, &lockedInStream.Pos)); - RINOK(inStream->Seek(startPos + packPositions[0], STREAM_SEEK_SET, &lockedInStreamSpec->Pos)); + RINOK(inStream->Seek((Int64)(startPos + packPositions[0]), STREAM_SEEK_SET, &lockedInStreamSpec->Pos)); lockedInStreamSpec->Stream = inStream; + #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST - if (_mixer->IsThere_ExternalCoder_in_PackTree(_mixer->MainCoderIndex)) - #endif + /* + For ST-mixer mode: + If parallel input stream reading from pack streams is possible, + we must use MT-lock for packed streams. + Internal decoders in 7-Zip will not read pack streams in parallel in ST-mixer mode. + So we force to needMtLock mode only if there is unknown (external) decoder. + */ + if (!needMtLock && _mixer->IsThere_ExternalCoder_in_PackTree(_mixer->MainCoderIndex)) needMtLock = true; + #endif + #endif } for (unsigned j = 0; j < folderInfo.PackStreams.Size(); j++) { CMyComPtr<ISequentialInStream> packStream; - UInt64 packPos = startPos + packPositions[j]; + const UInt64 packPos = startPos + packPositions[j]; if (folderInfo.PackStreams.Size() == 1) { - RINOK(inStream->Seek(packPos, STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek((Int64)packPos, STREAM_SEEK_SET, NULL)); packStream = inStream; } else { #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST - if (_useMixerMT || needMtLock) + if (needMtLock) #endif { CLockedSequentialInStreamMT *lockedStreamImpSpec = new CLockedSequentialInStreamMT; @@ -542,7 +556,7 @@ HRESULT CDecoder::Decode( streamSpec->Init(packPositions[j + 1] - packPositions[j]); } - unsigned num = inStreams.Size(); + const unsigned num = inStreams.Size(); CObjArray<ISequentialInStream *> inStreamPointers(num); for (i = 0; i < num; i++) inStreamPointers[i] = inStreams[i]; diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 7d8270f9..49963241 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -158,7 +158,7 @@ HRESULT CEncoder::CreateMixerCoder( { RINOK(CreateCoder_Index( EXTERNAL_CODECS_LOC_VARS - methodFull.CodecIndex, true, cod)); + (unsigned)methodFull.CodecIndex, true, cod)); } else { @@ -215,7 +215,7 @@ HRESULT CEncoder::CreateMixerCoder( if (cryptoSetPassword) { const unsigned sizeInBytes = _options.Password.Len() * 2; - CByteBuffer buffer(sizeInBytes); + CByteBuffer_Wipe buffer(sizeInBytes); for (unsigned i = 0; i < _options.Password.Len(); i++) { wchar_t c = _options.Password[i]; @@ -249,11 +249,12 @@ public: STDMETHODIMP CSequentialOutTempBufferImp2::Write(const void *data, UInt32 size, UInt32 *processed) { - if (!_buf->Write(data, size)) + HRESULT res = _buf->Write_HRESULT(data, size); + if (res != S_OK) { if (processed) *processed = 0; - return E_FAIL; + return res; } if (processed) *processed = size; @@ -309,7 +310,7 @@ HRESULT CEncoder::Encode( RINOK(CreateMixerCoder(EXTERNAL_CODECS_LOC_VARS inSizeForReduce)); } - _mixer->ReInit(); + RINOK(_mixer->ReInit2()); CMtEncMultiProgress *mtProgressSpec = NULL; CMyComPtr<ICompressProgressInfo> mtProgress; @@ -478,7 +479,7 @@ HRESULT CEncoder::Encode( unpackSize = streamSize; } else - streamSize = _mixer->GetBondStreamSize(bond); + streamSize = _mixer->GetBondStreamSize((unsigned)bond); coderUnpackSizes.Add(streamSize); } @@ -609,13 +610,13 @@ HRESULT CEncoder::EncoderConstr() int bond = _bindInfo.FindBond_for_PackStream(outIndex); if (bond >= 0) { - ci = _bindInfo.Bonds[bond].UnpackIndex; + ci = _bindInfo.Bonds[(unsigned)bond].UnpackIndex; continue; } int si = _bindInfo.FindStream_in_PackStreams(outIndex); if (si >= 0) - _bindInfo.PackStreams.MoveToFront(si); + _bindInfo.PackStreams.MoveToFront((unsigned)si); break; } } diff --git a/CPP/7zip/Archive/7z/7zEncode.h b/CPP/7zip/Archive/7z/7zEncode.h index f1a9b5ad..6ea7f276 100644 --- a/CPP/7zip/Archive/7z/7zEncode.h +++ b/CPP/7zip/Archive/7z/7zEncode.h @@ -41,7 +41,7 @@ public: STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; -class CEncoder +class CEncoder MY_UNCOPYABLE { #ifdef USE_MIXER_ST NCoderMixer2::CMixerST *_mixerST; diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index 9ffe2fdc..95eba9af 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -217,6 +217,10 @@ HRESULT CFolderOutStream::FlushCorrupted(Int32 callbackOperationResult) STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 testModeSpec, IArchiveExtractCallback *extractCallbackSpec) { + // for GCC + // CFolderOutStream *folderOutStream = new CFolderOutStream; + // CMyComPtr<ISequentialOutStream> outStream(folderOutStream); + COM_TRY_BEGIN CMyComPtr<IArchiveExtractCallback> extractCallback = extractCallbackSpec; @@ -350,7 +354,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, #ifndef _NO_CRYPTO bool isEncrypted = false; bool passwordIsDefined = false; - UString password; + UString_Wipe password; #endif @@ -411,7 +415,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, { RINOK(folderOutStream->FlushCorrupted(NExtract::NOperationResult::kDataError)); // continue; - return E_FAIL; + // return E_FAIL; + throw; } } diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index bbb892e7..9e344c34 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -107,34 +107,62 @@ static void ConvertMethodIdToString(AString &res, UInt64 id) res += s + len - ConvertMethodIdToString_Back(s + len, id); } -static unsigned GetStringForSizeValue(char *s, UInt32 val) + +static char *GetStringForSizeValue(char *s, UInt32 val) { unsigned i; for (i = 0; i <= 31; i++) if (((UInt32)1 << i) == val) { - if (i < 10) + if (i >= 10) { - s[0] = (char)('0' + i); - s[1] = 0; - return 1; + *s++= (char)('0' + i / 10); + i %= 10; } - if (i < 20) { s[0] = '1'; s[1] = (char)('0' + i - 10); } - else if (i < 30) { s[0] = '2'; s[1] = (char)('0' + i - 20); } - else { s[0] = '3'; s[1] = (char)('0' + i - 30); } - s[2] = 0; - return 2; + *s++ = (char)('0' + i); + *s = 0; + return s; } + char c = 'b'; if ((val & ((1 << 20) - 1)) == 0) { val >>= 20; c = 'm'; } else if ((val & ((1 << 10) - 1)) == 0) { val >>= 10; c = 'k'; } - ::ConvertUInt32ToString(val, s); - unsigned pos = MyStringLen(s); - s[pos++] = c; - s[pos] = 0; - return pos; + s = ConvertUInt32ToString(val, s); + *s++ = c; + *s = 0; + return s; } + +static void GetLzma2String(char *s, unsigned d) +{ + if (d > 40) + { + *s = 0; + return; + // s = MyStpCpy(s, "unsup"); + } + else if ((d & 1) == 0) + d = (d >> 1) + 12; + else + { + // s = GetStringForSizeValue(s, (UInt32)3 << ((d >> 1) + 11)); + d = (d >> 1) + 1; + char c = 'k'; + if (d >= 10) + { + c = 'm'; + d -= 10; + } + s = ConvertUInt32ToString((UInt32)3 << d, s); + *s++ = c; + *s = 0; + return; + } + ConvertUInt32ToString(d, s); +} + + /* static inline void AddHexToString(UString &res, Byte value) { @@ -147,8 +175,7 @@ static char *AddProp32(char *s, const char *name, UInt32 v) { *s++ = ':'; s = MyStpCpy(s, name); - ::ConvertUInt32ToString(v, s); - return s + MyStringLen(s); + return ConvertUInt32ToString(v, s); } void CHandler::AddMethodName(AString &s, UInt64 id) @@ -184,10 +211,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (id == k_LZMA2) { s += "LZMA2:"; - if ((pm.Lzma2Prop & 1) == 0) - ConvertUInt32ToString((pm.Lzma2Prop >> 1) + 12, temp); - else - GetStringForSizeValue(temp, 3 << ((pm.Lzma2Prop >> 1) + 11)); + GetLzma2String(temp, pm.Lzma2Prop); s += temp; } else if (id == k_LZMA) @@ -244,14 +268,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) break; } } - prop.Detach(value); - return S_OK; + return prop.Detach(value); #ifndef _SFX COM_TRY_END #endif } -static void SetFileTimeProp_From_UInt64Def(PROPVARIANT *prop, const CUInt64DefVector &v, int index) +static void SetFileTimeProp_From_UInt64Def(PROPVARIANT *prop, const CUInt64DefVector &v, unsigned index) { UInt64 value; if (v.GetItem(index, value)) @@ -416,7 +439,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const if (propsSize == 5) { UInt32 dicSize = GetUi32((const Byte *)props + 1); - char *dest = s + GetStringForSizeValue(s, dicSize); + char *dest = GetStringForSizeValue(s, dicSize); UInt32 d = props[0]; if (d != 0x5D) { @@ -434,24 +457,16 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const { name = "LZMA2"; if (propsSize == 1) - { - Byte d = props[0]; - if ((d & 1) == 0) - ConvertUInt32ToString((UInt32)((d >> 1) + 12), s); - else - GetStringForSizeValue(s, 3 << ((d >> 1) + 11)); - } + GetLzma2String(s, props[0]); } else if (id == k_PPMD) { name = "PPMD"; if (propsSize == 5) { - Byte order = *props; char *dest = s; *dest++ = 'o'; - ConvertUInt32ToString(order, dest); - dest += MyStringLen(dest); + dest = ConvertUInt32ToString(*props, dest); dest = MyStpCpy(dest, ":mem"); GetStringForSizeValue(dest, GetUi32(props + 1)); } @@ -534,7 +549,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value) { - PropVariant_Clear(value); + RINOK(PropVariant_Clear(value)); // COM_TRY_BEGIN // NCOM::CPropVariant prop; @@ -637,7 +652,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val #endif } - // prop.Detach(value); + // return prop.Detach(value); return S_OK; // COM_TRY_END } @@ -708,7 +723,7 @@ STDMETHODIMP CHandler::Close() #ifndef _NO_CRYPTO _isEncrypted = false; _passwordIsDefined = false; - _password.Empty(); + _password.Wipe_and_Empty(); #endif return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index 99942eb0..cad1ae61 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -129,6 +129,10 @@ public: DECL_ISetCompressCodecsInfo CHandler(); + ~CHandler() + { + Close(); + } private: CMyComPtr<IInStream> _inStream; @@ -137,7 +141,7 @@ private: #ifndef _NO_CRYPTO bool _isEncrypted; bool _passwordIsDefined; - UString _password; + UString _password; // _Wipe #endif #ifdef EXTRACT_ONLY diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index b6be06a8..b549d94a 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -111,7 +111,7 @@ HRESULT CHandler::SetMainMethod( } const UInt64 kSolidBytes_Min = (1 << 24); - const UInt64 kSolidBytes_Max = ((UInt64)1 << 32) - 1; + const UInt64 kSolidBytes_Max = ((UInt64)1 << 32); bool needSolid = false; @@ -140,26 +140,52 @@ HRESULT CHandler::SetMainMethod( case k_LZMA2: dicSize = oneMethodInfo.Get_Lzma_DicSize(); break; case k_PPMD: dicSize = oneMethodInfo.Get_Ppmd_MemSize(); break; case k_Deflate: dicSize = (UInt32)1 << 15; break; + case k_Deflate64: dicSize = (UInt32)1 << 16; break; case k_BZip2: dicSize = oneMethodInfo.Get_BZip2_BlockSize(); break; default: continue; } - - _numSolidBytes = (UInt64)dicSize << 7; - if (_numSolidBytes < kSolidBytes_Min) _numSolidBytes = kSolidBytes_Min; - if (_numSolidBytes > kSolidBytes_Max) _numSolidBytes = kSolidBytes_Max; + + if (methodFull.Id == k_LZMA2) + { + // he we calculate default chunk Size for LZMA2 as defined in LZMA2 encoder code + UInt64 cs = (UInt64)dicSize << 2; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + if (cs < kMinSize) cs = kMinSize; + if (cs > kMaxSize) cs = kMaxSize; + if (cs < dicSize) cs = dicSize; + cs += (kMinSize - 1); + cs &= ~(UInt64)(kMinSize - 1); + // we want to use at least 64 chunks (threads) per one solid block. + _numSolidBytes = cs << 6; + const UInt64 kSolidBytes_Lzma2_Max = ((UInt64)1 << 34); + if (_numSolidBytes > kSolidBytes_Lzma2_Max) + _numSolidBytes = kSolidBytes_Lzma2_Max; + } + else + { + _numSolidBytes = (UInt64)dicSize << 7; + if (_numSolidBytes > kSolidBytes_Max) + _numSolidBytes = kSolidBytes_Max; + } + + if (_numSolidBytes < kSolidBytes_Min) + _numSolidBytes = kSolidBytes_Min; _numSolidBytesDefined = true; } if (!_numSolidBytesDefined) + { if (needSolid) _numSolidBytes = kSolidBytes_Max; else _numSolidBytes = 0; + } _numSolidBytesDefined = true; return S_OK; } -static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, int index, PROPID propID, UInt64 &ft, bool &ftDefined) +static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, unsigned index, PROPID propID, UInt64 &ft, bool &ftDefined) { // ft = 0; // ftDefined = false; @@ -289,8 +315,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt bool need_CTime = (Write_CTime.Def && Write_CTime.Val); bool need_ATime = (Write_ATime.Def && Write_ATime.Val); - bool need_MTime = (Write_MTime.Def && Write_MTime.Val || !Write_MTime.Def); - bool need_Attrib = (Write_Attrib.Def && Write_Attrib.Val || !Write_Attrib.Def); + bool need_MTime = (Write_MTime.Def ? Write_MTime.Val : true); + bool need_Attrib = (Write_Attrib.Def ? Write_Attrib.Val : true); if (db && !db->Files.IsEmpty()) { @@ -313,7 +339,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CUpdateItem ui; ui.NewProps = IntToBool(newProps); ui.NewData = IntToBool(newData); - ui.IndexInArchive = indexInArchive; + ui.IndexInArchive = (int)indexInArchive; ui.IndexInClient = i; ui.IsAnti = false; ui.Size = 0; @@ -322,23 +348,23 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt // bool isAltStream = false; if (ui.IndexInArchive != -1) { - if (db == 0 || (unsigned)ui.IndexInArchive >= db->Files.Size()) + if (!db || (unsigned)ui.IndexInArchive >= db->Files.Size()) return E_INVALIDARG; - const CFileItem &fi = db->Files[ui.IndexInArchive]; + const CFileItem &fi = db->Files[(unsigned)ui.IndexInArchive]; if (!ui.NewProps) { - _db.GetPath(ui.IndexInArchive, name); + _db.GetPath((unsigned)ui.IndexInArchive, name); } ui.IsDir = fi.IsDir; ui.Size = fi.Size; // isAltStream = fi.IsAltStream; - ui.IsAnti = db->IsItemAnti(ui.IndexInArchive); + ui.IsAnti = db->IsItemAnti((unsigned)ui.IndexInArchive); if (!ui.NewProps) { - ui.CTimeDefined = db->CTime.GetItem(ui.IndexInArchive, ui.CTime); - ui.ATimeDefined = db->ATime.GetItem(ui.IndexInArchive, ui.ATime); - ui.MTimeDefined = db->MTime.GetItem(ui.IndexInArchive, ui.MTime); + ui.CTimeDefined = db->CTime.GetItem((unsigned)ui.IndexInArchive, ui.CTime); + ui.ATimeDefined = db->ATime.GetItem((unsigned)ui.IndexInArchive, ui.ATime); + ui.MTimeDefined = db->MTime.GetItem((unsigned)ui.IndexInArchive, ui.MTime); } } @@ -570,10 +596,10 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt updateCallback->QueryInterface(IID_ICryptoGetTextPassword2, (void **)&getPassword2); methodMode.PasswordIsDefined = false; - methodMode.Password.Empty(); + methodMode.Password.Wipe_and_Empty(); if (getPassword2) { - CMyComBSTR password; + CMyComBSTR_Wipe password; Int32 passwordIsDefined; RINOK(getPassword2->CryptoGetTextPassword2(&passwordIsDefined, &password)); methodMode.PasswordIsDefined = IntToBool(passwordIsDefined); @@ -751,7 +777,7 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) _solidExtension = true; continue; } - i += (int)(end - start); + i += (unsigned)(end - start); if (i == s2.Len()) return E_INVALIDARG; wchar_t c = s2[i++]; @@ -829,7 +855,7 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val } UInt32 number; - int index = ParseStringToUInt32(name, number); + unsigned index = ParseStringToUInt32(name, number); // UString realName = name.Ptr(index); if (index == 0) { diff --git a/CPP/7zip/Archive/7z/7zHeader.h b/CPP/7zip/Archive/7z/7zHeader.h index d7f0ae36..e1bbc0aa 100644 --- a/CPP/7zip/Archive/7z/7zHeader.h +++ b/CPP/7zip/Archive/7z/7zHeader.h @@ -108,8 +108,9 @@ const UInt32 k_SWAP4 = 0x20304; const UInt32 k_LZMA = 0x30101; const UInt32 k_PPMD = 0x30401; -const UInt32 k_Deflate = 0x40108; -const UInt32 k_BZip2 = 0x40202; +const UInt32 k_Deflate = 0x40108; +const UInt32 k_Deflate64 = 0x40109; +const UInt32 k_BZip2 = 0x40202; const UInt32 k_BCJ = 0x3030103; const UInt32 k_BCJ2 = 0x303011B; diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp index ab5b5de4..7134595c 100644 --- a/CPP/7zip/Archive/7z/7zIn.cpp +++ b/CPP/7zip/Archive/7z/7zIn.cpp @@ -11,6 +11,8 @@ #include "../../../../C/7zCrc.h" #include "../../../../C/CpuArch.h" +// #include "../../../Common/UTFConvert.h" + #include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" @@ -32,6 +34,7 @@ using namespace NCOM; namespace NArchive { namespace N7z { +unsigned BoolVector_CountSum(const CBoolVector &v); unsigned BoolVector_CountSum(const CBoolVector &v) { unsigned sum = 0; @@ -59,9 +62,13 @@ static void BoolVector_Fill_False(CBoolVector &v, unsigned size) class CInArchiveException {}; class CUnsupportedFeatureException: public CInArchiveException {}; +MY_ATTR_NORETURN static void ThrowException() { throw CInArchiveException(); } +MY_ATTR_NORETURN static inline void ThrowEndOfData() { ThrowException(); } +MY_ATTR_NORETURN static inline void ThrowUnsupported() { throw CUnsupportedFeatureException(); } +MY_ATTR_NORETURN static inline void ThrowIncorrect() { ThrowException(); } class CStreamSwitch @@ -328,7 +335,7 @@ HRESULT CInArchive::FindAndReadSignature(IInStream *stream, const UInt64 *search { memcpy(_header, p, kHeaderSize); _arhiveBeginStreamPosition += offset + pos; - return stream->Seek(_arhiveBeginStreamPosition + kHeaderSize, STREAM_SEEK_SET, NULL); + return stream->Seek((Int64)(_arhiveBeginStreamPosition + kHeaderSize), STREAM_SEEK_SET, NULL); } } @@ -344,7 +351,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit) Close(); RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_arhiveBeginStreamPosition)) RINOK(stream->Seek(0, STREAM_SEEK_END, &_fileEndPosition)) - RINOK(stream->Seek(_arhiveBeginStreamPosition, STREAM_SEEK_SET, NULL)) + RINOK(stream->Seek((Int64)_arhiveBeginStreamPosition, STREAM_SEEK_SET, NULL)) RINOK(FindAndReadSignature(stream, searchHeaderSizeLimit)); _stream = stream; return S_OK; @@ -478,7 +485,7 @@ void CDatabase::GetPath(unsigned index, UString &path) const #if defined(_WIN32) && defined(MY_CPU_LE) - wmemcpy(s, (const wchar_t *)p, size); + wmemcpy(s, (const wchar_t *)(const void *)p, size); #else @@ -506,10 +513,27 @@ HRESULT CDatabase::GetPath_Prop(unsigned index, PROPVARIANT *path) const throw() if (size >= (1 << 14)) return S_OK; - RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size - 1)); + // (size) includes null terminator + + /* + #if WCHAR_MAX > 0xffff + + const Byte *p = ((const Byte *)NamesBuf + offset * 2); + size = Utf16LE__Get_Num_WCHARs(p, size - 1); + // (size) doesn't include null terminator + RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size)); wchar_t *s = path->bstrVal; + wchar_t *sEnd = Utf16LE__To_WCHARs_Sep(p, size, s); + *sEnd = 0; + if (s + size != sEnd) return E_FAIL; + #else + */ + + RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size - 1)); + wchar_t *s = path->bstrVal; const Byte *p = ((const Byte *)NamesBuf + offset * 2); + // Utf16LE__To_WCHARs_Sep(p, size, s); for (size_t i = 0; i < size; i++) { @@ -518,10 +542,14 @@ HRESULT CDatabase::GetPath_Prop(unsigned index, PROPVARIANT *path) const throw() #if WCHAR_PATH_SEPARATOR != L'/' if (c == L'/') c = WCHAR_PATH_SEPARATOR; + else if (c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme #endif *s++ = c; } + // #endif + return S_OK; /* @@ -673,7 +701,7 @@ void CInArchive::ReadUnpackInfo( { UInt32 indexOfMainStream = 0; UInt32 numPackStreams = 0; - folders.FoCodersDataOffset[fo] = _inByteBack->GetPtr() - startBufPtr; + folders.FoCodersDataOffset[fo] = (size_t)(_inByteBack->GetPtr() - startBufPtr); CNum numInStreams = 0; CNum numCoders = inByte->ReadNum(); @@ -794,10 +822,10 @@ void CInArchive::ReadUnpackInfo( folders.FoToMainUnpackSizeIndex[fo] = (Byte)indexOfMainStream; } - size_t dataSize = _inByteBack->GetPtr() - startBufPtr; + const size_t dataSize = (size_t)(_inByteBack->GetPtr() - startBufPtr); folders.FoToCoderUnpackSizes[fo] = numCodersOutStreams; folders.FoStartPackStreamIndex[fo] = packStreamIndex; - folders.FoCodersDataOffset[fo] = _inByteBack->GetPtr() - startBufPtr; + folders.FoCodersDataOffset[fo] = (size_t)(_inByteBack->GetPtr() - startBufPtr); folders.CodersData.CopyFrom(startBufPtr, dataSize); // if (folders.NumPackStreams != packStreamIndex) ThrowUnsupported(); @@ -962,6 +990,8 @@ void CInArchive::ReadSubStreamsInfo( } } + + void CInArchive::ReadStreamsInfo( const CObjectVector<CByteBuffer> *dataVector, UInt64 &dataOffset, @@ -974,7 +1004,11 @@ void CInArchive::ReadStreamsInfo( if (type == NID::kPackInfo) { dataOffset = ReadNumber(); + if (dataOffset > _rangeLimit) + ThrowIncorrect(); ReadPackInfo(folders); + if (folders.PackPositions[folders.NumPackStreams] > _rangeLimit - dataOffset) + ThrowIncorrect(); type = ReadID(); } @@ -1029,7 +1063,7 @@ void CInArchive::ReadBoolVector(unsigned numItems, CBoolVector &v) mask = 0x80; } p[i] = ((b & mask) != 0); - mask >>= 1; + mask = (Byte)(mask >> 1); } } @@ -1090,8 +1124,8 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( for (CNum i = 0; i < folders.NumFolders; i++) { CByteBuffer &data = dataVector.AddNew(); - UInt64 unpackSize64 = folders.GetFolderUnpackSize(i); - size_t unpackSize = (size_t)unpackSize64; + const UInt64 unpackSize64 = folders.GetFolderUnpackSize(i); + const size_t unpackSize = (size_t)unpackSize64; if (unpackSize != unpackSize64) ThrowUnsupported(); data.Alloc(unpackSize); @@ -1106,7 +1140,7 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( EXTERNAL_CODECS_LOC_VARS _stream, baseOffset + dataOffset, folders, i, - NULL, // *unpackSize + NULL, // &unpackSize64 outStream, NULL, // *compressProgress @@ -1127,6 +1161,9 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( if (dataAfterEnd_Error) ThereIsHeaderError = true; + if (unpackSize != outStreamSpec->GetPos()) + ThrowIncorrect(); + if (folders.FolderCRCs.ValidAndDefined(i)) if (CrcCalc(data, unpackSize) != folders.FolderCRCs.Vals[i]) ThrowIncorrect(); @@ -1226,8 +1263,8 @@ HRESULT CInArchive::ReadHeader( unsigned i; for (i = 0; i < numFiles; i++) { - size_t curRem = (rem - pos) / 2; - const UInt16 *buf = (const UInt16 *)(db.NamesBuf + pos); + const size_t curRem = (rem - pos) / 2; + const UInt16 *buf = (const UInt16 *)(const void *)(db.NamesBuf + pos); size_t j; for (j = 0; j < curRem && buf[j] != 0; j++); if (j == curRem) @@ -1519,13 +1556,13 @@ HRESULT CInArchive::ReadDatabase2( const unsigned kCheckSize = 512; Byte buf[kCheckSize]; RINOK(_stream->Seek(0, STREAM_SEEK_END, &fileSize)); - UInt64 rem = fileSize - cur; + const UInt64 rem = fileSize - cur; unsigned checkSize = kCheckSize; if (rem < kCheckSize) checkSize = (unsigned)(rem); if (checkSize < 3) return S_FALSE; - RINOK(_stream->Seek(fileSize - checkSize, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)(fileSize - checkSize), STREAM_SEEK_SET, NULL)); RINOK(ReadStream_FALSE(_stream, buf, (size_t)checkSize)); if (buf[checkSize - 1] != 0) @@ -1534,8 +1571,8 @@ HRESULT CInArchive::ReadDatabase2( unsigned i; for (i = checkSize - 2;; i--) { - if (buf[i] == NID::kEncodedHeader && buf[i + 1] == NID::kPackInfo || - buf[i] == NID::kHeader && buf[i + 1] == NID::kMainStreamsInfo) + if ((buf[i] == NID::kEncodedHeader && buf[i + 1] == NID::kPackInfo) || + (buf[i] == NID::kHeader && buf[i + 1] == NID::kMainStreamsInfo)) break; if (i == 0) return S_FALSE; @@ -1543,7 +1580,7 @@ HRESULT CInArchive::ReadDatabase2( nextHeaderSize = checkSize - i; nextHeaderOffset = rem - nextHeaderSize; nextHeaderCRC = CrcCalc(buf + i, (size_t)nextHeaderSize); - RINOK(_stream->Seek(cur, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)cur, STREAM_SEEK_SET, NULL)); db.StartHeaderWasRecovered = true; } else @@ -1560,25 +1597,32 @@ HRESULT CInArchive::ReadDatabase2( if ((Int64)nextHeaderOffset < 0 || nextHeaderSize > ((UInt64)1 << 62)) return S_FALSE; + + HeadersSize = kHeaderSize; + if (nextHeaderSize == 0) { if (nextHeaderOffset != 0) return S_FALSE; db.IsArc = true; + db.HeadersSize = HeadersSize; return S_OK; } if (!db.StartHeaderWasRecovered) db.IsArc = true; - HeadersSize += kHeaderSize + nextHeaderSize; + HeadersSize += nextHeaderSize; + // db.EndHeaderOffset = nextHeaderOffset; + _rangeLimit = nextHeaderOffset; + db.PhySize = kHeaderSize + nextHeaderOffset + nextHeaderSize; if (_fileEndPosition - db.ArcInfo.StartPositionAfterHeader < nextHeaderOffset + nextHeaderSize) { db.UnexpectedEnd = true; return S_FALSE; } - RINOK(_stream->Seek(nextHeaderOffset, STREAM_SEEK_CUR, NULL)); + RINOK(_stream->Seek((Int64)nextHeaderOffset, STREAM_SEEK_CUR, NULL)); size_t nextHeaderSize_t = (size_t)nextHeaderSize; if (nextHeaderSize_t != nextHeaderSize) diff --git a/CPP/7zip/Archive/7z/7zIn.h b/CPP/7zip/Archive/7z/7zIn.h index 6a61d314..ffa1e4bc 100644 --- a/CPP/7zip/Archive/7z/7zIn.h +++ b/CPP/7zip/Archive/7z/7zIn.h @@ -174,13 +174,14 @@ struct CDatabase: public CFolders HRESULT GetPath_Prop(unsigned index, PROPVARIANT *path) const throw(); }; + struct CInArchiveInfo { CArchiveVersion Version; - UInt64 StartPosition; - UInt64 StartPositionAfterHeader; - UInt64 DataStartPosition; - UInt64 DataStartPosition2; + UInt64 StartPosition; // in stream + UInt64 StartPositionAfterHeader; // in stream + UInt64 DataStartPosition; // in stream + UInt64 DataStartPosition2; // in stream. it's for headers CRecordVector<UInt64> FileInfoPopIDs; void Clear() @@ -193,6 +194,7 @@ struct CInArchiveInfo } }; + struct CDbEx: public CDatabase { CInArchiveInfo ArcInfo; @@ -202,6 +204,7 @@ struct CDbEx: public CDatabase UInt64 HeadersSize; UInt64 PhySize; + // UInt64 EndHeaderOffset; // relative to position after StartHeader (32 bytes) /* CRecordVector<size_t> SecureOffsets; @@ -255,6 +258,7 @@ struct CDbEx: public CDatabase HeadersSize = 0; PhySize = 0; + // EndHeaderOffset = 0; } bool CanUpdate() const @@ -349,6 +353,8 @@ class CInArchive UInt64 _arhiveBeginStreamPosition; UInt64 _fileEndPosition; + UInt64 _rangeLimit; // relative to position after StartHeader (32 bytes) + Byte _header[kHeaderSize]; UInt64 HeadersSize; diff --git a/CPP/7zip/Archive/7z/7zItem.h b/CPP/7zip/Archive/7z/7zItem.h index ee4aed3f..0f9fdada 100644 --- a/CPP/7zip/Archive/7z/7zItem.h +++ b/CPP/7zip/Archive/7z/7zItem.h @@ -50,7 +50,7 @@ public: { FOR_VECTOR(i, PackStreams) if (PackStreams[i] == packStream) - return i; + return (int)i; return -1; } @@ -58,7 +58,7 @@ public: { FOR_VECTOR(i, Bonds) if (Bonds[i].PackIndex == packStream) - return i; + return (int)i; return -1; } diff --git a/CPP/7zip/Archive/7z/7zOut.cpp b/CPP/7zip/Archive/7z/7zOut.cpp index aa977292..2786bf28 100644 --- a/CPP/7zip/Archive/7z/7zOut.cpp +++ b/CPP/7zip/Archive/7z/7zOut.cpp @@ -5,6 +5,7 @@ #include "../../../../C/7zCrc.h" #include "../../../Common/AutoPtr.h" +// #include "../../../Common/UTFConvert.h" #include "../../Common/StreamObjects.h" @@ -196,7 +197,7 @@ void COutArchive::WriteNumber(UInt64 value) break; } firstByte |= mask; - mask >>= 1; + mask = (Byte)(mask >> 1); } WriteByte(firstByte); for (; i > 0; i--) @@ -206,9 +207,9 @@ void COutArchive::WriteNumber(UInt64 value) } } -static UInt32 GetBigNumberSize(UInt64 value) +static unsigned GetBigNumberSize(UInt64 value) { - int i; + unsigned i; for (i = 1; i < 9; i++) if (value < (((UInt64)1 << (i * 7)))) break; @@ -264,18 +265,18 @@ void COutArchive::WriteFolder(const CFolder &folder) for (idSize = 1; idSize < sizeof(id); idSize++) if ((id >> (8 * idSize)) == 0) break; - idSize &= 0xF; + // idSize &= 0xF; // idSize is smaller than 16 already Byte temp[16]; for (unsigned t = idSize; t != 0; t--, id >>= 8) temp[t] = (Byte)(id & 0xFF); - Byte b = (Byte)(idSize); - bool isComplex = !coder.IsSimpleCoder(); + unsigned b = idSize; + const bool isComplex = !coder.IsSimpleCoder(); b |= (isComplex ? 0x10 : 0); - size_t propsSize = coder.Props.Size(); + const size_t propsSize = coder.Props.Size(); b |= ((propsSize != 0) ? 0x20 : 0); - temp[0] = b; + temp[0] = (Byte)b; WriteBytes(temp, idSize + 1); if (isComplex) { @@ -309,7 +310,7 @@ void COutArchive::WriteBoolVector(const CBoolVector &boolVector) { if (boolVector[i]) b |= mask; - mask >>= 1; + mask = (Byte)(mask >> 1); if (mask == 0) { WriteByte(b); @@ -476,7 +477,7 @@ void COutArchive::WriteAlignedBools(const CBoolVector &v, unsigned numDefined, B { const unsigned bvSize = (numDefined == v.Size()) ? 0 : Bv_GetSizeInBytes(v); const UInt64 dataSize = ((UInt64)numDefined << itemSizeShifts) + bvSize + 2; - SkipToAligned(3 + (unsigned)bvSize + (unsigned)GetBigNumberSize(dataSize), itemSizeShifts); + SkipToAligned(3 + bvSize + GetBigNumberSize(dataSize), itemSizeShifts); WriteByte(type); WriteNumber(dataSize); @@ -545,7 +546,39 @@ void COutArchive::WriteHeader( WriteByte(NID::kHeader); - // Archive Properties + /* + { + // It's example for per archive properies writing + + WriteByte(NID::kArchiveProperties); + + // you must use random 40-bit number that will identify you + // then you can use same kDeveloperID for any properties and methods + const UInt64 kDeveloperID = 0x123456789A; // change that value to real random 40-bit number + + #define GENERATE_7Z_ID(developerID, subID) (((UInt64)0x3F << 56) | ((UInt64)developerID << 16) | subID) + + { + const UInt64 kSubID = 0x1; // you can use small number for subID + const UInt64 kID = GENERATE_7Z_ID(kDeveloperID, kSubID); + WriteNumber(kID); + const unsigned kPropsSize = 3; // it's example size + WriteNumber(kPropsSize); + for (unsigned i = 0; i < kPropsSize; i++) + WriteByte((Byte)(i & 0xFF)); + } + { + const UInt64 kSubID = 0x2; // you can use small number for subID + const UInt64 kID = GENERATE_7Z_ID(kDeveloperID, kSubID); + WriteNumber(kID); + const unsigned kPropsSize = 5; // it's example size + WriteNumber(kPropsSize); + for (unsigned i = 0; i < kPropsSize; i++) + WriteByte((Byte)(i + 16)); + } + WriteByte(NID::kEnd); + } + */ if (db.Folders.Size() > 0) { @@ -637,7 +670,15 @@ void COutArchive::WriteHeader( const UString &name = db.Names[i]; if (!name.IsEmpty()) numDefined++; - namesDataSize += (name.Len() + 1) * 2; + const size_t numUtfChars = + /* + #if WCHAR_MAX > 0xffff + Get_Num_Utf16_chars_from_wchar_string(name.Ptr()); + #else + */ + name.Len(); + // #endif + namesDataSize += (numUtfChars + 1) * 2; } if (numDefined > 0) @@ -654,6 +695,25 @@ void COutArchive::WriteHeader( for (unsigned t = 0; t <= name.Len(); t++) { wchar_t c = name[t]; + + /* + #if WCHAR_MAX > 0xffff + if (c >= 0x10000) + { + c -= 0x10000; + if (c < (1 << 20)) + { + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + WriteByte((Byte)c0); + WriteByte((Byte)(c0 >> 8)); + c = 0xdc00 + (c & 0x3FF); + } + else + c = '_'; // we change character unsupported by UTF16 + } + #endif + */ + WriteByte((Byte)c); WriteByte((Byte)(c >> 8)); } @@ -855,7 +915,7 @@ HRESULT COutArchive::WriteDatabase( h.NextHeaderSize = headerSize; h.NextHeaderCRC = headerCRC; h.NextHeaderOffset = headerOffset; - RINOK(Stream->Seek(_prefixHeaderPos, STREAM_SEEK_SET, NULL)); + RINOK(Stream->Seek((Int64)_prefixHeaderPos, STREAM_SEEK_SET, NULL)); return WriteStartHeader(h); } } diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 6705fc00..b641d93f 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -55,8 +55,8 @@ struct CFilterMode #define PE_SIG 0x00004550 #define PE_OptHeader_Magic_32 0x10B #define PE_OptHeader_Magic_64 0x20B -#define PE_SectHeaderSize 40 -#define PE_SECT_EXECUTE 0x20000000 +// #define PE_SectHeaderSize 40 +// #define PE_SECT_EXECUTE 0x20000000 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) { @@ -254,10 +254,12 @@ static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1) return False; - unsigned numChannels = GetUi16(buf + 0x16); - unsigned bitsPerSample = GetUi16(buf + 0x22); - - if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256) + const unsigned numChannels = GetUi16(buf + 0x16); + const unsigned bitsPerSample = GetUi16(buf + 0x22); + if ((bitsPerSample & 0x7) != 0) + return False; + const UInt32 delta = (UInt32)numChannels * (bitsPerSample >> 3); + if (delta == 0 || delta > 256) return False; pos = 0x14 + subChunkSize; @@ -271,9 +273,6 @@ static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) subChunkSize = GetUi32(buf + pos + 4); if (GetUi32(buf + pos) == WAV_SUBCHUNK_data) { - unsigned delta = numChannels * (bitsPerSample >> 3); - if (delta >= 256) - return False; filterMode->Id = k_Delta; filterMode->Delta = delta; return True; @@ -418,7 +417,7 @@ static unsigned Get_FilterGroup_for_Folder( static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream, UInt64 position, UInt64 size, ICompressProgressInfo *progress) { - RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0)); + RINOK(inStream->Seek((Int64)position, STREAM_SEEK_SET, 0)); CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec); streamSpec->SetStream(inStream); @@ -642,13 +641,13 @@ struct CRefItem if (sortByType) { int slashPos = ui.Name.ReverseFind_PathSepar(); - NamePos = slashPos + 1; + NamePos = (unsigned)(slashPos + 1); int dotPos = ui.Name.ReverseFind_Dot(); if (dotPos <= slashPos) ExtensionPos = ui.Name.Len(); else { - ExtensionPos = dotPos + 1; + ExtensionPos = (unsigned)(dotPos + 1); if (ExtensionPos != ui.Name.Len()) { AString s; @@ -836,7 +835,7 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo { const wchar_t *ext; if (dotPos > slashPos) - ext = ui.Name.Ptr(dotPos + 1); + ext = ui.Name.Ptr((unsigned)(dotPos + 1)); else ext = ui.Name.RightPtr(0); @@ -1071,12 +1070,12 @@ static HRESULT MakeExeMethod(CCompressionMethodMode &mode, if (alignBits != 0) { if (alignBits > 2 || filterMode.Id == k_Delta) - nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits); + nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits); unsigned lc = 0; if (alignBits < 3) - lc = 3 - alignBits; + lc = (unsigned)(3 - alignBits); nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc); - nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits); + nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits); } } } @@ -1453,7 +1452,7 @@ public: UInt64 StartPos; const CFolders *Folders; - int FolderIndex; + unsigned FolderIndex; // bool send_UnpackSize; // UInt64 UnpackSize; @@ -1609,7 +1608,10 @@ HRESULT Update( CRecordVector<CFilterMode2> filters; CObjectVector<CSolidGroup> groups; + + #ifndef _7ZIP_ST bool thereAreRepacks = false; + #endif bool useFilters = options.UseFilters; if (useFilters) @@ -1636,7 +1638,7 @@ HRESULT Update( { int index = updateItems[i].IndexInArchive; if (index != -1) - fileIndexToUpdateIndexMap[(unsigned)index] = i; + fileIndexToUpdateIndexMap[(unsigned)index] = (int)i; } for (i = 0; i < db->NumFolders; i++) @@ -1656,7 +1658,7 @@ HRESULT Update( { indexInFolder++; int updateIndex = fileIndexToUpdateIndexMap[fi]; - if (updateIndex >= 0 && !updateItems[updateIndex].NewData) + if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData) { numCopyItems++; repackSize += file.Size; @@ -1688,7 +1690,9 @@ HRESULT Update( complexity += db->GetFolderFullPackSize(i); else { + #ifndef _7ZIP_ST thereAreRepacks = true; + #endif complexity += repackSize; if (inSizeForReduce2 < repackSize) inSizeForReduce2 = repackSize; @@ -1727,10 +1731,12 @@ HRESULT Update( #ifndef _7ZIP_ST CStreamBinder sb; + /* if (options.MultiThreadMixer) { RINOK(sb.CreateEvents()); } + */ #endif @@ -1742,7 +1748,9 @@ HRESULT Update( #ifdef EXTERNAL_CODECS threadDecoder.__externalCodecs = __externalCodecs; #endif - RINOK(threadDecoder.Create()); + WRes wres = threadDecoder.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } #endif @@ -1887,7 +1895,7 @@ HRESULT Update( if (ui.HasStream()) continue; } - else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream) + else if (ui.IndexInArchive != -1 && db->Files[(unsigned)ui.IndexInArchive].HasStream) continue; /* if (ui.TreeFolderIndex >= 0) @@ -1912,8 +1920,8 @@ HRESULT Update( } else { - GetFile(*db, ui.IndexInArchive, file, file2); - db->GetPath(ui.IndexInArchive, name); + GetFile(*db, (unsigned)ui.IndexInArchive, file, file2); + db->GetPath((unsigned)ui.IndexInArchive, name); } /* @@ -2065,7 +2073,7 @@ HRESULT Update( { indexInFolder++; int updateIndex = fileIndexToUpdateIndexMap[fi]; - if (updateIndex >= 0 && !updateItems[updateIndex].NewData) + if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData) needExtract = true; // decodeSize += file.Size; } @@ -2099,8 +2107,8 @@ HRESULT Update( { repackBase = threadDecoder.FosSpec; CMyComPtr<ISequentialOutStream> sbOutStream; - sb.CreateStreams(&sbInStream, &sbOutStream); - sb.ReInit(); + sb.CreateStreams2(sbInStream, sbOutStream); + RINOK(sb.Create_ReInit()); threadDecoder.FosSpec->_stream = sbOutStream; @@ -2171,7 +2179,9 @@ HRESULT Update( #ifndef _7ZIP_ST if (options.MultiThreadMixer) { - threadDecoder.Start(); + WRes wres = threadDecoder.Start(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } #endif } @@ -2197,7 +2207,11 @@ HRESULT Update( inStreamSizeCount.Release(); sbInStream.Release(); - threadDecoder.WaitExecuteFinish(); + { + WRes wres = threadDecoder.WaitExecuteFinish(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } HRESULT decodeRes = threadDecoder.Result; // if (res == k_My_HRESULT_CRC_ERROR) @@ -2259,7 +2273,7 @@ HRESULT Update( int updateIndex = fileIndexToUpdateIndexMap[fi]; if (updateIndex >= 0) { - const CUpdateItem &ui = updateItems[updateIndex]; + const CUpdateItem &ui = updateItems[(unsigned)updateIndex]; if (ui.NewData) continue; @@ -2347,7 +2361,7 @@ HRESULT Update( { int slashPos = ui.Name.ReverseFind_PathSepar(); int dotPos = ui.Name.ReverseFind_Dot(); - const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1); + const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : (unsigned)(dotPos + 1)); if (numSubFiles == 0) prevExtension = ext; else if (!StringsAreEqualNoCase(ext, prevExtension)) @@ -2403,8 +2417,8 @@ HRESULT Update( } else { - GetFile(*db, ui.IndexInArchive, file, file2); - db->GetPath(ui.IndexInArchive, name); + GetFile(*db, (unsigned)ui.IndexInArchive, file, file2); + db->GetPath((unsigned)ui.IndexInArchive, name); } if (file2.IsAnti || file.IsDir) return E_FAIL; diff --git a/CPP/7zip/Archive/7z/7zUpdate.h b/CPP/7zip/Archive/7z/7zUpdate.h index a7abf779..7c0f78a8 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.h +++ b/CPP/7zip/Archive/7z/7zUpdate.h @@ -31,7 +31,7 @@ struct CTreeFolder struct CUpdateItem { int IndexInArchive; - int IndexInClient; + unsigned IndexInClient; UInt64 CTime; UInt64 ATime; diff --git a/CPP/7zip/Archive/ApmHandler.cpp b/CPP/7zip/Archive/ApmHandler.cpp index 4d3f2728..73e5fcb6 100644 --- a/CPP/7zip/Archive/ApmHandler.cpp +++ b/CPP/7zip/Archive/ApmHandler.cpp @@ -240,7 +240,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) mainIndex = -1; break; } - mainIndex = i; + mainIndex = (int)i; } } if (mainIndex >= 0) diff --git a/CPP/7zip/Archive/ArHandler.cpp b/CPP/7zip/Archive/ArHandler.cpp index 09a62201..0bea8b4e 100644 --- a/CPP/7zip/Archive/ArHandler.cpp +++ b/CPP/7zip/Archive/ArHandler.cpp @@ -618,13 +618,15 @@ STDMETHODIMP CHandler::Open(IInStream *stream, _items.DeleteFrontal(1); for (unsigned i = 0; i < _items.Size(); i++) if (_items[i].Name.IsPrefixedBy("data.tar.")) + { if (_mainSubfile < 0) - _mainSubfile = i; + _mainSubfile = (int)i; else { _mainSubfile = -1; break; } + } } else { @@ -845,7 +847,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } REGISTER_ARC_I( - "Ar", "ar a deb lib", 0, 0xEC, + "Ar", "ar a deb udeb lib", 0, 0xEC, kSignature, 0, 0, diff --git a/CPP/7zip/Archive/ArchiveExports.cpp b/CPP/7zip/Archive/ArchiveExports.cpp index 28e9946d..6549b3d2 100644 --- a/CPP/7zip/Archive/ArchiveExports.cpp +++ b/CPP/7zip/Archive/ArchiveExports.cpp @@ -46,7 +46,7 @@ static inline HRESULT SetPropGUID(const GUID &guid, PROPVARIANT *value) return SetPropStrFromBin((const char *)&guid, sizeof(guid), value); } -int FindFormatCalssId(const GUID *clsid) +static int FindFormatCalssId(const GUID *clsid) { GUID cls = *clsid; CLS_ARC_ID_ITEM(cls) = 0; @@ -59,6 +59,7 @@ int FindFormatCalssId(const GUID *clsid) return -1; } +STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject) { COM_TRY_BEGIN @@ -89,6 +90,7 @@ STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject) return S_OK; } +STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value); STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN @@ -130,17 +132,20 @@ STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value COM_TRY_END } +STDAPI GetHandlerProperty(PROPID propID, PROPVARIANT *value); STDAPI GetHandlerProperty(PROPID propID, PROPVARIANT *value) { return GetHandlerProperty2(g_DefaultArcIndex, propID, value); } +STDAPI GetNumberOfFormats(UINT32 *numFormats); STDAPI GetNumberOfFormats(UINT32 *numFormats) { *numFormats = g_NumArcs; return S_OK; } +STDAPI GetIsArc(UInt32 formatIndex, Func_IsArc *isArc); STDAPI GetIsArc(UInt32 formatIndex, Func_IsArc *isArc) { *isArc = NULL; diff --git a/CPP/7zip/Archive/ArjHandler.cpp b/CPP/7zip/Archive/ArjHandler.cpp index fb9e3e7a..0e353dca 100644 --- a/CPP/7zip/Archive/ArjHandler.cpp +++ b/CPP/7zip/Archive/ArjHandler.cpp @@ -235,13 +235,13 @@ namespace NFileType namespace NFlags { const Byte kGarbled = 1 << 0; - const Byte kAnsiPage = 1 << 1; // or (OLD_SECURED_FLAG) obsolete + // const Byte kAnsiPage = 1 << 1; // or (OLD_SECURED_FLAG) obsolete const Byte kVolume = 1 << 2; const Byte kExtFile = 1 << 3; - const Byte kPathSym = 1 << 4; - const Byte kBackup = 1 << 5; // obsolete - const Byte kSecured = 1 << 6; - const Byte kDualName = 1 << 7; + // const Byte kPathSym = 1 << 4; + // const Byte kBackup = 1 << 5; // obsolete + // const Byte kSecured = 1 << 6; + // const Byte kDualName = 1 << 7; } namespace NHostOS @@ -375,6 +375,32 @@ HRESULT CArcHeader::Parse(const Byte *p, unsigned size) return S_OK; } + +struct CExtendedInfo +{ + UInt64 Size; + bool CrcError; + + void Clear() + { + Size = 0; + CrcError = false; + } + void ParseToPropVar(NCOM::CPropVariant &prop) const + { + if (Size != 0) + { + AString s; + s += "Extended:"; + s.Add_UInt32((UInt32)Size); + if (CrcError) + s += ":CRC_ERROR"; + prop = s; + } + } +}; + + struct CItem { AString Name; @@ -399,6 +425,8 @@ struct CItem // Byte LastChapter; UInt64 DataPosition; + + CExtendedInfo ExtendedInfo; bool IsEncrypted() const { return (Flags & NFlags::kGarbled) != 0; } bool IsDir() const { return (FileType == NFileType::kDirectory); } @@ -462,7 +490,7 @@ enum EErrorType { k_ErrorType_OK, k_ErrorType_Corrupted, - k_ErrorType_UnexpectedEnd, + k_ErrorType_UnexpectedEnd }; class CArc @@ -476,19 +504,22 @@ public: UInt64 NumFiles; CArcHeader Header; + CExtendedInfo ExtendedInfo; + HRESULT Open(); HRESULT GetNextItem(CItem &item, bool &filled); void Close() { IsArc = false; Error = k_ErrorType_OK; + ExtendedInfo.Clear(); } private: - UInt32 _blockSize; - Byte _block[kBlockSizeMax + 4]; + unsigned _blockSize; + CByteBuffer _block; - HRESULT ReadBlock(bool &filled, bool readSignature); - HRESULT SkipExtendedHeaders(); + HRESULT ReadBlock(bool &filled, CExtendedInfo *extendedInfo); + HRESULT SkipExtendedHeaders(CExtendedInfo &extendedInfo); HRESULT Read(void *data, size_t *size); }; @@ -503,14 +534,14 @@ HRESULT CArc::Read(void *data, size_t *size) { size_t _processed_ = (_size_); RINOK(Read(_dest_, &_processed_)); \ if (_processed_ != (_size_)) { Error = k_ErrorType_UnexpectedEnd; return S_OK; } } -HRESULT CArc::ReadBlock(bool &filled, bool readSignature) +HRESULT CArc::ReadBlock(bool &filled, CExtendedInfo *extendedInfo) { Error = k_ErrorType_OK; filled = false; Byte buf[4]; - unsigned signSize = readSignature ? 2 : 0; + const unsigned signSize = extendedInfo ? 0 : 2; READ_STREAM(buf, signSize + 2) - if (readSignature) + if (!extendedInfo) if (buf[0] != kSig0 || buf[1] != kSig1) { Error = k_ErrorType_Corrupted; @@ -519,28 +550,48 @@ HRESULT CArc::ReadBlock(bool &filled, bool readSignature) _blockSize = Get16(buf + signSize); if (_blockSize == 0) // end of archive return S_OK; - if (_blockSize < kBlockSizeMin || - _blockSize > kBlockSizeMax) + + if (!extendedInfo) + if (_blockSize < kBlockSizeMin || _blockSize > kBlockSizeMax) + { + Error = k_ErrorType_Corrupted; + return S_OK; + } + + const size_t readSize = _blockSize + 4; + if (readSize > _block.Size()) { - Error = k_ErrorType_Corrupted; - return S_OK; + // extended data size is limited by (64 KB) + // _blockSize is less than 64 KB + const size_t upSize = (_blockSize > kBlockSizeMax ? (1 << 16) : kBlockSizeMax); + _block.Alloc(upSize + 4); } - READ_STREAM(_block, _blockSize + 4); + + if (extendedInfo) + extendedInfo->Size += _blockSize; + + READ_STREAM(_block, readSize); if (Get32(_block + _blockSize) != CrcCalc(_block, _blockSize)) { - Error = k_ErrorType_Corrupted; - return S_OK; + if (extendedInfo) + extendedInfo->CrcError = true; + else + { + Error = k_ErrorType_Corrupted; + return S_OK; + } } filled = true; return S_OK; } -HRESULT CArc::SkipExtendedHeaders() +HRESULT CArc::SkipExtendedHeaders(CExtendedInfo &extendedInfo) { + extendedInfo.Clear(); for (UInt32 i = 0;; i++) { bool filled; - RINOK(ReadBlock(filled, false)); + RINOK(ReadBlock(filled, &extendedInfo)); if (!filled) return S_OK; if (Callback && (i & 0xFF) == 0) @@ -551,17 +602,17 @@ HRESULT CArc::SkipExtendedHeaders() HRESULT CArc::Open() { bool filled; - RINOK(ReadBlock(filled, true)); + RINOK(ReadBlock(filled, NULL)); // (extendedInfo = NULL) if (!filled) return S_FALSE; RINOK(Header.Parse(_block, _blockSize)); IsArc = true; - return SkipExtendedHeaders(); + return SkipExtendedHeaders(ExtendedInfo); } HRESULT CArc::GetNextItem(CItem &item, bool &filled) { - RINOK(ReadBlock(filled, true)); + RINOK(ReadBlock(filled, NULL)); // (extendedInfo = NULL) if (!filled) return S_OK; filled = false; @@ -576,7 +627,7 @@ HRESULT CArc::GetNextItem(CItem &item, bool &filled) extraData = GetUi32(_block + pos); */ - RINOK(SkipExtendedHeaders()); + RINOK(SkipExtendedHeaders(item.ExtendedInfo)); filled = true; return S_OK; } @@ -603,7 +654,8 @@ static const Byte kArcProps[] = kpidCTime, kpidMTime, kpidHostOS, - kpidComment + kpidComment, + kpidCharacts }; static const Byte kProps[] = @@ -619,7 +671,8 @@ static const Byte kProps[] = kpidCRC, kpidMethod, kpidHostOS, - kpidComment + kpidComment, + kpidCharacts }; IMP_IInArchive_Props @@ -671,10 +724,14 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { case k_ErrorType_UnexpectedEnd: v |= kpv_ErrorFlags_UnexpectedEnd; break; case k_ErrorType_Corrupted: v |= kpv_ErrorFlags_HeadersError; break; + case k_ErrorType_OK: + default: + break; } prop = v; break; } + case kpidCharacts: _arc.ExtendedInfo.ParseToPropVar(prop); break; } prop.Detach(value); return S_OK; @@ -706,6 +763,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidHostOS: SetHostOS(item.HostOS, prop); break; case kpidMTime: SetTime(item.MTime, prop); break; case kpidComment: SetUnicodeString(item.Comment, prop); break; + case kpidCharacts: item.ExtendedInfo.ParseToPropVar(prop); break; } prop.Detach(value); return S_OK; diff --git a/CPP/7zip/Archive/Base64Handler.cpp b/CPP/7zip/Archive/Base64Handler.cpp new file mode 100644 index 00000000..63b4552e --- /dev/null +++ b/CPP/7zip/Archive/Base64Handler.cpp @@ -0,0 +1,511 @@ +// Base64Handler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../../Common/ComTry.h" +#include "../../Common/MyBuffer.h" +#include "../../Common/IntToString.h" +#include "../../Common/MyVector.h" + +#include "../../Windows/PropVariant.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" +#include "../Common/InBuffer.h" + +/* +spaces: + 9(TAB),10(LF),13(CR),32(SPACE) + Non-breaking space: + 0xa0 : Unicode, Windows code pages 1250-1258 + 0xff (unused): DOS code pages + +end of stream markers: '=' (0x3d): + "=" , if numBytes (% 3 == 2) + "==" , if numBytes (% 3 == 1) +*/ + + +static const Byte k_Base64Table[256] = +{ + 66,77,77,77,77,77,77,77,77,65,65,77,77,65,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 65,77,77,77,77,77,77,77,77,77,77,62,77,77,77,63, + 52,53,54,55,56,57,58,59,60,61,77,77,77,64,77,77, + 77, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,77,77,77,77,77, + 77,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 65,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, + 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77 +}; + +static const unsigned k_Code_Equals = 64; +static const unsigned k_Code_Space = 65; +static const unsigned k_Code_Zero = 66; + +API_FUNC_static_IsArc IsArc_Base64(const Byte *p, size_t size) +{ + size_t num = 0; + size_t firstSpace = 0; + + for (;;) + { + if (size == 0) + return k_IsArc_Res_NEED_MORE; + UInt32 c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c < 64) + { + num++; + continue; + } + + if (c == k_Code_Space) + { + if (p[-1] == ' ' && firstSpace == 0) + firstSpace = num; + continue; + } + + if (c != k_Code_Equals) + return k_IsArc_Res_NO; + break; + } + + { + // we try to redece false positive detection here. + // we don't expect space character in starting base64 line + const unsigned kNumExpectedNonSpaceSyms = 20; + if (firstSpace != 0 && firstSpace < num && firstSpace < kNumExpectedNonSpaceSyms) + return k_IsArc_Res_NO; + } + + num &= 3; + + if (num <= 1) + return k_IsArc_Res_NO; + if (num != 3) + { + if (size == 0) + return k_IsArc_Res_NEED_MORE; + UInt32 c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c != k_Code_Equals) + return k_IsArc_Res_NO; + } + + for (;;) + { + if (size == 0) + return k_IsArc_Res_YES; + UInt32 c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c == k_Code_Space) + continue; + return k_IsArc_Res_NO; + } +} +} + + +enum EBase64Res +{ + k_Base64_RES_MaybeFinished, + k_Base64_RES_Finished, + k_Base64_RES_NeedMoreInput, + k_Base64_RES_UnexpectedChar +}; + + +static EBase64Res Base64ToBin(Byte *p, size_t size, const Byte **srcEnd, Byte **destEnd) +{ + Byte *dest = p; + UInt32 val = 1; + EBase64Res res = k_Base64_RES_NeedMoreInput; + + for (;;) + { + if (size == 0) + { + if (val == 1) + res = k_Base64_RES_MaybeFinished; + break; + } + UInt32 c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c < 64) + { + val = (val << 6) | c; + if ((val & ((UInt32)1 << 24)) == 0) + continue; + dest[0] = (Byte)(val >> 16); + dest[1] = (Byte)(val >> 8); + dest[2] = (Byte)(val); + dest += 3; + val = 1; + continue; + } + + if (c == k_Code_Space) + continue; + + if (c == k_Code_Equals) + { + if (val >= (1 << 12)) + { + if (val & (1 << 18)) + { + res = k_Base64_RES_Finished; + break; + } + if (size == 0) + break; + c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c == k_Code_Equals) + { + res = k_Base64_RES_Finished; + break; + } + } + } + + p--; + res = k_Base64_RES_UnexpectedChar; + break; + } + + if (val >= ((UInt32)1 << 12)) + { + if (val & (1 << 18)) + { + *dest++ = (Byte)(val >> 10); + val <<= 2; + } + *dest++ = (Byte)(val >> 4); + } + + *srcEnd = p; + *destEnd = dest; + return res; +} + + +static const Byte *Base64_SkipSpaces(const Byte *p, size_t size) +{ + for (;;) + { + if (size == 0) + return p; + UInt32 c = k_Base64Table[(Byte)(*p++)]; + size--; + if (c == k_Code_Space) + continue; + return p - 1; + } +} + + +// the following function is used by DmgHandler.cpp + +Byte *Base64ToBin(Byte *dest, const char *src); +Byte *Base64ToBin(Byte *dest, const char *src) +{ + UInt32 val = 1; + + for (;;) + { + UInt32 c = k_Base64Table[(Byte)(*src++)]; + + if (c < 64) + { + val = (val << 6) | c; + if ((val & ((UInt32)1 << 24)) == 0) + continue; + dest[0] = (Byte)(val >> 16); + dest[1] = (Byte)(val >> 8); + dest[2] = (Byte)(val); + dest += 3; + val = 1; + continue; + } + + if (c == k_Code_Space) + continue; + + if (c == k_Code_Equals) + break; + + if (c == k_Code_Zero && val == 1) // end of string + return dest; + + return NULL; + } + + if (val < (1 << 12)) + return NULL; + + if (val & (1 << 18)) + { + *dest++ = (Byte)(val >> 10); + val <<= 2; + } + else if (k_Base64Table[(Byte)(*src++)] != k_Code_Equals) + return NULL; + *dest++ = (Byte)(val >> 4); + + for (;;) + { + Byte c = k_Base64Table[(Byte)(*src++)]; + if (c == k_Code_Space) + continue; + if (c == k_Code_Zero) + return dest; + return NULL; + } +} + + +namespace NArchive { +namespace NBase64 { + +class CHandler: + public IInArchive, + public CMyUnknownImp +{ + bool _isArc; + UInt64 _phySize; + size_t _size; + EBase64Res _sres; + CByteBuffer _data; +public: + MY_UNKNOWN_IMP1(IInArchive) + INTERFACE_IInArchive(;) +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize, +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps_NO_Table + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) +{ + NWindows::NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_phySize != 0) prop = _phySize; break; + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + if (_sres == k_Base64_RES_NeedMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + if (v != 0) + prop = v; + break; + } + } + prop.Detach(value); + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + // COM_TRY_BEGIN + NWindows::NCOM::CPropVariant prop; + switch (propID) + { + case kpidSize: prop = (UInt64)_size; break; + case kpidPackSize: prop = _phySize; break; + } + prop.Detach(value); + return S_OK; + // COM_TRY_END +} + + +static HRESULT ReadStream_OpenProgress(ISequentialInStream *stream, void *data, size_t size, IArchiveOpenCallback *openCallback) throw() +{ + UInt64 bytes = 0; + while (size != 0) + { + const UInt32 kBlockSize = ((UInt32)1 << 24); + UInt32 curSize = (size < kBlockSize) ? (UInt32)size : kBlockSize; + UInt32 processedSizeLoc; + RINOK(stream->Read(data, curSize, &processedSizeLoc)); + if (processedSizeLoc == 0) + return E_FAIL; + data = (void *)((Byte *)data + processedSizeLoc); + size -= processedSizeLoc; + bytes += processedSizeLoc; + const UInt64 files = 1; + RINOK(openCallback->SetCompleted(&files, &bytes)); + } + return S_OK; +} + + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openCallback) +{ + COM_TRY_BEGIN + { + Close(); + { + const unsigned kStartSize = 1 << 12; + _data.Alloc(kStartSize); + size_t size = kStartSize; + RINOK(ReadStream(stream, _data, &size)); + UInt32 isArcRes = IsArc_Base64(_data, size); + if (isArcRes == k_IsArc_Res_NO) + return S_FALSE; + } + _isArc = true; + + UInt64 packSize64; + RINOK(stream->Seek(0, STREAM_SEEK_END, &packSize64)); + + if (packSize64 == 0) + return S_FALSE; + + size_t curSize = 1 << 16; + if (curSize > packSize64) + curSize = (size_t)packSize64; + const unsigned kLogStep = 4; + + for (;;) + { + RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL)); + + _data.Alloc(curSize); + RINOK(ReadStream_OpenProgress(stream, _data, curSize, openCallback)); + + const Byte *srcEnd; + Byte *dest; + _sres = Base64ToBin(_data, curSize, &srcEnd, &dest); + _size = dest - _data; + size_t mainSize = srcEnd - _data; + _phySize = mainSize; + if (_sres == k_Base64_RES_UnexpectedChar) + break; + if (curSize != mainSize) + { + const Byte *end2 = Base64_SkipSpaces(srcEnd, curSize - mainSize); + if ((size_t)(end2 - _data) != curSize) + break; + _phySize = curSize; + } + + if (curSize == packSize64) + break; + + UInt64 curSize64 = packSize64; + if (curSize < (packSize64 >> kLogStep)) + curSize64 = (UInt64)curSize << kLogStep; + curSize = (size_t)curSize64; + if (curSize != curSize64) + return E_OUTOFMEMORY; + } + if (_size == 0) + return S_FALSE; + return S_OK; + } + COM_TRY_END +} + +STDMETHODIMP CHandler::Close() +{ + _phySize = 0; + _size = 0; + _isArc = false; + _sres = k_Base64_RES_MaybeFinished; + _data.Free(); + return S_OK; +} + + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = 1; + if (numItems == 0) + return S_OK; + if (numItems != 1 || *indices != 0) + return E_INVALIDARG; + + RINOK(extractCallback->SetTotal(_size)); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr<ICompressProgressInfo> progress = lps; + lps->Init(extractCallback, false); + + { + lps->InSize = lps->OutSize = 0; + RINOK(lps->SetCur()); + + CMyComPtr<ISequentialOutStream> realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + if (realOutStream) + { + RINOK(WriteStream(realOutStream, (const Byte *)_data, _size)); + realOutStream.Release(); + } + + Int32 opRes = NExtract::NOperationResult::kOK; + + if (_sres != k_Base64_RES_Finished) + { + if (_sres == k_Base64_RES_NeedMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_sres == k_Base64_RES_UnexpectedChar) + opRes = NExtract::NOperationResult::kDataError; + } + + RINOK(extractCallback->SetOperationResult(opRes)); + } + + lps->InSize = _phySize; + lps->OutSize = _size; + return lps->SetCur(); + + COM_TRY_END +} + +REGISTER_ARC_I_NO_SIG( + "Base64", "b64", 0, 0xC5, + 0, + NArcInfoFlags::kKeepName | NArcInfoFlags::kStartOpen | NArcInfoFlags::kByExtOnlyOpen, + IsArc_Base64) + +}} diff --git a/CPP/7zip/Archive/Cab/CabHandler.cpp b/CPP/7zip/Archive/Cab/CabHandler.cpp index 4395ae1a..9f50d3aa 100644 --- a/CPP/7zip/Archive/Cab/CabHandler.cpp +++ b/CPP/7zip/Archive/Cab/CabHandler.cpp @@ -309,11 +309,14 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMethod: { - UInt32 realFolderIndex = item.GetFolderIndex(db.Folders.Size()); - const CFolder &folder = db.Folders[realFolderIndex]; - char s[kMethodNameBufSize];; - SetMethodName(s, folder.GetMethod(), folder.MethodMinor); - prop = s; + const int realFolderIndex = item.GetFolderIndex(db.Folders.Size()); + if (realFolderIndex >= 0) + { + const CFolder &folder = db.Folders[(unsigned)realFolderIndex]; + char s[kMethodNameBufSize];; + SetMethodName(s, folder.GetMethod(), folder.MethodMinor); + prop = s; + } break; } @@ -341,7 +344,8 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, CInArchive archive; CMyComPtr<IArchiveOpenVolumeCallback> openVolumeCallback; - callback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&openVolumeCallback); + if (callback) + callback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&openVolumeCallback); CMyComPtr<IInStream> nextStream = inStream; bool prevChecked = false; @@ -420,7 +424,10 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, } } - RINOK(callback->SetCompleted(&numItems, NULL)); + if (callback) + { + RINOK(callback->SetCompleted(&numItems, NULL)); + } nextStream = NULL; @@ -1007,7 +1014,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; } - unsigned startIndex2 = m_Database.FolderStartFileIndex[folderIndex]; + const unsigned startIndex2 = m_Database.FolderStartFileIndex[(unsigned)folderIndex]; unsigned startIndex = startIndex2; extractStatuses.Clear(); for (; startIndex < index; startIndex++) @@ -1037,8 +1044,10 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, CFolderOutStream *cabFolderOutStream = new CFolderOutStream; CMyComPtr<ISequentialOutStream> outStream(cabFolderOutStream); - unsigned folderIndex2 = item.GetFolderIndex(db.Folders.Size()); - const CFolder &folder = db.Folders[folderIndex2]; + const int folderIndex2 = item.GetFolderIndex(db.Folders.Size()); + if (folderIndex2 < 0) + return E_FAIL; + const CFolder &folder = db.Folders[(unsigned)folderIndex2]; cabFolderOutStream->Init(&m_Database, &extractStatuses, startIndex2, curUnpack, extractCallback, testMode); @@ -1107,12 +1116,14 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, } const CDatabaseEx &db2 = m_Database.Volumes[volIndex]; - const CFolder &folder2 = db2.Folders[locFolderIndex]; + if (locFolderIndex < 0) + return E_FAIL; + const CFolder &folder2 = db2.Folders[(unsigned)locFolderIndex]; if (bl == 0) { cabBlockInStreamSpec->ReservedSize = db2.ArcInfo.GetDataBlockReserveSize(); - RINOK(db2.Stream->Seek(db2.StartPosition + folder2.DataStart, STREAM_SEEK_SET, NULL)); + RINOK(db2.Stream->Seek((Int64)(db2.StartPosition + folder2.DataStart), STREAM_SEEK_SET, NULL)); } if (bl == folder2.NumDataBlocks) @@ -1242,7 +1253,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (res != S_OK || cabFolderOutStream->NeedMoreWrite()) { - RINOK(cabFolderOutStream->FlushCorrupted(folderIndex2)); + RINOK(cabFolderOutStream->FlushCorrupted((unsigned)folderIndex2)); } totalUnPacked += curUnpack; diff --git a/CPP/7zip/Archive/Cab/CabIn.cpp b/CPP/7zip/Archive/Cab/CabIn.cpp index ca0052bf..e11ce9d0 100644 --- a/CPP/7zip/Archive/Cab/CabIn.cpp +++ b/CPP/7zip/Archive/Cab/CabIn.cpp @@ -114,8 +114,8 @@ HRESULT CSignatureFinder::Find() Byte b = Signature[0]; for (;;) { - if (*p == b) break; p++; - if (*p == b) break; p++; + if (*p == b) { break; } p++; + if (*p == b) { break; } p++; } Pos = (UInt32)(p - Buf); if (End - Pos < _HeaderSize) @@ -311,7 +311,7 @@ HRESULT CInArchive::Open2(CDatabaseEx &db, const UInt64 *searchHeaderSizeLimit) { // printf("\n!!! Seek Error !!!!\n"); // fflush(stdout); - RINOK(db.Stream->Seek(db.StartPosition + ai.FileHeadersOffset, STREAM_SEEK_SET, NULL)); + RINOK(db.Stream->Seek((Int64)(db.StartPosition + ai.FileHeadersOffset), STREAM_SEEK_SET, NULL)); limitedStreamSpec->Init(ai.Size - ai.FileHeadersOffset); _inBuffer.Init(); } diff --git a/CPP/7zip/Archive/Cab/CabIn.h b/CPP/7zip/Archive/Cab/CabIn.h index a1fc6bdc..39586d12 100644 --- a/CPP/7zip/Archive/Cab/CabIn.h +++ b/CPP/7zip/Archive/Cab/CabIn.h @@ -100,7 +100,7 @@ struct CDatabase int GetNumberOfNewFolders() const { - int res = Folders.Size(); + int res = (int)Folders.Size(); if (IsTherePrevFolder()) res--; return res; diff --git a/CPP/7zip/Archive/Cab/CabItem.h b/CPP/7zip/Archive/Cab/CabItem.h index 9b513202..9a912d5e 100644 --- a/CPP/7zip/Archive/Cab/CabItem.h +++ b/CPP/7zip/Archive/Cab/CabItem.h @@ -56,8 +56,8 @@ struct CItem if (ContinuedFromPrev()) return 0; if (ContinuedToNext()) - return numFolders - 1; - return FolderIndex; + return (int)numFolders - 1; + return (int)FolderIndex; } }; diff --git a/CPP/7zip/Archive/Chm/ChmHandler.cpp b/CPP/7zip/Archive/Chm/ChmHandler.cpp index 7ffdafe0..03e7ddd2 100644 --- a/CPP/7zip/Archive/Chm/ChmHandler.cpp +++ b/CPP/7zip/Archive/Chm/ChmHandler.cpp @@ -145,10 +145,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMethod: { if (!item.IsDir()) + { if (item.Section == 0) prop = "Copy"; else if (item.Section < m_Database.Sections.Size()) prop = m_Database.Sections[(unsigned)item.Section].GetMethodName(); + } break; } case kpidBlock: diff --git a/CPP/7zip/Archive/Chm/ChmIn.cpp b/CPP/7zip/Archive/Chm/ChmIn.cpp index 7e3f155b..f4916b68 100644 --- a/CPP/7zip/Archive/Chm/ChmIn.cpp +++ b/CPP/7zip/Archive/Chm/ChmIn.cpp @@ -13,7 +13,6 @@ #include "ChmIn.h" -#define Get16(p) GetUi16(p) #define Get32(p) GetUi32(p) #define Get64(p) GetUi64(p) @@ -117,7 +116,7 @@ UString CSectionInfo::GetMethodName() const if (!IsLzx()) { UString temp; - if (ConvertUTF8ToUnicode(Name, temp)) + ConvertUTF8ToUnicode(Name, temp); s += temp; s += ": "; } diff --git a/CPP/7zip/Archive/ComHandler.cpp b/CPP/7zip/Archive/ComHandler.cpp index d2dc6c5f..a1f643b7 100644 --- a/CPP/7zip/Archive/ComHandler.cpp +++ b/CPP/7zip/Archive/ComHandler.cpp @@ -36,7 +36,7 @@ enum EType k_Type_Msp, k_Type_Doc, k_Type_Ppt, - k_Type_Xls, + k_Type_Xls }; static const char * const kExtensions[] = @@ -51,10 +51,10 @@ static const char * const kExtensions[] = namespace NFatID { - static const UInt32 kFree = 0xFFFFFFFF; + // static const UInt32 kFree = 0xFFFFFFFF; static const UInt32 kEndOfChain = 0xFFFFFFFE; - static const UInt32 kFatSector = 0xFFFFFFFD; - static const UInt32 kMatSector = 0xFFFFFFFC; + // static const UInt32 kFatSector = 0xFFFFFFFD; + // static const UInt32 kMatSector = 0xFFFFFFFC; static const UInt32 kMaxValue = 0xFFFFFFFA; } @@ -62,9 +62,9 @@ namespace NItemType { static const Byte kEmpty = 0; static const Byte kStorage = 1; - static const Byte kStream = 2; - static const Byte kLockBytes = 3; - static const Byte kProperty = 4; + // static const Byte kStream = 2; + // static const Byte kLockBytes = 3; + // static const Byte kProperty = 4; static const Byte kRootStorage = 5; } @@ -298,7 +298,7 @@ static bool CompoundMsiNameToFileName(const UString &name, UString &res) for (unsigned i = 0; i < name.Len(); i++) { wchar_t c = name[i]; - if (c < k_Msi_StartUnicodeChar || c > k_Msi_StartUnicodeChar + k_Msi_UnicodeRange) + if (c < (wchar_t)k_Msi_StartUnicodeChar || c > (wchar_t)(k_Msi_StartUnicodeChar + k_Msi_UnicodeRange)) return false; /* if (i == 0) @@ -578,11 +578,10 @@ HRESULT CDatabase::Open(IInStream *inStream) { // bool isThereExt = (msiName.Find(L'.') >= 0); bool isMsiSpec = (msiName[0] == k_Msi_SpecChar); - if (msiName.Len() >= 4 && StringsAreEqualNoCase_Ascii(msiName.RightPtr(4), ".cab") - || !isMsiSpec && msiName.Len() >= 3 && StringsAreEqualNoCase_Ascii(msiName.RightPtr(3), "exe") - // || !isMsiSpec && !isThereExt + if ((msiName.Len() >= 4 && StringsAreEqualNoCase_Ascii(msiName.RightPtr(4), ".cab")) + || (!isMsiSpec && msiName.Len() >= 3 && StringsAreEqualNoCase_Ascii(msiName.RightPtr(3), "exe")) + // || (!isMsiSpec && !isThereExt) ) - { numCabs++; MainSubfile = i; diff --git a/CPP/7zip/Archive/Common/CoderMixer2.cpp b/CPP/7zip/Archive/Common/CoderMixer2.cpp index c7002121..c8b67bd4 100644 --- a/CPP/7zip/Archive/Common/CoderMixer2.cpp +++ b/CPP/7zip/Archive/Common/CoderMixer2.cpp @@ -137,7 +137,7 @@ bool CBondsChecks::CheckCoder(unsigned coderIndex) return false; _coderUsed[coderIndex] = true; - UInt32 start = BindInfo->Coder_to_Stream[coderIndex]; + const UInt32 start = BindInfo->Coder_to_Stream[coderIndex]; for (unsigned i = 0; i < coder.NumStreams; i++) { @@ -146,10 +146,10 @@ bool CBondsChecks::CheckCoder(unsigned coderIndex) if (BindInfo->IsStream_in_PackStreams(ind)) continue; - int bond = BindInfo->FindBond_for_PackStream(ind); + const int bond = BindInfo->FindBond_for_PackStream(ind); if (bond < 0) return false; - if (!CheckCoder(BindInfo->Bonds[bond].UnpackIndex)) + if (!CheckCoder(BindInfo->Bonds[(unsigned)bond].UnpackIndex)) return false; } @@ -246,15 +246,15 @@ bool CMixer::Is_UnpackSize_Correct_for_Coder(UInt32 coderIndex) if (coderIndex == _bi.UnpackCoder) return true; - int bond = _bi.FindBond_for_UnpackStream(coderIndex); + const int bond = _bi.FindBond_for_UnpackStream(coderIndex); if (bond < 0) throw 20150213; /* UInt32 coderIndex, coderStreamIndex; - _bi.GetCoder_for_Stream(_bi.Bonds[bond].PackIndex, coderIndex, coderStreamIndex); + _bi.GetCoder_for_Stream(_bi.Bonds[(unsigned)bond].PackIndex, coderIndex, coderStreamIndex); */ - UInt32 nextCoder = _bi.Stream_to_Coder[_bi.Bonds[bond].PackIndex]; + const UInt32 nextCoder = _bi.Stream_to_Coder[_bi.Bonds[(unsigned)bond].PackIndex]; if (!IsFilter_Vector[nextCoder]) return false; @@ -267,11 +267,11 @@ bool CMixer::Is_PackSize_Correct_for_Stream(UInt32 streamIndex) if (_bi.IsStream_in_PackStreams(streamIndex)) return true; - int bond = _bi.FindBond_for_PackStream(streamIndex); + const int bond = _bi.FindBond_for_PackStream(streamIndex); if (bond < 0) throw 20150213; - UInt32 nextCoder = _bi.Bonds[bond].UnpackIndex; + const UInt32 nextCoder = _bi.Bonds[(unsigned)bond].UnpackIndex; if (!IsFilter_Vector[nextCoder]) return false; @@ -281,8 +281,8 @@ bool CMixer::Is_PackSize_Correct_for_Stream(UInt32 streamIndex) bool CMixer::Is_PackSize_Correct_for_Coder(UInt32 coderIndex) { - UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; - UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; + const UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; + const UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; for (UInt32 i = 0; i < numStreams; i++) if (!Is_PackSize_Correct_for_Stream(startIndex + i)) return false; @@ -293,19 +293,19 @@ bool CMixer::IsThere_ExternalCoder_in_PackTree(UInt32 coderIndex) { if (IsExternal_Vector[coderIndex]) return true; - UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; - UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; + const UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; + const UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; for (UInt32 i = 0; i < numStreams; i++) { - UInt32 si = startIndex + i; + const UInt32 si = startIndex + i; if (_bi.IsStream_in_PackStreams(si)) continue; - int bond = _bi.FindBond_for_PackStream(si); + const int bond = _bi.FindBond_for_PackStream(si); if (bond < 0) throw 20150213; - if (IsThere_ExternalCoder_in_PackTree(_bi.Bonds[bond].UnpackIndex)) + if (IsThere_ExternalCoder_in_PackTree(_bi.Bonds[(unsigned)bond].UnpackIndex)) return true; } return false; @@ -360,7 +360,7 @@ CCoder &CMixerST::GetCoder(unsigned index) return _coders[index]; } -void CMixerST::ReInit() {} +HRESULT CMixerST::ReInit2() { return S_OK; } HRESULT CMixerST::GetInStream2( ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ @@ -444,18 +444,18 @@ HRESULT CMixerST::GetInStream( } } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( true, // forInputStream inStreamIndex); if (bond < 0) return E_INVALIDARG; RINOK(GetInStream2(inStreams, /* inSizes, */ - _bi.Bonds[bond].Get_OutIndex(EncodeMode), &seqInStream)); + _bi.Bonds[(unsigned)bond].Get_OutIndex(EncodeMode), &seqInStream)); while (_binderStreams.Size() <= (unsigned)bond) _binderStreams.AddNew(); - CStBinderStream &bs = _binderStreams[bond]; + CStBinderStream &bs = _binderStreams[(unsigned)bond]; if (bs.StreamRef || bs.InStreamSpec) return E_NOTIMPL; @@ -498,13 +498,13 @@ HRESULT CMixerST::GetOutStream( } } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( false, // forInputStream outStreamIndex); if (bond < 0) return E_INVALIDARG; - UInt32 inStreamIndex = _bi.Bonds[bond].Get_InIndex(EncodeMode); + UInt32 inStreamIndex = _bi.Bonds[(unsigned)bond].Get_InIndex(EncodeMode); UInt32 coderIndex = inStreamIndex; UInt32 coderStreamIndex = 0; @@ -560,7 +560,7 @@ HRESULT CMixerST::GetOutStream( while (_binderStreams.Size() <= (unsigned)bond) _binderStreams.AddNew(); - CStBinderStream &bs = _binderStreams[bond]; + CStBinderStream &bs = _binderStreams[(unsigned)bond]; if (bs.StreamRef || bs.OutStreamSpec) return E_NOTIMPL; @@ -610,13 +610,13 @@ HRESULT CMixerST::FinishStream(UInt32 streamIndex) return S_OK; } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( false, // forInputStream streamIndex); if (bond < 0) return E_INVALIDARG; - UInt32 inStreamIndex = _bi.Bonds[bond].Get_InIndex(EncodeMode); + UInt32 inStreamIndex = _bi.Bonds[(unsigned)bond].Get_InIndex(EncodeMode); UInt32 coderIndex = inStreamIndex; UInt32 coderStreamIndex = 0; @@ -654,7 +654,7 @@ void CMixerST::SelectMainCoder(bool useFirst) unsigned ci = _bi.UnpackCoder; int firstNonFilter = -1; - int firstAllowed = ci; + unsigned firstAllowed = ci; for (;;) { @@ -674,7 +674,7 @@ void CMixerST::SelectMainCoder(bool useFirst) UInt32 st = _bi.Coder_to_Stream[ci]; if (_bi.IsStream_in_PackStreams(st)) break; - int bond = _bi.FindBond_for_PackStream(st); + const int bond = _bi.FindBond_for_PackStream(st); if (bond < 0) throw 20150213; @@ -682,15 +682,15 @@ void CMixerST::SelectMainCoder(bool useFirst) break; if (firstNonFilter == -1 && !IsFilter_Vector[ci]) - firstNonFilter = ci; + firstNonFilter = (int)ci; - ci = _bi.Bonds[bond].UnpackIndex; + ci = _bi.Bonds[(unsigned)bond].UnpackIndex; } if (useFirst) ci = firstAllowed; else if (firstNonFilter >= 0) - ci = firstNonFilter; + ci = (unsigned)firstNonFilter; MainCoderIndex = ci; } @@ -919,7 +919,8 @@ HRESULT CMixerMT::SetBindInfo(const CBindInfo &bindInfo) _streamBinders.Clear(); FOR_VECTOR (i, _bi.Bonds) { - RINOK(_streamBinders.AddNew().CreateEvents()); + // RINOK(_streamBinders.AddNew().CreateEvents()); + _streamBinders.AddNew(); } return S_OK; } @@ -941,10 +942,13 @@ CCoder &CMixerMT::GetCoder(unsigned index) return _coders[index]; } -void CMixerMT::ReInit() +HRESULT CMixerMT::ReInit2() { FOR_VECTOR (i, _streamBinders) - _streamBinders[i].ReInit(); + { + RINOK(_streamBinders[i].Create_ReInit()); + } + return S_OK; } void CMixerMT::SelectMainCoder(bool useFirst) @@ -962,10 +966,10 @@ void CMixerMT::SelectMainCoder(bool useFirst) UInt32 st = _bi.Coder_to_Stream[ci]; if (_bi.IsStream_in_PackStreams(st)) break; - int bond = _bi.FindBond_for_PackStream(st); + const int bond = _bi.FindBond_for_PackStream(st); if (bond < 0) throw 20150213; - ci = _bi.Bonds[bond].UnpackIndex; + ci = _bi.Bonds[(unsigned)bond].UnpackIndex; } MainCoderIndex = ci; @@ -1012,9 +1016,9 @@ HRESULT CMixerMT::Init(ISequentialInStream * const *inStreams, ISequentialOutStr outCoderStreamIndex = EncodeMode ? coderStreamIndex : 0; } - _streamBinders[i].CreateStreams( - &_coders[inCoderIndex].InStreams[inCoderStreamIndex], - &_coders[outCoderIndex].OutStreams[outCoderStreamIndex]); + _streamBinders[i].CreateStreams2( + _coders[inCoderIndex].InStreams[inCoderStreamIndex], + _coders[outCoderIndex].OutStreams[outCoderStreamIndex]); CMyComPtr<ICompressSetBufSize> inSetSize, outSetSize; _coders[inCoderIndex].QueryInterface(IID_ICompressSetBufSize, (void **)&inSetSize); @@ -1072,18 +1076,31 @@ HRESULT CMixerMT::Code( for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) { - RINOK(_coders[i].Create()); + const WRes wres = _coders[i].Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) - _coders[i].Start(); + { + const WRes wres = _coders[i].Start(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } _coders[MainCoderIndex].Code(progress); + WRes wres = 0; for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) - _coders[i].WaitExecuteFinish(); + { + WRes wres2 = _coders[i].WaitExecuteFinish(); + if (wres == 0) + wres = wres2; + } + if (wres != 0) + return HRESULT_FROM_WIN32(wres); RINOK(ReturnIfError(E_ABORT)); RINOK(ReturnIfError(E_OUTOFMEMORY)); diff --git a/CPP/7zip/Archive/Common/CoderMixer2.h b/CPP/7zip/Archive/Common/CoderMixer2.h index 798411ab..f099ac3e 100644 --- a/CPP/7zip/Archive/Common/CoderMixer2.h +++ b/CPP/7zip/Archive/Common/CoderMixer2.h @@ -107,7 +107,7 @@ struct CBindInfo { FOR_VECTOR (i, Bonds) if (Bonds[i].PackIndex == packStream) - return i; + return (int)i; return -1; } @@ -115,7 +115,7 @@ struct CBindInfo { FOR_VECTOR (i, Bonds) if (Bonds[i].UnpackIndex == unpackStream) - return i; + return (int)i; return -1; } @@ -144,7 +144,7 @@ struct CBindInfo { FOR_VECTOR(i, PackStreams) if (PackStreams[i] == streamIndex) - return i; + return (int)i; return -1; } @@ -251,6 +251,7 @@ public: // , InternalPackSizeError(false) {} + virtual ~CMixer() {}; /* Sequence of calling: @@ -279,7 +280,7 @@ public: virtual void AddCoder(const CCreatedCoder &cod) = 0; virtual CCoder &GetCoder(unsigned index) = 0; virtual void SelectMainCoder(bool useFirst) = 0; - virtual void ReInit() = 0; + virtual HRESULT ReInit2() = 0; virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) = 0; virtual HRESULT Code( ISequentialInStream * const *inStreams, @@ -322,6 +323,8 @@ class CMixerST: public CMixer, public CMyUnknownImp { + CLASS_NO_COPY(CMixerST) + HRESULT GetInStream2(ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ UInt32 outStreamIndex, ISequentialInStream **inStreamRes); HRESULT GetInStream(ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ @@ -345,7 +348,7 @@ public: virtual void AddCoder(const CCreatedCoder &cod); virtual CCoder &GetCoder(unsigned index); virtual void SelectMainCoder(bool useFirst); - virtual void ReInit(); + virtual HRESULT ReInit2(); virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes, finish); } virtual HRESULT Code( @@ -402,7 +405,7 @@ public: }; CCoderMT(): EncodeMode(false) {} - ~CCoderMT() { CVirtThread::WaitThreadFinish(); } + virtual ~CCoderMT() { CVirtThread::WaitThreadFinish(); } void Code(ICompressProgressInfo *progress); }; @@ -413,11 +416,14 @@ class CMixerMT: public CMixer, public CMyUnknownImp { + CLASS_NO_COPY(CMixerMT) + CObjectVector<CStreamBinder> _streamBinders; HRESULT Init(ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams); HRESULT ReturnIfError(HRESULT code); + // virtual ~CMixerMT() {}; public: CObjectVector<CCoderMT> _coders; @@ -427,7 +433,7 @@ public: virtual void AddCoder(const CCreatedCoder &cod); virtual CCoder &GetCoder(unsigned index); virtual void SelectMainCoder(bool useFirst); - virtual void ReInit(); + virtual HRESULT ReInit2(); virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes, finish); } virtual HRESULT Code( diff --git a/CPP/7zip/Archive/Common/HandlerOut.cpp b/CPP/7zip/Archive/Common/HandlerOut.cpp index 77a35c74..972a766a 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -179,8 +179,8 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN } if (number > 64) return E_FAIL; - for (int j = _methods.Size(); j <= (int)number; j++) - _methods.Add(COneMethodInfo()); + for (unsigned j = _methods.Size(); j <= number; j++) + _methods.AddNew(); return _methods[number].ParseMethodFromPROPVARIANT(realName, value); } diff --git a/CPP/7zip/Archive/Common/ItemNameUtils.cpp b/CPP/7zip/Archive/Common/ItemNameUtils.cpp index d5093a24..905a863d 100644 --- a/CPP/7zip/Archive/Common/ItemNameUtils.cpp +++ b/CPP/7zip/Archive/Common/ItemNameUtils.cpp @@ -8,7 +8,10 @@ namespace NArchive { namespace NItemName { static const wchar_t kOsPathSepar = WCHAR_PATH_SEPARATOR; + +#if WCHAR_PATH_SEPARATOR != L'/' static const wchar_t kUnixPathSepar = L'/'; +#endif void ReplaceSlashes_OsToUnix #if WCHAR_PATH_SEPARATOR != L'/' @@ -44,17 +47,35 @@ UString GetOsPath_Remove_TailSlash(const UString &name) } -void ReplaceToOsSlashes_Remove_TailSlash(UString &name) -{ - if (!name.IsEmpty()) - { +void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool #if WCHAR_PATH_SEPARATOR != L'/' - name.Replace(kUnixPathSepar, kOsPathSepar); + useBackslashReplacement #endif - - if (name.Back() == kOsPathSepar) - name.DeleteBack(); + ) +{ + if (name.IsEmpty()) + return; + + #if WCHAR_PATH_SEPARATOR != L'/' + { + // name.Replace(kUnixPathSepar, kOsPathSepar); + const unsigned len = name.Len(); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = name[i]; + if (c == L'/') + c = WCHAR_PATH_SEPARATOR; + else if (useBackslashReplacement && c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme + else + continue; + name.ReplaceOneCharAtPos(i, c); + } } + #endif + + if (name.Back() == kOsPathSepar) + name.DeleteBack(); } @@ -66,12 +87,15 @@ bool HasTailSlash(const AString &name, UINT { if (name.IsEmpty()) return false; - char c = + char c; #if defined(_WIN32) && !defined(UNDER_CE) - *CharPrevExA((WORD)codePage, name, name.Ptr(name.Len()), 0); - #else - name.Back(); + if (codePage != CP_UTF8) + c = *CharPrevExA((WORD)codePage, name, name.Ptr(name.Len()), 0); + else #endif + { + c = name.Back(); + } return (c == '/'); } diff --git a/CPP/7zip/Archive/Common/ItemNameUtils.h b/CPP/7zip/Archive/Common/ItemNameUtils.h index 31150864..6a4d6c71 100644 --- a/CPP/7zip/Archive/Common/ItemNameUtils.h +++ b/CPP/7zip/Archive/Common/ItemNameUtils.h @@ -13,7 +13,7 @@ void ReplaceSlashes_OsToUnix(UString &name); UString GetOsPath(const UString &name); UString GetOsPath_Remove_TailSlash(const UString &name); -void ReplaceToOsSlashes_Remove_TailSlash(UString &name); +void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool useBackslashReplacement = false); bool HasTailSlash(const AString &name, UINT codePage); diff --git a/CPP/7zip/Archive/Common/MultiStream.cpp b/CPP/7zip/Archive/Common/MultiStream.cpp index 1de74afe..162fc928 100644 --- a/CPP/7zip/Archive/Common/MultiStream.cpp +++ b/CPP/7zip/Archive/Common/MultiStream.cpp @@ -36,7 +36,7 @@ STDMETHODIMP CMultiStream::Read(void *data, UInt32 size, UInt32 *processedSize) UInt64 localPos = _pos - s.GlobalOffset; if (localPos != s.LocalPos) { - RINOK(s.Stream->Seek(localPos, STREAM_SEEK_SET, &s.LocalPos)); + RINOK(s.Stream->Seek((Int64)localPos, STREAM_SEEK_SET, &s.LocalPos)); } UInt64 rem = s.Size - localPos; if (size > rem) @@ -60,9 +60,9 @@ STDMETHODIMP CMultiStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosi } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } diff --git a/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp b/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp index 77252938..ac26edf7 100644 --- a/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp +++ b/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp @@ -10,7 +10,7 @@ STDMETHODIMP COutStreamWithSha1::Write(const void *data, UInt32 size, UInt32 *pr if (_stream) result = _stream->Write(data, size, &size); if (_calculate) - Sha1_Update(&_sha, (const Byte *)data, size); + Sha1_Update(Sha(), (const Byte *)data, size); _size += size; if (processedSize) *processedSize = size; diff --git a/CPP/7zip/Archive/Common/OutStreamWithSha1.h b/CPP/7zip/Archive/Common/OutStreamWithSha1.h index 41a84cd6..5a7bfef3 100644 --- a/CPP/7zip/Archive/Common/OutStreamWithSha1.h +++ b/CPP/7zip/Archive/Common/OutStreamWithSha1.h @@ -5,6 +5,7 @@ #include "../../../../C/Sha1.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/MyCom.h" #include "../../IStream.h" @@ -15,10 +16,16 @@ class COutStreamWithSha1: { CMyComPtr<ISequentialOutStream> _stream; UInt64 _size; - CSha1 _sha; + // CSha1 _sha; bool _calculate; + CAlignedBuffer _sha; + + CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_sha; } public: MY_UNKNOWN_IMP + + COutStreamWithSha1(): _sha(sizeof(CSha1)) {} + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); void SetStream(ISequentialOutStream *stream) { _stream = stream; } void ReleaseStream() { _stream.Release(); } @@ -26,11 +33,11 @@ public: { _size = 0; _calculate = calculate; - Sha1_Init(&_sha); + Sha1_Init(Sha()); } - void InitSha1() { Sha1_Init(&_sha); } + void InitSha1() { Sha1_Init(Sha()); } UInt64 GetSize() const { return _size; } - void Final(Byte *digest) { Sha1_Final(&_sha, digest); } + void Final(Byte *digest) { Sha1_Final(Sha(), digest); } }; #endif diff --git a/CPP/7zip/Archive/CpioHandler.cpp b/CPP/7zip/Archive/CpioHandler.cpp index b8fb530f..ffdab16c 100644 --- a/CPP/7zip/Archive/CpioHandler.cpp +++ b/CPP/7zip/Archive/CpioHandler.cpp @@ -131,7 +131,7 @@ enum EErrorType { k_ErrorType_OK, k_ErrorType_Corrupted, - k_ErrorType_UnexpectedEnd, + k_ErrorType_UnexpectedEnd }; struct CInArchive @@ -218,7 +218,7 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) { if (size < k_OctRecord_Size) return k_IsArc_Res_NEED_MORE; - for (int i = 6; i < k_OctRecord_Size; i++) + for (unsigned i = 6; i < k_OctRecord_Size; i++) { char c = p[i]; if (c < '0' || c > '7') @@ -231,7 +231,7 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) { if (size < k_HexRecord_Size) return k_IsArc_Res_NEED_MORE; - for (int i = 6; i < k_HexRecord_Size; i++) + for (unsigned i = 6; i < k_HexRecord_Size; i++) { char c = p[i]; if ((c < '0' || c > '9') && @@ -268,7 +268,9 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) if (nameSize > (1 << 8)) return k_IsArc_Res_NO; } - if (numLinks == 0 || numLinks >= (1 << 10)) + // 20.03: some cpio files have (numLinks == 0). + // if (numLinks == 0) return k_IsArc_Res_NO; + if (numLinks >= (1 << 10)) return k_IsArc_Res_NO; if (nameSize == 0 || nameSize > kNameSizeMax) return k_IsArc_Res_NO; @@ -462,6 +464,9 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { case k_ErrorType_UnexpectedEnd: v |= kpv_ErrorFlags_UnexpectedEnd; break; case k_ErrorType_Corrupted: v |= kpv_ErrorFlags_HeadersError; break; + case k_ErrorType_OK: + default: + break; } prop = v; break; @@ -565,23 +570,30 @@ STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb { // Read tailing zeros. // Most of cpio files use 512-bytes aligned zeros - UInt64 pos = arc.Processed; - const UInt32 kTailSize_MAX = 1 << 9; + // rare case: 4K/8K aligment is possible also + const unsigned kTailSize_MAX = 1 << 9; Byte buf[kTailSize_MAX]; - UInt32 rem = (kTailSize_MAX - (UInt32)pos) & (kTailSize_MAX - 1); - if (rem != 0) + unsigned pos = (unsigned)arc.Processed & (kTailSize_MAX - 1); + if (pos != 0) // use this check to support 512 bytes alignment only + for (;;) { - rem++; // we need to see that it's end of file + unsigned rem = kTailSize_MAX - pos; size_t processed = rem; - RINOK(ReadStream(stream, buf, &processed)); - if (processed < rem) - { - unsigned i; - for (i = 0; i < processed && buf[i] == 0; i++); - if (i == processed) - _phySize += processed; - } + RINOK(ReadStream(stream, buf + pos, &processed)); + if (processed != rem) + break; + + for (; pos < kTailSize_MAX && buf[pos] == 0; pos++) + {} + if (pos != kTailSize_MAX) + break; + _phySize += processed; + pos = 0; + + // use break to support 512 bytes alignment zero tail + // don't use break to support 512*n bytes alignment zero tail + break; } } @@ -622,7 +634,9 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val UString res; bool needConvert = true; #ifdef _WIN32 - if (ConvertUTF8ToUnicode(item.Name, res)) + // if ( + ConvertUTF8ToUnicode(item.Name, res); + // ) needConvert = false; #endif if (needConvert) diff --git a/CPP/7zip/Archive/DllExports2.cpp b/CPP/7zip/Archive/DllExports2.cpp index 10889e75..967a7cbf 100644 --- a/CPP/7zip/Archive/DllExports2.cpp +++ b/CPP/7zip/Archive/DllExports2.cpp @@ -22,11 +22,24 @@ #include "IArchive.h" -HINSTANCE g_hInstance; +#ifdef _WIN32 + +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION return FALSE; +#endif + +HINSTANCE g_hInstance; + +extern "C" +BOOL WINAPI DllMain( + #ifdef UNDER_CE + HANDLE + #else + HINSTANCE + #endif + hInstance, DWORD dwReason, LPVOID /*lpReserved*/); -#ifdef _WIN32 extern "C" BOOL WINAPI DllMain( #ifdef UNDER_CE @@ -50,7 +63,22 @@ BOOL WINAPI DllMain( */ return TRUE; } -#endif + +#else // _WIN32 + +#include "../../Common/StringConvert.h" +// #include <stdio.h> + +// STDAPI LibStartup(); +static __attribute__((constructor)) void Init_ForceToUTF8(); +static __attribute__((constructor)) void Init_ForceToUTF8() +{ + g_ForceToUTF8 = IsNativeUTF8(); + // printf("\nDLLExports2.cpp::Init_ForceToUTF8 =%d\n", g_ForceToUTF8 ? 1 : 0); +} + +#endif // _WIN32 + DEFINE_GUID(CLSID_CArchiveHandler, k_7zip_GUID_Data1, @@ -62,6 +90,7 @@ STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateHasher(const GUID *clsid, IHasher **hasher); STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject); +STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject) { // COM_TRY_BEGIN @@ -76,16 +105,20 @@ STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject) // COM_TRY_END } +STDAPI SetLargePageMode(); STDAPI SetLargePageMode() { #if defined(_7ZIP_LARGE_PAGES) + #ifdef _WIN32 SetLargePageSize(); #endif + #endif return S_OK; } extern bool g_CaseSensitive; +STDAPI SetCaseSensitive(Int32 caseSensitive); STDAPI SetCaseSensitive(Int32 caseSensitive) { g_CaseSensitive = (caseSensitive != 0); @@ -96,6 +129,7 @@ STDAPI SetCaseSensitive(Int32 caseSensitive) CExternalCodecs g_ExternalCodecs; +STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo); STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo) { COM_TRY_BEGIN @@ -114,6 +148,7 @@ STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo) #else +STDAPI SetCodecs(ICompressCodecsInfo *); STDAPI SetCodecs(ICompressCodecsInfo *) { return S_OK; diff --git a/CPP/7zip/Archive/DmgHandler.cpp b/CPP/7zip/Archive/DmgHandler.cpp index d9fe6016..a08dcac1 100644 --- a/CPP/7zip/Archive/DmgHandler.cpp +++ b/CPP/7zip/Archive/DmgHandler.cpp @@ -33,99 +33,22 @@ #define Get32(p) GetBe32(p) #define Get64(p) GetBe64(p) -static const Byte k_Base64Table[256] = -{ - 66,77,77,77,77,77,77,77,77,65,65,77,77,65,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 65,77,77,77,77,77,77,77,77,77,77,62,77,77,77,63, - 52,53,54,55,56,57,58,59,60,61,77,77,77,64,77,77, - 77, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, - 15,16,17,18,19,20,21,22,23,24,25,77,77,77,77,77, - 77,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, - 41,42,43,44,45,46,47,48,49,50,51,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77, - 77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77 -}; - -static Byte *Base64ToBin(Byte *dest, const char *src) -{ - UInt32 val = 1; - - for (;;) - { - UInt32 c = k_Base64Table[(Byte)(*src++)]; - - if (c < 64) - { - val = (val << 6) | c; - if ((val & ((UInt32)1 << 24)) == 0) - continue; - dest[0] = (Byte)(val >> 16); - dest[1] = (Byte)(val >> 8); - dest[2] = (Byte)(val); - dest += 3; - val = 1; - continue; - } - - if (c == 65) // space - continue; - - if (c == 64) // '=' - break; - - if (c == 66 && val == 1) // end of string - return dest; - - return NULL; - } - - if (val < (1 << 12)) - return NULL; - - if (val & (1 << 18)) - { - *dest++ = (Byte)(val >> 10); - *dest++ = (Byte)(val >> 2); - } - else if (k_Base64Table[(Byte)(*src++)] != 64) // '=' - return NULL; - else - *dest++ = (Byte)(val >> 4); - - for (;;) - { - Byte c = k_Base64Table[(Byte)(*src++)]; - if (c == 65) // space - continue; - if (c == 66) // end of string - return dest; - return NULL; - } -} - +Byte *Base64ToBin(Byte *dest, const char *src); namespace NArchive { namespace NDmg { -enum -{ - METHOD_ZERO_0 = 0, - METHOD_COPY = 1, - METHOD_ZERO_2 = 2, // without file CRC calculation - METHOD_ADC = 0x80000004, - METHOD_ZLIB = 0x80000005, - METHOD_BZIP2 = 0x80000006, - METHOD_LZFSE = 0x80000007, - METHOD_COMMENT = 0x7FFFFFFE, // is used to comment "+beg" and "+end" in extra field. - METHOD_END = 0xFFFFFFFF -}; + +static const UInt32 METHOD_ZERO_0 = 0; +static const UInt32 METHOD_COPY = 1; +static const UInt32 METHOD_ZERO_2 = 2; // without file CRC calculation +static const UInt32 METHOD_ADC = 0x80000004; +static const UInt32 METHOD_ZLIB = 0x80000005; +static const UInt32 METHOD_BZIP2 = 0x80000006; +static const UInt32 METHOD_LZFSE = 0x80000007; +static const UInt32 METHOD_COMMENT = 0x7FFFFFFE; // is used to comment "+beg" and "+end" in extra field. +static const UInt32 METHOD_END = 0xFFFFFFFF; + struct CBlock { @@ -266,7 +189,7 @@ void CMethods::GetString(AString &res) const for (i = 0; i < Types.Size(); i++) { - UInt32 type = Types[i]; + const UInt32 type = Types[i]; if (type == METHOD_COMMENT || type == METHOD_END) continue; char buf[16]; @@ -407,6 +330,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidWarning: if (_masterCrcError) prop = "Master CRC error"; + break; case kpidWarningFlags: { @@ -727,7 +651,8 @@ HRESULT CHandler::Open2(IInStream *stream) if (xmlPair2.Len > len) xmlPair2.Len = len; CByteBuffer buf2; - if (ReadData(stream, xmlPair2, buf2) != S_OK + if (xmlPair2.Len < len + || ReadData(stream, xmlPair2, buf2) != S_OK || memcmp(buf2, sz, len) != 0) { // if absolute offset is not OK, probably it's archive with offset @@ -1054,7 +979,9 @@ STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) return S_OK; } +#ifdef DMG_SHOW_RAW #define RAW_PREFIX "raw" STRING_PATH_SEPARATOR +#endif STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value) { @@ -1608,7 +1535,7 @@ public: }; -unsigned FindBlock(const CRecordVector<CBlock> &blocks, UInt64 pos) +static unsigned FindBlock(const CRecordVector<CBlock> &blocks, UInt64 pos) { unsigned left = 0, right = blocks.Size(); for (;;) @@ -1762,8 +1689,8 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) } const CBlock &block = File->Blocks[_latestBlock]; - UInt64 offset = _virtPos - block.UnpPos; - UInt64 rem = block.UnpSize - offset; + const UInt64 offset = _virtPos - block.UnpPos; + const UInt64 rem = block.UnpSize - offset; if (size > rem) size = (UInt32)rem; @@ -1777,7 +1704,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) else if (block.IsZeroMethod()) memset(data, 0, size); else if (size != 0) - memcpy(data, _chunks[_latestChunk].Buf + offset, size); + memcpy(data, _chunks[_latestChunk].Buf + (size_t)offset, size); _virtPos += size; if (processedSize) diff --git a/CPP/7zip/Archive/ElfHandler.cpp b/CPP/7zip/Archive/ElfHandler.cpp index c5ce279e..efcde95d 100644 --- a/CPP/7zip/Archive/ElfHandler.cpp +++ b/CPP/7zip/Archive/ElfHandler.cpp @@ -226,6 +226,7 @@ void CSegment::Parse(const Byte *p, bool mode64, bool be) // Section types +/* #define SHT_NULL 0 #define SHT_PROGBITS 1 #define SHT_SYMTAB 2 @@ -234,7 +235,9 @@ void CSegment::Parse(const Byte *p, bool mode64, bool be) #define SHT_HASH 5 #define SHT_DYNAMIC 6 #define SHT_NOTE 7 +*/ #define SHT_NOBITS 8 +/* #define SHT_REL 9 #define SHT_SHLIB 10 #define SHT_DYNSYM 11 @@ -245,7 +248,7 @@ void CSegment::Parse(const Byte *p, bool mode64, bool be) #define SHT_PREINIT_ARRAY 16 #define SHT_GROUP 17 #define SHT_SYMTAB_SHNDX 18 - +*/ static const CUInt32PCharPair g_SectTypes[] = { @@ -633,11 +636,11 @@ static const CUInt32PCharPair g_MIPS_Flags[] = }; -#define ET_NONE 0 +// #define ET_NONE 0 #define ET_REL 1 -#define ET_EXEC 2 +// #define ET_EXEC 2 #define ET_DYN 3 -#define ET_CORE 4 +// #define ET_CORE 4 static const char * const g_Types[] = { diff --git a/CPP/7zip/Archive/ExtHandler.cpp b/CPP/7zip/Archive/ExtHandler.cpp index db65fbea..e9808aac 100644 --- a/CPP/7zip/Archive/ExtHandler.cpp +++ b/CPP/7zip/Archive/ExtHandler.cpp @@ -99,6 +99,8 @@ static UInt32 Crc16Calc(Byte const *data, size_t size) #define EXT4_GOOD_OLD_INODE_SIZE 128 +#define EXT_NODE_SIZE_MIN 128 + // inodes numbers @@ -436,9 +438,9 @@ bool CHeader::Parse(const Byte *p) LE_16 (0x58, InodeSize); if (FirstInode < k_INODE_GOOD_OLD_FIRST) return false; - if (InodeSize > (UInt32)1 << BlockBits) - return false; - if (GetLog(InodeSize) < 0) + if (InodeSize > ((UInt32)1 << BlockBits) + || InodeSize < EXT_NODE_SIZE_MIN + || GetLog(InodeSize) < 0) return false; } @@ -603,7 +605,7 @@ struct CExtent if (Len > (UInt32)0x8000) { IsInited = false; - Len -= (UInt32)0x8000; + Len = (UInt16)(Len - (UInt32)0x8000); } LE_32 (0x08, PhyStart); UInt16 hi; @@ -628,8 +630,8 @@ struct CNode int DirIndex; // in _dirs[] UInt16 Mode; - UInt16 Uid; - UInt16 Gid; + UInt32 Uid; // fixed 21.02 + UInt32 Gid; // fixed 21.02 // UInt16 Checksum; UInt64 FileSize; @@ -730,6 +732,8 @@ bool CNode::Parse(const Byte *p, const CHeader &_h) if (_h.InodeSize > 128) { + // InodeSize is power of 2, so the following check is not required: + // if (_h.InodeSize < 128 + 2) return false; UInt16 extra_isize; LE_16 (0x80, extra_isize); if (128 + extra_isize > _h.InodeSize) @@ -842,7 +846,7 @@ class CHandler: } - const int GetParentAux(const CItem &item) const + int GetParentAux(const CItem &item) const { if (item.Node < _h.FirstInode && _auxSysIndex >= 0) return _auxSysIndex; @@ -931,7 +935,7 @@ HRESULT CHandler::ParseDir(const Byte *p, size_t size, unsigned iNodeDir) return S_FALSE; if (_isUTF) - _isUTF = CheckUTF8(item.Name); + _isUTF = CheckUTF8_AString(item.Name); if (iNode == 0) { @@ -1201,7 +1205,7 @@ HRESULT CHandler::Open2(IInStream *inStream) UInt32 numNodes = _h.InodesPerGroup; if (numNodes > _h.NumInodes) numNodes = _h.NumInodes; - size_t nodesDataSize = (size_t)numNodes * _h.InodeSize; + const size_t nodesDataSize = (size_t)numNodes * _h.InodeSize; if (nodesDataSize / _h.InodeSize != numNodes) return S_FALSE; @@ -1213,7 +1217,7 @@ HRESULT CHandler::Open2(IInStream *inStream) return S_FALSE; } - UInt32 numReserveInodes = _h.NumInodes - _h.NumFreeInodes + 1; + const UInt32 numReserveInodes = _h.NumInodes - _h.NumFreeInodes + 1; // numReserveInodes = _h.NumInodes + 1; if (numReserveInodes != 0) { @@ -1348,7 +1352,8 @@ HRESULT CHandler::Open2(IInStream *inStream) RINOK(CheckProgress()); } - if (_nodes[_refs[k_INODE_ROOT]].ParentNode != k_INODE_ROOT) + int ref = _refs[k_INODE_ROOT]; + if (ref < 0 || _nodes[ref].ParentNode != k_INODE_ROOT) return S_FALSE; } diff --git a/CPP/7zip/Archive/FatHandler.cpp b/CPP/7zip/Archive/FatHandler.cpp index bf6053e0..1cbc8508 100644 --- a/CPP/7zip/Archive/FatHandler.cpp +++ b/CPP/7zip/Archive/FatHandler.cpp @@ -682,7 +682,7 @@ HRESULT CDatabase::Open() RINOK(ReadStream_FALSE(InStream, byteBuf, readSize)); NumCurUsedBytes += readSize; - const UInt32 *src = (const UInt32 *)(const Byte *)byteBuf; + const UInt32 *src = (const UInt32 *)(const void *)(const Byte *)byteBuf; UInt32 *dest = Fat + i; if (numFreeClustersDefined) for (UInt32 j = 0; j < size; j++) diff --git a/CPP/7zip/Archive/FlvHandler.cpp b/CPP/7zip/Archive/FlvHandler.cpp index 1f52f60b..97a7c268 100644 --- a/CPP/7zip/Archive/FlvHandler.cpp +++ b/CPP/7zip/Archive/FlvHandler.cpp @@ -23,14 +23,14 @@ ((UInt32)((const Byte *)(p))[1] << 8) | \ ((const Byte *)(p))[2] ) -#define Get16(p) GetBe16(p) +// #define Get16(p) GetBe16(p) #define Get24(p) GetBe24(p) #define Get32(p) GetBe32(p) namespace NArchive { namespace NFlv { -static const UInt32 kFileSizeMax = (UInt32)1 << 30; +// static const UInt32 kFileSizeMax = (UInt32)1 << 30; static const UInt32 kNumChunksMax = (UInt32)1 << 23; static const UInt32 kTagHeaderSize = 11; diff --git a/CPP/7zip/Archive/GptHandler.cpp b/CPP/7zip/Archive/GptHandler.cpp index a86ad37c..7b3d23bb 100644 --- a/CPP/7zip/Archive/GptHandler.cpp +++ b/CPP/7zip/Archive/GptHandler.cpp @@ -333,16 +333,25 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { case kpidPath: { + // Windows BDP partitions can have identical names. + // So we add the partition number at front UString s; - for (unsigned i = 0; i < kNameLen; i++) + s.Add_UInt32(index); { - wchar_t c = (wchar_t)Get16(item.Name + i * 2); - if (c == 0) - break; - s += c; + UString s2; + for (unsigned i = 0; i < kNameLen; i++) + { + wchar_t c = (wchar_t)Get16(item.Name + i * 2); + if (c == 0) + break; + s2 += c; + } + if (!s2.IsEmpty()) + { + s += '.'; + s += s2; + } } - if (s.IsEmpty()) - s.Add_UInt32(index); { s += '.'; const char *ext = NULL; diff --git a/CPP/7zip/Archive/GzHandler.cpp b/CPP/7zip/Archive/GzHandler.cpp index 130f8b35..0054840d 100644 --- a/CPP/7zip/Archive/GzHandler.cpp +++ b/CPP/7zip/Archive/GzHandler.cpp @@ -45,7 +45,7 @@ namespace NGz { namespace NFlags { - const Byte kIsText = 1 << 0; + // const Byte kIsText = 1 << 0; const Byte kCrc = 1 << 1; const Byte kExtra = 1 << 2; const Byte kName = 1 << 3; @@ -234,7 +234,8 @@ static UInt32 Is_Deflate(const Byte *p, size_t size) return k_IsArc_Res_NO; if (size < 4) return k_IsArc_Res_NEED_MORE; - if (GetUi16(p) != (UInt16)~GetUi16(p + 2)) + UInt16 r = (UInt16)~GetUi16(p + 2); + if (GetUi16(p) != r) return k_IsArc_Res_NO; } else if (type == 2) @@ -248,8 +249,8 @@ static UInt32 Is_Deflate(const Byte *p, size_t size) return k_IsArc_Res_YES; } -static unsigned kNameMaxLen = 1 << 12; -static unsigned kCommentMaxLen = 1 << 16; +static const unsigned kNameMaxLen = 1 << 12; +static const unsigned kCommentMaxLen = 1 << 16; API_FUNC_static_IsArc IsArc_Gz(const Byte *p, size_t size) { @@ -962,7 +963,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt UString name = prop.bstrVal; int slashPos = name.ReverseFind_PathSepar(); if (slashPos >= 0) - name.DeleteFrontal(slashPos + 1); + name.DeleteFrontal((unsigned)(slashPos + 1)); newItem.Name = UnicodeStringToMultiByte(name, CP_ACP); if (!newItem.Name.IsEmpty()) newItem.Flags |= NFlags::kName; @@ -1019,7 +1020,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt newItem.WriteHeader(outStream); offset += _headerSize; } - RINOK(_stream->Seek(offset, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)offset, STREAM_SEEK_SET, NULL)); return NCompress::CopyStream(_stream, outStream, progress); diff --git a/CPP/7zip/Archive/HandlerCont.cpp b/CPP/7zip/Archive/HandlerCont.cpp index c2d5c70c..22a8c4a9 100644 --- a/CPP/7zip/Archive/HandlerCont.cpp +++ b/CPP/7zip/Archive/HandlerCont.cpp @@ -269,7 +269,7 @@ HRESULT ReadZeroTail(ISequentialInStream *stream, bool &areThereNonZeros, UInt64 for (;;) { UInt32 size = 0; - HRESULT(stream->Read(buf, kBufSize, &size)); + RINOK(stream->Read(buf, kBufSize, &size)); if (size == 0) return S_OK; for (UInt32 i = 0; i < size; i++) diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index ca1d7bda..57313280 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -275,7 +275,7 @@ struct CItem CItem(): UseAttr(false), UseInlineData(false) {} bool IsDir() const { return Type == RECORD_TYPE_FOLDER; } - const CFork &GetFork(bool isResource) const { return (CFork & )*(isResource ? &ResourceFork: &DataFork ); } + const CFork &GetFork(bool isResource) const { return (const CFork & )*(isResource ? &ResourceFork: &DataFork ); } }; struct CAttr @@ -320,12 +320,14 @@ public: UInt64 SpecOffset; UInt64 PhySize; UInt64 PhySize2; + UInt64 ArcFileSize; void Clear() { SpecOffset = 0; PhySize = 0; PhySize2 = 0; + ArcFileSize = 0; HeadersError = false; ThereAreAltStreams = false; // CaseSensetive = false; @@ -440,7 +442,10 @@ HRESULT CDatabase::ReadFile(const CFork &fork, CByteBuffer &buf, IInStream *inSt { if (fork.NumBlocks >= Header.NumBlocks) return S_FALSE; - size_t totalSize = (size_t)fork.NumBlocks << Header.BlockSizeLog; + if ((ArcFileSize >> Header.BlockSizeLog) + 1 < fork.NumBlocks) + return S_FALSE; + + const size_t totalSize = (size_t)fork.NumBlocks << Header.BlockSizeLog; if ((totalSize >> Header.BlockSizeLog) != fork.NumBlocks) return S_FALSE; buf.Alloc(totalSize); @@ -473,20 +478,36 @@ struct CNodeDescriptor // Byte Height; unsigned NumRecords; - bool CheckNumRecords(unsigned nodeSizeLog) - { - return (kNodeDescriptor_Size + ((UInt32)NumRecords + 1) * 2 <= ((UInt32)1 << nodeSizeLog)); - } - void Parse(const Byte *p); + bool Parse(const Byte *p, unsigned nodeSizeLog); }; -void CNodeDescriptor::Parse(const Byte *p) + +bool CNodeDescriptor::Parse(const Byte *p, unsigned nodeSizeLog) { fLink = Get32(p); // bLink = Get32(p + 4); Kind = p[8]; // Height = p[9]; NumRecords = Get16(p + 10); + + const size_t nodeSize = (size_t)1 << nodeSizeLog; + if (kNodeDescriptor_Size + ((UInt32)NumRecords + 1) * 2 > nodeSize) + return false; + const size_t limit = nodeSize - ((UInt32)NumRecords + 1) * 2; + + p += nodeSize - 2; + + for (unsigned i = 0; i < NumRecords; i++) + { + const UInt32 offs = Get16(p); + p -= 2; + const UInt32 offsNext = Get16(p); + if (offs < kNodeDescriptor_Size + || offs >= offsNext + || offsNext > limit) + return false; + } + return true; } struct CHeaderRec @@ -576,7 +597,7 @@ HRESULT CDatabase::LoadExtentFile(const CFork &fork, IInStream *inStream, CObjec if (node == 0) return S_OK; - CByteBuffer usedBuf(hr.TotalNodes); + CByteArr usedBuf(hr.TotalNodes); memset(usedBuf, 0, hr.TotalNodes); while (node != 0) @@ -585,10 +606,9 @@ HRESULT CDatabase::LoadExtentFile(const CFork &fork, IInStream *inStream, CObjec return S_FALSE; usedBuf[node] = 1; - size_t nodeOffset = (size_t)node << hr.NodeSizeLog; + const size_t nodeOffset = (size_t)node << hr.NodeSizeLog; CNodeDescriptor desc; - desc.Parse(p + nodeOffset); - if (!desc.CheckNumRecords(hr.NodeSizeLog)) + if (!desc.Parse(p + nodeOffset, hr.NodeSizeLog)) return S_FALSE; if (desc.Kind != kNodeType_Leaf) return S_FALSE; @@ -597,18 +617,16 @@ HRESULT CDatabase::LoadExtentFile(const CFork &fork, IInStream *inStream, CObjec for (unsigned i = 0; i < desc.NumRecords; i++) { - const UInt32 nodeSize = (UInt32)1 << hr.NodeSizeLog; - const UInt32 offs = Get16(p + nodeOffset + nodeSize - (i + 1) * 2); - const UInt32 offsNext = Get16(p + nodeOffset + nodeSize - (i + 2) * 2); - if (offs > nodeSize || offsNext > nodeSize) - return S_FALSE; - UInt32 recSize = offsNext - offs; + const UInt32 nodeSize = (1 << hr.NodeSizeLog); + const Byte *r = p + nodeOffset + nodeSize - i * 2; + const UInt32 offs = Get16(r - 2); + UInt32 recSize = Get16(r - 4) - offs; const unsigned kKeyLen = 10; if (recSize != 2 + kKeyLen + kNumFixedExtents * 8) return S_FALSE; - const Byte *r = p + nodeOffset + offs; + r = p + nodeOffset + offs; if (Get16(r) != kKeyLen) return S_FALSE; @@ -717,7 +735,7 @@ HRESULT CDatabase::LoadAttrs(const CFork &fork, IInStream *inStream, IArchiveOpe if (node == 0) return S_OK; - CByteBuffer usedBuf(hr.TotalNodes); + CByteArr usedBuf(hr.TotalNodes); memset(usedBuf, 0, hr.TotalNodes); CFork resFork; @@ -728,10 +746,9 @@ HRESULT CDatabase::LoadAttrs(const CFork &fork, IInStream *inStream, IArchiveOpe return S_FALSE; usedBuf[node] = 1; - size_t nodeOffset = (size_t)node << hr.NodeSizeLog; + const size_t nodeOffset = (size_t)node << hr.NodeSizeLog; CNodeDescriptor desc; - desc.Parse(p + nodeOffset); - if (!desc.CheckNumRecords(hr.NodeSizeLog)) + if (!desc.Parse(p + nodeOffset, hr.NodeSizeLog)) return S_FALSE; if (desc.Kind != kNodeType_Leaf) return S_FALSE; @@ -739,19 +756,14 @@ HRESULT CDatabase::LoadAttrs(const CFork &fork, IInStream *inStream, IArchiveOpe for (unsigned i = 0; i < desc.NumRecords; i++) { const UInt32 nodeSize = (1 << hr.NodeSizeLog); - const UInt32 offs = Get16(p + nodeOffset + nodeSize - (i + 1) * 2); - const UInt32 offsNext = Get16(p + nodeOffset + nodeSize - (i + 2) * 2); - UInt32 recSize = offsNext - offs; - if (offs >= nodeSize - || offsNext > nodeSize - || offsNext < offs) - return S_FALSE; - + const Byte *r = p + nodeOffset + nodeSize - i * 2; + const UInt32 offs = Get16(r - 2); + UInt32 recSize = Get16(r - 4) - offs; const unsigned kHeadSize = 14; if (recSize < kHeadSize) return S_FALSE; - const Byte *r = p + nodeOffset + offs; + r = p + nodeOffset + offs; UInt32 keyLen = Get16(r); // UInt16 pad = Get16(r + 2); @@ -805,7 +817,7 @@ HRESULT CDatabase::LoadAttrs(const CFork &fork, IInStream *inStream, IArchiveOpe if (progress && (i & 0xFFF) == 0) { - UInt64 numFiles = 0; + const UInt64 numFiles = 0; RINOK(progress->SetCompleted(&numFiles, NULL)); } } @@ -873,13 +885,6 @@ bool CDatabase::Parse_decmpgfs(const CAttr &attr, CItem &item, bool &skip) HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents> *overflowExtentsArray, IInStream *inStream, IArchiveOpenCallback *progress) { - unsigned reserveSize = (unsigned)(Header.NumFolders + 1 + Header.NumFiles); - Items.ClearAndReserve(reserveSize); - Refs.ClearAndReserve(reserveSize); - - CRecordVector<CIdIndexPair> IdToIndexMap; - IdToIndexMap.ClearAndReserve(reserveSize); - CByteBuffer buf; RINOK(ReadFile(fork, buf, inStream)); const Byte *p = (const Byte *)buf; @@ -888,10 +893,24 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents // nodeDesc.Parse(p); CHeaderRec hr; RINOK(hr.Parse2(buf)); - + + CRecordVector<CIdIndexPair> IdToIndexMap; + + const unsigned reserveSize = (unsigned)(Header.NumFolders + 1 + Header.NumFiles); + + const unsigned kBasicRecSize = 0x58; + const unsigned kMinRecSize = kBasicRecSize + 10; + + if ((UInt64)reserveSize * kMinRecSize < buf.Size()) + { + Items.ClearAndReserve(reserveSize); + Refs.ClearAndReserve(reserveSize); + IdToIndexMap.ClearAndReserve(reserveSize); + } + // CaseSensetive = (Header.IsHfsX() && hr.KeyCompareType == 0xBC); - CByteBuffer usedBuf(hr.TotalNodes); + CByteArr usedBuf(hr.TotalNodes); memset(usedBuf, 0, hr.TotalNodes); CFork resFork; @@ -908,8 +927,7 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents const size_t nodeOffset = (size_t)node << hr.NodeSizeLog; CNodeDescriptor desc; - desc.Parse(p + nodeOffset); - if (!desc.CheckNumRecords(hr.NodeSizeLog)) + if (!desc.Parse(p + nodeOffset, hr.NodeSizeLog)) return S_FALSE; if (desc.Kind != kNodeType_Leaf) return S_FALSE; @@ -917,16 +935,13 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents for (unsigned i = 0; i < desc.NumRecords; i++) { const UInt32 nodeSize = (1 << hr.NodeSizeLog); - const UInt32 offs = Get16(p + nodeOffset + nodeSize - (i + 1) * 2); - const UInt32 offsNext = Get16(p + nodeOffset + nodeSize - (i + 2) * 2); - UInt32 recSize = offsNext - offs; - if (offs >= nodeSize - || offsNext > nodeSize - || offsNext < offs - || recSize < 6) + const Byte *r = p + nodeOffset + nodeSize - i * 2; + const UInt32 offs = Get16(r - 2); + UInt32 recSize = Get16(r - 4) - offs; + if (recSize < 6) return S_FALSE; - const Byte *r = p + nodeOffset + offs; + r = p + nodeOffset + offs; UInt32 keyLen = Get16(r); UInt32 parentID = Get32(r + 2); if (keyLen < 6 || (keyLen & 1) != 0 || keyLen + 2 > recSize) @@ -952,7 +967,6 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents type != RECORD_TYPE_FILE) continue; - const unsigned kBasicRecSize = 0x58; if (recSize < kBasicRecSize) return S_FALSE; @@ -1042,7 +1056,7 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector<CIdExtents } if (progress && (Items.Size() & 0xFFF) == 0) { - UInt64 numItems = Items.Size(); + const UInt64 numItems = Items.Size(); RINOK(progress->SetCompleted(&numItems, NULL)); } } @@ -1286,9 +1300,12 @@ HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) if (h.NumFolders > ((UInt32)1 << 29) || h.NumFiles > ((UInt32)1 << 30)) return S_FALSE; + + RINOK(inStream->Seek(0, STREAM_SEEK_END, &ArcFileSize)); + if (progress) { - UInt64 numFiles = (UInt64)h.NumFiles + h.NumFolders + 1; + const UInt64 numFiles = (UInt64)h.NumFiles + h.NumFolders + 1; RINOK(progress->SetTotal(&numFiles, NULL)); } @@ -1313,13 +1330,6 @@ HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) h.VolID = Get64(p + 0x68); */ - /* - UInt64 endPos; - RINOK(inStream->Seek(0, STREAM_SEEK_END, &endPos)); - if ((endPos >> h.BlockSizeLog) < h.NumBlocks) - return S_FALSE; - */ - ResFileName = kResFileName; CFork extentsFork, catalogFork, attrFork; @@ -1499,7 +1509,7 @@ STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data else s = &Items[ref.ItemIndex].Name; *data = (const wchar_t *)(*s); - *dataSize = (s->Len() + 1) * sizeof(wchar_t); + *dataSize = (s->Len() + 1) * (UInt32)sizeof(wchar_t); *propType = PROP_DATA_TYPE_wchar_t_PTR_Z_LE; return S_OK; } @@ -1700,7 +1710,7 @@ HRESULT CHandler::ExtractZlibFile( } outPos += blockSize; - UInt64 progressPos = progressStart + outPos; + const UInt64 progressPos = progressStart + outPos; RINOK(extractCallback->SetCompleted(&progressPos)); } @@ -1884,7 +1894,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, } pos += cur; extentRem -= cur; - UInt64 processed = currentTotalSize + pos; + const UInt64 processed = currentTotalSize + pos; RINOK(extractCallback->SetCompleted(&processed)); } } diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 7d7256c9..8290c735 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -10,6 +10,30 @@ #define ARCHIVE_INTERFACE_SUB(i, base, x) DECL_INTERFACE_SUB(i, base, 6, x) #define ARCHIVE_INTERFACE(i, x) ARCHIVE_INTERFACE_SUB(i, IUnknown, x) +/* +How the function in 7-Zip returns object for output parameter via pointer + +1) The caller sets the value of variable before function call: + PROPVARIANT : vt = VT_EMPTY + BSTR : NULL + IUnknown* and derived interfaces : NULL + another scalar types : any non-initialized value is allowed + +2) The callee in current 7-Zip code now can free input object for output parameter: + PROPVARIANT : the callee calls VariantClear(propvaiant_ptr) for input + value stored in variable + another types : the callee ignores stored value. + +3) The callee writes new value to variable for output parameter and + returns execution to caller. + +4) The caller must free or release object returned by the callee: + PROPVARIANT : VariantClear(&propvaiant) + BSTR : SysFreeString(bstr) + IUnknown* and derived interfaces : if (ptr) ptr->Relase() +*/ + + namespace NFileTimeType { enum EEnum @@ -34,6 +58,7 @@ namespace NArcInfoFlags const UInt32 kPreArc = 1 << 9; // such archive can be stored before real archive (like SFX stub) const UInt32 kSymLinks = 1 << 10; // the handler supports symbolic links const UInt32 kHardLinks = 1 << 11; // the handler supports hard links + const UInt32 kByExtOnlyOpen = 1 << 12; // call handler only if file extension matches } namespace NArchive @@ -106,7 +131,7 @@ namespace NArchive enum { kOK = 0 - , // kError + // , kError }; } } @@ -137,13 +162,13 @@ IArchiveExtractCallback::GetStream() Int32 askExtractMode (Extract::NAskMode) if (askMode != NExtract::NAskMode::kExtract) { - then the callee can not real stream: (*inStream == NULL) + then the callee doesn't write data to stream: (*outStream == NULL) } Out: - (*inStream == NULL) - for directories - (*inStream == NULL) - if link (hard link or symbolic link) was created - if (*inStream == NULL && askMode == NExtract::NAskMode::kExtract) + (*outStream == NULL) - for directories + (*outStream == NULL) - if link (hard link or symbolic link) was created + if (*outStream == NULL && askMode == NExtract::NAskMode::kExtract) { then the caller must skip extracting of that file. } diff --git a/CPP/7zip/Archive/IhexHandler.cpp b/CPP/7zip/Archive/IhexHandler.cpp index 00ff80a7..05453ee6 100644 --- a/CPP/7zip/Archive/IhexHandler.cpp +++ b/CPP/7zip/Archive/IhexHandler.cpp @@ -121,9 +121,9 @@ static int Parse(const Byte *p) #define kType_Data 0 #define kType_Eof 1 #define kType_Seg 2 -#define kType_CsIp 3 +// #define kType_CsIp 3 #define kType_High 4 -#define kType_Ip32 5 +// #define kType_Ip32 5 #define kType_MAX 5 diff --git a/CPP/7zip/Archive/Iso/IsoHandler.cpp b/CPP/7zip/Archive/Iso/IsoHandler.cpp index 2230cd23..87f4aa3b 100644 --- a/CPP/7zip/Archive/Iso/IsoHandler.cpp +++ b/CPP/7zip/Archive/Iso/IsoHandler.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include "../../../Common/ComTry.h" +#include "../../../Common/MyLinux.h" #include "../../../Common/StringConvert.h" #include "../../../Windows/PropVariant.h" @@ -221,16 +222,15 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidSymLink: if (_archive.IsSusp) { - UString s; UInt32 mode; if (item.GetPx(_archive.SuspSkipSize, k_Px_Mode, mode)) { - if (((mode >> 12) & 0xF) == 10) + if (MY_LIN_S_ISLNK(mode)) { AString s8; if (item.GetSymLink(_archive.SuspSkipSize, s8)) { - s = MultiByteToUnicodeString(s8, CP_OEMCP); + UString s = MultiByteToUnicodeString(s8, CP_OEMCP); prop = s; } } diff --git a/CPP/7zip/Archive/Iso/IsoIn.cpp b/CPP/7zip/Archive/Iso/IsoIn.cpp index b0bfb164..211b3eea 100644 --- a/CPP/7zip/Archive/Iso/IsoIn.cpp +++ b/CPP/7zip/Archive/Iso/IsoIn.cpp @@ -299,10 +299,12 @@ void CInArchive::ReadVolumeDescriptor(CVolumeDescriptor &d) static const Byte kSig_CD001[5] = { 'C', 'D', '0', '0', '1' }; +/* static const Byte kSig_NSR02[5] = { 'N', 'S', 'R', '0', '2' }; static const Byte kSig_NSR03[5] = { 'N', 'S', 'R', '0', '3' }; static const Byte kSig_BEA01[5] = { 'B', 'E', 'A', '0', '1' }; static const Byte kSig_TEA01[5] = { 'T', 'E', 'A', '0', '1' }; +*/ static inline bool CheckSignature(const Byte *sig, const Byte *data) { diff --git a/CPP/7zip/Archive/Iso/IsoItem.h b/CPP/7zip/Archive/Iso/IsoItem.h index 5ae13a60..a42ae039 100644 --- a/CPP/7zip/Archive/Iso/IsoItem.h +++ b/CPP/7zip/Archive/Iso/IsoItem.h @@ -149,7 +149,7 @@ struct CDirRecord } - const bool GetSymLink(int skipSize, AString &link) const + bool GetSymLink(int skipSize, AString &link) const { link.Empty(); const Byte *p = NULL; @@ -208,7 +208,7 @@ struct CDirRecord return true; } - static const bool GetLe32Be32(const Byte *p, UInt32 &dest) + static bool GetLe32Be32(const Byte *p, UInt32 &dest) { UInt32 v1 = GetUi32(p); UInt32 v2 = GetBe32(p + 4); @@ -221,7 +221,7 @@ struct CDirRecord } - const bool GetPx(int skipSize, unsigned pxType, UInt32 &val) const + bool GetPx(int skipSize, unsigned pxType, UInt32 &val) const { val = 0; const Byte *p = NULL; @@ -237,7 +237,7 @@ struct CDirRecord } /* - const bool GetTf(int skipSize, unsigned pxType, CRecordingDateTime &t) const + bool GetTf(int skipSize, unsigned pxType, CRecordingDateTime &t) const { const Byte *p = NULL; unsigned len = 0; diff --git a/CPP/7zip/Archive/LzhHandler.cpp b/CPP/7zip/Archive/LzhHandler.cpp index 53d69db7..e1984d28 100644 --- a/CPP/7zip/Archive/LzhHandler.cpp +++ b/CPP/7zip/Archive/LzhHandler.cpp @@ -41,6 +41,7 @@ static UInt16 g_LzhCrc16Table[256]; #define CRC16_UPDATE_BYTE(crc, b) (g_LzhCrc16Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt32 LzhCrc16Update(UInt32 crc, const void *data, size_t size); UInt32 LzhCrc16Update(UInt32 crc, const void *data, size_t size) { const Byte *p = (const Byte *)data; @@ -83,13 +84,7 @@ struct CExtension AString GetString() const { AString s; - for (size_t i = 0; i < Data.Size(); i++) - { - char c = (char)Data[i]; - if (c == 0) - break; - s += c; - } + s.SetFrom_CalcLen((const char *)(const Byte *)Data, (unsigned)Data.Size()); return s; } }; @@ -184,11 +179,13 @@ struct CItem return i; return -1; } + bool GetUnixTime(UInt32 &value) const { value = 0; int index = FindExt(kExtIdUnixTime); - if (index < 0) + if (index < 0 + || Extensions[index].Data.Size() < 4) { if (Level == 2) { @@ -220,13 +217,14 @@ struct CItem AString GetName() const { - AString dirName (GetDirName()); + AString s (GetDirName()); const char kDirSeparator = '\\'; // check kDirSeparator in Linux - dirName.Replace((char)(unsigned char)0xFF, kDirSeparator); - if (!dirName.IsEmpty() && dirName.Back() != kDirSeparator) - dirName += kDirSeparator; - return dirName + GetFileName(); + s.Replace((char)(unsigned char)0xFF, kDirSeparator); + if (!s.IsEmpty() && s.Back() != kDirSeparator) + s += kDirSeparator; + s += GetFileName(); + return s; } }; @@ -332,7 +330,7 @@ static HRESULT GetNextItem(ISequentialInStream *stream, bool &filled, CItem &ite return S_FALSE; CExtension ext; RINOK(ReadStream_FALSE(stream, &ext.Type, 1)) - nextSize -= 3; + nextSize = (UInt16)(nextSize - 3); ext.Data.Alloc(nextSize); RINOK(ReadStream_FALSE(stream, (Byte *)ext.Data, nextSize)) item.Extensions.Add(ext); diff --git a/CPP/7zip/Archive/LzmaHandler.cpp b/CPP/7zip/Archive/LzmaHandler.cpp index 9eac3ca1..ba547c83 100644 --- a/CPP/7zip/Archive/LzmaHandler.cpp +++ b/CPP/7zip/Archive/LzmaHandler.cpp @@ -242,29 +242,25 @@ STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) } -static void DictSizeToString(UInt32 val, char *s) +static char * DictSizeToString(UInt32 val, char *s) { for (unsigned i = 0; i <= 31; i++) if (((UInt32)1 << i) == val) - { - ::ConvertUInt32ToString(i, s); - return; - } + return ::ConvertUInt32ToString(i, s); char c = 'b'; if ((val & ((1 << 20) - 1)) == 0) { val >>= 20; c = 'm'; } else if ((val & ((1 << 10) - 1)) == 0) { val >>= 10; c = 'k'; } - ::ConvertUInt32ToString(val, s); - s += MyStringLen(s); + s = ::ConvertUInt32ToString(val, s); *s++ = c; *s = 0; + return s; } static char *AddProp32(char *s, const char *name, UInt32 v) { *s++ = ':'; s = MyStpCpy(s, name); - ::ConvertUInt32ToString(v, s); - return s + MyStringLen(s); + return ::ConvertUInt32ToString(v, s); } void CHandler::GetMethod(NCOM::CPropVariant &prop) @@ -277,8 +273,7 @@ void CHandler::GetMethod(NCOM::CPropVariant &prop) if (_header.FilterID != 0) s = MyStpCpy(s, "BCJ "); s = MyStpCpy(s, "LZMA:"); - DictSizeToString(_header.GetDicSize(), s); - s += strlen(s); + s = DictSizeToString(_header.GetDicSize(), s); UInt32 d = _header.GetProp(); // if (d != 0x5D) @@ -315,10 +310,10 @@ API_FUNC_static_IsArc IsArc_Lzma(const Byte *p, size_t size) return k_IsArc_Res_NEED_MORE; if (p[0] >= 5 * 5 * 9) return k_IsArc_Res_NO; - UInt64 unpackSize = GetUi64(p + 1 + 4); + const UInt64 unpackSize = GetUi64(p + 1 + 4); if (unpackSize != (UInt64)(Int64)-1) { - if (size >= ((UInt64)1 << 56)) + if (unpackSize >= ((UInt64)1 << 56)) return k_IsArc_Res_NO; } if (unpackSize != 0) diff --git a/CPP/7zip/Archive/MachoHandler.cpp b/CPP/7zip/Archive/MachoHandler.cpp index 1f65574d..bc8ba223 100644 --- a/CPP/7zip/Archive/MachoHandler.cpp +++ b/CPP/7zip/Archive/MachoHandler.cpp @@ -35,11 +35,11 @@ namespace NMacho { #define CPU_SUBTYPE_I386_ALL 3 -#define CPU_TYPE_PPC64 (CPU_ARCH_ABI64 | CPU_TYPE_PPC) +// #define CPU_TYPE_PPC64 (CPU_ARCH_ABI64 | CPU_TYPE_PPC) #define CPU_TYPE_AMD64 (CPU_ARCH_ABI64 | CPU_TYPE_386) #define CPU_TYPE_ARM64 (CPU_ARCH_ABI64 | CPU_TYPE_ARM) -#define CPU_SUBTYPE_LIB64 (1 << 31) +#define CPU_SUBTYPE_LIB64 ((UInt32)1 << 31) #define CPU_SUBTYPE_POWERPC_970 100 diff --git a/CPP/7zip/Archive/MbrHandler.cpp b/CPP/7zip/Archive/MbrHandler.cpp index 14a1224c..026696f3 100644 --- a/CPP/7zip/Archive/MbrHandler.cpp +++ b/CPP/7zip/Archive/MbrHandler.cpp @@ -51,10 +51,10 @@ struct CChs bool Check() const { return GetSector() > 0; } }; -#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } // Chs in some MBRs contains only low bits of "Cyl number". So we disable check. /* +#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } static int CompareChs(const CChs &c1, const CChs &c2) { RINOZ(MyCompare(c1.GetCyl(), c2.GetCyl())); diff --git a/CPP/7zip/Archive/MubHandler.cpp b/CPP/7zip/Archive/MubHandler.cpp index 6d054356..c790265d 100644 --- a/CPP/7zip/Archive/MubHandler.cpp +++ b/CPP/7zip/Archive/MubHandler.cpp @@ -33,7 +33,7 @@ namespace NMub { #define MACH_CPU_TYPE_AMD64 (MACH_CPU_ARCH_ABI64 | MACH_CPU_TYPE_386) #define MACH_CPU_TYPE_ARM64 (MACH_CPU_ARCH_ABI64 | MACH_CPU_TYPE_ARM) -#define MACH_CPU_SUBTYPE_LIB64 (1 << 31) +#define MACH_CPU_SUBTYPE_LIB64 ((UInt32)1 << 31) #define MACH_CPU_SUBTYPE_I386_ALL 3 @@ -124,14 +124,15 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } if (ext) strcpy(temp, ext); - if (item.SubType != 0 && ( - item.Type != MACH_CPU_TYPE_386 && - item.Type != MACH_CPU_TYPE_AMD64 || - (item.SubType & ~(UInt32)MACH_CPU_SUBTYPE_LIB64) != MACH_CPU_SUBTYPE_I386_ALL)) + if (item.SubType != 0) + if ((item.Type != MACH_CPU_TYPE_386 && + item.Type != MACH_CPU_TYPE_AMD64) + || (item.SubType & ~(UInt32)MACH_CPU_SUBTYPE_LIB64) != MACH_CPU_SUBTYPE_I386_ALL + ) { unsigned pos = MyStringLen(temp); temp[pos++] = '-'; - ConvertUInt32ToString(item.SubType, temp + pos); + ConvertUInt32ToString(item.SubType, temp + pos); } return PropVarEm_Set_Str(value, temp); } diff --git a/CPP/7zip/Archive/Nsis/NsisHandler.cpp b/CPP/7zip/Archive/Nsis/NsisHandler.cpp index 095105fe..aa0a9175 100644 --- a/CPP/7zip/Archive/Nsis/NsisHandler.cpp +++ b/CPP/7zip/Archive/Nsis/NsisHandler.cpp @@ -50,6 +50,7 @@ static const Byte kArcProps[] = { kpidMethod, kpidSolid, + kpidBit64, kpidHeadersSize, kpidEmbeddedStubSize, kpidSubType @@ -134,6 +135,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) break; } + case kpidBit64: if (_archive.Is64Bit) prop = true; break; case kpidMethod: prop = _methodString; break; case kpidSolid: prop = _archive.IsSolid; break; case kpidOffset: prop = _archive.StartOffset; break; diff --git a/CPP/7zip/Archive/Nsis/NsisIn.cpp b/CPP/7zip/Archive/Nsis/NsisIn.cpp index f4f9ab04..3c1a0f17 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -155,7 +155,7 @@ enum kNumCmds }; -static const unsigned kNumAdditionalParkCmds = 3; + struct CCommandInfo { @@ -506,18 +506,20 @@ void CInArchive::AddLicense(UInt32 param, Int32 langID) #endif -#define kVar_CMDLINE 20 +// #define kVar_CMDLINE 20 #define kVar_INSTDIR 21 #define kVar_OUTDIR 22 #define kVar_EXEDIR 23 -#define kVar_LANGUAGE 24 +// #define kVar_LANGUAGE 24 #define kVar_TEMP 25 #define kVar_PLUGINSDIR 26 #define kVar_EXEPATH 27 // NSIS 2.26+ -#define kVar_EXEFILE 28 // NSIS 2.26+ +// #define kVar_EXEFILE 28 // NSIS 2.26+ #define kVar_HWNDPARENT_225 27 +#ifdef NSIS_SCRIPT #define kVar_HWNDPARENT 29 +#endif // #define kVar__CLICK 30 #define kVar_Spec_OUTDIR_225 29 // NSIS 2.04 - 2.25 @@ -606,9 +608,9 @@ void CInArchive::AddParam_UInt(UInt32 value) #define NS_CODE_SKIP 252 #define NS_CODE_VAR 253 #define NS_CODE_SHELL 254 -#define NS_CODE_LANG 255 +// #define NS_CODE_LANG 255 -#define NS_3_CODE_LANG 1 +// #define NS_3_CODE_LANG 1 #define NS_3_CODE_SHELL 2 #define NS_3_CODE_VAR 3 #define NS_3_CODE_SKIP 4 @@ -1364,7 +1366,7 @@ void CInArchive::ReadString2(AString &s, UInt32 pos) #ifdef NSIS_SCRIPT -#define DEL_DIR 1 +// #define DEL_DIR 1 #define DEL_RECURSE 2 #define DEL_REBOOT 4 // #define DEL_SIMPLE 8 @@ -1445,9 +1447,11 @@ static void FlagsToString2(CDynLimBuf &s, const char * const *table, unsigned nu static bool DoesNeedQuotes(const char *s) { - char c = s[0]; - if (c == 0 || c == '#' || c == ';' || (c == '/' && s[1] == '*')) - return true; + { + char c = s[0]; + if (c == 0 || c == '#' || c == ';' || (c == '/' && s[1] == '*')) + return true; + } for (;;) { char c = *s++; @@ -1737,12 +1741,12 @@ void CNsis_CtlColors::Parse(const Byte *p) // Win32 constants #define MY__TRANSPARENT 1 -#define MY__OPAQUE 2 +// #define MY__OPAQUE 2 -#define MY__GENERIC_READ (1 << 31) -#define MY__GENERIC_WRITE (1 << 30) -#define MY__GENERIC_EXECUTE (1 << 29) -#define MY__GENERIC_ALL (1 << 28) +#define MY__GENERIC_READ ((UInt32)1 << 31) +#define MY__GENERIC_WRITE ((UInt32)1 << 30) +#define MY__GENERIC_EXECUTE ((UInt32)1 << 29) +#define MY__GENERIC_ALL ((UInt32)1 << 28) #define MY__CREATE_NEW 1 #define MY__CREATE_ALWAYS 2 @@ -2068,9 +2072,11 @@ void CSection::Parse(const Byte *p) #define SF_BOLD (1 << 3) #define SF_RO (1 << 4) #define SF_EXPAND (1 << 5) +/* #define SF_PSELECTED (1 << 6) #define SF_TOGGLED (1 << 7) #define SF_NAMECHG (1 << 8) +*/ bool CInArchive::PrintSectionBegin(const CSection §, unsigned index) { @@ -2115,7 +2121,7 @@ bool CInArchive::PrintSectionBegin(const CSection §, unsigned index) Script += ' '; else */ - SmallSpaceComment(); + SmallSpaceComment(); Script += "Section_"; Add_UInt(index); @@ -2290,14 +2296,15 @@ bool CInArchive::CompareCommands(const Byte *rawCmds, const Byte *sequence, size return true; } -#endif static const UInt32 kSectionSize_base = 6 * 4; -static const UInt32 kSectionSize_8bit = kSectionSize_base + 1024; -static const UInt32 kSectionSize_16bit = kSectionSize_base + 1024 * 2; -static const UInt32 kSectionSize_16bit_Big = kSectionSize_base + 8196 * 2; +// static const UInt32 kSectionSize_8bit = kSectionSize_base + 1024; +// static const UInt32 kSectionSize_16bit = kSectionSize_base + 1024 * 2; +// static const UInt32 kSectionSize_16bit_Big = kSectionSize_base + 8196 * 2; // 8196 is default string length in NSIS-Unicode since 2.37.3 +#endif + static void AddString(AString &dest, const char *src) { @@ -2330,8 +2337,13 @@ AString CInArchive::GetFormatDescription() const if (IsUnicode) AddString(s, "Unicode"); + + if (Is64Bit) + AddString(s, "64-bit"); + if (LogCmdIsEnabled) AddString(s, "log"); + if (BadCmd >= 0) { AddString(s, "BadCmd="); @@ -2342,9 +2354,11 @@ AString CInArchive::GetFormatDescription() const #ifdef NSIS_SCRIPT +static const unsigned kNumAdditionalParkCmds = 3; + unsigned CInArchive::GetNumSupportedCommands() const { - unsigned numCmds = IsPark() ? kNumCmds : kNumCmds - kNumAdditionalParkCmds; + unsigned numCmds = IsPark() ? (unsigned)kNumCmds : (unsigned)(kNumCmds) - kNumAdditionalParkCmds; if (!LogCmdIsEnabled) numCmds--; if (!IsUnicode) @@ -2792,26 +2806,20 @@ bool CInArchive::IsAbsolutePathVar(UInt32 strPos) const return false; } -#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z') +#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) // We use same check as in NSIS decoder -bool IsDrivePath(const wchar_t *s) { return IS_LETTER_CHAR(s[0]) && s[1] == ':' /* && s[2] == '\\' */ ; } -bool IsDrivePath(const char *s) { return IS_LETTER_CHAR(s[0]) && s[1] == ':' /* && s[2] == '\\' */ ; } +static bool IsDrivePath(const wchar_t *s) { return IS_LETTER_CHAR(s[0]) && s[1] == ':' /* && s[2] == '\\' */ ; } +static bool IsDrivePath(const char *s) { return IS_LETTER_CHAR(s[0]) && s[1] == ':' /* && s[2] == '\\' */ ; } static bool IsAbsolutePath(const wchar_t *s) { - return - s[0] == WCHAR_PATH_SEPARATOR && - s[1] == WCHAR_PATH_SEPARATOR || - IsDrivePath(s); + return (s[0] == WCHAR_PATH_SEPARATOR && s[1] == WCHAR_PATH_SEPARATOR) || IsDrivePath(s); } static bool IsAbsolutePath(const char *s) { - return - s[0] == CHAR_PATH_SEPARATOR && - s[1] == CHAR_PATH_SEPARATOR || - IsDrivePath(s); + return (s[0] == CHAR_PATH_SEPARATOR && s[1] == CHAR_PATH_SEPARATOR) || IsDrivePath(s); } void CInArchive::SetItemName(CItem &item, UInt32 strPos) @@ -3748,8 +3756,8 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) for (UInt32 j = i - 1; j >= kkk + 3; j--) { - const Byte *pCmd = p + kCmdSize * (j - kkk); - AddParam(GET_CMD_PARAM(pCmd, 0)); + const Byte *pCmd2 = p + kCmdSize * (j - kkk); + AddParam(GET_CMD_PARAM(pCmd2, 0)); } NewLine(); Tab(true); @@ -3850,13 +3858,13 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) { case k_ExecFlags_AutoClose: case k_ExecFlags_RebootFlag: - if (v < 2) s2 = (v == 0) ? "false" : "true"; break; + if (v < 2) { s2 = (v == 0) ? "false" : "true"; } break; case k_ExecFlags_ShellVarContext: - if (v < 2) s2 = (v == 0) ? "current" : "all"; break; + if (v < 2) { s2 = (v == 0) ? "current" : "all"; } break; case k_ExecFlags_Silent: - if (v < 2) s2 = (v == 0) ? "normal" : "silent"; break; + if (v < 2) { s2 = (v == 0) ? "normal" : "silent"; } break; case k_ExecFlags_RegView: - if (v == 0) s2 = "32"; + if (v == 0) s2 = "32"; else if (v == 256) s2 = "64"; break; case k_ExecFlags_DetailsPrint: @@ -3864,6 +3872,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) else if (v == 2) s2 = "textonly"; else if (v == 4) s2 = "listonly"; else if (v == 6) s2 = "none"; + break; } if (s2) { @@ -4400,7 +4409,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) s += 'F'; Add_UInt(key - kMy_VK_F1 + 1); } - else if (key >= 'A' && key <= 'Z' || key >= '0' && key <= '9') + else if ((key >= 'A' && key <= 'Z') || (key >= '0' && key <= '9')) s += (char)key; else { @@ -4687,6 +4696,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) s += "Text"; AddParam(params[1]); } + break; } case EW_SECTIONSET: @@ -4829,9 +4839,6 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) } else { - if (curSectionIndex == 49) - curSectionIndex = curSectionIndex; - if (PrintSectionBegin(sect, curSectionIndex)) curSectionIndex++; else @@ -4846,8 +4853,8 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh) static int CompareItems(void *const *p1, void *const *p2, void *param) { - const CItem &i1 = **(CItem **)p1; - const CItem &i2 = **(CItem **)p2; + const CItem &i1 = **(const CItem *const *)p1; + const CItem &i2 = **(const CItem *const *)p2; RINOZ(MyCompare(i1.Pos, i2.Pos)); const CInArchive *inArchive = (const CInArchive *)param; if (inArchive->IsUnicode) @@ -4970,14 +4977,15 @@ HRESULT CInArchive::SortItems() return S_OK; } +#ifdef NSIS_SCRIPT // Flags for common_header.flags -#define CH_FLAGS_DETAILS_SHOWDETAILS 1 -#define CH_FLAGS_DETAILS_NEVERSHOW 2 +// #define CH_FLAGS_DETAILS_SHOWDETAILS 1 +// #define CH_FLAGS_DETAILS_NEVERSHOW 2 #define CH_FLAGS_PROGRESS_COLORED 4 #define CH_FLAGS_SILENT 8 #define CH_FLAGS_SILENT_LOG 16 #define CH_FLAGS_AUTO_CLOSE 32 -#define CH_FLAGS_DIR_NO_SHOW 64 // unused now +// #define CH_FLAGS_DIR_NO_SHOW 64 // unused now #define CH_FLAGS_NO_ROOT_DIR 128 #define CH_FLAGS_COMP_ONLY_ON_CUSTOM 256 #define CH_FLAGS_NO_CUSTOM 512 @@ -4989,29 +4997,59 @@ static const char * const k_PostStrings[] = , "uninstcmd" // NSIS 2.25+, used by uninstaller: , "wininit" // NSIS 2.25+, used by move file on reboot }; +#endif + + +void CBlockHeader::Parse(const Byte *p, unsigned bhoSize) +{ + if (bhoSize == 12) + { + // UInt64 a = GetUi64(p); + if (GetUi32(p + 4) != 0) + throw 1; + } + Offset = GetUi32(p); + Num = GetUi32(p + bhoSize - 4); +} + +#define PARSE_BH(k, bh) bh.Parse (p1 + 4 + bhoSize * k, bhoSize) + HRESULT CInArchive::Parse() { // UInt32 offset = ReadUInt32(); // ???? offset == FirstHeader.HeaderSize - const Byte *p = _data; + const Byte * const p1 = _data; + + if (_size < 4 + 12 * 8) + Is64Bit = false; + else + { + Is64Bit = true; + // here we test high 32-bit of possible UInt64 CBlockHeader::Offset field + for (int k = 0; k < 8; k++) + if (GetUi32(p1 + 4 + 12 * k + 4) != 0) + Is64Bit = false; + } - if (_size < 4 + 8 * 8) + const unsigned bhoSize = Is64Bit ? 12 : 8; + if (_size < 4 + bhoSize * 8) return S_FALSE; CBlockHeader bhEntries, bhStrings, bhLangTables; - bhEntries.Parse(p + 4 + 8 * 2); - bhStrings.Parse(p + 4 + 8 * 3); - bhLangTables.Parse(p + 4 + 8 * 4); + + PARSE_BH (2, bhEntries); + PARSE_BH (3, bhStrings); + PARSE_BH (4, bhLangTables); #ifdef NSIS_SCRIPT CBlockHeader bhFont; - bhPages.Parse(p + 4 + 8 * 0); - bhSections.Parse(p + 4 + 8 * 1); - bhCtlColors.Parse(p + 4 + 8 * 5); - bhFont.Parse(p + 4 + 8 * 6); - bhData.Parse(p + 4 + 8 * 7); + PARSE_BH (0, bhPages); + PARSE_BH (1, bhSections); + PARSE_BH (5, bhCtlColors); + PARSE_BH (6, bhFont); + PARSE_BH (7, bhData); #endif @@ -5084,6 +5122,7 @@ HRESULT CInArchive::Parse() case NMethodType::kDeflate: m = "zlib"; break; case NMethodType::kBZip2: m = "bzip2"; break; case NMethodType::kLZMA: m = "lzma"; break; + default: break; } Script += "SetCompressor"; if (IsSolid) @@ -5167,7 +5206,7 @@ HRESULT CInArchive::Parse() memset(strUsed, 0, NumStringChars); { - UInt32 ehFlags = Get32(p); + UInt32 ehFlags = Get32(p1); UInt32 showDetails = ehFlags & 3;// CH_FLAGS_DETAILS_SHOWDETAILS & CH_FLAGS_DETAILS_NEVERSHOW; if (showDetails >= 1 && showDetails <= 2) { @@ -5208,11 +5247,16 @@ HRESULT CInArchive::Parse() } } - unsigned paramsOffset = 4 + 8 * 8; - if (bhPages.Offset == 276) - paramsOffset -= 8; + unsigned paramsOffset; + { + unsigned numBhs = 8; + // probably its for old NSIS? + if (bhoSize == 8 && bhPages.Offset == 276) + numBhs = 7; + paramsOffset = 4 + bhoSize * numBhs; + } - const Byte *p2 = p + paramsOffset; + const Byte *p2 = p1 + paramsOffset; { UInt32 rootKey = Get32(p2); // (rootKey = -1) in uninstaller by default (the bug in NSIS) @@ -5261,7 +5305,8 @@ HRESULT CInArchive::Parse() } UInt32 license_bg = Get32(p2 + 36); - if (license_bg != (UInt32)(Int32)-1 && license_bg != -15) // COLOR_BTNFACE + if (license_bg != (UInt32)(Int32)-1 && + license_bg != (UInt32)(Int32)-15) // COLOR_BTNFACE { Script += "LicenseBkColor"; if ((Int32)license_bg == -5) // COLOR_WINDOW @@ -5275,13 +5320,19 @@ HRESULT CInArchive::Parse() AddLF(); } - UInt32 langtable_size = Get32(p2 + 32); if (bhLangTables.Num > 0) { + const UInt32 langtable_size = Get32(p2 + 32); + if (langtable_size == (UInt32)(Int32)-1) return E_NOTIMPL; // maybe it's old NSIS archive() - UInt32 numStrings = (langtable_size - 10) / 4; + if (langtable_size < 10) + return S_FALSE; + if (bhLangTables.Num > (_size - bhLangTables.Offset) / langtable_size) + return S_FALSE; + + const UInt32 numStrings = (langtable_size - 10) / 4; _numLangStrings = numStrings; AddLF(); Separator(); @@ -5289,12 +5340,12 @@ HRESULT CInArchive::Parse() PrintNumComment("LANG STRINGS", numStrings); AddLF(); - if (licenseLangIndex >= 0) + if (licenseLangIndex >= 0 && (unsigned)licenseLangIndex < numStrings) { for (UInt32 i = 0; i < bhLangTables.Num; i++) { - const Byte *p = _data + bhLangTables.Offset + langtable_size * i; - LANGID langID = Get16(p); + const Byte * const p = _data + bhLangTables.Offset + langtable_size * i; + const UInt16 langID = Get16(p); UInt32 val = Get32(p + 10 + (UInt32)licenseLangIndex * 4); if (val != 0) { @@ -5309,33 +5360,24 @@ HRESULT CInArchive::Parse() AddLF(); } - UInt32 brandingText = 0; - UInt32 caption = 0; - UInt32 name = 0; + UInt32 names[3] = { 0 }; + UInt32 i; for (i = 0; i < bhLangTables.Num; i++) { - const Byte *p = _data + bhLangTables.Offset + langtable_size * i; - LANGID langID = Get16(p); + const Byte * const p = _data + bhLangTables.Offset + langtable_size * i; + const UInt16 langID = Get16(p); if (i == 0 || langID == 1033) _mainLang = p + 10; + for (unsigned k = 0; k < ARRAY_SIZE(names) && k < numStrings; k++) { - UInt32 v = Get32(p + 10 + 0 * 4); - if (v != 0 && (langID == 1033 || brandingText == 0)) - brandingText = v; - } - { - UInt32 v = Get32(p + 10 + 1 * 4); - if (v != 0 && (langID == 1033 || caption == 0)) - caption = v; - } - { - UInt32 v = Get32(p + 10 + 2 * 4); - if (v != 0 && (langID == 1033 || name == 0)) - name = v; + UInt32 v = Get32(p + 10 + k * 4); + if (v != 0 && (langID == 1033 || names[k] == 0)) + names[k] = v; } } - + + const UInt32 name = names[2]; if (name != 0) { Script += "Name"; @@ -5346,6 +5388,7 @@ HRESULT CInArchive::Parse() } /* + const UInt32 caption = names[1]; if (caption != 0) { Script += "Caption"; @@ -5354,6 +5397,7 @@ HRESULT CInArchive::Parse() } */ + const UInt32 brandingText = names[0]; if (brandingText != 0) { Script += "BrandingText"; @@ -5365,8 +5409,8 @@ HRESULT CInArchive::Parse() for (i = 0; i < bhLangTables.Num; i++) { - const Byte *p = _data + bhLangTables.Offset + langtable_size * i; - LANGID langID = Get16(p); + const Byte * const p = _data + bhLangTables.Offset + langtable_size * i; + const UInt16 langID = Get16(p); AddLF(); AddCommentAndString("LANG:"); @@ -5641,10 +5685,12 @@ HRESULT CInArchive::Open2(const Byte *sig, size_t size) if (Get32((const Byte *)buf) != FirstHeader.HeaderSize) return S_FALSE; } - size_t processedSize = FirstHeader.HeaderSize; - RINOK(Decoder.Read(_data, &processedSize)); - if (processedSize != FirstHeader.HeaderSize) - return S_FALSE; + { + size_t processedSize = FirstHeader.HeaderSize; + RINOK(Decoder.Read(_data, &processedSize)); + if (processedSize != FirstHeader.HeaderSize) + return S_FALSE; + } #ifdef NSIS_SCRIPT if (IsSolid) @@ -5824,14 +5870,25 @@ HRESULT CInArchive::Open(IInStream *inStream, const UInt64 *maxCheckStartPositio DataStreamOffset = pos + kStartHeaderSize; FirstHeader.Flags = Get32(buf); if ((FirstHeader.Flags & (~kFlagsMask)) != 0) + { + // return E_NOTIMPL; return S_FALSE; + } IsInstaller = (FirstHeader.Flags & NFlags::kUninstall) == 0; FirstHeader.HeaderSize = Get32(buf + kSignatureSize + 4); FirstHeader.ArcSize = Get32(buf + kSignatureSize + 8); if (FirstHeader.ArcSize <= kStartHeaderSize) return S_FALSE; - + + /* + if ((FirstHeader.Flags & NFlags::k_BI_ExternalFileSupport) != 0) + { + UInt32 datablock_low = Get32(buf + kSignatureSize + 12); + UInt32 datablock_high = Get32(buf + kSignatureSize + 16); + } + */ + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_fileSize)); IsArc = true; @@ -5872,7 +5929,8 @@ UString CInArchive::ConvertToUnicode(const AString &s) const if (IsUnicode) { UString res; - if (ConvertUTF8ToUnicode(s, res)) + // if ( + ConvertUTF8ToUnicode(s, res); return res; } return MultiByteToUnicodeString(s); @@ -5886,6 +5944,7 @@ void CInArchive::Clear2() IsNsis200 = false; LogCmdIsEnabled = false; BadCmd = -1; + Is64Bit = false; #ifdef NSIS_SCRIPT Name.Empty(); diff --git a/CPP/7zip/Archive/Nsis/NsisIn.h b/CPP/7zip/Archive/Nsis/NsisIn.h index 028e4a5d..5c88188d 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.h +++ b/CPP/7zip/Archive/Nsis/NsisIn.h @@ -34,6 +34,11 @@ namespace NFlags const UInt32 kSilent = 2; const UInt32 kNoCrc = 4; const UInt32 kForceCrc = 8; + // NSISBI fork flags: + const UInt32 k_BI_LongOffset = 16; + const UInt32 k_BI_ExternalFileSupport = 32; + const UInt32 k_BI_ExternalFile = 64; + const UInt32 k_BI_IsStubInstaller = 128; } struct CFirstHeader @@ -58,11 +63,7 @@ struct CBlockHeader UInt32 Offset; UInt32 Num; - void Parse(const Byte *p) - { - Offset = GetUi32(p); - Num = GetUi32(p + 4); - } + void Parse(const Byte *p, unsigned bhoSize); }; struct CItem @@ -159,6 +160,7 @@ public: CByteBuffer _data; CObjectVector<CItem> Items; bool IsUnicode; + bool Is64Bit; private: UInt32 _stringsPos; // relative to _data UInt32 NumStringChars; @@ -170,7 +172,6 @@ private: ENsisType NsisType; bool IsNsis200; // NSIS 2.03 and before bool IsNsis225; // NSIS 2.25 and before - bool LogCmdIsEnabled; int BadCmd; // -1: no bad command; in another cases lowest bad command id diff --git a/CPP/7zip/Archive/NtfsHandler.cpp b/CPP/7zip/Archive/NtfsHandler.cpp index 21372033..daa01fef 100644 --- a/CPP/7zip/Archive/NtfsHandler.cpp +++ b/CPP/7zip/Archive/NtfsHandler.cpp @@ -208,7 +208,7 @@ enum Posix name can be after or before Win32 name */ -static const Byte kFileNameType_Posix = 0; // for hard links +// static const Byte kFileNameType_Posix = 0; // for hard links static const Byte kFileNameType_Win32 = 1; // after Dos name static const Byte kFileNameType_Dos = 2; // short name static const Byte kFileNameType_Win32Dos = 3; // short and full name are same @@ -386,8 +386,8 @@ struct CAttr static int CompareAttr(void *const *elem1, void *const *elem2, void *) { - const CAttr &a1 = *(*((const CAttr **)elem1)); - const CAttr &a2 = *(*((const CAttr **)elem2)); + const CAttr &a1 = *(*((const CAttr *const *)elem1)); + const CAttr &a2 = *(*((const CAttr *const *)elem2)); RINOZ(MyCompare(a1.Type, a2.Type)); if (a1.Name.IsEmpty()) { @@ -717,12 +717,16 @@ static size_t Lznt1Dec(Byte *dest, size_t outBufLim, size_t destLen, const Byte UInt32 dist = (v >> (16 - numDistBits)); if (dist >= sbOffset) return 0; - Int32 offs = -1 - dist; - Byte *p = dest + destSize; - for (UInt32 t = 0; t < len; t++) - p[t] = p[t + offs]; + const size_t offs = 1 + dist; + Byte *p = dest + destSize - offs; destSize += len; sbOffset += len; + const Byte *lim = p + len; + p[offs] = *p; ++p; + p[offs] = *p; ++p; + do + p[offs] = *p; + while (++p != lim); } } } @@ -1094,7 +1098,7 @@ struct CMftRec void CMftRec::ParseDataNames() { DataRefs.Clear(); - DataAttrs.Sort(CompareAttr, 0); + DataAttrs.Sort(CompareAttr, NULL); for (unsigned i = 0; i < DataAttrs.Size();) { @@ -2188,7 +2192,7 @@ STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data *data = (const wchar_t *)EmptyString; else *data = s->GetRawPtr(); - *dataSize = (s->Len() + 1) * sizeof(wchar_t); + *dataSize = (s->Len() + 1) * (UInt32)sizeof(wchar_t); *propType = PROP_DATA_TYPE_wchar_t_PTR_Z_LE; #endif return S_OK; diff --git a/CPP/7zip/Archive/PeHandler.cpp b/CPP/7zip/Archive/PeHandler.cpp index 675293ba..ee265571 100644 --- a/CPP/7zip/Archive/PeHandler.cpp +++ b/CPP/7zip/Archive/PeHandler.cpp @@ -363,8 +363,8 @@ struct CSection CSection(): IsRealSect(false), IsDebug(false), IsAdditionalSection(false) {} - const UInt32 GetSizeExtract() const { return PSize; } - const UInt32 GetSizeMin() const { return MyMin(PSize, VSize); } + UInt32 GetSizeExtract() const { return PSize; } + UInt32 GetSizeMin() const { return MyMin(PSize, VSize); } void UpdateTotalSize(UInt32 &totalSize) const { @@ -768,15 +768,15 @@ class CHandler: bool _oneLang; UString _resourcesPrefix; CUsedBitmap _usedRes; - bool _parseResources; + // bool _parseResources; bool _checksumError; bool IsOpt() const { return _header.OptHeaderSize != 0; } COptHeader _optHeader; - bool _allowTail; bool _coffMode; + bool _allowTail; HRESULT LoadDebugSections(IInStream *stream, bool &thereIsSection); HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback); @@ -2005,10 +2005,12 @@ HRESULT CHandler::OpenResources(unsigned sectionIndex, IInStream *stream, IArchi const UInt32 mask = ((UInt32)1 << numBits) - 1; const size_t end = (size_t)((sect.VSize + mask) & (UInt32)~mask); if (end > sect.VSize) + { if (end <= sect.PSize) fileSize = end; else fileSize = sect.PSize; + } } } @@ -2459,7 +2461,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) return S_FALSE; } - _parseResources = true; + bool _parseResources = true; // _parseResources = false; UInt64 mainSize = 0, mainSize2 = 0; @@ -2467,11 +2469,11 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) for (i = 0; i < _sections.Size(); i++) { const CSection § = _sections[i]; - CMixItem mixItem; - mixItem.SectionIndex = i; if (IsOpt()) - if (_parseResources && sect.Name == ".rsrc" && _items.IsEmpty()) + if (_parseResources && sect.Name == ".rsrc") { + // 20.01: we try to parse only first copy of .rsrc section. + _parseResources = false; const unsigned numMixItems = _mixItems.Size(); HRESULT res = OpenResources(i, stream, callback); if (res == S_OK) @@ -2483,6 +2485,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) const CResItem &item = _items[j]; if (item.Enabled) { + CMixItem mixItem; + mixItem.SectionIndex = i; mixItem.ResourceIndex = j; if (item.IsRcDataOrUnknown()) { @@ -2531,6 +2535,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) _mixItems.DeleteFrom(numMixItems); CloseResources(); } + if (sect.IsAdditionalSection) { if (sect.PSize >= mainSize) @@ -2542,6 +2547,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) else if (sect.PSize >= mainSize2) mainSize2 = sect.PSize; } + + CMixItem mixItem; + mixItem.SectionIndex = i; _mixItems.Add(mixItem); } @@ -3017,7 +3025,7 @@ static const Byte kProps[] = enum { - kpidSubSystem = kpidUserDefined, + kpidSubSystem = kpidUserDefined // , kpidImageBase }; diff --git a/CPP/7zip/Archive/PpmdHandler.cpp b/CPP/7zip/Archive/PpmdHandler.cpp index c80400bc..05a07e53 100644 --- a/CPP/7zip/Archive/PpmdHandler.cpp +++ b/CPP/7zip/Archive/PpmdHandler.cpp @@ -1,5 +1,5 @@ /* PpmdHandler.cpp -- PPMd format handler -2015-11-30 : Igor Pavlov : Public domain +2020 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001) / var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -59,7 +59,7 @@ struct CItem unsigned Restor; HRESULT ReadHeader(ISequentialInStream *s, UInt32 &headerSize); - bool IsSupported() const { return Ver == 7 || (Ver == 8 && Restor <= 1); } + bool IsSupported() const { return Ver == 7 || (Ver == 8 && Restor < PPMD8_RESTORE_METHOD_UNSUPPPORTED); } }; HRESULT CItem::ReadHeader(ISequentialInStream *s, UInt32 &headerSize) @@ -218,91 +218,11 @@ STDMETHODIMP CHandler::Close() return S_OK; } -static const UInt32 kTopValue = (1 << 24); -static const UInt32 kBot = (1 << 15); -struct CRangeDecoder -{ - IPpmd7_RangeDec vt; - UInt32 Range; - UInt32 Code; - UInt32 Low; - CByteInBufWrap *Stream; - -public: - bool Init() - { - Code = 0; - Low = 0; - Range = 0xFFFFFFFF; - for (int i = 0; i < 4; i++) - Code = (Code << 8) | Stream->ReadByte(); - return Code < 0xFFFFFFFF; - } - - void Normalize() - { - while ((Low ^ (Low + Range)) < kTopValue || - Range < kBot && ((Range = (0 - Low) & (kBot - 1)), 1)) - { - Code = (Code << 8) | Stream->ReadByte(); - Range <<= 8; - Low <<= 8; - } - } - - CRangeDecoder(); -}; - - -extern "C" { - -#define GET_RangeDecoder CRangeDecoder *p = CONTAINER_FROM_VTBL(pp, CRangeDecoder, vt); - -static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) -{ - GET_RangeDecoder - return p->Code / (p->Range /= total); -} - -static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) -{ - GET_RangeDecoder - start *= p->Range; - p->Low += start; - p->Code -= start; - p->Range *= size; - p->Normalize(); -} - -static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) -{ - GET_RangeDecoder - if (p->Code / (p->Range >>= 14) < size0) - { - Range_Decode(&p->vt, 0, size0); - return 0; - } - else - { - Range_Decode(&p->vt, size0, (1 << 14) - size0); - return 1; - } -} - -} - -CRangeDecoder::CRangeDecoder() -{ - vt.GetThreshold = Range_GetThreshold; - vt.Decode = Range_Decode; - vt.DecodeBit = Range_DecodeBit; -} struct CPpmdCpp { unsigned Ver; - CRangeDecoder _rc; CPpmd7 _ppmd7; CPpmd8 _ppmd8; @@ -339,20 +259,20 @@ struct CPpmdCpp { if (Ver == 7) { - _rc.Stream = inStream; - return _rc.Init(); + _ppmd7.rc.dec.Stream = &inStream->vt; + return (Ppmd7a_RangeDec_Init(&_ppmd7.rc.dec) != 0); } else { _ppmd8.Stream.In = &inStream->vt; - return Ppmd8_RangeDec_Init(&_ppmd8) != 0; + return Ppmd8_Init_RangeDec(&_ppmd8) != 0; } } bool IsFinishedOK() { if (Ver == 7) - return Ppmd7z_RangeDec_IsFinishedOK(&_rc); + return Ppmd7z_RangeDec_IsFinishedOK(&_ppmd7.rc.dec); return Ppmd8_RangeDec_IsFinishedOK(&_ppmd8); } }; @@ -416,14 +336,15 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, size_t i; int sym = 0; + Byte *buf = outBuf.Buf; if (ppmd.Ver == 7) { for (i = 0; i < kBufSize; i++) { - sym = Ppmd7_DecodeSymbol(&ppmd._ppmd7, &ppmd._rc.vt); + sym = Ppmd7a_DecodeSymbol(&ppmd._ppmd7); if (inBuf.Extra || sym < 0) break; - outBuf.Buf[i] = (Byte)sym; + buf[i] = (Byte)sym; } } else @@ -433,7 +354,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, sym = Ppmd8_DecodeSymbol(&ppmd._ppmd8); if (inBuf.Extra || sym < 0) break; - outBuf.Buf[i] = (Byte)sym; + buf[i] = (Byte)sym; } } diff --git a/CPP/7zip/Archive/QcowHandler.cpp b/CPP/7zip/Archive/QcowHandler.cpp index 065f59b3..200ec62d 100644 --- a/CPP/7zip/Archive/QcowHandler.cpp +++ b/CPP/7zip/Archive/QcowHandler.cpp @@ -8,6 +8,7 @@ #include "../../Common/ComTry.h" #include "../../Common/IntToString.h" +#include "../../Common/MyBuffer2.h" #include "../../Windows/PropVariant.h" @@ -31,13 +32,21 @@ namespace NQcow { static const Byte k_Signature[] = SIGNATURE; +/* +VA to PA maps: + high bits (L1) : : in L1 Table : the reference to L1 Table + mid bits (L2) : _numMidBits : in L2 Table : the reference to cluster + low bits : _clusterBits +*/ + class CHandler: public CHandlerImg { unsigned _clusterBits; unsigned _numMidBits; UInt64 _compressedFlag; - CObjectVector<CByteBuffer> _tables; + CObjArray2<UInt32> _dir; + CAlignedBuffer _table; UInt64 _cacheCluster; CByteBuffer _cache; CByteBuffer _cacheCompressed; @@ -63,7 +72,7 @@ class CHandler: public CHandlerImg UInt32 _version; UInt32 _cryptMethod; - HRESULT Seek(UInt64 offset) + HRESULT Seek2(UInt64 offset) { _posInArc = offset; return Stream->Seek(offset, STREAM_SEEK_SET, NULL); @@ -72,7 +81,7 @@ class CHandler: public CHandlerImg HRESULT InitAndSeek() { _virtPos = 0; - return Seek(0); + return Seek2(0); } HRESULT Open2(IInStream *stream, IArchiveOpenCallback *openCallback); @@ -85,10 +94,15 @@ public: }; +static const UInt32 kEmptyDirItem = (UInt32)0 - 1; + STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) { if (processedSize) *processedSize = 0; + + // printf("\nRead _virtPos = %6d size = %6d\n", (UInt32)_virtPos, size); + if (_virtPos >= _size) return S_OK; { @@ -101,9 +115,9 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) for (;;) { - UInt64 cluster = _virtPos >> _clusterBits; - size_t clusterSize = (size_t)1 << _clusterBits; - size_t lowBits = (size_t)_virtPos & (clusterSize - 1); + const UInt64 cluster = _virtPos >> _clusterBits; + const size_t clusterSize = (size_t)1 << _clusterBits; + const size_t lowBits = (size_t)_virtPos & (clusterSize - 1); { size_t rem = clusterSize - lowBits; if (size > rem) @@ -113,21 +127,19 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) if (cluster == _cacheCluster) { memcpy(data, _cache + lowBits, size); - _virtPos += size; - if (processedSize) - *processedSize = size; - return S_OK; + break; } - UInt64 high = cluster >> _numMidBits; + const UInt64 high = cluster >> _numMidBits; - if (high < _tables.Size()) + if (high < _dir.Size()) { - const CByteBuffer &buffer = _tables[(unsigned)high]; + const UInt32 tabl = _dir[(unsigned)high]; - if (buffer.Size() != 0) + if (tabl != kEmptyDirItem) { - size_t midBits = (size_t)cluster & (((size_t)1 << _numMidBits) - 1); + const Byte *buffer = _table + ((size_t)tabl << (_numMidBits + 3)); + const size_t midBits = (size_t)cluster & (((size_t)1 << _numMidBits) - 1); const Byte *p = (const Byte *)buffer + (midBits << 3); UInt64 v = Get64(p); @@ -137,19 +149,32 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) { if (_version <= 1) return E_FAIL; - unsigned numOffsetBits = (62 - (_clusterBits - 8)); - UInt64 offset = v & (((UInt64)1 << 62) - 1); + + /* + the example of table record for 12-bit clusters (4KB uncompressed). + 2 bits : isCompressed status + 4 bits : num_sectors_minus1; packSize = (num_sectors_minus1 + 1) * 512; + it uses one additional bit over unpacked cluster_bits + 49 bits : offset of 512-sector + 9 bits : offset in 512-sector + */ + + const unsigned numOffsetBits = (62 - (_clusterBits - 9 + 1)); + const UInt64 offset = v & (((UInt64)1 << 62) - 1); const size_t dataSize = ((size_t)(offset >> numOffsetBits) + 1) << 9; - offset &= ((UInt64)1 << numOffsetBits) - 1; - UInt64 sectorOffset = offset >> 9 << 9; - UInt64 offset2inCache = sectorOffset - _comprPos; + UInt64 sectorOffset = offset & (((UInt64)1 << numOffsetBits) - (1 << 9)); + const UInt64 offset2inCache = sectorOffset - _comprPos; + // _comprPos is aligned for 512-bytes + // we try to use previous _cacheCompressed that contains compressed data + // that was read for previous unpacking + if (sectorOffset >= _comprPos && offset2inCache < _comprSize) { if (offset2inCache != 0) { _comprSize -= (size_t)offset2inCache; - memmove(_cacheCompressed, _cacheCompressed + offset2inCache, _comprSize); + memmove(_cacheCompressed, _cacheCompressed + (size_t)offset2inCache, _comprSize); _comprPos = sectorOffset; } sectorOffset += _comprSize; @@ -160,25 +185,27 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) _comprSize = 0; } - // printf("\nDeflate"); - if (sectorOffset != _posInArc) + if (dataSize > _comprSize) { - // printf("\nDeflate %12I64x %12I64x\n", sectorOffset, sectorOffset - _posInArc); - RINOK(Seek(sectorOffset)); + if (sectorOffset != _posInArc) + { + // printf("\nDeflate-Seek %12I64x %12I64x\n", sectorOffset, sectorOffset - _posInArc); + RINOK(Seek2(sectorOffset)); + } + if (_cacheCompressed.Size() < dataSize) + return E_FAIL; + const size_t dataSize3 = dataSize - _comprSize; + size_t dataSize2 = dataSize3; + // printf("\n\n=======\nReadStream = %6d _comprPos = %6d \n", (UInt32)dataSize2, (UInt32)_comprPos); + RINOK(ReadStream(Stream, _cacheCompressed + _comprSize, &dataSize2)); + _posInArc += dataSize2; + if (dataSize2 != dataSize3) + return E_FAIL; + _comprSize += dataSize2; } - if (_cacheCompressed.Size() < dataSize) - return E_FAIL; - size_t dataSize3 = dataSize - _comprSize; - size_t dataSize2 = dataSize3; - RINOK(ReadStream(Stream, _cacheCompressed + _comprSize, &dataSize2)); - _posInArc += dataSize2; - if (dataSize2 != dataSize3) - return E_FAIL; - _comprSize += dataSize2; - const size_t kSectorMask = (1 << 9) - 1; - size_t offsetInSector = ((size_t)offset & kSectorMask); + const size_t offsetInSector = ((size_t)offset & kSectorMask); _bufInStreamSpec->Init(_cacheCompressed + offsetInSector, dataSize - offsetInSector); _cacheCluster = (UInt64)(Int64)-1; @@ -187,7 +214,7 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) _bufOutStreamSpec->Init(_cache, clusterSize); // Do we need to use smaller block than clusterSize for last cluster? - UInt64 blockSize64 = clusterSize; + const UInt64 blockSize64 = clusterSize; HRESULT res = _deflateDecoderSpec->Code(_bufInStream, _bufOutStream, NULL, &blockSize64, NULL); /* @@ -206,10 +233,7 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) continue; /* memcpy(data, _cache + lowBits, size); - _virtPos += size; - if (processedSize) - *processedSize = size; - return S_OK; + break; */ } @@ -221,7 +245,7 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) if (v != _posInArc) { // printf("\n%12I64x\n", v - _posInArc); - RINOK(Seek(v)); + RINOK(Seek2(v)); } HRESULT res = Stream->Read(data, size, &size); _posInArc += size; @@ -235,11 +259,13 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) } memset(data, 0, size); - _virtPos += size; - if (processedSize) - *processedSize = size; - return S_OK; + break; } + + _virtPos += size; + if (processedSize) + *processedSize = size; + return S_OK; } @@ -347,8 +373,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) const UInt64 backOffset = Get64(buf + 8); // UInt32 backSize = Get32(buf + 0x10); - UInt64 l1Offset = 0; - UInt32 l1Size = 0; + UInt64 l1Offset; + UInt32 l1Size; if (_version == 1) { @@ -364,8 +390,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) l1Offset = Get64(buf + 0x28); if (l1Offset < 0x30) return S_FALSE; - unsigned numBits2 = (_clusterBits + _numMidBits); - UInt64 l1Size64 = (_size + (((UInt64)1 << numBits2) - 1)) >> numBits2; + const unsigned numBits2 = (_clusterBits + _numMidBits); + const UInt64 l1Size64 = (_size + (((UInt64)1 << numBits2) - 1)) >> numBits2; if (l1Size64 > ((UInt32)1 << 31)) return S_FALSE; l1Size = (UInt32)l1Size64; @@ -381,8 +407,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) l1Size = Get32(buf + 0x24); l1Offset = Get64(buf + 0x28); // must be aligned for cluster - UInt64 refOffset = Get64(buf + 0x30); // must be aligned for cluster - UInt32 refClusters = Get32(buf + 0x38); + const UInt64 refOffset = Get64(buf + 0x30); // must be aligned for cluster + const UInt32 refClusters = Get32(buf + 0x38); // UInt32 numSnapshots = Get32(buf + 0x3C); // UInt64 snapshotsOffset = Get64(buf + 0x40); // must be aligned for cluster @@ -393,14 +419,14 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) if (refClusters != 0) { - size_t numBytes = refClusters << _clusterBits; + const size_t numBytes = refClusters << _clusterBits; /* CByteBuffer refs; refs.Alloc(numBytes); RINOK(stream->Seek(refOffset, STREAM_SEEK_SET, NULL)); RINOK(ReadStream_FALSE(stream, refs, numBytes)); */ - UInt64 end = refOffset + numBytes; + const UInt64 end = refOffset + numBytes; if (_phySize < end) _phySize = end; /* @@ -426,7 +452,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) CByteBuffer table; { - size_t t1SizeBytes = (size_t)l1Size << 3; + const size_t t1SizeBytes = (size_t)l1Size << 3; if ((t1SizeBytes >> 3) != l1Size) return S_FALSE; table.Alloc(t1SizeBytes); @@ -442,41 +468,72 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) } } - if (openCallback) + _compressedFlag = (_version <= 1) ? ((UInt64)1 << 63) : ((UInt64)1 << 62); + const UInt64 offsetMask = _compressedFlag - 1; + + UInt32 numTables = 0; + UInt32 i; + + for (i = 0; i < l1Size; i++) { - UInt64 totalBytes = (UInt64)l1Size << (_numMidBits + 3); - RINOK(openCallback->SetTotal(NULL, &totalBytes)); + const UInt64 v = Get64((const Byte *)table + (size_t)i * 8) & offsetMask; + if (v != 0) + numTables++; } - _compressedFlag = (_version <= 1) ? ((UInt64)1 << 63) : ((UInt64)1 << 62); - const UInt64 offsetMask = _compressedFlag - 1; + if (numTables != 0) + { + const size_t size = (size_t)numTables << (_numMidBits + 3); + if (size >> (_numMidBits + 3) != numTables) + return E_OUTOFMEMORY; + _table.Alloc(size); + if (!_table.IsAllocated()) + return E_OUTOFMEMORY; + } + + _dir.SetSize(l1Size); - for (UInt32 i = 0; i < l1Size; i++) + UInt32 curTable = 0; + + if (openCallback) { - if (openCallback) - { - UInt64 numBytes = (UInt64)i << (_numMidBits + 3); - RINOK(openCallback->SetCompleted(NULL, &numBytes)); - } + const UInt64 totalBytes = (UInt64)numTables << (_numMidBits + 3); + RINOK(openCallback->SetTotal(NULL, &totalBytes)); + } - CByteBuffer &buf2 = _tables.AddNew(); - + for (i = 0; i < l1Size; i++) + { + Byte *buf2; + const size_t midSize = (size_t)1 << (_numMidBits + 3); + { - UInt64 v = Get64((const Byte *)table + (size_t)i * 8); - v &= offsetMask; + const UInt64 v = Get64((const Byte *)table + (size_t)i * 8) & offsetMask; if (v == 0) + { + _dir[i] = kEmptyDirItem; continue; + } + + _dir[i] = curTable; + const size_t tableOffset = ((size_t)curTable << (_numMidBits + 3)); + buf2 = (Byte *)_table + tableOffset; + curTable++; + + if (openCallback && (tableOffset & 0xFFFFF) == 0) + { + const UInt64 numBytes = tableOffset; + RINOK(openCallback->SetCompleted(NULL, &numBytes)); + } - buf2.Alloc((size_t)1 << (_numMidBits + 3)); RINOK(stream->Seek(v, STREAM_SEEK_SET, NULL)); - RINOK(ReadStream_FALSE(stream, buf2, clusterSize)); + RINOK(ReadStream_FALSE(stream, buf2, midSize)); - const UInt64 end = v + clusterSize; + const UInt64 end = v + midSize; if (_phySize < end) _phySize = end; } - for (size_t k = 0; k < clusterSize; k += 8) + for (size_t k = 0; k < midSize; k += 8) { const UInt64 v = Get64((const Byte *)buf2 + (size_t)k); if (v == 0) @@ -519,12 +576,15 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) } } - UInt64 end = offset + dataSize; + const UInt64 end = offset + dataSize; if (_phySize < end) _phySize = end; } } + if (curTable != numTables) + return E_FAIL; + if (_cryptMethod != 0) _unsupported = true; @@ -538,7 +598,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) STDMETHODIMP CHandler::Close() { - _tables.Clear(); + _table.Free(); + _dir.Free(); _phySize = 0; _size = 0; @@ -588,7 +649,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 /* index */, ISequentialInStream **strea _deflateDecoderSpec->Set_NeedFinishInput(true); } - size_t clusterSize = (size_t)1 << _clusterBits; + const size_t clusterSize = (size_t)1 << _clusterBits; _cache.AllocAtLeast(clusterSize); _cacheCompressed.AllocAtLeast(clusterSize * 2); } diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index 320771d5..bb8a2edb 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -124,19 +124,13 @@ static unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val) bool CLinkInfo::Parse(const Byte *p, unsigned size) { const Byte *pStart = p; - unsigned num = ReadVarInt(p, size, &Type); - if (num == 0) return false; p += num; size -= num; - - num = ReadVarInt(p, size, &Flags); - if (num == 0) return false; p += num; size -= num; - + unsigned num; UInt64 len; - num = ReadVarInt(p, size, &len); - if (num == 0) return false; p += num; size -= num; - + num = ReadVarInt(p, size, &Type); if (num == 0) { return false; } p += num; size -= num; + num = ReadVarInt(p, size, &Flags); if (num == 0) { return false; } p += num; size -= num; + num = ReadVarInt(p, size, &len); if (num == 0) { return false; } p += num; size -= num; if (size != len) return false; - NameLen = (unsigned)len; NameOffset = (unsigned)(p - pStart); return true; @@ -319,10 +313,10 @@ bool CCryptoInfo::Parse(const Byte *p, size_t size) Cnt = 0; unsigned num = ReadVarInt(p, size, &Algo); - if (num == 0) return false; p += num; size -= num; + if (num == 0) { return false; } p += num; size -= num; num = ReadVarInt(p, size, &Flags); - if (num == 0) return false; p += num; size -= num; + if (num == 0) { return false; } p += num; size -= num; if (size > 0) Cnt = p[0]; @@ -344,10 +338,10 @@ bool CItem::FindExtra_Version(UInt64 &version) const UInt64 flags; unsigned num = ReadVarInt(p, size, &flags); - if (num == 0) return false; p += num; size -= num; + if (num == 0) { return false; } p += num; size -= num; num = ReadVarInt(p, size, &version); - if (num == 0) return false; p += num; size -= num; + if (num == 0) { return false; } p += num; size -= num; return size == 0; } @@ -406,8 +400,8 @@ void CItem::Link_to_Prop(unsigned linkType, NWindows::NCOM::CPropVariant &prop) s.SetFrom_CalcLen((const char *)(Extra + link.NameOffset), link.NameLen); UString unicode; - if (ConvertUTF8ToUnicode(s, unicode)) - prop = NItemName::GetOsPath(unicode); + ConvertUTF8ToUnicode(s, unicode); + prop = NItemName::GetOsPath(unicode); } bool CItem::GetAltStreamName(AString &name) const @@ -596,11 +590,12 @@ public: static HRESULT MySetPassword(ICryptoGetTextPassword *getTextPassword, NCrypto::NRar5::CDecoder *cryptoDecoderSpec) { - CMyComBSTR password; + CMyComBSTR_Wipe password; RINOK(getTextPassword->CryptoGetTextPassword(&password)); - AString utf8; + AString_Wipe utf8; const unsigned kPasswordLen_MAX = 127; - UString unicode = (LPCOLESTR)password; + UString_Wipe unicode; + unicode.SetFromBstr(password); if (unicode.Len() > kPasswordLen_MAX) unicode.DeleteFrom(kPasswordLen_MAX); ConvertUnicodeToUTF8(unicode, utf8); @@ -1153,7 +1148,7 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz } else { - res = res; + // res = res; } if (isCryptoMode) @@ -1434,8 +1429,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) AString s; s.SetFrom_CalcLen((const char *)(const Byte *)cmt, (unsigned)cmt.Size()); UString unicode; - if (ConvertUTF8ToUnicode(s, unicode)) - prop = unicode; + ConvertUTF8ToUnicode(s, unicode); + prop = unicode; } } break; @@ -1686,13 +1681,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val if (name[0] != ':') s += ':'; s += name; - if (!ConvertUTF8ToUnicode(s, unicodeName)) - break; + ConvertUTF8ToUnicode(s, unicodeName); } else { - if (!ConvertUTF8ToUnicode(item.Name, unicodeName)) - break; + ConvertUTF8ToUnicode(item.Name, unicodeName); + if (item.Version_Defined) { char temp[32]; @@ -1752,8 +1746,8 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { name.DeleteFrontal(1); UString unicodeName; - if (ConvertUTF8ToUnicode(name, unicodeName)) - prop = unicodeName; + ConvertUTF8ToUnicode(name, unicodeName); + prop = unicodeName; } } break; diff --git a/CPP/7zip/Archive/Rar/RarHandler.cpp b/CPP/7zip/Archive/Rar/RarHandler.cpp index dd78e312..7491c50b 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.cpp +++ b/CPP/7zip/Archive/Rar/RarHandler.cpp @@ -466,7 +466,7 @@ bool CInArchive::ReadHeaderReal(const Byte *p, unsigned size, CItem &item) for (unsigned i = 0; i < sizeof(item.Salt); i++) item.Salt[i] = p[i]; p += sizeof(item.Salt); - size -= sizeof(item.Salt); + size -= (unsigned)sizeof(item.Salt); } // some rar archives have HasExtTime flag without field. @@ -526,31 +526,36 @@ HRESULT CInArchive::GetNextItem(CItem &item, ICryptoGetTextPassword *getTextPass } // m_RarAESSpec->SetRar350Mode(ArcInfo.IsEncryptOld()); - // Salt - const UInt32 kSaltSize = 8; - Byte salt[kSaltSize]; - if (!ReadBytesAndTestSize(salt, kSaltSize)) - return S_FALSE; - m_Position += kSaltSize; - RINOK(m_RarAESSpec->SetDecoderProperties2(salt, kSaltSize)) - // Password - CMyComBSTR password; - RINOK(getTextPassword->CryptoGetTextPassword(&password)) - unsigned len = 0; - if (password) - len = MyStringLen(password); - if (len > kPasswordLen_MAX) - len = kPasswordLen_MAX; - - CByteArr buffer(len * 2); - for (unsigned i = 0; i < len; i++) { - wchar_t c = password[i]; - ((Byte *)buffer)[i * 2] = (Byte)c; - ((Byte *)buffer)[i * 2 + 1] = (Byte)(c >> 8); + // Salt + const UInt32 kSaltSize = 8; + Byte salt[kSaltSize]; + if (!ReadBytesAndTestSize(salt, kSaltSize)) + return S_FALSE; + m_Position += kSaltSize; + RINOK(m_RarAESSpec->SetDecoderProperties2(salt, kSaltSize)) } - m_RarAESSpec->SetPassword((const Byte *)buffer, len * 2); + { + // Password + CMyComBSTR_Wipe password; + RINOK(getTextPassword->CryptoGetTextPassword(&password)) + unsigned len = 0; + if (password) + len = MyStringLen(password); + if (len > kPasswordLen_MAX) + len = kPasswordLen_MAX; + + CByteBuffer_Wipe buffer(len * 2); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = password[i]; + ((Byte *)buffer)[i * 2] = (Byte)c; + ((Byte *)buffer)[i * 2 + 1] = (Byte)(c >> 8); + } + + m_RarAESSpec->SetPassword((const Byte *)buffer, len * 2); + } const UInt32 kDecryptedBufferSize = (1 << 12); if (m_DecryptedDataAligned.Size() == 0) @@ -1621,7 +1626,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, // if (getTextPassword) { - CMyComBSTR password; + CMyComBSTR_Wipe password; RINOK(getTextPassword->CryptoGetTextPassword(&password)); if (item.UnPackVersion >= 29) @@ -1631,7 +1636,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, len = MyStringLen(password); if (len > kPasswordLen_MAX) len = kPasswordLen_MAX; - CByteArr buffer(len * 2); + CByteBuffer_Wipe buffer(len * 2); for (unsigned k = 0; k < len; k++) { wchar_t c = password[k]; @@ -1642,13 +1647,14 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, } else { - AString oemPassword; + AString_Wipe oemPassword; if (password) { - UString unicode = (LPCOLESTR)password; + UString_Wipe unicode; + unicode.SetFromBstr(password); if (unicode.Len() > kPasswordLen_MAX) unicode.DeleteFrom(kPasswordLen_MAX); - oemPassword = UnicodeStringToMultiByte(unicode, CP_OEMCP); + UnicodeStringToMultiByte2(oemPassword, unicode, CP_OEMCP); } rar20CryptoDecoderSpec->SetPassword((const Byte *)(const char *)oemPassword, oemPassword.Len()); } diff --git a/CPP/7zip/Archive/Rar/RarHandler.h b/CPP/7zip/Archive/Rar/RarHandler.h index e444bd77..a62b60cd 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.h +++ b/CPP/7zip/Archive/Rar/RarHandler.h @@ -26,7 +26,7 @@ struct CInArcInfo UInt32 DataCRC; bool EndOfArchive_was_Read; - CInArcInfo(): EndFlags(0), EndOfArchive_was_Read(false), VolNumber(0) {} + CInArcInfo(): EndFlags(0), VolNumber(0), EndOfArchive_was_Read(false) {} UInt64 GetPhySize() const { return EndPos - StartPos; } diff --git a/CPP/7zip/Archive/SplitHandler.cpp b/CPP/7zip/Archive/SplitHandler.cpp index f4a10b1d..6705aee0 100644 --- a/CPP/7zip/Archive/SplitHandler.cpp +++ b/CPP/7zip/Archive/SplitHandler.cpp @@ -142,8 +142,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) } int dotPos = name.ReverseFind_Dot(); - const UString prefix = name.Left(dotPos + 1); - const UString ext = name.Ptr(dotPos + 1); + const UString prefix = name.Left((unsigned)(dotPos + 1)); + const UString ext = name.Ptr((unsigned)(dotPos + 1)); UString ext2 = ext; ext2.MakeLower_Ascii(); diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index 89de660f..74bc8fb8 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -41,9 +41,9 @@ static const unsigned kNumDirLevelsMax = (1 << 10); #define Get64(p) (be ? GetBe64(p) : GetUi64(p)) */ -UInt16 Get16b(const Byte *p, bool be) { return be ? GetBe16(p) : GetUi16(p); } -UInt32 Get32b(const Byte *p, bool be) { return be ? GetBe32(p) : GetUi32(p); } -UInt64 Get64b(const Byte *p, bool be) { return be ? GetBe64(p) : GetUi64(p); } +static UInt16 Get16b(const Byte *p, bool be) { return be ? GetBe16(p) : GetUi16(p); } +static UInt32 Get32b(const Byte *p, bool be) { return be ? GetBe32(p) : GetUi32(p); } +static UInt64 Get64b(const Byte *p, bool be) { return be ? GetBe64(p) : GetUi64(p); } #define Get16(p) Get16b(p, be) #define Get32(p) Get32b(p, be) @@ -121,6 +121,10 @@ static const char * const k_Flags[] = , "ALWAYS_FRAGMENTS" , "DUPLICATES_REMOVED" , "EXPORTABLE" + , "UNCOMPRESSED_XATTRS" + , "NO_XATTRS" + , "COMPRESSOR_OPTIONS" + , "UNCOMPRESSED_IDS" }; static const UInt32 kNotCompressedBit16 = (1 << 15); @@ -129,10 +133,10 @@ static const UInt32 kNotCompressedBit32 = (1 << 24); #define GET_COMPRESSED_BLOCK_SIZE(size) ((size) & ~kNotCompressedBit32) #define IS_COMPRESSED_BLOCK(size) (((size) & kNotCompressedBit32) == 0) -static const UInt32 kHeaderSize1 = 0x33; -static const UInt32 kHeaderSize2 = 0x3F; +// static const UInt32 kHeaderSize1 = 0x33; +// static const UInt32 kHeaderSize2 = 0x3F; static const UInt32 kHeaderSize3 = 0x77; -static const UInt32 kHeaderSize4 = 0x60; +// static const UInt32 kHeaderSize4 = 0x60; struct CHeader { @@ -1243,7 +1247,8 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool &status, &g_Alloc); if (res != 0) return SResToHRESULT(res); - if (status != LZMA_STATUS_FINISHED_WITH_MARK) + if (status != LZMA_STATUS_FINISHED_WITH_MARK + && status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) return S_FALSE; } else @@ -1310,7 +1315,7 @@ HRESULT CHandler::ReadData(CData &data, UInt64 start, UInt64 end) { if (end < start || end - start >= ((UInt64)1 << 32)) return S_FALSE; - UInt32 size = (UInt32)(end - start); + const UInt32 size = (UInt32)(end - start); RINOK(_stream->Seek(start, STREAM_SEEK_SET, NULL)); _dynOutStreamSpec->Init(); UInt32 packPos = 0; @@ -1322,8 +1327,11 @@ HRESULT CHandler::ReadData(CData &data, UInt64 start, UInt64 end) return S_FALSE; UInt32 packSize = size - packPos; RINOK(ReadMetadataBlock(packSize)); - if (_dynOutStreamSpec->GetSize() >= ((UInt64)1 << 32)) - return S_FALSE; + { + const size_t tSize = _dynOutStreamSpec->GetSize(); + if (tSize != (UInt32)tSize) + return S_FALSE; + } packPos += packSize; } data.UnpackPos.Add((UInt32)_dynOutStreamSpec->GetSize()); @@ -1487,7 +1495,7 @@ HRESULT CHandler::OpenDir(int parent, UInt32 startBlock, UInt32 offset, unsigned if (_openCodePage == CP_UTF8) { tempString.SetFrom_CalcLen((const char *)p, size); - if (!CheckUTF8(tempString)) + if (!CheckUTF8_AString(tempString)) _openCodePage = CP_OEMCP; } @@ -1608,11 +1616,14 @@ HRESULT CHandler::Open2(IInStream *inStream) { UInt32 pos = 0; UInt32 totalSize = (UInt32)_inodesData.Data.Size(); + const unsigned kMinNodeParseSize = 4; + if (_h.NumInodes > totalSize / kMinNodeParseSize) + return S_FALSE; _nodesPos.ClearAndReserve(_h.NumInodes); _nodes.ClearAndReserve(_h.NumInodes); // we use _blockToNode for binary search seed optimizations _blockToNode.ClearAndReserve(_inodesData.GetNumBlocks() + 1); - int curBlock = 0; + unsigned curBlock = 0; for (UInt32 i = 0; i < _h.NumInodes; i++) { CNode n; @@ -2090,9 +2101,9 @@ HRESULT CHandler::ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) bool compressed; if (blockIndex < _blockCompressed.Size()) { - compressed = _blockCompressed[(int)blockIndex]; - blockOffset = _blockOffsets[(int)blockIndex]; - packBlockSize = (UInt32)(_blockOffsets[(int)blockIndex + 1] - blockOffset); + compressed = _blockCompressed[(unsigned)blockIndex]; + blockOffset = _blockOffsets[(unsigned)blockIndex]; + packBlockSize = (UInt32)(_blockOffsets[(unsigned)blockIndex + 1] - blockOffset); blockOffset += node.StartBlock; } else @@ -2126,14 +2137,16 @@ HRESULT CHandler::ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) bool outBufWasWritten; UInt32 outBufWasWrittenSize; HRESULT res = Decompress(_outStream, _cachedBlock, &outBufWasWritten, &outBufWasWrittenSize, packBlockSize, _h.BlockSize); + RINOK(res); if (outBufWasWritten) _cachedUnpackBlockSize = outBufWasWrittenSize; else _cachedUnpackBlockSize = (UInt32)_outStreamSpec->GetPos(); - RINOK(res); } else { + if (packBlockSize > _h.BlockSize) + return S_FALSE; RINOK(ReadStream_FALSE(_limitedInStream, _cachedBlock, packBlockSize)); _cachedUnpackBlockSize = packBlockSize; } diff --git a/CPP/7zip/Archive/SwfHandler.cpp b/CPP/7zip/Archive/SwfHandler.cpp index ebb6acfd..9666c69d 100644 --- a/CPP/7zip/Archive/SwfHandler.cpp +++ b/CPP/7zip/Archive/SwfHandler.cpp @@ -776,21 +776,21 @@ STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb static UInt16 Read16(CInBuffer &stream) { - UInt16 res = 0; - for (int i = 0; i < 2; i++) + UInt32 res = 0; + for (unsigned i = 0; i < 2; i++) { Byte b; if (!stream.ReadByte(b)) throw 1; - res |= (UInt16)b << (i * 8); + res |= (UInt32)b << (i * 8); } - return res; + return (UInt16)res; } static UInt32 Read32(CInBuffer &stream) { UInt32 res = 0; - for (int i = 0; i < 4; i++) + for (unsigned i = 0; i < 4; i++) { Byte b; if (!stream.ReadByte(b)) @@ -826,7 +826,7 @@ UInt32 CBitReader::ReadBits(unsigned numBits) res <<= numBits; NumBits -= numBits; res |= (Val >> NumBits); - Val &= (1 << NumBits) - 1; + Val = (Byte)(Val & (((unsigned)1 << NumBits) - 1)); break; } else diff --git a/CPP/7zip/Archive/Tar/TarHandler.cpp b/CPP/7zip/Archive/Tar/TarHandler.cpp index 72fbf74e..bc00e3fc 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.cpp +++ b/CPP/7zip/Archive/Tar/TarHandler.cpp @@ -24,7 +24,9 @@ using namespace NWindows; namespace NArchive { namespace NTar { -static const UINT k_DefaultCodePage = CP_OEMCP; // it uses it if UTF8 check in names shows error +// 21.02: we use UTF8 code page by default, even if some files show error +// before 21.02 : CP_OEMCP; +// static const UINT k_DefaultCodePage = CP_UTF8; static const Byte kProps[] = @@ -39,13 +41,15 @@ static const Byte kProps[] = kpidGroup, kpidSymLink, kpidHardLink, + kpidCharacts // kpidLinkType }; static const Byte kArcProps[] = { kpidHeadersSize, - kpidCodePage + kpidCodePage, + kpidCharacts }; IMP_IInArchive_Props @@ -67,8 +71,12 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { case k_ErrorType_UnexpectedEnd: flags = kpv_ErrorFlags_UnexpectedEnd; break; case k_ErrorType_Corrupted: flags = kpv_ErrorFlags_HeadersError; break; + // case k_ErrorType_OK: break; + // case k_ErrorType_Warning: break; + default: break; } - prop = flags; + if (flags != 0) + prop = flags; break; } @@ -96,6 +104,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = name; break; } + + case kpidCharacts: + { + AString s = _encodingCharacts.GetCharactsString(); + prop = s; + break; + } } prop.Detach(value); return S_OK; @@ -119,12 +134,64 @@ HRESULT CHandler::ReadItem2(ISequentialInStream *stream, bool &filled, CItemEx & */ if (item.IsPaxExtendedHeader()) _thereIsPaxExtendedHeader = true; + if (item.IsThereWarning()) + _warning = true; } _phySize += item.HeaderSize; _headersSize += item.HeaderSize; return S_OK; } + +void CEncodingCharacts::Check(const AString &s) +{ + IsAscii = s.IsAscii(); + if (!IsAscii) + { + /* + { + Oem_Checked = true; + UString u; + MultiByteToUnicodeString2(u, s, CP_OEMCP); + Oem_Ok = (u.Find((wchar_t)0xfffd) <= 0); + } + Utf_Checked = true; + */ + UtfCheck.Check_AString(s); + } +} + + +AString CEncodingCharacts::GetCharactsString() const +{ + AString s; + if (IsAscii) + { + s += "ASCII"; + } + /* + if (Oem_Checked) + { + s.Add_Space_if_NotEmpty(); + s += (Oem_Ok ? "oem-ok" : "oem-error"); + } + if (Utf_Checked) + */ + else + { + s.Add_Space_if_NotEmpty(); + s += (UtfCheck.IsOK() ? "UTF8" : "UTF8-ERROR"); // "UTF8-error" + { + AString s2; + UtfCheck.PrintStatus(s2); + s.Add_Space_if_NotEmpty(); + s += s2; + } + } + return s; +} + + HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) { UInt64 endPos = 0; @@ -135,12 +202,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) _phySizeDefined = true; - bool utf8_OK = true; - if (!_forceCodePage) - { - if (!utf8_OK) - _curCodePage = k_DefaultCodePage; - } + // bool utf8_OK = true; for (;;) { @@ -151,8 +213,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) break; _isArc = true; - _items.Add(item); + /* if (!_forceCodePage) { if (utf8_OK) utf8_OK = CheckUTF8(item.Name, item.NameCouldBeReduced); @@ -160,8 +222,14 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) if (utf8_OK) utf8_OK = CheckUTF8(item.User); if (utf8_OK) utf8_OK = CheckUTF8(item.Group); } - - RINOK(stream->Seek(item.GetPackSizeAligned(), STREAM_SEEK_CUR, &_phySize)); + */ + + item.EncodingCharacts.Check(item.Name); + _encodingCharacts.Update(item.EncodingCharacts); + + _items.Add(item); + + RINOK(stream->Seek((Int64)item.GetPackSizeAligned(), STREAM_SEEK_CUR, &_phySize)); if (_phySize > endPos) { _error = k_ErrorType_UnexpectedEnd; @@ -188,11 +256,13 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) } } + /* if (!_forceCodePage) { if (!utf8_OK) _curCodePage = k_DefaultCodePage; } + */ _openCodePage = _curCodePage; if (_items.Size() == 0) @@ -255,6 +325,7 @@ STDMETHODIMP CHandler::Close() _latestIsRead = false; // _isSparse = false; _thereIsPaxExtendedHeader = false; + _encodingCharacts.Clear(); _items.Clear(); _seqStream.Release(); _stream.Release(); @@ -315,7 +386,8 @@ void CHandler::TarStringToUnicode(const AString &s, NWindows::NCOM::CPropVariant else MultiByteToUnicodeString2(dest, s, _curCodePage); if (toOs) - NItemName::ReplaceToOsSlashes_Remove_TailSlash(dest); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(dest, + true); // useBackslashReplacement prop = dest; } @@ -358,6 +430,17 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidSymLink: if (item->LinkFlag == NFileHeader::NLinkFlag::kSymLink && !item->LinkName.IsEmpty()) TarStringToUnicode(item->LinkName, prop); break; case kpidHardLink: if (item->LinkFlag == NFileHeader::NLinkFlag::kHardLink && !item->LinkName.IsEmpty()) TarStringToUnicode(item->LinkName, prop); break; // case kpidLinkType: prop = (int)item->LinkFlag; break; + case kpidCharacts: + { + AString s = item->EncodingCharacts.GetCharactsString(); + if (item->IsThereWarning()) + { + s.Add_Space_if_NotEmpty(); + s += "HEADER_ERROR"; + } + prop = s; + break; + } } prop.Detach(value); return S_OK; @@ -407,7 +490,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; - Int32 index = allFilesMode ? i : indices[i]; + const UInt32 index = allFilesMode ? i : indices[i]; const CItemEx *item; if (seqMode) { @@ -475,7 +558,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, { if (!seqMode) { - RINOK(_stream->Seek(item->GetDataPosition(), STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)item->GetDataPosition(), STREAM_SEEK_SET, NULL)); } streamSpec->Init(item->GetPackSizeAligned()); RINOK(copyCoder->Code(inStream2, outStream, NULL, NULL, progress)); @@ -566,7 +649,7 @@ STDMETHODIMP CSparseStream::Read(void *data, UInt32 size, UInt32 *processedSize) UInt64 phyPos = PhyOffsets[left] + relat; if (_needStartSeek || _phyPos != phyPos) { - RINOK(Handler->_stream->Seek(item.GetDataPosition() + phyPos, STREAM_SEEK_SET, NULL)); + RINOK(Handler->_stream->Seek((Int64)(item.GetDataPosition() + phyPos), STREAM_SEEK_SET, NULL)); _needStartSeek = false; _phyPos = phyPos; } @@ -604,7 +687,7 @@ STDMETHODIMP CSparseStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -650,7 +733,6 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) void CHandler::Init() { _forceCodePage = false; - // _codePage = CP_OEMCP; _curCodePage = _specifiedCodePage = CP_UTF8; // CP_OEMCP; _thereIsPaxExtendedHeader = false; } diff --git a/CPP/7zip/Archive/Tar/TarHandler.h b/CPP/7zip/Archive/Tar/TarHandler.h index eb9c049e..4834c2a7 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.h +++ b/CPP/7zip/Archive/Tar/TarHandler.h @@ -48,6 +48,8 @@ private: UInt32 _curCodePage; UInt32 _openCodePage; + CEncodingCharacts _encodingCharacts; + NCompress::CCopyCoder *copyCoderSpec; CMyComPtr<ICompressCoder> copyCoder; diff --git a/CPP/7zip/Archive/Tar/TarHandlerOut.cpp b/CPP/7zip/Archive/Tar/TarHandlerOut.cpp index 41934339..7a7a2cba 100644 --- a/CPP/7zip/Archive/Tar/TarHandlerOut.cpp +++ b/CPP/7zip/Archive/Tar/TarHandlerOut.cpp @@ -25,8 +25,8 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) return S_OK; } -HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, - AString &res, UINT codePage, bool convertSlash = false) +HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, AString &res, + UINT codePage, unsigned utfFlags, bool convertSlash) { NCOM::CPropVariant prop; RINOK(callback->GetProperty(index, propId, &prop)); @@ -39,7 +39,7 @@ HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID pro if (codePage == CP_UTF8) { - ConvertUnicodeToUTF8(s, res); + ConvertUnicodeToUTF8_Flags(s, res, utfFlags); // if (!ConvertUnicodeToUTF8(s, res)) // return E_INVALIDARG; } else @@ -56,8 +56,8 @@ HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID pro static int CompareUpdateItems(void *const *p1, void *const *p2, void *) { - const CUpdateItem &u1 = *(*((const CUpdateItem **)p1)); - const CUpdateItem &u2 = *(*((const CUpdateItem **)p2)); + const CUpdateItem &u1 = *(*((const CUpdateItem *const *)p1)); + const CUpdateItem &u2 = *(*((const CUpdateItem *const *)p2)); if (!u1.NewProps) { if (u2.NewProps) @@ -78,8 +78,15 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if ((_stream && (_error != k_ErrorType_OK || _warning /* || _isSparse */)) || _seqStream) return E_NOTIMPL; CObjectVector<CUpdateItem> updateItems; - UINT codePage = (_forceCodePage ? _specifiedCodePage : _openCodePage); - + const UINT codePage = (_forceCodePage ? _specifiedCodePage : _openCodePage); + const unsigned utfFlags = g_Unicode_To_UTF8_Flags; + /* + // for debug only: + unsigned utfFlags = 0; + utfFlags |= UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE; + utfFlags |= UTF_FLAG__TO_UTF8__SURROGATE_ERROR; + */ + for (UInt32 i = 0; i < numItems; i++) { CUpdateItem ui; @@ -94,7 +101,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt ui.NewProps = IntToBool(newProps); ui.NewData = IntToBool(newData); - ui.IndexInArc = indexInArc; + ui.IndexInArc = (int)indexInArc; ui.IndexInClient = i; if (IntToBool(newProps)) @@ -138,11 +145,11 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt ui.MTime = NTime::FileTimeToUnixTime64(prop.filetime); } - RINOK(GetPropString(callback, i, kpidPath, ui.Name, codePage, true)); + RINOK(GetPropString(callback, i, kpidPath, ui.Name, codePage, utfFlags, true)); if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') ui.Name += '/'; - RINOK(GetPropString(callback, i, kpidUser, ui.User, codePage)); - RINOK(GetPropString(callback, i, kpidGroup, ui.Group, codePage)); + RINOK(GetPropString(callback, i, kpidUser, ui.User, codePage, utfFlags, false)); + RINOK(GetPropString(callback, i, kpidGroup, ui.Group, codePage, utfFlags, false)); } if (IntToBool(newData)) @@ -168,7 +175,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt updateItems.Sort(CompareUpdateItems, NULL); } - return UpdateArchive(_stream, outStream, _items, updateItems, codePage, callback); + return UpdateArchive(_stream, outStream, _items, updateItems, codePage, utfFlags, callback); COM_TRY_END } diff --git a/CPP/7zip/Archive/Tar/TarHeader.h b/CPP/7zip/Archive/Tar/TarHeader.h index 47971b58..249368f6 100644 --- a/CPP/7zip/Archive/Tar/TarHeader.h +++ b/CPP/7zip/Archive/Tar/TarHeader.h @@ -58,6 +58,7 @@ namespace NFileHeader const char kGnu_LongLink = 'K'; const char kGnu_LongName = 'L'; const char kSparse = 'S'; + const char kLabel = 'V'; const char kDumpDir = 'D'; /* GNUTYPE_DUMPDIR. data: list of files created by the --incremental (-G) option Each file name is preceded by either diff --git a/CPP/7zip/Archive/Tar/TarIn.cpp b/CPP/7zip/Archive/Tar/TarIn.cpp index 32761658..0099e6f7 100644 --- a/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/CPP/7zip/Archive/Tar/TarIn.cpp @@ -81,14 +81,14 @@ static void ReadString(const char *s, unsigned size, AString &result) static bool ParseInt64(const char *p, Int64 &val) { UInt32 h = GetBe32(p); - val = GetBe64(p + 4); + val = (Int64)GetBe64(p + 4); if (h == (UInt32)1 << 31) return ((val >> 63) & 1) == 0; if (h == (UInt32)(Int32)-1) return ((val >> 63) & 1) != 0; UInt64 uv; bool res = OctalToNumber(p, 12, uv); - val = uv; + val = (Int64)uv; return res; } @@ -112,7 +112,9 @@ static bool ParseSize(const char *p, UInt64 &val) val = GetBe64(p + 4); return ((val >> 63) & 1) == 0; } - return OctalToNumber(p, 12, val); + return OctalToNumber(p, 12, val, + true // 20.03: allow empty size for 'V' Label entry + ); } #define CHECK(x) { if (!(x)) return k_IsArc_Res_NO; } @@ -201,8 +203,8 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE // we allow empty Mode value for LongName prefix items RIF(OctalToNumber32(p, 8, item.Mode, true)); p += 8; - if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0; p += 8; - if (!OctalToNumber32(p, 8, item.GID)) item.GID = 0; p += 8; + if (!OctalToNumber32(p, 8, item.UID)) { item.UID = 0; } p += 8; + if (!OctalToNumber32(p, 8, item.GID)) { item.GID = 0; } p += 8; RIF(ParseSize(p, item.PackSize)); item.Size = item.PackSize; @@ -245,6 +247,15 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE item.PackSize = 0; item.Size = 0; } + + if (item.LinkFlag == NFileHeader::NLinkFlag::kDirectory) + { + // GNU tar ignores Size field, if LinkFlag is kDirectory + // 21.02 : we set PackSize = 0 to be more compatible with GNU tar + item.PackSize = 0; + // item.Size = 0; + } + /* TAR standard requires sum of unsigned byte values. But some TAR programs use sum of signed byte values. @@ -269,7 +280,7 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE if (item.LinkFlag == NFileHeader::NLinkFlag::kSparse) { - Byte isExtended = buf[482]; + Byte isExtended = (Byte)buf[482]; if (isExtended != 0 && isExtended != 1) return S_OK; RIF(ParseSize(buf + 483, item.Size)); @@ -309,7 +320,7 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE } item.HeaderSize += NFileHeader::kRecordSize; - isExtended = buf[21 * 24]; + isExtended = (Byte)buf[21 * 24]; if (isExtended != 0 && isExtended != 1) return S_OK; for (unsigned i = 0; i < 21; i++) @@ -442,9 +453,16 @@ HRESULT ReadItem(ISequentialInStream *stream, bool &filled, CItemEx &item, EErro case 'x': case 'X': { - // pax Extended Header - if (item.Name.IsPrefixedBy("PaxHeader/") - || item.Name.Find("PaxHeaders.4467/") >= 0) + const char *s = item.Name.Ptr(); + if (IsString1PrefixedByString2(s, "./")) + s += 2; + if (IsString1PrefixedByString2(s, "./")) + s += 2; + if ( IsString1PrefixedByString2(s, "PaxHeader/") + || IsString1PrefixedByString2(s, "PaxHeaders.X/") + || IsString1PrefixedByString2(s, "PaxHeaders.4467/") + || StringsAreEqual_Ascii(s, "@PaxHeader") + ) { RINOK(ReadDataToString(stream, item, pax, error)); if (error != k_ErrorType_OK) diff --git a/CPP/7zip/Archive/Tar/TarItem.h b/CPP/7zip/Archive/Tar/TarItem.h index bc3b4084..8c34b841 100644 --- a/CPP/7zip/Archive/Tar/TarItem.h +++ b/CPP/7zip/Archive/Tar/TarItem.h @@ -4,6 +4,7 @@ #define __ARCHIVE_TAR_ITEM_H #include "../../../Common/MyLinux.h" +#include "../../../Common/UTFConvert.h" #include "../Common/ItemNameUtils.h" @@ -108,8 +109,52 @@ struct CItem } UInt64 GetPackSizeAligned() const { return (PackSize + 0x1FF) & (~((UInt64)0x1FF)); } + + bool IsThereWarning() const + { + // that Header Warning is possible if (Size != 0) for dir item + return (PackSize < Size) && (LinkFlag == NFileHeader::NLinkFlag::kDirectory); + } }; + + +struct CEncodingCharacts +{ + bool IsAscii; + // bool Oem_Checked; + // bool Oem_Ok; + // bool Utf_Checked; + CUtf8Check UtfCheck; + + void Clear() + { + IsAscii = true; + // Oem_Checked = false; + // Oem_Ok = false; + // Utf_Checked = false; + UtfCheck.Clear(); + } + + void Update(const CEncodingCharacts &ec) + { + if (!ec.IsAscii) + IsAscii = false; + + // if (ec.Utf_Checked) + { + UtfCheck.Update(ec.UtfCheck); + // Utf_Checked = true; + } + } + + CEncodingCharacts() { Clear(); } + void Check(const AString &s); + AString GetCharactsString() const; +}; + + + struct CItemEx: public CItem { UInt64 HeaderPos; @@ -117,6 +162,8 @@ struct CItemEx: public CItem bool NameCouldBeReduced; bool LinkNameCouldBeReduced; + CEncodingCharacts EncodingCharacts; + UInt64 GetDataPosition() const { return HeaderPos + HeaderSize; } UInt64 GetFullSize() const { return HeaderSize + PackSize; } }; diff --git a/CPP/7zip/Archive/Tar/TarOut.cpp b/CPP/7zip/Archive/Tar/TarOut.cpp index 51081e8b..e2da3238 100644 --- a/CPP/7zip/Archive/Tar/TarOut.cpp +++ b/CPP/7zip/Archive/Tar/TarOut.cpp @@ -62,7 +62,7 @@ static void WriteOctal_12_Signed(char *s, Int64 val) { if (val >= 0) { - WriteOctal_12(s, val); + WriteOctal_12(s, (UInt64)val); return; } s[0] = s[1] = s[2] = s[3] = (char)(Byte)0xFF; diff --git a/CPP/7zip/Archive/Tar/TarUpdate.cpp b/CPP/7zip/Archive/Tar/TarUpdate.cpp index 0cdb30d1..c7598f8d 100644 --- a/CPP/7zip/Archive/Tar/TarUpdate.cpp +++ b/CPP/7zip/Archive/Tar/TarUpdate.cpp @@ -15,13 +15,10 @@ namespace NArchive { namespace NTar { -HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, - AString &res, UINT codePage, bool convertSlash = false); - HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, const CObjectVector<NArchive::NTar::CItemEx> &inputItems, const CObjectVector<CUpdateItem> &updateItems, - UINT codePage, + UINT codePage, unsigned utfFlags, IArchiveUpdateCallback *updateCallback) { COutArchive outArchive; @@ -43,7 +40,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, if (ui.NewData) complexity += ui.Size; else - complexity += inputItems[ui.IndexInArc].GetFullSize(); + complexity += inputItems[(unsigned)ui.IndexInArc].GetFullSize(); } RINOK(updateCallback->SetTotal(complexity)); @@ -95,12 +92,12 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, memcpy(item.Magic, NFileHeader::NMagic::kUsTar_00, 8); } else - item = inputItems[ui.IndexInArc]; + item = inputItems[(unsigned)ui.IndexInArc]; AString symLink; if (ui.NewData || ui.NewProps) { - RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidSymLink, symLink, codePage, true)); + RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidSymLink, symLink, codePage, utfFlags, true)); if (!symLink.IsEmpty()) { item.LinkFlag = NFileHeader::NLinkFlag::kSymLink; @@ -159,7 +156,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, { AString hardLink; - RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidHardLink, hardLink, codePage, true)); + RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidHardLink, hardLink, codePage, utfFlags, true)); if (!hardLink.IsEmpty()) { item.LinkFlag = NFileHeader::NLinkFlag::kHardLink; @@ -189,7 +186,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, outArchive.Pos = fileHeaderStartPos; item.PackSize = copyCoderSpec->TotalSize; RINOK(outArchive.WriteHeader(item)); - RINOK(outSeekStream->Seek(item.PackSize, STREAM_SEEK_CUR, NULL)); + RINOK(outSeekStream->Seek((Int64)item.PackSize, STREAM_SEEK_CUR, NULL)); outArchive.Pos += item.PackSize; } RINOK(outArchive.FillDataResidual(item.PackSize)); @@ -201,7 +198,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, } else { - const CItemEx &existItem = inputItems[ui.IndexInArc]; + const CItemEx &existItem = inputItems[(unsigned)ui.IndexInArc]; UInt64 size; if (ui.NewProps) @@ -231,12 +228,12 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, item.GID = existItem.GID; RINOK(outArchive.WriteHeader(item)); - RINOK(inStream->Seek(existItem.GetDataPosition(), STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek((Int64)existItem.GetDataPosition(), STREAM_SEEK_SET, NULL)); size = existItem.PackSize; } else { - RINOK(inStream->Seek(existItem.HeaderPos, STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek((Int64)existItem.HeaderPos, STREAM_SEEK_SET, NULL)); size = existItem.GetFullSize(); } diff --git a/CPP/7zip/Archive/Tar/TarUpdate.h b/CPP/7zip/Archive/Tar/TarUpdate.h index b758635f..1e3d0217 100644 --- a/CPP/7zip/Archive/Tar/TarUpdate.h +++ b/CPP/7zip/Archive/Tar/TarUpdate.h @@ -13,7 +13,7 @@ namespace NTar { struct CUpdateItem { int IndexInArc; - int IndexInClient; + unsigned IndexInClient; UInt64 Size; Int64 MTime; UInt32 Mode; @@ -30,9 +30,12 @@ struct CUpdateItem HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, const CObjectVector<CItemEx> &inputItems, const CObjectVector<CUpdateItem> &updateItems, - UINT codePage, + UINT codePage, unsigned utfFlags, IArchiveUpdateCallback *updateCallback); +HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, AString &res, + UINT codePage, unsigned utfFlags, bool convertSlash); + }} #endif diff --git a/CPP/7zip/Archive/Udf/UdfIn.cpp b/CPP/7zip/Archive/Udf/UdfIn.cpp index cfe6c5ad..520ceeea 100644 --- a/CPP/7zip/Archive/Udf/UdfIn.cpp +++ b/CPP/7zip/Archive/Udf/UdfIn.cpp @@ -39,13 +39,13 @@ static const UInt64 kFileNameLengthTotalMax = (UInt64)1 << 33; static const UInt64 kInlineExtentsSizeMax = (UInt64)1 << 33; #define CRC16_INIT_VAL 0 -#define CRC16_GET_DIGEST(crc) (crc) +// #define CRC16_GET_DIGEST(crc) (crc) #define CRC16_UPDATE_BYTE(crc, b) ((UInt16)(g_Crc16Table[(((crc) >> 8) ^ (b)) & 0xFF] ^ ((crc) << 8))) #define kCrc16Poly 0x1021 static UInt16 g_Crc16Table[256]; -void MY_FAST_CALL Crc16GenerateTable(void) +static void MY_FAST_CALL Crc16GenerateTable(void) { UInt32 i; for (i = 0; i < 256; i++) @@ -57,7 +57,7 @@ void MY_FAST_CALL Crc16GenerateTable(void) } } -UInt32 MY_FAST_CALL Crc16_Update(UInt32 v, const void *data, size_t size) +static UInt32 MY_FAST_CALL Crc16_Update(UInt32 v, const void *data, size_t size) { const Byte *p = (const Byte *)data; for (; size > 0 ; size--, p++) @@ -65,7 +65,7 @@ UInt32 MY_FAST_CALL Crc16_Update(UInt32 v, const void *data, size_t size) return v; } -UInt32 MY_FAST_CALL Crc16Calc(const void *data, size_t size) +static UInt32 MY_FAST_CALL Crc16Calc(const void *data, size_t size) { return Crc16_Update(CRC16_INIT_VAL, data, size); } diff --git a/CPP/7zip/Archive/Udf/UdfIn.h b/CPP/7zip/Archive/Udf/UdfIn.h index f7379401..c26f6099 100644 --- a/CPP/7zip/Archive/Udf/UdfIn.h +++ b/CPP/7zip/Archive/Udf/UdfIn.h @@ -351,7 +351,7 @@ class CInArchive UInt64 _processedProgressBytes; UInt64 _fileNameLengthTotal; - int _numRefs; + unsigned _numRefs; UInt32 _numExtents; UInt64 _inlineExtentsSize; bool CheckExtent(int volIndex, int partitionRef, UInt32 blockPos, UInt32 len) const; diff --git a/CPP/7zip/Archive/UefiHandler.cpp b/CPP/7zip/Archive/UefiHandler.cpp index 5892d568..67fe795a 100644 --- a/CPP/7zip/Archive/UefiHandler.cpp +++ b/CPP/7zip/Archive/UefiHandler.cpp @@ -52,7 +52,7 @@ static const Byte k_IntelMeSignature[] = 0x5A, 0xA5, 0xF0, 0x0F }; -bool IsIntelMe(const Byte *p) +static bool IsIntelMe(const Byte *p) { return memcmp(p, k_IntelMeSignature, sizeof(k_IntelMeSignature)) == 0; } @@ -318,19 +318,19 @@ static const CUInt32PCharPair g_FFS_FILE_ATTRIBUTES[] = // SECTION_TYPE -#define SECTION_ALL 0x00 +// #define SECTION_ALL 0x00 #define SECTION_COMPRESSION 0x01 #define SECTION_GUID_DEFINED 0x02 // Leaf section Type values -#define SECTION_PE32 0x10 -#define SECTION_PIC 0x11 -#define SECTION_TE 0x12 +// #define SECTION_PE32 0x10 +// #define SECTION_PIC 0x11 +// #define SECTION_TE 0x12 #define SECTION_DXE_DEPEX 0x13 #define SECTION_VERSION 0x14 #define SECTION_USER_INTERFACE 0x15 -#define SECTION_COMPATIBILITY16 0x16 +// #define SECTION_COMPATIBILITY16 0x16 #define SECTION_FIRMWARE_VOLUME_IMAGE 0x17 #define SECTION_FREEFORM_SUBTYPE_GUID 0x18 #define SECTION_RAW 0x19 @@ -471,7 +471,7 @@ public: bool Parse(const Byte *p) { - int i; + unsigned i; for (i = 0; i < kFileHeaderSize; i++) if (p[i] != 0xFF) break; @@ -719,11 +719,11 @@ class CHandler: HRESULT ParseIntelMe(int bufIndex, UInt32 posBase, UInt32 exactSize, UInt32 limitSize, - int parent, int method, int level); + int parent, int method, unsigned level); HRESULT ParseVolume(int bufIndex, UInt32 posBase, UInt32 exactSize, UInt32 limitSize, - int parent, int method, int level); + int parent, int method, unsigned level); HRESULT OpenCapsule(IInStream *stream); HRESULT OpenFv(IInStream *stream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback); @@ -857,13 +857,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) } #ifdef SHOW_DEBUG_INFO -static void PrintLevel(int level) +static void PrintLevel(unsigned level) { PRF(printf("\n")); - for (int i = 0; i < level; i++) + for (unsigned i = 0; i < level; i++) PRF(printf(" ")); } -static void MyPrint(UInt32 posBase, UInt32 size, int level, const char *name) +static void MyPrint(UInt32 posBase, UInt32 size, unsigned level, const char *name) { PrintLevel(level); PRF(printf("%s, pos = %6x, size = %6x", name, posBase, size)); @@ -1312,7 +1312,7 @@ bool CVolFfsHeader::Parse(const Byte *p) HRESULT CHandler::ParseVolume( int bufIndex, UInt32 posBase, UInt32 exactSize, UInt32 limitSize, - int parent, int method, int level) + int parent, int method, unsigned level) { if (level > kLevelMax) return S_FALSE; @@ -1333,7 +1333,7 @@ HRESULT CHandler::ParseVolume( if (!Is_FF_Stream(p + kFfsGuidOffset, 16)) item.SetGuid(p + kFfsGuidOffset); // if (item.Name.IsEmpty()) - item.Name += "[VOL]"; + item.Name += "[VOL]"; AddItem(item); return S_OK; } @@ -1512,7 +1512,7 @@ static const char * const kRegionName[] = HRESULT CHandler::ParseIntelMe( int bufIndex, UInt32 posBase, UInt32 exactSize, UInt32 limitSize, - int parent, int method, int level) + int parent, int method, unsigned level) { UNUSED_VAR(limitSize) level++; diff --git a/CPP/7zip/Archive/VdiHandler.cpp b/CPP/7zip/Archive/VdiHandler.cpp index b8ef35bb..66cf39cd 100644 --- a/CPP/7zip/Archive/VdiHandler.cpp +++ b/CPP/7zip/Archive/VdiHandler.cpp @@ -18,7 +18,6 @@ #include "HandlerCont.h" -#define Get16(p) GetUi16(p) #define Get32(p) GetUi32(p) #define Get64(p) GetUi64(p) @@ -87,7 +86,7 @@ class CHandler: public CHandlerImg Byte Guids[kNumGuids][16]; - HRESULT Seek(UInt64 offset) + HRESULT Seek2(UInt64 offset) { _posInArc = offset; return Stream->Seek(offset, STREAM_SEEK_SET, NULL); @@ -96,7 +95,7 @@ class CHandler: public CHandlerImg HRESULT InitAndSeek() { _virtPos = 0; - return Seek(0); + return Seek2(0); } HRESULT Open2(IInStream *stream, IArchiveOpenCallback *openCallback); @@ -143,7 +142,7 @@ STDMETHODIMP CHandler::Read(void *data, UInt32 size, UInt32 *processedSize) offset += lowBits; if (offset != _posInArc) { - RINOK(Seek(offset)); + RINOK(Seek2(offset)); } HRESULT res = Stream->Read(data, size, &size); _posInArc += size; diff --git a/CPP/7zip/Archive/VhdHandler.cpp b/CPP/7zip/Archive/VhdHandler.cpp index d79ae907..c70d667e 100644 --- a/CPP/7zip/Archive/VhdHandler.cpp +++ b/CPP/7zip/Archive/VhdHandler.cpp @@ -248,7 +248,7 @@ class CHandler: public CHandlerImg } void Reset_PosInArc() { _posInArc = (UInt64)0 - 1; } - HRESULT Seek(UInt64 offset); + HRESULT Seek2(UInt64 offset); HRESULT InitAndSeek(); HRESULT ReadPhy(UInt64 offset, void *data, UInt32 size); @@ -316,7 +316,7 @@ public: STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); }; -HRESULT CHandler::Seek(UInt64 offset) { return Stream->Seek(_startOffset + offset, STREAM_SEEK_SET, NULL); } +HRESULT CHandler::Seek2(UInt64 offset) { return Stream->Seek(_startOffset + offset, STREAM_SEEK_SET, NULL); } HRESULT CHandler::InitAndSeek() { @@ -327,7 +327,7 @@ HRESULT CHandler::InitAndSeek() _virtPos = _posInArc = 0; BitMapTag = kUnusedBlock; BitMap.Alloc(Dyn.NumBitMapSectors() << kSectorSize_Log); - return Seek(0); + return Seek2(0); } HRESULT CHandler::ReadPhy(UInt64 offset, void *data, UInt32 size) @@ -337,7 +337,7 @@ HRESULT CHandler::ReadPhy(UInt64 offset, void *data, UInt32 size) if (offset != _posInArc) { _posInArc = offset; - RINOK(Seek(offset)); + RINOK(Seek2(offset)); } HRESULT res = ReadStream_FALSE(Stream, data, size); if (res == S_OK) diff --git a/CPP/7zip/Archive/VmdkHandler.cpp b/CPP/7zip/Archive/VmdkHandler.cpp index 942bd792..096bd103 100644 --- a/CPP/7zip/Archive/VmdkHandler.cpp +++ b/CPP/7zip/Archive/VmdkHandler.cpp @@ -41,7 +41,7 @@ namespace NVmdk { static const Byte k_Signature[] = SIGNATURE; static const UInt32 k_Flags_NL = (UInt32)1 << 0; -static const UInt32 k_Flags_RGD = (UInt32)1 << 1; +// static const UInt32 k_Flags_RGD = (UInt32)1 << 1; static const UInt32 k_Flags_ZeroGrain = (UInt32)1 << 2; static const UInt32 k_Flags_Compressed = (UInt32)1 << 16; static const UInt32 k_Flags_Marker = (UInt32)1 << 17; @@ -855,11 +855,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) } case kpidId: + { if (desc && !desc->CID.IsEmpty()) { prop = desc->CID; - break; } + break; + } case kpidName: { diff --git a/CPP/7zip/Archive/Wim/WimHandler.cpp b/CPP/7zip/Archive/Wim/WimHandler.cpp index 927a0b38..2553c175 100644 --- a/CPP/7zip/Archive/Wim/WimHandler.cpp +++ b/CPP/7zip/Archive/Wim/WimHandler.cpp @@ -350,7 +350,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) COM_TRY_END } -void GetFileTime(const Byte *p, NCOM::CPropVariant &prop) +static void GetFileTime(const Byte *p, NCOM::CPropVariant &prop) { prop.vt = VT_FILETIME; prop.filetime.dwLowDateTime = Get32(p); diff --git a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp index 18740c70..6b4497fe 100644 --- a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp +++ b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp @@ -4,6 +4,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/StringToInt.h" #include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" @@ -93,9 +94,15 @@ struct CMetaItem CByteBuffer Reparse; unsigned GetNumAltStreams() const { return AltStreams.Size() - NumSkipAltStreams; } - CMetaItem(): UpdateIndex(-1), HashIndex(-1), SecurityId(-1), - FileID(0), VolID(0), - Skip(false), NumSkipAltStreams(0) {} + CMetaItem(): + UpdateIndex(-1) + , HashIndex(-1) + , FileID(0) + , VolID(0) + , SecurityId(-1) + , Skip(false) + , NumSkipAltStreams(0) + {} }; @@ -320,20 +327,23 @@ class CInStreamWithSha1: { CMyComPtr<ISequentialInStream> _stream; UInt64 _size; - NCrypto::NSha1::CContext _sha; + // NCrypto::NSha1::CContext _sha; + CAlignedBuffer _sha; + CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_sha; } public: MY_UNKNOWN_IMP1(IInStream) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); + CInStreamWithSha1(): _sha(sizeof(CSha1)) {} void SetStream(ISequentialInStream *stream) { _stream = stream; } void Init() { _size = 0; - _sha.Init(); + Sha1_Init(Sha()); } void ReleaseStream() { _stream.Release(); } UInt64 GetSize() const { return _size; } - void Final(Byte *digest) { _sha.Final(digest); } + void Final(Byte *digest) { Sha1_Final(Sha(), digest); } }; STDMETHODIMP CInStreamWithSha1::Read(void *data, UInt32 size, UInt32 *processedSize) @@ -341,7 +351,7 @@ STDMETHODIMP CInStreamWithSha1::Read(void *data, UInt32 size, UInt32 *processedS UInt32 realProcessedSize; HRESULT result = _stream->Read(data, size, &realProcessedSize); _size += realProcessedSize; - _sha.Update((const Byte *)data, realProcessedSize); + Sha1_Update(Sha(), (const Byte *)data, realProcessedSize); if (processedSize) *processedSize = realProcessedSize; return result; @@ -427,9 +437,9 @@ static size_t WriteItem(const CStreamInfo *streams, const CMetaItem &item, Byte Set16(p + 0x64, (UInt16)fileNameLen); unsigned i; for (i = 0; i * 2 < fileNameLen; i++) - Set16(p + kDirRecordSize + i * 2, item.Name[i]); + Set16(p + kDirRecordSize + i * 2, (UInt16)item.Name[i]); for (i = 0; i * 2 < shortNameLen; i++) - Set16(p + kDirRecordSize + fileNameLen2 + i * 2, item.ShortName[i]); + Set16(p + kDirRecordSize + fileNameLen2 + i * 2, (UInt16)item.ShortName[i]); if (item.GetNumAltStreams() == 0) { @@ -468,7 +478,7 @@ static size_t WriteItem(const CStreamInfo *streams, const CMetaItem &item, Byte memcpy(p + 0x10, streams[ss.HashIndex].Hash, kHashSize); Set16(p + 0x24, (UInt16)fileNameLen); for (i = 0; i * 2 < fileNameLen; i++) - Set16(p + 0x26 + i * 2, ss.Name[i]); + Set16(p + 0x26 + i * 2, (UInt16)ss.Name[i]); totalLen += curLen; p += curLen; } @@ -690,7 +700,7 @@ static void AddTrees(CObjectVector<CDir> &trees, CObjectVector<CMetaItem> &metaI } -#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z') +#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) @@ -1024,7 +1034,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu imageIndex = (int)val - 1; if (imageIndex < (int)isChangedImage.Size()) - if (!isChangedImage[imageIndex]) + if (!isChangedImage[imageIndex]) return E_FAIL; AddTrees(trees, db.MetaItems, ri, imageIndex); @@ -1079,7 +1089,27 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu fileName.Empty(); } else + { + /* + #if WCHAR_MAX > 0xffff + if (c >= 0x10000) + { + c -= 0x10000; + + if (c < (1 << 20)) + { + wchar_t c0 = 0xd800 + ((c >> 10) & 0x3FF); + fileName += c0; + c = 0xdc00 + (c & 0x3FF); + } + else + c = '_'; // we change character unsupported by UTF16 + } + #endif + */ + fileName += c; + } } if (isAltStream) @@ -1838,7 +1868,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu if (_xmls.Size() == 1) { const CWimXml &_oldXml = _xmls[0]; - if ((int)i < _oldXml.Images.Size()) + if (i < _oldXml.Images.Size()) { // int ttt = _oldXml.Images[i].ItemIndexInXml; item = _oldXml.Xml.Root.SubItems[_oldXml.Images[i].ItemIndexInXml]; @@ -1880,7 +1910,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu CByteArr xmlBuf(xmlSize); Set16((Byte *)xmlBuf, 0xFEFF); for (i = 0; i < (unsigned)utf16.Len(); i++) - Set16((Byte *)xmlBuf + 2 + i * 2, utf16[i]); + Set16((Byte *)xmlBuf + 2 + i * 2, (UInt16)utf16[i]); RINOK(WriteStream(outStream, (const Byte *)xmlBuf, xmlSize)); } diff --git a/CPP/7zip/Archive/Wim/WimIn.cpp b/CPP/7zip/Archive/Wim/WimIn.cpp index 3c712084..fef6b34f 100644 --- a/CPP/7zip/Archive/Wim/WimIn.cpp +++ b/CPP/7zip/Archive/Wim/WimIn.cpp @@ -302,12 +302,12 @@ HRESULT CUnpacker::Unpack2( UInt64 packDataSize; size_t numChunks; { - UInt64 numChunks64 = (unpackSize + (((UInt32)1 << chunkSizeBits) - 1)) >> chunkSizeBits; - UInt64 sizesBufSize64 = (numChunks64 - 1) << entrySizeShifts; + const UInt64 numChunks64 = (unpackSize + (((UInt32)1 << chunkSizeBits) - 1)) >> chunkSizeBits; + const UInt64 sizesBufSize64 = (numChunks64 - 1) << entrySizeShifts; if (sizesBufSize64 > resource.PackSize) return S_FALSE; packDataSize = resource.PackSize - sizesBufSize64; - size_t sizesBufSize = (size_t)sizesBufSize64; + const size_t sizesBufSize = (size_t)sizesBufSize64; if (sizesBufSize != sizesBufSize64) return E_OUTOFMEMORY; sizesBuf.AllocAtLeast(sizesBufSize); @@ -639,10 +639,10 @@ HRESULT CDatabase::ParseDirItem(size_t pos, int parent) p += dirRecordSize; { - if (*(const UInt16 *)(p + fileNameLen) != 0) + if (*(const UInt16 *)(const void *)(p + fileNameLen) != 0) return S_FALSE; for (UInt32 j = 0; j < fileNameLen; j += 2) - if (*(const UInt16 *)(p + j) == 0) + if (*(const UInt16 *)(const void *)(p + j) == 0) return S_FALSE; } @@ -652,10 +652,10 @@ HRESULT CDatabase::ParseDirItem(size_t pos, int parent) { // empty shortName has no ZERO at the end ? const Byte *p2 = p + fileNameLen2; - if (*(const UInt16 *)(p2 + shortNameLen) != 0) + if (*(const UInt16 *)(const void *)(p2 + shortNameLen) != 0) return S_FALSE; for (UInt32 j = 0; j < shortNameLen; j += 2) - if (*(const UInt16 *)(p2 + j) == 0) + if (*(const UInt16 *)(const void *)(p2 + j) == 0) return S_FALSE; } @@ -703,10 +703,10 @@ HRESULT CDatabase::ParseDirItem(size_t pos, int parent) { const Byte *p3 = p2 + extraOffset + 2; - if (*(const UInt16 *)(p3 + fileNameLen111) != 0) + if (*(const UInt16 *)(const void *)(p3 + fileNameLen111) != 0) return S_FALSE; for (UInt32 j = 0; j < fileNameLen111; j += 2) - if (*(const UInt16 *)(p3 + j) == 0) + if (*(const UInt16 *)(const void *)(p3 + j) == 0) return S_FALSE; // PRF(printf("\n %S", p3)); @@ -1790,7 +1790,8 @@ void CImageInfo::Parse(const CXmlItem &item) { CTimeDefined = ParseTime(item, CTime, "CREATIONTIME"); MTimeDefined = ParseTime(item, MTime, "LASTMODIFICATIONTIME"); - NameDefined = ConvertUTF8ToUnicode(item.GetSubStringForTag("NAME"), Name); + NameDefined = true; + ConvertUTF8ToUnicode(item.GetSubStringForTag("NAME"), Name); ParseNumber64(item.GetSubStringForTag("DIRCOUNT"), DirCount); ParseNumber64(item.GetSubStringForTag("FILECOUNT"), FileCount); diff --git a/CPP/7zip/Archive/XarHandler.cpp b/CPP/7zip/Archive/XarHandler.cpp index f20b1eb9..35dd607a 100644 --- a/CPP/7zip/Archive/XarHandler.cpp +++ b/CPP/7zip/Archive/XarHandler.cpp @@ -82,11 +82,13 @@ struct CFile int Parent; - CFile(): IsDir(false), HasData(false), ModeDefined(false), Sha1IsDefined(false), - /* packSha1IsDefined(false), */ - Parent(-1), + CFile(): Size(0), PackSize(0), Offset(0), - CTime(0), MTime(0), ATime(0), Mode(0) {} + CTime(0), MTime(0), ATime(0), Mode(0), + IsDir(false), HasData(false), ModeDefined(false), Sha1IsDefined(false), + /* packSha1IsDefined(false), */ + Parent(-1) + {} bool IsCopyMethod() const { @@ -435,8 +437,8 @@ static void Utf8StringToProp(const AString &s, NCOM::CPropVariant &prop) if (!s.IsEmpty()) { UString us; - if (ConvertUTF8ToUnicode(s, us)) - prop = us; + ConvertUTF8ToUnicode(s, us); + prop = us; } } diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 801619fb..2803f6a8 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -61,20 +61,29 @@ class CHandler: public CCommonMethodProps #endif { - CXzStatInfo _stat; - SRes MainDecodeSRes; + CXzStatInfo _stat; // it's stat from backward parsing + CXzStatInfo _stat2; // it's data from forward parsing, if the decoder was called + SRes _stat2_decode_SRes; + bool _stat_defined; + bool _stat2_defined; + + const CXzStatInfo *GetStat() const + { + if (_stat_defined) return &_stat; + if (_stat2_defined) return &_stat2; + return NULL; + } bool _isArc; bool _needSeekToStart; - bool _phySize_Defined; bool _firstBlockWasRead; AString _methodsString; + #ifndef EXTRACT_ONLY UInt32 _filterId; - UInt64 _numSolidBytes; void InitXz() @@ -85,6 +94,7 @@ class CHandler: #endif + void Init() { #ifndef EXTRACT_ONLY @@ -109,18 +119,24 @@ class CHandler: #endif decoder._memUsage = _memUsage; - MainDecodeSRes = SZ_OK; - - RINOK(decoder.Decode(seqInStream, outStream, + HRESULT hres = decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit true, // finishStream - progress)); + progress); - _stat = decoder.Stat; - MainDecodeSRes = decoder.MainDecodeSRes; + if (decoder.MainDecodeSRes_wasUsed + && decoder.MainDecodeSRes != SZ_ERROR_MEM + && decoder.MainDecodeSRes != SZ_ERROR_UNSUPPORTED) + { + // if (!_stat2_defined) + { + _stat2_decode_SRes = decoder.MainDecodeSRes; + _stat2 = decoder.Stat; + _stat2_defined = true; + } + } - _phySize_Defined = true; - return S_OK; + return hres; } public: @@ -143,8 +159,8 @@ public: INTERFACE_IOutArchive(;) #endif - size_t _blocksArraySize; CBlockInfo *_blocks; + size_t _blocksArraySize; UInt64 _maxBlocksSize; CMyComPtr<IInStream> _stream; CMyComPtr<ISequentialInStream> _seqStream; @@ -156,7 +172,7 @@ public: HRESULT SeekToPackPos(UInt64 pos) { - return _stream->Seek(pos, STREAM_SEEK_SET, NULL); + return _stream->Seek((Int64)pos, STREAM_SEEK_SET, NULL); } }; @@ -325,13 +341,16 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN NCOM::CPropVariant prop; + + const CXzStatInfo *stat = GetStat(); + switch (propID) { - case kpidPhySize: if (_phySize_Defined) prop = _stat.InSize; break; - case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; - case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; - case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidClusterSize: if (_stat.NumBlocks_Defined && _stat.NumBlocks > 1) prop = _maxBlocksSize; break; + case kpidPhySize: if (stat) prop = stat->InSize; break; + case kpidNumStreams: if (stat && stat->NumStreams_Defined) prop = stat->NumStreams; break; + case kpidNumBlocks: if (stat && stat->NumBlocks_Defined) prop = stat->NumBlocks; break; + case kpidUnpackSize: if (stat && stat->UnpackSize_Defined) prop = stat->OutSize; break; + case kpidClusterSize: if (_stat_defined && _stat.NumBlocks_Defined && stat->NumBlocks > 1) prop = _maxBlocksSize; break; case kpidCharacts: if (_firstBlockWasRead) { @@ -350,14 +369,14 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidErrorFlags: { UInt32 v = 0; - SRes sres = MainDecodeSRes; // _stat.DecodeRes2; // - if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; - if (/*_stat.UnexpectedEnd */ sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; - if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; - if (/* _stat.HeadersError */ sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; - if (/* _stat.Unsupported */ sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; - if (/* _stat.DataError */ sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; - if (/* _stat.CrcError */ sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; + SRes sres = _stat2_decode_SRes; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + if (sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; + if (_stat2_defined && _stat2.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; + if (sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; + if (sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; + if (sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; + if (sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; if (v != 0) prop = v; break; @@ -384,11 +403,12 @@ STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN + const CXzStatInfo *stat = GetStat(); NCOM::CPropVariant prop; switch (propID) { - case kpidSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidPackSize: if (_phySize_Defined) prop = _stat.InSize; break; + case kpidSize: if (stat && stat->UnpackSize_Defined) prop = stat->OutSize; break; + case kpidPackSize: if (stat) prop = stat->InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; } prop.Detach(value); @@ -402,7 +422,9 @@ struct COpenCallbackWrap ICompressProgress vt; IArchiveOpenCallback *OpenCallback; HRESULT Res; - COpenCallbackWrap(IArchiveOpenCallback *progress); + + // new clang shows "non-POD" warning for offsetof(), if we use constructor instead of Init() + void Init(IArchiveOpenCallback *progress); }; static SRes OpenCallbackProgress(const ICompressProgress *pp, UInt64 inSize, UInt64 /* outSize */) @@ -413,7 +435,7 @@ static SRes OpenCallbackProgress(const ICompressProgress *pp, UInt64 inSize, UIn return HRESULT_To_SRes(p->Res, SZ_ERROR_PROGRESS); } -COpenCallbackWrap::COpenCallbackWrap(IArchiveOpenCallback *callback) +void COpenCallbackWrap::Init(IArchiveOpenCallback *callback) { vt.Progress = OpenCallbackProgress; OpenCallback = callback; @@ -482,7 +504,11 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal CSeqInStreamWrap inStreamWrap; inStreamWrap.Init(inStream); + SRes res = Xz_ReadHeader(&st, &inStreamWrap.vt); + + if (inStreamWrap.Res != S_OK) + return inStreamWrap.Res; if (res != SZ_OK) return SRes_to_Open_HRESULT(res); @@ -490,8 +516,27 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal CXzBlock block; BoolInt isIndex; UInt32 headerSizeRes; + SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes); - if (res2 == SZ_OK && !isIndex) + + if (inStreamWrap.Res != S_OK) + return inStreamWrap.Res; + + if (res2 != SZ_OK) + { + if (res2 == SZ_ERROR_INPUT_EOF) + { + _stat2_decode_SRes = res2; + _stream = inStream; + _seqStream = inStream; + _isArc = true; + return S_OK; + } + + if (res2 == SZ_ERROR_ARCHIVE) + return S_FALSE; + } + else if (!isIndex) { _firstBlockWasRead = true; _firstBlock = block; @@ -526,7 +571,8 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal lookStream.realStream = &inStreamImp.vt; LookToRead2_Init(&lookStream); - COpenCallbackWrap openWrap(callback); + COpenCallbackWrap openWrap; + openWrap.Init(callback); CXzsCPP xzs; Int64 startPosition; @@ -539,7 +585,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal */ if (res == SZ_OK && startPosition == 0) { - _phySize_Defined = true; + _stat_defined = true; _stat.OutSize = Xzs_GetUnpackSize(&xzs.p); _stat.UnpackSize_Defined = true; @@ -615,6 +661,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal } RINOK(SRes_to_Open_HRESULT(res)); + _stream = inStream; _seqStream = inStream; _isArc = true; @@ -645,12 +692,15 @@ STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) STDMETHODIMP CHandler::Close() { XzStatInfo_Clear(&_stat); + XzStatInfo_Clear(&_stat2); + _stat_defined = false; + _stat2_defined = false; + _stat2_decode_SRes = SZ_OK; _isArc = false; _needSeekToStart = false; - _phySize_Defined = false; _firstBlockWasRead = false; - + _methodsString.Empty(); _stream.Release(); _seqStream.Release(); @@ -660,8 +710,6 @@ STDMETHODIMP CHandler::Close() _blocksArraySize = 0; _maxBlocksSize = 0; - MainDecodeSRes = SZ_OK; - return S_OK; } @@ -729,7 +777,7 @@ CInStream::~CInStream() } -size_t FindBlock(const CBlockInfo *blocks, size_t numBlocks, UInt64 pos) +static size_t FindBlock(const CBlockInfo *blocks, size_t numBlocks, UInt64 pos) { size_t left = 0, right = numBlocks; for (;;) @@ -900,9 +948,9 @@ STDMETHODIMP CInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPositio } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -951,7 +999,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) static Int32 Get_Extract_OperationResult(const NCompress::NXz::CDecoder &decoder) { Int32 opRes; - SRes sres = decoder.MainDecodeSRes; // decoder.Stat.DecodeRes2; + SRes sres = decoder.MainDecodeSRes; if (sres == SZ_ERROR_NO_ARCHIVE) // (!IsArc) opRes = NExtract::NOperationResult::kIsNotArc; else if (sres == SZ_ERROR_INPUT_EOF) // (UnexpectedEnd) @@ -985,8 +1033,10 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) return E_INVALIDARG; - if (_phySize_Defined) - extractCallback->SetTotal(_stat.InSize); + const CXzStatInfo *stat = GetStat(); + + if (stat) + extractCallback->SetTotal(stat->InSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)); @@ -1106,7 +1156,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt */ #ifndef _7ZIP_ST - xzProps.numTotalThreads = _numThreads; + xzProps.numTotalThreads = (int)_numThreads; #endif xzProps.blockSize = _numSolidBytes; @@ -1161,7 +1211,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CMyComPtr<ICompressProgressInfo> progress = lps; lps->Init(updateCallback, true); - return encoderSpec->Code(fileInStream, outStream, NULL, NULL, progress); + RINOK(encoderSpec->Code(fileInStream, outStream, NULL, NULL, progress)); + + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); } if (indexInArchive != 0) @@ -1176,8 +1228,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) { - if (_phySize_Defined) - RINOK(updateCallback->SetTotal(_stat.InSize)); + const CXzStatInfo *stat = GetStat(); + if (stat) + RINOK(updateCallback->SetTotal(stat->InSize)); RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); } diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 1ee7e22f..2bb57d5c 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -88,14 +88,18 @@ STDMETHODIMP CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStr } -CAddCommon::CAddCommon(const CCompressionMethodMode &options): - _options(options), +CAddCommon::CAddCommon(): _copyCoderSpec(NULL), + _isLzmaEos(false), _cryptoStreamSpec(NULL), - _buf(NULL), - _isLzmaEos(false) + _buf(NULL) {} +void CAddCommon::SetOptions(const CCompressionMethodMode &options) +{ + _options = options; +} + CAddCommon::~CAddCommon() { MidFree(_buf); @@ -230,6 +234,11 @@ HRESULT CAddCommon::Compress( unsigned numTestMethods = _options.MethodSequence.Size(); bool descriptorMode = outSeqMode; + + // ZipCrypto without descriptor requires additional reading pass for + // inStream to calculate CRC for password check field. + // The descriptor allows to use ZipCrypto check field without CRC (InfoZip's modification). + if (!outSeqMode) if (inSeqMode && _options.PasswordIsDefined && !_options.IsAesMode) descriptorMode = true; @@ -262,6 +271,15 @@ HRESULT CAddCommon::Compress( RINOK(outStream->SetSize(0)); RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL)); } + + method = _options.MethodSequence[i]; + if (method == NCompressionMethod::kStore && descriptorMode) + { + // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% + return E_NOTIMPL; + } + + bool needCode = true; if (_options.PasswordIsDefined) { @@ -314,23 +332,25 @@ HRESULT CAddCommon::Compress( RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check)); } - RINOK(_cryptoStreamSpec->SetOutStream(outStream)); - RINOK(_cryptoStreamSpec->InitEncoder()); - outStreamReleaser.FilterCoder = _cryptoStreamSpec; + if (method == NCompressionMethod::kStore) + { + needCode = false; + RINOK(_cryptoStreamSpec->Code(inCrcStream, outStream, NULL, NULL, progress)); + } + else + { + RINOK(_cryptoStreamSpec->SetOutStream(outStream)); + RINOK(_cryptoStreamSpec->InitEncoder()); + outStreamReleaser.FilterCoder = _cryptoStreamSpec; + } } - method = _options.MethodSequence[i]; - - switch (method) + if (needCode) { + switch (method) + { case NCompressionMethod::kStore: { - if (descriptorMode) - { - // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% - return E_NOTIMPL; - } - if (!_copyCoderSpec) { _copyCoderSpec = new NCompress::CCopyCoder; @@ -438,15 +458,21 @@ HRESULT CAddCommon::Compress( } } + try { RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress)); + } catch (...) { return E_FAIL; } break; } + } // switch end + + if (_options.PasswordIsDefined) + { + RINOK(_cryptoStreamSpec->OutStreamFinish()); + } } if (_options.PasswordIsDefined) { - RINOK(_cryptoStreamSpec->OutStreamFinish()); - if (_options.IsAesMode) { RINOK(_filterAesSpec->WriteFooter(outStream)); diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.h b/CPP/7zip/Archive/Zip/ZipAddCommon.h index ff3251db..0aa44adf 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.h +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.h @@ -28,9 +28,15 @@ struct CCompressingResult Byte ExtractVersion; bool DescriptorMode; bool LzmaEos; + + CCompressingResult() + { + // for GCC: + UnpackSize = 0; + } }; -class CAddCommon +class CAddCommon MY_UNCOPYABLE { CCompressionMethodMode _options; NCompress::CCopyCoder *_copyCoderSpec; @@ -50,7 +56,9 @@ class CAddCommon HRESULT CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultCRC); public: - CAddCommon(const CCompressionMethodMode &options); + // CAddCommon(const CCompressionMethodMode &options); + CAddCommon(); + void SetOptions(const CCompressionMethodMode &options); ~CAddCommon(); HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, diff --git a/CPP/7zip/Archive/Zip/ZipCompressionMode.h b/CPP/7zip/Archive/Zip/ZipCompressionMode.h index 1125f6ed..842991c4 100644 --- a/CPP/7zip/Archive/Zip/ZipCompressionMode.h +++ b/CPP/7zip/Archive/Zip/ZipCompressionMode.h @@ -35,7 +35,7 @@ struct CCompressionMethodMode: public CBaseProps { CRecordVector<Byte> MethodSequence; bool PasswordIsDefined; - AString Password; + AString Password; // _Wipe UInt64 _dataSizeReduce; bool _dataSizeReduceDefined; @@ -47,6 +47,8 @@ struct CCompressionMethodMode: public CBaseProps _dataSizeReduceDefined = false; _dataSizeReduce = 0; } + + ~CCompressionMethodMode() { Password.Wipe_and_Empty(); } }; }} diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index a4794f51..72a77cb7 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -18,6 +18,17 @@ #include "../../Common/StreamUtils.h" #include "../../Compress/CopyCoder.h" + +#ifdef EXTERNAL_CODECS +#ifndef SUPPORT_LZFSE +#define SUPPORT_LZFSE +#endif +#endif + +#ifdef SUPPORT_LZFSE +#include "../../Compress/LzfseDecoder.h" +#endif + #include "../../Compress/LzmaDecoder.h" #include "../../Compress/ImplodeDecoder.h" #include "../../Compress/PpmdZip.h" @@ -81,16 +92,24 @@ const char * const kMethodNames1[kNumMethodNames1] = , "BZip2" , NULL , "LZMA" + , NULL + , NULL + , NULL + , NULL + , NULL + , "zstd-pk" }; const char * const kMethodNames2[kNumMethodNames2] = { - "xz" + "zstd-wz" + , "MP3" + , "xz" , "Jpeg" , "WavPack" , "PPMd" - , "WzAES" + , "LZFSE" // , "WzAES" }; #define kMethod_AES "AES" @@ -240,6 +259,12 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (m_Archive.IsZip64) s.Add_OptSpaced("Zip64"); + if (m_Archive.IsCdUnsorted) + s.Add_OptSpaced("Unsorted_CD"); + + if (m_Archive.IsApk) + s.Add_OptSpaced("apk"); + if (m_Archive.ExtraMinorError) s.Add_OptSpaced("Minor_Extra_ERROR"); @@ -312,9 +337,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) // case kpidIsAltStream: prop = true; break; } - prop.Detach(value); + return prop.Detach(value); COM_TRY_END - return S_OK; } STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) @@ -336,7 +360,9 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { UString res; item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); - NItemName::ReplaceToOsSlashes_Remove_TailSlash(res); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(res, + item.Is_MadeBy_Unix() // useBackslashReplacement + ); /* if (item.ParentOfAltStream >= 0) { @@ -359,7 +385,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidIsDir: prop = item.IsDir(); break; case kpidSize: { - if (item.FromCentral || !item.FromLocal || !item.HasDescriptor() || item.DescriptorWasRead) + if (!item.IsBadDescriptor()) prop = item.Size; break; } @@ -467,23 +493,27 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMethod: { - unsigned id = item.Method; AString m; - - if (item.IsEncrypted()) + bool isWzAes = false; + unsigned id = item.Method; + + if (id == NFileHeader::NCompressionMethod::kWzAES) { - if (id == NFileHeader::NCompressionMethod::kWzAES) + CWzAesExtra aesField; + if (extra.GetWzAes(aesField)) { m += kMethod_AES; - CWzAesExtra aesField; - if (extra.GetWzAes(aesField)) - { - m += '-'; - m.Add_UInt32(((unsigned)aesField.Strength + 1) * 64); - id = aesField.Method; - } + m += '-'; + m.Add_UInt32(((unsigned)aesField.Strength + 1) * 64); + id = aesField.Method; + isWzAes = true; } - else if (item.IsStrongEncrypted()) + } + + if (item.IsEncrypted()) + if (!isWzAes) + { + if (item.IsStrongEncrypted()) { CStrongCryptoExtra f; f.AlgId = 0; @@ -506,8 +536,9 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } else m += kMethod_ZipCrypto; - m += ' '; } + + m.Add_Space_if_NotEmpty(); { const char *s = NULL; @@ -516,7 +547,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val else { int id2 = (int)id - (int)kMethodNames2Start; - if (id2 >= 0 && id2 < kNumMethodNames2) + if (id2 >= 0 && (unsigned)id2 < kNumMethodNames2) s = kMethodNames2[id2]; } if (s) @@ -532,7 +563,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { if (level & 1) m += ":eos"; - level &= ~1; + level &= ~(unsigned)1; } else if (id == NFileHeader::NCompressionMethod::kDeflate) { @@ -576,7 +607,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } UInt32 flags = item.Flags; - flags &= ~(6); // we don't need compression related bits here. + flags &= ~(unsigned)6; // we don't need compression related bits here. if (flags != 0) { @@ -589,7 +620,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } } - if (!item.FromCentral && item.FromLocal && item.HasDescriptor() && !item.DescriptorWasRead) + if (item.IsBadDescriptor()) s.Add_OptSpaced("Descriptor_ERROR"); if (!s.IsEmpty()) @@ -634,8 +665,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val */ } - prop.Detach(value); - return S_OK; + return prop.Detach(value); COM_TRY_END } @@ -860,11 +890,14 @@ public: }; -static HRESULT SkipStreamData(ISequentialInStream *stream, bool &thereAreData) +static HRESULT SkipStreamData(ISequentialInStream *stream, + ICompressProgressInfo *progress, UInt64 packSize, UInt64 unpackSize, + bool &thereAreData) { thereAreData = false; const size_t kBufSize = 1 << 12; Byte buf[kBufSize]; + UInt64 prev = packSize; for (;;) { size_t size = kBufSize; @@ -872,10 +905,80 @@ static HRESULT SkipStreamData(ISequentialInStream *stream, bool &thereAreData) if (size == 0) return S_OK; thereAreData = true; + packSize += size; + if ((packSize - prev) >= (1 << 22)) + { + prev = packSize; + RINOK(progress->SetRatioInfo(&packSize, &unpackSize)); + } } } + +class COutStreamWithPadPKCS7: + public ISequentialOutStream, + public CMyUnknownImp +{ + CMyComPtr<ISequentialOutStream> _stream; + UInt64 _size; + UInt64 _padPos; + UInt32 _padSize; + bool _padFailure; +public: + MY_UNKNOWN_IMP + STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); + void SetStream(ISequentialOutStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + + // padSize == 0 means (no_pad Mode) + void Init(UInt64 padPos, UInt32 padSize) + { + _padPos = padPos; + _padSize = padSize; + _size = 0; + _padFailure = false; + } + UInt64 GetSize() const { return _size; } + bool WasPadFailure() const { return _padFailure; } +}; + + +STDMETHODIMP COutStreamWithPadPKCS7::Write(const void *data, UInt32 size, UInt32 *processedSize) +{ + UInt32 written = 0; + HRESULT result = S_OK; + if (_size < _padPos) + { + const UInt64 rem = _padPos - _size; + UInt32 num = size; + if (num > rem) + num = (UInt32)rem; + result = _stream->Write(data, num, &written); + _size += written; + if (processedSize) + *processedSize = written; + if (_size != _padPos || result != S_OK) + return result; + size -= written; + data = ((const Byte *)data) + written; + } + _size += size; + written += size; + if (processedSize) + *processedSize = written; + if (_padSize != 0) + for (; size != 0; size--) + { + if (*(const Byte *)data != _padSize) + _padFailure = true; + data = ((const Byte *)data) + 1; + } + return result; +} + + + HRESULT CZipDecoder::Decode( DECL_EXTERNAL_CODECS_LOC_VARS CInArchive &archive, const CItemEx &item, @@ -895,9 +998,32 @@ HRESULT CZipDecoder::Decode( bool needCRC = true; bool wzAesMode = false; bool pkAesMode = false; + + bool badDescriptor = item.IsBadDescriptor(); + if (badDescriptor) + needCRC = false; + unsigned id = item.Method; + CWzAesExtra aesField; + // LZFSE and WinZip's AES use same id - kWzAES. + + if (id == NFileHeader::NCompressionMethod::kWzAES) + { + if (item.GetMainExtra().GetWzAes(aesField)) + { + if (!item.IsEncrypted()) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + wzAesMode = true; + needCRC = aesField.NeedCrc(); + } + } + + if (!wzAesMode) if (item.IsEncrypted()) { if (item.IsStrongEncrypted()) @@ -910,14 +1036,6 @@ HRESULT CZipDecoder::Decode( } pkAesMode = true; } - else if (id == NFileHeader::NCompressionMethod::kWzAES) - { - CWzAesExtra aesField; - if (!item.GetMainExtra().GetWzAes(aesField)) - return S_OK; - wzAesMode = true; - needCRC = aesField.NeedCrc(); - } } COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; @@ -957,9 +1075,6 @@ HRESULT CZipDecoder::Decode( { if (wzAesMode) { - CWzAesExtra aesField; - if (!item.GetMainExtra().GetWzAes(aesField)) - return S_OK; id = aesField.Method; if (!_wzAesDecoder) { @@ -1002,12 +1117,12 @@ HRESULT CZipDecoder::Decode( if (getTextPassword) { - CMyComBSTR password; + CMyComBSTR_Wipe password; RINOK(getTextPassword->CryptoGetTextPassword(&password)); - AString charPassword; + AString_Wipe charPassword; if (password) { - UnicodeStringToMultiByte2(charPassword, (const wchar_t *)password, CP_ACP); + UnicodeStringToMultiByte2(charPassword, (LPCOLESTR)password, CP_ACP); /* if (wzAesMode || pkAesMode) { @@ -1063,6 +1178,10 @@ HRESULT CZipDecoder::Decode( mi.Coder = new NCompress::NXz::CComDecoder; else if (id == NFileHeader::NCompressionMethod::kPPMd) mi.Coder = new NCompress::NPpmdZip::CDecoder(true); + #ifdef SUPPORT_LZFSE + else if (id == NFileHeader::NCompressionMethod::kWzAES) + mi.Coder = new NCompress::NLzfse::CDecoder; + #endif else { CMethodId szMethodID; @@ -1089,7 +1208,8 @@ HRESULT CZipDecoder::Decode( m = methodItems.Add(mi); } - ICompressCoder *coder = methodItems[m].Coder; + const CMethodItem &mi = methodItems[m]; + ICompressCoder *coder = mi.Coder; #ifndef _7ZIP_ST @@ -1123,14 +1243,22 @@ HRESULT CZipDecoder::Decode( } - CMyComPtr<ISequentialInStream> inStreamNew; - bool isFullStreamExpected = (!item.HasDescriptor() || item.PackSize != 0); bool needReminderCheck = false; bool dataAfterEnd = false; bool truncatedError = false; bool lzmaEosError = false; + bool headersError = false; + bool padError = false; + bool readFromFilter = false; + + const bool useUnpackLimit = (id == NFileHeader::NCompressionMethod::kStore + || !item.HasDescriptor() + || item.Size >= ((UInt64)1 << 32) + || item.LocalExtra.IsZip64 + || item.CentralExtra.IsZip64 + ); { HRESULT result = S_OK; @@ -1198,23 +1326,7 @@ HRESULT CZipDecoder::Decode( } } } - - if (result == S_OK) - { - inStreamReleaser.FilterCoder = filterStreamSpec; - RINOK(filterStreamSpec->SetInStream(inStream)); - - /* IFilter::Init() does nothing in all zip crypto filters. - So we can call any Initialize function in CFilterCoder. */ - - RINOK(filterStreamSpec->Init_NoSubFilterInit()); - // RINOK(filterStreamSpec->SetOutStreamSize(NULL)); - - inStreamNew = filterStream; - } } - else - inStreamNew = inStream; if (result == S_OK) { @@ -1222,26 +1334,84 @@ HRESULT CZipDecoder::Decode( coder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); if (setFinishMode) { - RINOK(setFinishMode->SetFinishMode(BoolToInt(true))); + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))); } const UInt64 coderPackSize = limitedStreamSpec->GetRem(); - bool useUnpackLimit = (id == 0 - || !item.HasDescriptor() - || item.Size >= ((UInt64)1 << 32) - || item.LocalExtra.IsZip64 - || item.CentralExtra.IsZip64 - ); + if (id == NFileHeader::NCompressionMethod::kStore && item.IsEncrypted()) + { + readFromFilter = false; + + COutStreamWithPadPKCS7 *padStreamSpec = NULL; + CMyComPtr<ISequentialOutStream> padStream; + UInt32 padSize = 0; + + if (pkAesMode) + { + padStreamSpec = new COutStreamWithPadPKCS7; + padStream = padStreamSpec; + padSize = _pkAesDecoderSpec->GetPadSize((UInt32)item.Size); + padStreamSpec->SetStream(outStream); + padStreamSpec->Init(item.Size, padSize); + } + + // Here we decode minimal required size, including padding + const UInt64 expectedSize = item.Size + padSize; + UInt64 size = coderPackSize; + if (item.Size > coderPackSize) + headersError = true; + else if (expectedSize != coderPackSize) + { + headersError = true; + if (coderPackSize > expectedSize) + size = expectedSize; + } - result = coder->Code(inStreamNew, outStream, - isFullStreamExpected ? &coderPackSize : NULL, - // NULL, - useUnpackLimit ? &item.Size : NULL, - compressProgress); + result = filterStreamSpec->Code(inStream, padStream ? + (ISequentialOutStream *)padStream : + (ISequentialOutStream *)outStream, + NULL, &size, compressProgress); - if (result == S_OK) + if (outStreamSpec->GetSize() != item.Size) + truncatedError = true; + + if (pkAesMode) + { + if (padStreamSpec->GetSize() != size) + truncatedError = true; + if (padStreamSpec->WasPadFailure()) + padError = true; + } + } + else { + if (item.IsEncrypted()) + { + readFromFilter = true; + inStreamReleaser.FilterCoder = filterStreamSpec; + RINOK(filterStreamSpec->SetInStream(inStream)); + + /* IFilter::Init() does nothing in all zip crypto filters. + So we can call any Initialize function in CFilterCoder. */ + + RINOK(filterStreamSpec->Init_NoSubFilterInit()); + // RINOK(filterStreamSpec->SetOutStreamSize(NULL)); + } + + try { + result = coder->Code(readFromFilter ? + (ISequentialInStream *)filterStream : + (ISequentialInStream *)inStream, + outStream, + isFullStreamExpected ? &coderPackSize : NULL, + // NULL, + useUnpackLimit ? &item.Size : NULL, + compressProgress); + } catch (...) { return E_FAIL; } + + if (result == S_OK) + { CMyComPtr<ICompressGetInStreamProcessedSize> getInStreamProcessedSize; coder->QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); if (getInStreamProcessedSize && setFinishMode) @@ -1259,7 +1429,32 @@ HRESULT CZipDecoder::Decode( { if (processed + padSize < coderPackSize) dataAfterEnd = true; - // also here we can check PKCS7 padding data from reminder (it can be inside stream buffer in coder). + else + { + // here we can PKCS7 padding data from reminder (it can be inside stream buffer in coder). + CMyComPtr<ICompressReadUnusedFromInBuf> readInStream; + coder->QueryInterface(IID_ICompressReadUnusedFromInBuf, (void **)&readInStream); + if (readInStream) + { + // change pad size, it we support another block size in ZipStron + // here we request more to detect error with data after end. + const UInt32 kBufSize = NCrypto::NZipStrong::kAesPadAllign + 16; + Byte buf[kBufSize]; + UInt32 processedSize; + RINOK(readInStream->ReadUnusedFromInBuf(buf, kBufSize, &processedSize)); + if (processedSize > padSize) + dataAfterEnd = true; + else + { + if (ReadStream_FALSE(filterStream, buf + processedSize, padSize - processedSize) != S_OK) + padError = true; + else + for (unsigned i = 0; i < padSize; i++) + if (buf[i] != padSize) + padError = true; + } + } + } } } else @@ -1270,11 +1465,15 @@ HRESULT CZipDecoder::Decode( dataAfterEnd = true; } else if (processed > coderPackSize) + { + // that case is additional check, that can show the bugs in code (coder) truncatedError = true; + } needReminderCheck = isFullStreamExpected; } } } + } } if (result == S_OK && id == NFileHeader::NCompressionMethod::kLZMA) @@ -1298,19 +1497,33 @@ HRESULT CZipDecoder::Decode( bool authOk = true; if (needCRC) crcOK = (outStreamSpec->GetCRC() == item.Crc); + + if (useUnpackLimit) + if (outStreamSpec->GetSize() != item.Size) + truncatedError = true; if (wzAesMode) { + const UInt64 unpackSize = outStreamSpec->GetSize(); + const UInt64 packSize = limitedStreamSpec->GetSize(); bool thereAreData = false; - if (SkipStreamData(inStreamNew, thereAreData) != S_OK) + // read to the end from filter or from packed stream + if (SkipStreamData(readFromFilter ? + (ISequentialInStream *)filterStream : + (ISequentialInStream *)inStream, + compressProgress, packSize, unpackSize, thereAreData) != S_OK) authOk = false; - if (needReminderCheck && thereAreData) dataAfterEnd = true; - - limitedStreamSpec->Init(NCrypto::NWzAes::kMacSize); - if (_wzAesDecoderSpec->CheckMac(inStream, authOk) != S_OK) - authOk = false; + + if (limitedStreamSpec->GetRem() != 0) + truncatedError = true; + else + { + limitedStreamSpec->Init(NCrypto::NWzAes::kMacSize); + if (_wzAesDecoderSpec->CheckMac(inStream, authOk) != S_OK) + authOk = false; + } } res = NExtract::NOperationResult::kCRCError; @@ -1321,10 +1534,16 @@ HRESULT CZipDecoder::Decode( if (dataAfterEnd) res = NExtract::NOperationResult::kDataAfterEnd; + else if (padError) + res = NExtract::NOperationResult::kCRCError; else if (truncatedError) res = NExtract::NOperationResult::kUnexpectedEnd; + else if (headersError) + res = NExtract::NOperationResult::kHeadersError; else if (lzmaEosError) res = NExtract::NOperationResult::kHeadersError; + else if (badDescriptor) + res = NExtract::NOperationResult::kUnexpectedEnd; // CheckDescriptor() supports only data descriptor with signature and // it doesn't support "old" pkzip's data descriptor without signature. diff --git a/CPP/7zip/Archive/Zip/ZipHandler.h b/CPP/7zip/Archive/Zip/ZipHandler.h index bee57c00..3043e41c 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.h +++ b/CPP/7zip/Archive/Zip/ZipHandler.h @@ -15,8 +15,8 @@ namespace NArchive { namespace NZip { -const unsigned kNumMethodNames1 = NFileHeader::NCompressionMethod::kLZMA + 1; -const unsigned kMethodNames2Start = NFileHeader::NCompressionMethod::kXz; +const unsigned kNumMethodNames1 = NFileHeader::NCompressionMethod::kZstdPk + 1; +const unsigned kMethodNames2Start = NFileHeader::NCompressionMethod::kZstdWz; const unsigned kNumMethodNames2 = NFileHeader::NCompressionMethod::kWzAES + 1 - kMethodNames2Start; extern const char * const kMethodNames1[kNumMethodNames1]; diff --git a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index c21b5605..a9b3eae5 100644 --- a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -53,7 +53,7 @@ static int FindZipMethod(const char *s, const char * const *names, unsigned num) { const char *name = names[i]; if (name && StringsAreEqualNoCase_Ascii(s, name)) - return i; + return (int)i; } return -1; } @@ -65,7 +65,7 @@ static int FindZipMethod(const char *s) return k; k = FindZipMethod(s, kMethodNames2, kNumMethodNames2); if (k >= 0) - return kMethodNames2Start + k; + return (int)kMethodNames2Start + k; return -1; } @@ -75,7 +75,7 @@ static int FindZipMethod(const char *s) catch(const CSystemException &e) { return e.ErrorCode; } \ catch(...) { return E_OUTOFMEMORY; } -static HRESULT GetTime(IArchiveUpdateCallback *callback, int index, PROPID propID, FILETIME &filetime) +static HRESULT GetTime(IArchiveUpdateCallback *callback, unsigned index, PROPID propID, FILETIME &filetime) { filetime.dwHighDateTime = filetime.dwLowDateTime = 0; NCOM::CPropVariant prop; @@ -106,6 +106,10 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt UInt64 largestSize = 0; bool largestSizeDefined = false; + #ifdef _WIN32 + const UINT oemCP = GetOEMCP(); + #endif + UString name; CUpdateItem ui; @@ -125,7 +129,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt ui.NewProps = IntToBool(newProps); ui.NewData = IntToBool(newData); - ui.IndexInArc = indexInArc; + ui.IndexInArc = (int)indexInArc; ui.IndexInClient = i; bool existInArchive = (indexInArc != (UInt32)(Int32)-1); @@ -240,10 +244,25 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (needSlash) name += kSlash; - UINT codePage = _forceCodePage ? _specifiedCodePage : CP_OEMCP; - + const UINT codePage = _forceCodePage ? _specifiedCodePage : CP_OEMCP; bool tryUtf8 = true; - if ((m_ForceLocal || !m_ForceUtf8) && codePage != CP_UTF8) + + /* + Windows 10 allows users to set UTF-8 in Region Settings via option: + "Beta: Use Unicode UTF-8 for worldwide language support" + In that case Windows uses CP_UTF8 when we use CP_OEMCP. + 21.02 fixed: + we set UTF-8 mark for non-latin files for such UTF-8 mode in Windows. + we write additional Info-Zip Utf-8 FileName Extra for non-latin names/ + */ + + if ((codePage != CP_UTF8) && + #ifdef _WIN32 + (m_ForceLocal || !m_ForceUtf8) && (oemCP != CP_UTF8) + #else + (m_ForceLocal && !m_ForceUtf8) + #endif + ) { bool defaultCharWasUsed; ui.Name = UnicodeStringToMultiByte(name, codePage, '_', defaultCharWasUsed); @@ -251,13 +270,26 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt MultiByteToUnicodeString(ui.Name, codePage) != name)); } + const bool isNonLatin = !name.IsAscii(); + if (tryUtf8) { - ui.IsUtf8 = !name.IsAscii(); + ui.IsUtf8 = isNonLatin; ConvertUnicodeToUTF8(name, ui.Name); + + #ifndef _WIN32 + if (ui.IsUtf8 && !CheckUTF8_AString(ui.Name)) + { + // if it's non-Windows and there are non-UTF8 characters we clear UTF8-flag + ui.IsUtf8 = false; + } + #endif } + else if (isNonLatin) + Convert_Unicode_To_UTF8_Buf(name, ui.Name_Utf); - if (ui.Name.Len() >= (1 << 16)) + if (ui.Name.Len() >= (1 << 16) + || ui.Name_Utf.Size() >= (1 << 16) - 128) return E_INVALIDARG; { @@ -337,10 +369,10 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt options._dataSizeReduceDefined = largestSizeDefined; options.PasswordIsDefined = false; - options.Password.Empty(); + options.Password.Wipe_and_Empty(); if (getTextPassword) { - CMyComBSTR password; + CMyComBSTR_Wipe password; Int32 passwordIsDefined; RINOK(getTextPassword->CryptoGetTextPassword2(&passwordIsDefined, &password)); options.PasswordIsDefined = IntToBool(passwordIsDefined); @@ -352,7 +384,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (!IsSimpleAsciiString(password)) return E_INVALIDARG; if (password) - options.Password = UnicodeStringToMultiByte((LPCOLESTR)password, CP_OEMCP); + UnicodeStringToMultiByte2(options.Password, (LPCOLESTR)password, CP_OEMCP); if (options.IsAesMode) { if (options.Password.Len() > NCrypto::NWzAes::kPasswordSizeMax) @@ -496,7 +528,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR UInt32 id = prop.ulVal; if (id > 0xFF) return E_INVALIDARG; - m_MainMethod = id; + m_MainMethod = (int)id; } else { @@ -518,7 +550,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR const char *end; UInt32 id = ConvertStringToUInt32(methodName, &end); if (*end == 0 && id <= 0xFF) - m_MainMethod = id; + m_MainMethod = (int)id; else if (methodName.IsEqualTo_Ascii_NoCase("Copy")) // it's alias for "Store" m_MainMethod = 0; } diff --git a/CPP/7zip/Archive/Zip/ZipHeader.h b/CPP/7zip/Archive/Zip/ZipHeader.h index 5e6f00e4..c5c7166e 100644 --- a/CPP/7zip/Archive/Zip/ZipHeader.h +++ b/CPP/7zip/Archive/Zip/ZipHeader.h @@ -56,7 +56,10 @@ namespace NFileHeader kTerse = 18, kLz77 = 19, + kZstdPk = 20, + kZstdWz = 93, + kMP3 = 94, kXz = 95, kJpeg = 96, kWavPack = 97, @@ -90,7 +93,10 @@ namespace NFileHeader kUnixExtra = 0x5855, kIzUnicodeComment = 0x6375, kIzUnicodeName = 0x7075, - kWzAES = 0x9901 + kUnix2Extra = 0x7855, + kUnix3Extra = 0x7875, + kWzAES = 0x9901, + kApkAlign = 0xD935 }; } diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 509753c2..880ff218 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -152,6 +152,9 @@ void CInArchive::Close() IsArc = false; IsZip64 = false; + IsApk = false; + IsCdUnsorted = false; + HeadersError = false; HeadersWarning = false; ExtraMinorError = false; @@ -169,7 +172,7 @@ void CInArchive::Close() IsMultiVol = false; UseDisk_in_SingleVol = false; EcdVolIndex = 0; - + ArcInfo.Clear(); ClearRefs(); @@ -181,7 +184,7 @@ HRESULT CInArchive::Seek_SavePos(UInt64 offset) { // InitBuf(); // if (!Stream) return S_FALSE; - return Stream->Seek(offset, STREAM_SEEK_SET, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_SET, &_streamPos); } HRESULT CInArchive::SeekToVol(int volIndex, UInt64 offset) @@ -193,9 +196,9 @@ HRESULT CInArchive::SeekToVol(int volIndex, UInt64 offset) { if ((unsigned)volIndex >= Vols.Streams.Size()) return S_FALSE; - if (!Vols.Streams[volIndex].Stream) + if (!Vols.Streams[(unsigned)volIndex].Stream) return S_FALSE; - Stream = Vols.Streams[volIndex].Stream; + Stream = Vols.Streams[(unsigned)volIndex].Stream; } else if (volIndex == -2) { @@ -277,11 +280,11 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed } else { - UInt32 cur = 0; - result = Stream->Read(data, size, &cur); + size_t cur = size; + result = ReadStream(Stream, data, &cur); data += cur; - size -= cur; - processed += cur; + size -= (unsigned)cur; + processed += (unsigned)cur; _streamPos += cur; _cnt += cur; if (cur != 0) @@ -299,7 +302,7 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) break; - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) break; result = s.SeekToStart(); @@ -316,6 +319,16 @@ HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed } +HRESULT CInArchive::ReadFromCache_FALSE(Byte *data, unsigned size) +{ + unsigned processed; + HRESULT res = ReadFromCache(data, size, processed); + if (res == S_OK && size != processed) + return S_FALSE; + return res; +} + + static bool CheckDosTime(UInt32 dosTime) { if (dosTime == 0) @@ -412,8 +425,12 @@ API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size) const unsigned nameSize = Get16(p + 22); unsigned extraSize = Get16(p + 24); const UInt32 extraOffset = kLocalHeaderSize + (UInt32)nameSize; + + /* + // 21.02: fixed. we don't use the following check if (extraOffset + extraSize > (1 << 16)) return k_IsArc_Res_NO; + */ p -= 4; @@ -498,12 +515,9 @@ static const Byte *FindPK(const Byte *p, const Byte *limit) { for (;;) { - Byte b0 = p[0]; - if (p >= limit) - return p; - p++; - if (b0 == 0x50) - break; + Byte b0; + b0 = p[0]; if (p >= limit) return p; p++; if (b0 == 0x50) break; + b0 = p[0]; if (p >= limit) return p; p++; if (b0 == 0x50) break; } if (p[0] == 0x4B) return p - 1; @@ -540,10 +554,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if (searchLimit && *searchLimit == 0) { Byte startBuf[kMarkerSize]; - unsigned processed; - RINOK(ReadFromCache(startBuf, kMarkerSize, processed)); - if (processed != kMarkerSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)); UInt32 marker = Get32(startBuf); _signature = marker; @@ -551,9 +562,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if ( marker == NSignature::kNoSpan || marker == NSignature::kSpan) { - RINOK(ReadFromCache(startBuf, kMarkerSize, processed)); - if (processed != kMarkerSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)); _signature = Get32(startBuf); } @@ -605,7 +614,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) SkipLookahed(avail); - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) break; @@ -645,14 +654,14 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) p = FindPK(p, limit); if (p >= limit) break; - const size_t rem = pStart + avail - p; + const size_t rem = (size_t)(pStart + avail - p); UInt32 res = IsArc_Zip_2(p, rem, isFinished); if (res != k_IsArc_Res_NO) { if (rem < kMarkerSize) return S_FALSE; _signature = Get32(p); - SkipLookahed(p - pStart); + SkipLookahed((size_t)(p - pStart)); ArcInfo.MarkerVolIndex = Vols.StreamIndex; ArcInfo.MarkerPos = GetVirtStreamPos(); ArcInfo.MarkerPos2 = ArcInfo.MarkerPos; @@ -674,7 +683,7 @@ HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) if (!IsMultiVol && isFinished) break; - SkipLookahed(p - pStart); + SkipLookahed((size_t)(p - pStart)); if (Callback && (_cnt - progressPrev) >= ((UInt32)1 << 23)) { @@ -728,7 +737,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) if (!IsMultiVol) { _cnt += offset; - return Stream->Seek(offset, STREAM_SEEK_CUR, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); } for (;;) @@ -744,7 +753,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) return S_OK; } { - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex]; if (!s.Stream) { isFinished = true; @@ -756,7 +765,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) if ((UInt64)offset <= rem) { _cnt += offset; - return Stream->Seek(offset, STREAM_SEEK_CUR, &_streamPos); + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); } RINOK(Seek_SavePos(s.Size)); offset -= rem; @@ -771,7 +780,7 @@ HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) isFinished = true; return S_OK; } - const CVols::CSubStreamInfo &s2 = Vols.Streams[Vols.StreamIndex]; + const CVols::CSubStreamInfo &s2 = Vols.Streams[(unsigned)Vols.StreamIndex]; if (!s2.Stream) { isFinished = true; @@ -834,7 +843,7 @@ HRESULT CInArchive::LookAhead(size_t minRequired) || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) return S_OK; - const CVols::CSubStreamInfo &s = Vols.Streams[Vols.StreamIndex + 1]; + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; if (!s.Stream) return S_OK; @@ -979,7 +988,7 @@ bool CInArchive::ReadFileName(unsigned size, AString &s) #define ZIP64_IS_16_MAX(n) ((n) == 0xFFFF) -bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, +bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk) { extra.Clear(); @@ -1010,16 +1019,16 @@ bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, bool isOK = true; if (ZIP64_IS_32_MAX(unpackSize)) - if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} if (isOK && ZIP64_IS_32_MAX(packSize)) - if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} if (isOK && ZIP64_IS_32_MAX(localOffset)) - if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); } + { if (size < 8) isOK = false; else { size -= 8; localOffset = ReadUInt64(); }} if (isOK && ZIP64_IS_16_MAX(disk)) - if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); } + { if (size < 4) isOK = false; else { size -= 4; disk = ReadUInt32(); }} if (!isOK || size != 0) { @@ -1033,6 +1042,11 @@ bool CInArchive::ReadExtra(unsigned extraSize, CExtraBlock &extra, { ReadBuffer(subBlock.Data, size); extra.SubBlocks.Add(subBlock); + if (subBlock.ID == NFileHeader::NExtraID::kIzUnicodeName) + { + if (!subBlock.CheckIzUnicode(item.Name)) + extra.Error = true; + } } } @@ -1054,7 +1068,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) { item.Disk = 0; if (IsMultiVol && Vols.StreamIndex >= 0) - item.Disk = Vols.StreamIndex; + item.Disk = (UInt32)Vols.StreamIndex; const unsigned kPureHeaderSize = kLocalHeaderSize - 4; Byte p[kPureHeaderSize]; SafeRead(p, kPureHeaderSize); @@ -1088,7 +1102,7 @@ bool CInArchive::ReadLocalItem(CItemEx &item) { UInt64 localOffset = 0; UInt32 disk = 0; - if (!ReadExtra(extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) + if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, localOffset, disk)) { /* Most of archives are OK for Extra. But there are some rare cases that have error. And if error in first item, it can't open archive. @@ -1111,33 +1125,39 @@ bool CInArchive::ReadLocalItem(CItemEx &item) HeadersWarning = true; } - return item.LocalFullHeaderSize <= ((UInt32)1 << 16); + // return item.LocalFullHeaderSize <= ((UInt32)1 << 16); + return true; } -static bool FlagsAreSame(const CItem &i1, const CItem &i2) +static bool FlagsAreSame(const CItem &i1, const CItem &i2_cd) { - if (i1.Method != i2.Method) + if (i1.Method != i2_cd.Method) return false; - if (i1.Flags == i2.Flags) + + UInt32 mask = i1.Flags ^ i2_cd.Flags; + if (mask == 0) return true; - UInt32 mask = 0xFFFF; switch (i1.Method) { case NFileHeader::NCompressionMethod::kDeflate: - mask = 0x7FF9; + mask &= 0x7FF9; break; default: if (i1.Method <= NFileHeader::NCompressionMethod::kImplode) - mask = 0x7FFF; + mask &= 0x7FFF; } // we can ignore utf8 flag, if name is ascii - if ((i1.Flags ^ i2.Flags) & NFileHeader::NFlags::kUtf8) - if (i1.Name.IsAscii() && i2.Name.IsAscii()) + if (mask & NFileHeader::NFlags::kUtf8) + if (i1.Name.IsAscii() && i2_cd.Name.IsAscii()) mask &= ~NFileHeader::NFlags::kUtf8; + + // some bad archive in rare case can use descriptor without descriptor flag in Central Dir + // if (i1.HasDescriptor()) + mask &= ~NFileHeader::NFlags::kDescriptorUsedMask; - return ((i1.Flags & mask) == (i2.Flags & mask)); + return (mask == 0); } @@ -1167,13 +1187,13 @@ static bool AreEqualPaths_IgnoreSlashes(const char *s1, const char *s2) static bool AreItemsEqual(const CItemEx &localItem, const CItemEx &cdItem) { - if (!FlagsAreSame(cdItem, localItem)) + if (!FlagsAreSame(localItem, cdItem)) return false; if (!localItem.HasDescriptor()) { if (cdItem.PackSize != localItem.PackSize || cdItem.Size != localItem.Size - || cdItem.Crc != localItem.Crc && cdItem.Crc != 0) // some program writes 0 to crc field in central directory + || (cdItem.Crc != localItem.Crc && cdItem.Crc != 0)) // some program writes 0 to crc field in central directory return false; } /* pkzip 2.50 creates incorrect archives. It uses @@ -1235,7 +1255,7 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool return S_FALSE; } Stream = Vols.Streams[item.Disk].Stream; - Vols.StreamIndex = item.Disk; + Vols.StreamIndex = (int)item.Disk; if (!Stream) { isAvail = false; @@ -1251,7 +1271,7 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool } Stream = StreamRef; - offset += ArcInfo.Base; + offset = (UInt64)((Int64)offset + ArcInfo.Base); if (ArcInfo.Base < 0 && (Int64)offset < 0) { isAvail = false; @@ -1281,6 +1301,11 @@ HRESULT CInArchive::ReadLocalItemAfterCdItem(CItemEx &item, bool &isAvail, bool item.Crc = localItem.Crc; headersError = true; } + if ((item.Flags ^ localItem.Flags) & NFileHeader::NFlags::kDescriptorUsedMask) + { + item.Flags = (UInt16)(item.Flags ^ NFileHeader::NFlags::kDescriptorUsedMask); + headersError = true; + } item.FromLocal = true; } catch(...) { return S_FALSE; } @@ -1351,8 +1376,11 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) { // we write to packSize all these available bytes. // later it's simpler to work with such value than with 0 - if (item.PackSize == 0) + // if (item.PackSize == 0) item.PackSize = packedSize + avail; + if (item.Method == 0) + item.Size = item.PackSize; + SkipLookahed(avail); return S_OK; } @@ -1384,7 +1412,7 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) && sig != NSignature::kCentralFileHeader) continue; - const UInt64 packSizeCur = packedSize + (p - pStart); + const UInt64 packSizeCur = packedSize + (size_t)(p - pStart); if (descriptorSize4 == kDataDescriptorSize64 + kNextSignatureSize) // if (item.LocalExtra.IsZip64) { const UInt64 descriptorPackSize = Get64(p + 8); @@ -1406,14 +1434,14 @@ HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) item.DescriptorWasRead = true; item.Crc = Get32(p + 4); - const size_t skip = (p - pStart) + descriptorSize4 - kNextSignatureSize; + const size_t skip = (size_t)(p - pStart) + descriptorSize4 - kNextSignatureSize; SkipLookahed(skip); return S_OK; } - const size_t skip = (p - pStart); + const size_t skip = (size_t)(p - pStart); SkipLookahed(skip); packedSize += skip; @@ -1529,7 +1557,7 @@ HRESULT CInArchive::ReadCdItem(CItemEx &item) ReadFileName(nameSize, item.Name); if (extraSize > 0) - ReadExtra(extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); + ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, item.LocalHeaderPos, item.Disk); // May be these strings must be deleted /* @@ -1549,11 +1577,7 @@ HRESULT CInArchive::TryEcd64(UInt64 offset, CCdInfo &cdInfo) Byte buf[kEcd64_FullSize]; RINOK(SeekToVol(Vols.StreamIndex, offset)); - unsigned processed = 0; - ReadFromCache(buf, kEcd64_FullSize, processed); - - if (processed != kEcd64_FullSize) - return S_FALSE; + RINOK(ReadFromCache_FALSE(buf, kEcd64_FullSize)); if (Get32(buf) != NSignature::kEcd64) return S_FALSE; @@ -1636,8 +1660,12 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) { CLocator locator; locator.Parse(buf + locatorIndex + 4); - if ((cdInfo.ThisDisk == locator.NumDisks - 1 || ZIP64_IS_16_MAX(cdInfo.ThisDisk)) - && locator.Ecd64Disk < locator.NumDisks) + UInt32 numDisks = locator.NumDisks; + // we ignore the error, where some zip creators use (NumDisks == 0) + if (numDisks == 0) + numDisks = 1; + if ((cdInfo.ThisDisk == numDisks - 1 || ZIP64_IS_16_MAX(cdInfo.ThisDisk)) + && locator.Ecd64Disk < numDisks) { if (locator.Ecd64Disk != cdInfo.ThisDisk && !ZIP64_IS_16_MAX(cdInfo.ThisDisk)) return E_NOTIMPL; @@ -1657,7 +1685,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) if (mainEcd64Size == kEcd64_MainSize) { cdInfo.ParseEcd64e(ecd64 + 12); - ArcInfo.Base = absEcd64 - locator.Ecd64Offset; + ArcInfo.Base = (Int64)(absEcd64 - locator.Ecd64Offset); // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; return S_OK; } @@ -1685,7 +1713,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) { if (TryEcd64(ArcInfo.MarkerPos + locator.Ecd64Offset, cdInfo) == S_OK) { - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; return S_OK; } @@ -1719,7 +1747,7 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) } else */ - ArcInfo.Base = absEcdPos - cdEnd; + ArcInfo.Base = (Int64)(absEcdPos - cdEnd); } return S_OK; } @@ -1730,11 +1758,12 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) HRESULT CInArchive::TryReadCd(CObjectVector<CItemEx> &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize) { items.Clear(); + IsCdUnsorted = false; // _startLocalFromCd_Disk = (UInt32)(Int32)-1; // _startLocalFromCd_Offset = (UInt64)(Int64)-1; - RINOK(SeekToVol(IsMultiVol ? cdInfo.CdDisk : -1, cdOffset)); + RINOK(SeekToVol(IsMultiVol ? (int)cdInfo.CdDisk : -1, cdOffset)); _inBufMode = true; _cnt = 0; @@ -1767,6 +1796,15 @@ HRESULT CInArchive::TryReadCd(CObjectVector<CItemEx> &items, const CCdInfo &cdIn } */ + if (items.Size() > 0 && !IsCdUnsorted) + { + const CItemEx &prev = items.Back(); + if (cdItem.Disk < prev.Disk + || (cdItem.Disk == prev.Disk && + cdItem.LocalHeaderPos < prev.LocalHeaderPos)) + IsCdUnsorted = true; + } + items.Add(cdItem); } if (Callback && (items.Size() & 0xFFF) == 0) @@ -1793,6 +1831,22 @@ HRESULT CInArchive::TryReadCd(CObjectVector<CItemEx> &items, const CCdInfo &cdIn } +/* +static int CompareCdItems(void *const *elem1, void *const *elem2, void *) +{ + const CItemEx *i1 = *(const CItemEx **)elem1; + const CItemEx *i2 = *(const CItemEx **)elem2; + + if (i1->Disk < i2->Disk) return -1; + if (i1->Disk > i2->Disk) return 1; + if (i1->LocalHeaderPos < i2->LocalHeaderPos) return -1; + if (i1->LocalHeaderPos > i2->LocalHeaderPos) return 1; + if (i1 < i2) return -1; + if (i1 > i2) return 1; + return 0; +} +*/ + HRESULT CInArchive::ReadCd(CObjectVector<CItemEx> &items, UInt32 &cdDisk, UInt64 &cdOffset, UInt64 &cdSize) { bool checkOffsetMode = true; @@ -1801,7 +1855,7 @@ HRESULT CInArchive::ReadCd(CObjectVector<CItemEx> &items, UInt32 &cdDisk, UInt64 { if (Vols.EndVolIndex == -1) return S_FALSE; - Stream = Vols.Streams[Vols.EndVolIndex].Stream; + Stream = Vols.Streams[(unsigned)Vols.EndVolIndex].Stream; if (!Vols.StartIsZip) checkOffsetMode = false; } @@ -1827,7 +1881,7 @@ HRESULT CInArchive::ReadCd(CObjectVector<CItemEx> &items, UInt32 &cdDisk, UInt64 return S_FALSE; } - const UInt64 base = (IsMultiVol ? 0 : ArcInfo.Base); + const UInt64 base = (IsMultiVol ? 0 : (UInt64)ArcInfo.Base); res = TryReadCd(items, cdInfo, base + cdOffset, cdSize); if (res == S_FALSE && !IsMultiVol && base != ArcInfo.MarkerPos) @@ -1835,9 +1889,11 @@ HRESULT CInArchive::ReadCd(CObjectVector<CItemEx> &items, UInt32 &cdDisk, UInt64 // do we need that additional attempt to read cd? res = TryReadCd(items, cdInfo, ArcInfo.MarkerPos + cdOffset, cdSize); if (res == S_OK) - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; } + // Some rare case files are unsorted + // items.Sort(CompareCdItems, NULL); return res; } @@ -1849,14 +1905,14 @@ static int FindItem(const CObjectVector<CItemEx> &items, const CItemEx &item) { if (left >= right) return -1; - unsigned index = (left + right) / 2; + const unsigned index = (left + right) / 2; const CItemEx &item2 = items[index]; if (item.Disk < item2.Disk) right = index; else if (item.Disk > item2.Disk) left = index + 1; else if (item.LocalHeaderPos == item2.LocalHeaderPos) - return index; + return (int)index; else if (item.LocalHeaderPos < item2.LocalHeaderPos) right = index; else @@ -1921,7 +1977,7 @@ HRESULT CInArchive::ReadLocals(CObjectVector<CItemEx> &items) item.LocalHeaderPos = GetVirtStreamPos() - 4; if (!IsMultiVol) - item.LocalHeaderPos -= ArcInfo.Base; + item.LocalHeaderPos = (UInt64)((Int64)item.LocalHeaderPos - ArcInfo.Base); try { @@ -1950,7 +2006,7 @@ HRESULT CInArchive::ReadLocals(CObjectVector<CItemEx> &items) } catch (CUnexpectEnd &) { - if (items.IsEmpty() || items.Size() == 1 && IsStrangeItem(items[0])) + if (items.IsEmpty() || (items.Size() == 1 && IsStrangeItem(items[0]))) return S_FALSE; throw; } @@ -1986,11 +2042,11 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) name = prop.bstrVal; } - int dotPos = name.ReverseFind_Dot(); + const int dotPos = name.ReverseFind_Dot(); if (dotPos < 0) return S_OK; - const UString ext = name.Ptr(dotPos + 1); - name.DeleteFrom(dotPos + 1); + const UString ext = name.Ptr((unsigned)(dotPos + 1)); + name.DeleteFrom((unsigned)(dotPos + 1)); StartVolIndex = (Int32)(-1); @@ -2047,7 +2103,7 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) UInt32 volNum = ConvertStringToUInt32(ext.Ptr(1), &end); if (*end != 0 || volNum < 1 || volNum > ((UInt32)1 << 30)) return S_OK; - StartVolIndex = volNum - 1; + StartVolIndex = (Int32)(volNum - 1); BaseName = name; StartIsZ = true; } @@ -2147,7 +2203,7 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, UInt64 pos; RINOK(stream->Seek(0, STREAM_SEEK_CUR, &pos)); RINOK(stream->Seek(0, STREAM_SEEK_END, &size)); - RINOK(stream->Seek(pos, STREAM_SEEK_SET, NULL)); + RINOK(stream->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); while (i >= Vols.Streams.Size()) Vols.Streams.AddNew(); @@ -2161,7 +2217,7 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, if ((int)i == zipDisk) { - Vols.EndVolIndex = Vols.Streams.Size() - 1; + Vols.EndVolIndex = (int)(Vols.Streams.Size() - 1); break; } } @@ -2211,7 +2267,7 @@ HRESULT CInArchive::ReadVols() CCdInfo &ecd = Vols.ecd; if (res == S_OK) { - zipDisk = ecd.ThisDisk; + zipDisk = (int)ecd.ThisDisk; Vols.ecd_wasRead = true; // if is not multivol or bad multivol, we return to main single stream code @@ -2220,9 +2276,9 @@ HRESULT CInArchive::ReadVols() || ecd.ThisDisk < ecd.CdDisk) return S_OK; - cdDisk = ecd.CdDisk; + cdDisk = (int)ecd.CdDisk; if (Vols.StartVolIndex < 0) - Vols.StartVolIndex = ecd.ThisDisk; + Vols.StartVolIndex = (Int32)ecd.ThisDisk; else if ((UInt32)Vols.StartVolIndex >= ecd.ThisDisk) return S_OK; @@ -2232,7 +2288,7 @@ HRESULT CInArchive::ReadVols() if (cdDisk != zipDisk) { // get volumes required for cd. - RINOK(ReadVols2(volCallback, cdDisk, zipDisk, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)cdDisk, zipDisk, zipDisk, 0, numMissingVols)); if (numMissingVols != 0) { // cdOK = false; @@ -2269,10 +2325,10 @@ HRESULT CInArchive::ReadVols() if (Vols.StartVolIndex > (1 << 20)) return S_OK; if ((unsigned)Vols.StartVolIndex >= Vols.Streams.Size() - || !Vols.Streams[Vols.StartVolIndex].Stream) + || !Vols.Streams[(unsigned)Vols.StartVolIndex].Stream) { // we get volumes starting from StartVolIndex, if they we not requested before know the volume index (if FindCd() was ok) - RINOK(ReadVols2(volCallback, Vols.StartVolIndex, zipDisk, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)Vols.StartVolIndex, zipDisk, zipDisk, 0, numMissingVols)); } } @@ -2285,7 +2341,7 @@ HRESULT CInArchive::ReadVols() if (zipDisk >= 0) { // we create item in Streams for ZipStream, if we know the volume index (if FindCd() was ok) - RINOK(ReadVols2(volCallback, zipDisk, zipDisk + 1, zipDisk, 0, numMissingVols)); + RINOK(ReadVols2(volCallback, (unsigned)zipDisk, zipDisk + 1, zipDisk, 0, numMissingVols)); } } @@ -2331,7 +2387,7 @@ HRESULT CVols::Read(void *data, UInt32 size, UInt32 *processedSize) return S_OK; if ((unsigned)StreamIndex >= Streams.Size()) return S_OK; - const CVols::CSubStreamInfo &s = Streams[StreamIndex]; + const CVols::CSubStreamInfo &s = Streams[(unsigned)StreamIndex]; if (!s.Stream) return S_FALSE; if (NeedSeek) @@ -2473,7 +2529,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) if (!ecd.IsEmptyArc()) return S_FALSE; - ArcInfo.Base = ArcInfo.MarkerPos; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; IsArc = true; // check it: we need more tests? RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)); @@ -2514,16 +2570,44 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) res = S_FALSE; else { - firstItem.LocalHeaderPos = ArcInfo.MarkerPos2 - ArcInfo.Base; - int index = FindItem(items, firstItem); + firstItem.LocalHeaderPos = (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); + int index = -1; + + UInt32 min_Disk = (UInt32)(Int32)-1; + UInt64 min_LocalHeaderPos = (UInt64)(Int64)-1; + + if (!IsCdUnsorted) + index = FindItem(items, firstItem); + else + { + FOR_VECTOR (i, items) + { + const CItemEx &cdItem = items[i]; + if (cdItem.Disk == firstItem.Disk + && (cdItem.LocalHeaderPos == firstItem.LocalHeaderPos)) + index = (int)i; + + if (i == 0 + || cdItem.Disk < min_Disk + || (cdItem.Disk == min_Disk && cdItem.LocalHeaderPos < min_LocalHeaderPos)) + { + min_Disk = cdItem.Disk; + min_LocalHeaderPos = cdItem.LocalHeaderPos; + } + } + } + if (index == -1) res = S_FALSE; - else if (!AreItemsEqual(firstItem, items[index])) + else if (!AreItemsEqual(firstItem, items[(unsigned)index])) res = S_FALSE; else { ArcInfo.CdWasRead = true; - ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; + if (IsCdUnsorted) + ArcInfo.FirstItemRelatOffset = min_LocalHeaderPos; + else + ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; // ArcInfo.FirstItemRelatOffset = _startLocalFromCd_Offset; } @@ -2588,7 +2672,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) The (Base) can be corrected later after ECD reading. But sfx volume with stub and (No)Span-marker in (!IsMultiVol) mode will have incorrect (Base) here. */ - ArcInfo.Base = ArcInfo.MarkerPos2; + ArcInfo.Base = (Int64)ArcInfo.MarkerPos2; } RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)); @@ -2607,15 +2691,42 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) // GetVirtStreamPos() - 4 if (items.IsEmpty()) return S_FALSE; - NoCentralDir = true; - HeadersError = true; - return S_OK; + + bool isError = true; + + const UInt32 apkSize = _signature; + const unsigned kApkFooterSize = 16 + 8; + if (apkSize >= kApkFooterSize && apkSize <= (1 << 20)) + { + if (ReadUInt32() == 0) + { + CByteBuffer apk; + apk.Alloc(apkSize); + SafeRead(apk, apkSize); + ReadSignature(); + const Byte *footer = apk + apkSize - kApkFooterSize; + if (_signature == NSignature::kCentralFileHeader) + if (GetUi64(footer) == apkSize) + if (memcmp(footer + 8, "APK Sig Block 42", 16) == 0) + { + isError = false; + IsApk = true; + } + } + } + + if (isError) + { + NoCentralDir = true; + HeadersError = true; + return S_OK; + } } _inBufMode = true; cdAbsOffset = GetVirtStreamPos() - 4; - cdDisk = Vols.StreamIndex; + cdDisk = (UInt32)Vols.StreamIndex; #ifdef ZIP_SELF_CHECK if (!IsMultiVol && _cnt != GetVirtStreamPos() - ArcInfo.MarkerPos2) @@ -2656,7 +2767,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) needSetBase = true; numCdItems = cdItems.Size(); - cdRelatOffset = cdAbsOffset - ArcInfo.Base; + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - ArcInfo.Base); if (!cdItems.IsEmpty()) { @@ -2712,6 +2823,8 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) Byte buf[kBufSize]; SafeRead(buf, kBufSize); locator.Parse(buf); + // we ignore the error, where some zip creators use (NumDisks == 0) + // if (locator.NumDisks == 0) HeadersWarning = true; } ReadSignature(); @@ -2764,12 +2877,12 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) if (IsMultiVol) { - if (cdDisk != (int)cdInfo.CdDisk) + if (cdDisk != cdInfo.CdDisk) HeadersError = true; } else if (needSetBase && cdOK) { - const UInt64 oldBase = ArcInfo.Base; + const UInt64 oldBase = (UInt64)ArcInfo.Base; // localsWereRead == true // ArcInfo.Base == ArcInfo.MarkerPos2 // cdRelatOffset == (cdAbsOffset - ArcInfo.Base) @@ -2778,13 +2891,13 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) { if (ecd64Disk == Vols.StartVolIndex) { - const Int64 newBase = (Int64)ecd64AbsOffset - locator.Ecd64Offset; + const Int64 newBase = (Int64)ecd64AbsOffset - (Int64)locator.Ecd64Offset; if (newBase <= (Int64)ecd64AbsOffset) { if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) { ArcInfo.Base = newBase; - cdRelatOffset = cdAbsOffset - newBase; + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - newBase); } else cdOK = false; @@ -2795,7 +2908,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) { if ((int)cdDisk == Vols.StartVolIndex) { - const Int64 newBase = (Int64)cdAbsOffset - cdInfo.Offset; + const Int64 newBase = (Int64)cdAbsOffset - (Int64)cdInfo.Offset; if (newBase <= (Int64)cdAbsOffset) { if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) @@ -2828,7 +2941,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) if (localsWereRead) { - const UInt64 delta = oldBase - ArcInfo.Base; + const UInt64 delta = (UInt64)((Int64)oldBase - ArcInfo.Base); if (delta != 0) { FOR_VECTOR (i, items) @@ -2864,7 +2977,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) if (isZip64) { - if (cdInfo.ThisDisk == 0 && ecd64AbsOffset != ArcInfo.Base + locator.Ecd64Offset + if ((cdInfo.ThisDisk == 0 && ecd64AbsOffset != (UInt64)(ArcInfo.Base + (Int64)locator.Ecd64Offset)) // || cdInfo.NumEntries_in_ThisDisk != numCdItems || cdInfo.NumEntries != numCdItems || cdInfo.Size != cdSize @@ -2902,10 +3015,10 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) { if ((unsigned)nextLocalIndex < items.Size()) { - CItemEx &item = items[nextLocalIndex]; + CItemEx &item = items[(unsigned)nextLocalIndex]; if (item.Disk == cdItem.Disk && (item.LocalHeaderPos == cdItem.LocalHeaderPos - || Overflow32bit && (UInt32)item.LocalHeaderPos == cdItem.LocalHeaderPos)) + || (Overflow32bit && (UInt32)item.LocalHeaderPos == cdItem.LocalHeaderPos))) index = nextLocalIndex++; else nextLocalIndex = -1; @@ -2924,7 +3037,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) continue; } - CItemEx &item = items[index]; + CItemEx &item = items[(unsigned)index]; if (item.Name != cdItem.Name // || item.Name.Len() != cdItem.Name.Len() || item.PackSize != cdItem.PackSize @@ -2965,7 +3078,7 @@ HRESULT CInArchive::ReadHeaders(CObjectVector<CItemEx> &items) if (isZip64) { if (cdInfo.NumEntries != items.Size() - || ecd.NumEntries != items.Size() && ecd.NumEntries != 0xFFFF) + || (ecd.NumEntries != items.Size() && ecd.NumEntries != 0xFFFF)) HeadersError = true; } else @@ -3069,7 +3182,9 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, else { // printf("\nOpen offset = %u\n", (unsigned)startPos); - if (IsMultiVol && (unsigned)Vols.StartParsingVol < Vols.Streams.Size() && Vols.Streams[Vols.StartParsingVol].Stream) + if (IsMultiVol + && (unsigned)Vols.StartParsingVol < Vols.Streams.Size() + && Vols.Streams[(unsigned)Vols.StartParsingVol].Stream) { RINOK(SeekToVol(Vols.StartParsingVol, Vols.StreamIndex == Vols.StartVolIndex ? startPos : 0)); } @@ -3117,7 +3232,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, { if ((unsigned)Vols.StartVolIndex < Vols.Streams.Size()) { - Stream = Vols.Streams[Vols.StartVolIndex].Stream; + Stream = Vols.Streams[(unsigned)Vols.StartVolIndex].Stream; if (Stream) { RINOK(Seek_SavePos(curPos)); @@ -3173,7 +3288,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, { ArcInfo.FinishPos = ArcInfo.FileEndPos; if ((unsigned)Vols.StreamIndex < Vols.Streams.Size()) - if (GetVirtStreamPos() < Vols.Streams[Vols.StreamIndex].Size) + if (GetVirtStreamPos() < Vols.Streams[(unsigned)Vols.StreamIndex].Size) ArcInfo.ThereIsTail = true; } else @@ -3204,8 +3319,8 @@ HRESULT CInArchive::GetItemStream(const CItemEx &item, bool seekPackData, CMyCom { if (UseDisk_in_SingleVol && item.Disk != EcdVolIndex) return S_OK; - pos += ArcInfo.Base; - RINOK(StreamRef->Seek(pos, STREAM_SEEK_SET, NULL)); + pos = (UInt64)((Int64)pos + ArcInfo.Base); + RINOK(StreamRef->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); stream = StreamRef; return S_OK; } @@ -3216,10 +3331,10 @@ HRESULT CInArchive::GetItemStream(const CItemEx &item, bool seekPackData, CMyCom IInStream *str2 = Vols.Streams[item.Disk].Stream; if (!str2) return S_OK; - RINOK(str2->Seek(pos, STREAM_SEEK_SET, NULL)); + RINOK(str2->Seek((Int64)pos, STREAM_SEEK_SET, NULL)); Vols.NeedSeek = false; - Vols.StreamIndex = item.Disk; + Vols.StreamIndex = (int)item.Disk; CVolStream *volsStreamSpec = new CVolStream; volsStreamSpec->Vols = &Vols; diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index f46f1f07..31e524b6 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -32,6 +32,11 @@ public: { return LocalFullHeaderSize + GetPackSizeWithDescriptor(); } UInt64 GetDataPosition() const { return LocalHeaderPos + LocalFullHeaderSize; } + + bool IsBadDescriptor() const + { + return !FromCentral && FromLocal && HasDescriptor() && !DescriptorWasRead; + } }; @@ -282,6 +287,7 @@ class CInArchive HRESULT SeekToVol(int volIndex, UInt64 offset); HRESULT ReadFromCache(Byte *data, unsigned size, unsigned &processed); + HRESULT ReadFromCache_FALSE(Byte *data, unsigned size); HRESULT ReadVols2(IArchiveOpenVolumeCallback *volCallback, unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols); @@ -305,7 +311,7 @@ class CInArchive bool ReadFileName(unsigned nameSize, AString &dest); - bool ReadExtra(unsigned extraSize, CExtraBlock &extra, + bool ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, UInt64 &unpackSize, UInt64 &packSize, UInt64 &localOffset, UInt32 &disk); bool ReadLocalItem(CItemEx &item); HRESULT FindDescriptor(CItemEx &item, unsigned numFiles); @@ -325,6 +331,9 @@ public: bool IsArc; bool IsZip64; + + bool IsApk; + bool IsCdUnsorted; bool HeadersError; bool HeadersWarning; @@ -345,14 +354,19 @@ public: CVols Vols; - CInArchive(): Stream(NULL), StartStream(NULL), Callback(NULL), IsArcOpen(false) {} + CInArchive(): + IsArcOpen(false), + Stream(NULL), + StartStream(NULL), + Callback(NULL) + {} UInt64 GetPhySize() const { if (IsMultiVol) return ArcInfo.FinishPos; else - return ArcInfo.FinishPos - ArcInfo.Base; + return (UInt64)((Int64)ArcInfo.FinishPos - ArcInfo.Base); } UInt64 GetOffset() const @@ -360,7 +374,7 @@ public: if (IsMultiVol) return 0; else - return ArcInfo.Base; + return (UInt64)ArcInfo.Base; } @@ -393,7 +407,7 @@ public: return ArcInfo.FirstItemRelatOffset; if (IsMultiVol) return 0; - return ArcInfo.MarkerPos2 - ArcInfo.Base; + return (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); } @@ -412,7 +426,9 @@ public: || ArcInfo.Base < 0 || (Int64)ArcInfo.MarkerPos2 < ArcInfo.Base || ArcInfo.ThereIsTail - || GetEmbeddedStubSize() != 0) + || GetEmbeddedStubSize() != 0 + || IsApk + || IsCdUnsorted) return false; // 7-zip probably can update archives with embedded stubs. diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index 5cff1735..38921dce 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -33,9 +33,12 @@ static const CUInt32PCharPair g_ExtraTypes[] = { NExtraID::kStrongEncrypt, "StrongCrypto" }, { NExtraID::kUnixTime, "UT" }, { NExtraID::kUnixExtra, "UX" }, + { NExtraID::kUnix2Extra, "Ux" }, + { NExtraID::kUnix3Extra, "ux" }, { NExtraID::kIzUnicodeComment, "uc" }, { NExtraID::kIzUnicodeName, "up" }, - { NExtraID::kWzAES, "WzAES" } + { NExtraID::kWzAES, "WzAES" }, + { NExtraID::kApkAlign, "ApkAlign" } }; void CExtraSubBlock::PrintInfo(AString &s) const @@ -46,6 +49,22 @@ void CExtraSubBlock::PrintInfo(AString &s) const if (pair.Value == ID) { s += pair.Name; + /* + if (ID == NExtraID::kApkAlign && Data.Size() >= 2) + { + char sz[32]; + sz[0] = ':'; + ConvertUInt32ToHex(GetUi16(Data), sz + 1); + s += sz; + for (unsigned j = 2; j < Data.Size(); j++) + { + char sz[32]; + sz[0] = '-'; + ConvertUInt32ToHex(Data[j], sz + 1); + s += sz; + } + } + */ return; } } @@ -209,6 +228,7 @@ bool CLocalItem::IsDir() const bool CItem::IsDir() const { + // FIXME: we can check InfoZip UTF-8 name at first. if (NItemName::HasTailSlash(Name, GetCodePage())) return true; @@ -315,10 +335,30 @@ bool CItem::GetPosixAttrib(UInt32 &attrib) const return false; } + +bool CExtraSubBlock::CheckIzUnicode(const AString &s) const +{ + size_t size = Data.Size(); + if (size < 1 + 4) + return false; + const Byte *p = (const Byte *)Data; + if (p[0] > 1) + return false; + if (CrcCalc(s, s.Len()) != GetUi32(p + 1)) + return false; + size -= 5; + p += 5; + for (size_t i = 0; i < size; i++) + if (p[i] == 0) + return false; + return Check_UTF8_Buf((const char *)(const void *)p, size, false); +} + + void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const { bool isUtf8 = IsUtf8(); - bool ignore_Utf8_Errors = true; + // bool ignore_Utf8_Errors = true; if (!isUtf8) { @@ -333,10 +373,14 @@ void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, boo const CExtraSubBlock &sb = subBlocks[i]; if (sb.ID == id) { - AString utf; - if (sb.ExtractIzUnicode(CrcCalc(s, s.Len()), utf)) - if (ConvertUTF8ToUnicode(utf, res)) + if (sb.CheckIzUnicode(s)) + { + // const unsigned kIzUnicodeHeaderSize = 5; + if (Convert_UTF8_Buf_To_Unicode( + (const char *)(const void *)(const Byte *)sb.Data + 5, + sb.Data.Size() - 5, res)) return; + } break; } } @@ -351,15 +395,21 @@ void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, boo We try to get name as UTF-8. Do we need to do it in POSIX version also? */ isUtf8 = true; - ignore_Utf8_Errors = false; + + /* 21.02: we want to ignore UTF-8 errors to support file paths that are mixed + of UTF-8 and non-UTF-8 characters. */ + // ignore_Utf8_Errors = false; + // ignore_Utf8_Errors = true; } #endif } if (isUtf8) - if (ConvertUTF8ToUnicode(s, res) || ignore_Utf8_Errors) - return; + { + ConvertUTF8ToUnicode(s, res); + return; + } MultiByteToUnicodeString2(res, s, useSpecifiedCodePage ? codePage : GetCodePage()); } diff --git a/CPP/7zip/Archive/Zip/ZipItem.h b/CPP/7zip/Archive/Zip/ZipItem.h index e5769711..6ee87658 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.h +++ b/CPP/7zip/Archive/Zip/ZipItem.h @@ -33,23 +33,8 @@ struct CExtraSubBlock bool ExtractNtfsTime(unsigned index, FILETIME &ft) const; bool ExtractUnixTime(bool isCentral, unsigned index, UInt32 &res) const; bool ExtractUnixExtraTime(unsigned index, UInt32 &res) const; - - bool ExtractIzUnicode(UInt32 crc, AString &name) const - { - unsigned size = (unsigned)Data.Size(); - if (size < 1 + 4) - return false; - const Byte *p = (const Byte *)Data; - if (p[0] > 1) - return false; - if (crc != GetUi32(p + 1)) - return false; - size -= 5; - name.SetFrom_CalcLen((const char *)p + 5, size); - if (size != name.Len()) - return false; - return CheckUTF8(name, false); - } + + bool CheckIzUnicode(const AString &s) const; void PrintInfo(AString &s) const; }; @@ -202,8 +187,14 @@ struct CExtraBlock for (unsigned i = SubBlocks.Size(); i != 0;) { i--; - if (SubBlocks[i].ID != NFileHeader::NExtraID::kWzAES) - SubBlocks.Delete(i); + switch (SubBlocks[i].ID) + { + case NFileHeader::NExtraID::kStrongEncrypt: + case NFileHeader::NExtraID::kWzAES: + break; + default: + SubBlocks.Delete(i); + } } } }; @@ -266,9 +257,9 @@ private: void SetFlag(unsigned bitMask, bool enable) { if (enable) - Flags |= bitMask; + Flags = (UInt16)(Flags | bitMask); else - Flags &= ~bitMask; + Flags = (UInt16)(Flags & ~bitMask); } public: @@ -279,7 +270,12 @@ public: // void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); } void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); } - UINT GetCodePage() const { return CP_OEMCP; } + UINT GetCodePage() const + { + if (IsUtf8()) + return CP_UTF8; + return CP_OEMCP; + } }; @@ -330,10 +326,19 @@ public: } return (Crc != 0 || !IsDir()); } + + bool Is_MadeBy_Unix() const + { + if (!FromCentral) + return false; + return (MadeByVersion.HostOS == NFileHeader::NHostOS::kUnix); + } UINT GetCodePage() const { // 18.06: now we use HostOS only from Central::MadeByVersion + if (IsUtf8()) + return CP_UTF8; if (!FromCentral) return CP_OEMCP; Byte hostOS = MadeByVersion.HostOS; diff --git a/CPP/7zip/Archive/Zip/ZipOut.cpp b/CPP/7zip/Archive/Zip/ZipOut.cpp index 945bd020..efed0a41 100644 --- a/CPP/7zip/Archive/Zip/ZipOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipOut.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../../C/7zCrc.h" + #include "../../Common/OffsetStream.h" #include "ZipOut.h" @@ -23,7 +25,7 @@ HRESULT COutArchive::Create(IOutStream *outStream) void COutArchive::SeekToCurPos() { - HRESULT res = m_Stream->Seek(m_Base + m_CurPos, STREAM_SEEK_SET, NULL); + HRESULT res = m_Stream->Seek((Int64)(m_Base + m_CurPos), STREAM_SEEK_SET, NULL); if (res != S_OK) throw CSystemException(res); } @@ -97,6 +99,17 @@ void COutArchive::WriteCommonItemInfo(const CLocalItem &item, bool isZip64) #define WRITE_32_VAL_SPEC(__v, __isZip64) Write32((__isZip64) ? 0xFFFFFFFF : (UInt32)(__v)); +void COutArchive::WriteUtfName(const CItemOut &item) +{ + if (item.Name_Utf.Size() == 0) + return; + Write16(NFileHeader::NExtraID::kIzUnicodeName); + Write16((UInt16)(5 + item.Name_Utf.Size())); + Write8(1); // (1 = version) of that extra field + Write32(CrcCalc(item.Name.Ptr(), item.Name.Len())); + WriteBytes(item.Name_Utf, (UInt16)item.Name_Utf.Size()); +} + void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) { m_LocalHeaderPos = m_CurPos; @@ -109,7 +122,10 @@ void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) if (needCheck && m_IsZip64) isZip64 = true; - const UInt32 localExtraSize = (UInt32)((isZip64 ? (4 + 8 + 8): 0) + item.LocalExtra.GetSize()); + const UInt32 localExtraSize = (UInt32)( + (isZip64 ? (4 + 8 + 8): 0) + + item.Get_UtfName_ExtraSize() + + item.LocalExtra.GetSize()); if ((UInt16)localExtraSize != localExtraSize) throw CSystemException(E_FAIL); if (needCheck && m_ExtraSize != localExtraSize) @@ -152,6 +168,8 @@ void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) Write64(packSize); } + WriteUtfName(item); + WriteExtra(item.LocalExtra); // Why don't we write NTFS timestamps to local header? @@ -230,14 +248,19 @@ void COutArchive::WriteCentralHeader(const CItemOut &item) Write16((UInt16)item.Name.Len()); - UInt16 zip64ExtraSize = (UInt16)((isUnPack64 ? 8: 0) + (isPack64 ? 8: 0) + (isPosition64 ? 8: 0)); + const UInt16 zip64ExtraSize = (UInt16)((isUnPack64 ? 8: 0) + (isPack64 ? 8: 0) + (isPosition64 ? 8: 0)); const UInt16 kNtfsExtraSize = 4 + 2 + 2 + (3 * 8); - const UInt16 centralExtraSize = (UInt16)( - (isZip64 ? 4 + zip64ExtraSize : 0) + - (item.NtfsTimeIsDefined ? 4 + kNtfsExtraSize : 0) + - item.CentralExtra.GetSize()); + const size_t centralExtraSize = + (isZip64 ? 4 + zip64ExtraSize : 0) + + (item.NtfsTimeIsDefined ? 4 + kNtfsExtraSize : 0) + + item.Get_UtfName_ExtraSize() + + item.CentralExtra.GetSize(); + + const UInt16 centralExtraSize16 = (UInt16)centralExtraSize; + if (centralExtraSize16 != centralExtraSize) + throw CSystemException(E_FAIL); - Write16(centralExtraSize); // test it; + Write16(centralExtraSize16); const UInt16 commentSize = (UInt16)item.Comment.Size(); @@ -271,6 +294,8 @@ void COutArchive::WriteCentralHeader(const CItemOut &item) WriteNtfsTime(item.Ntfs_ATime); WriteNtfsTime(item.Ntfs_CTime); } + + WriteUtfName(item); WriteExtra(item.CentralExtra); if (commentSize != 0) diff --git a/CPP/7zip/Archive/Zip/ZipOut.h b/CPP/7zip/Archive/Zip/ZipOut.h index 0a0ac0c8..3546411c 100644 --- a/CPP/7zip/Archive/Zip/ZipOut.h +++ b/CPP/7zip/Archive/Zip/ZipOut.h @@ -21,6 +21,16 @@ public: bool NtfsTimeIsDefined; // It's possible that NtfsTime is not defined, but there is NtfsTime in Extra. + + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra + + size_t Get_UtfName_ExtraSize() const + { + const size_t size = Name_Utf.Size(); + if (size == 0) + return 0; + return 4 + 5 + size; + } CItemOut(): NtfsTimeIsDefined(false) {} }; @@ -52,6 +62,7 @@ class COutArchive Write32(ft.dwHighDateTime); } + void WriteUtfName(const CItemOut &item); void WriteExtra(const CExtraBlock &extra); void WriteCommonItemInfo(const CLocalItem &item, bool isZip64); void WriteCentralHeader(const CItemOut &item); diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index e65c2b8b..4468c7c5 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -62,6 +62,21 @@ static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method) } +static void Copy_From_UpdateItem_To_ItemOut(const CUpdateItem &ui, CItemOut &item) +{ + item.Name = ui.Name; + item.Name_Utf = ui.Name_Utf; + item.Comment = ui.Comment; + item.SetUtf8(ui.IsUtf8); + // item.SetFlag_AltStream(ui.IsAltStream); + // item.ExternalAttrib = ui.Attrib; + item.Time = ui.Time; + item.Ntfs_MTime = ui.Ntfs_MTime; + item.Ntfs_ATime = ui.Ntfs_ATime; + item.Ntfs_CTime = ui.Ntfs_CTime; + item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; +} + static void SetFileHeader( const CCompressionMethodMode &options, const CUpdateItem &ui, @@ -69,22 +84,15 @@ static void SetFileHeader( CItemOut &item) { item.Size = ui.Size; - bool isDir = ui.IsDir; + const bool isDir = ui.IsDir; item.ClearFlags(); if (ui.NewProps) { - item.Name = ui.Name; - item.Comment = ui.Comment; - item.SetUtf8(ui.IsUtf8); + Copy_From_UpdateItem_To_ItemOut(ui, item); // item.SetFlag_AltStream(ui.IsAltStream); item.ExternalAttrib = ui.Attrib; - item.Time = ui.Time; - item.Ntfs_MTime = ui.Ntfs_MTime; - item.Ntfs_ATime = ui.Ntfs_ATime; - item.Ntfs_CTime = ui.Ntfs_CTime; - item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; } /* else @@ -148,6 +156,35 @@ static void SetItemInfoFromCompressingResult(const CCompressingResult &compressi #ifndef _7ZIP_ST +struct CMtSem +{ + NWindows::NSynchronization::CSemaphore Semaphore; + NWindows::NSynchronization::CCriticalSection CS; + CIntVector Indexes; + int Head; + + void ReleaseItem(unsigned index) + { + { + CCriticalSectionLock lock(CS); + Indexes[index] = Head; + Head = (int)index; + } + Semaphore.Release(); + } + + int GetFreeItem() + { + int i; + { + CCriticalSectionLock lock(CS); + i = Head; + Head = Indexes[(unsigned)i]; + } + return i; + } +}; + static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo); struct CThreadInfo @@ -156,7 +193,9 @@ struct CThreadInfo NWindows::CThread Thread; NWindows::NSynchronization::CAutoResetEvent CompressEvent; - NWindows::NSynchronization::CAutoResetEvent CompressionCompletedEvent; + CMtSem *MtSem; + unsigned ThreadIndex; + bool ExitThread; CMtCompressProgress *ProgressSpec; @@ -177,34 +216,43 @@ struct CThreadInfo UInt32 FileTime; UInt64 ExpectedDataSize; - CThreadInfo(const CCompressionMethodMode &options): + CThreadInfo(): ExitThread(false), - ProgressSpec(0), - OutStreamSpec(0), - Coder(options), + ProgressSpec(NULL), + OutStreamSpec(NULL), InSeqMode(false), OutSeqMode(false), FileTime(0), ExpectedDataSize((UInt64)(Int64)-1) {} + + void SetOptions(const CCompressionMethodMode &options) + { + Coder.SetOptions(options); + } HRESULT CreateEvents() { - RINOK(CompressEvent.CreateIfNotCreated()); - return CompressionCompletedEvent.CreateIfNotCreated(); + WRes wres = CompressEvent.CreateIfNotCreated_Reset(); + return HRESULT_FROM_WIN32(wres); + } + + HRESULT CreateThread() + { + WRes wres = Thread.Create(CoderThread, this); + return HRESULT_FROM_WIN32(wres); } - HRes CreateThread() { return Thread.Create(CoderThread, this); } void WaitAndCode(); - void StopWaitClose() + + void StopWait_Close() { ExitThread = true; - if (OutStreamSpec != 0) + if (OutStreamSpec) OutStreamSpec->StopWriting(E_ABORT); if (CompressEvent.IsCreated()) CompressEvent.Set(); - Thread.Wait(); - Thread.Close(); + Thread.Wait_Close(); } }; @@ -215,7 +263,7 @@ void CThreadInfo::WaitAndCode() CompressEvent.Lock(); if (ExitThread) return; - + Result = Coder.Compress( EXTERNAL_CODECS_LOC_VARS InStream, OutStream, @@ -224,7 +272,8 @@ void CThreadInfo::WaitAndCode() if (Result == S_OK && Progress) Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize); - CompressionCompletedEvent.Set(); + + MtSem->ReleaseItem(ThreadIndex); } } @@ -241,7 +290,7 @@ public: ~CThreads() { FOR_VECTOR (i, Threads) - Threads[i].StopWaitClose(); + Threads[i].StopWait_Close(); } }; @@ -253,7 +302,8 @@ struct CMemBlocks2: public CMemLockBlocks bool Finished; CCompressingResult CompressingResult; - CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false) {} + CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false), + CompressingResult() {} }; class CMemRefs @@ -359,7 +409,6 @@ STDMETHODIMP CMtProgressMixer::SetRatioInfo(const UInt64 *inSize, const UInt64 * #endif - static HRESULT UpdateItemOldData( COutArchive &archive, CInArchive *inArchive, @@ -385,21 +434,11 @@ static HRESULT UpdateItemOldData( if (item.HasDescriptor()) return E_NOTIMPL; - // use old name size. - // we keep ExternalAttrib and some another properties from old archive // item.ExternalAttrib = ui.Attrib; - // if we don't change Comment, we keep Comment from OldProperties - item.Comment = ui.Comment; - item.Name = ui.Name; - item.SetUtf8(ui.IsUtf8); + Copy_From_UpdateItem_To_ItemOut(ui, item); // item.SetFlag_AltStream(ui.IsAltStream); - item.Time = ui.Time; - item.Ntfs_MTime = ui.Ntfs_MTime; - item.Ntfs_ATime = ui.Ntfs_ATime; - item.Ntfs_CTime = ui.Ntfs_CTime; - item.NtfsTimeIsDefined = ui.NtfsTimeIsDefined; item.CentralExtra.RemoveUnknownSubBlocks(); item.LocalExtra.RemoveUnknownSubBlocks(); @@ -452,16 +491,16 @@ static void UpdatePropsFromStream(CUpdateItem &item, ISequentialInStream *fileIn FILETIME cTime, aTime, mTime; UInt64 size; - // UInt32 attrib; - if (getProps->GetProps(&size, &cTime, &aTime, &mTime, NULL) != S_OK) + UInt32 attrib; + if (getProps->GetProps(&size, &cTime, &aTime, &mTime, &attrib) != S_OK) return; if (size != item.Size && size != (UInt64)(Int64)-1) { - Int64 newComplexity = totalComplexity + ((Int64)size - (Int64)item.Size); + const Int64 newComplexity = (Int64)totalComplexity + ((Int64)size - (Int64)item.Size); if (newComplexity > 0) { - totalComplexity = newComplexity; + totalComplexity = (UInt64)newComplexity; updateCallback->SetTotal(totalComplexity); } item.Size = size; @@ -481,7 +520,7 @@ static void UpdatePropsFromStream(CUpdateItem &item, ISequentialInStream *fileIn if (!IsZero_FILETIME(cTime)) item.Ntfs_CTime = cTime; if (!IsZero_FILETIME(aTime)) item.Ntfs_ATime = aTime; - // item.Attrib = attrib; + item.Attrib = attrib; } @@ -501,7 +540,8 @@ static HRESULT Update2St( CMyComPtr<ICompressProgressInfo> progress = lps; lps->Init(updateCallback, true); - CAddCommon compressor(*options); + CAddCommon compressor; + compressor.SetOptions(*options); CObjectVector<CItemOut> items; UInt64 unpackSizeTotal = 0, packSizeTotal = 0; @@ -519,7 +559,7 @@ static HRESULT Update2St( { // Note: for (ui.NewProps && !ui.NewData) it copies Props from old archive, // But we will rewrite all important properties later. But we can keep some properties like Comment - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -659,7 +699,7 @@ static HRESULT Update2( } else { - CItemEx inputItem = inputItems[ui.IndexInArc]; + CItemEx inputItem = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(inputItem) != S_OK) return E_NOTIMPL; complexity += inputItem.GetLocalFullSize(); @@ -686,7 +726,8 @@ static HRESULT Update2( options2._methods.AddNew(); } - CAddCommon compressor(options2); + CAddCommon compressor; + compressor.SetOptions(options2); complexity = 0; @@ -715,15 +756,24 @@ static HRESULT Update2( UInt32 numThreads = options._numThreads; - const UInt32 kNumMaxThreads = 64; - if (numThreads > kNumMaxThreads) - numThreads = kNumMaxThreads; - if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows (is it 64 in all versions?) + { + const UInt32 kNumMaxThreads = + #ifdef _WIN32 + 64; // _WIN32 supports only 64 threads in one group. So no need for more threads here + #else + 128; + #endif + if (numThreads > kNumMaxThreads) + numThreads = kNumMaxThreads; + } + /* + if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows numThreads = MAXIMUM_WAIT_OBJECTS; + */ if (numThreads < 1) numThreads = 1; - const size_t kMemPerThread = (1 << 25); + const size_t kMemPerThread = (size_t)1 << 25; const size_t kBlockSize = 1 << 16; bool mtMode = (numThreads > 1); @@ -731,6 +781,8 @@ static HRESULT Update2( if (numFilesToCompress <= 1) mtMode = false; + // mtMode = true; // debug: to test mtMode + if (!mtMode) { FOR_VECTOR (mi, options2._methods) @@ -788,7 +840,7 @@ static HRESULT Update2( if (t > numThreads) t = numThreads; oneMethodMain->AddProp_NumThreads(t); - numXzThreads = t; + numXzThreads = (int)t; } numThreads /= (unsigned)numXzThreads; } @@ -830,8 +882,16 @@ static HRESULT Update2( CMemBlockManagerMt memManager(kBlockSize); CMemRefs refs(&memManager); + CMtSem mtSem; CThreads threads; - CRecordVector<HANDLE> compressingCompletedEvents; + mtSem.Head = -1; + mtSem.Indexes.ClearAndSetSize(numThreads); + { + WRes wres = mtSem.Semaphore.Create(0, numThreads); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + CUIntVector threadIndices; // list threads in order of updateItems { @@ -840,26 +900,32 @@ static HRESULT Update2( refs.Refs.Add(CMemBlocks2()); for (i = 0; i < numThreads; i++) - threads.Threads.Add(CThreadInfo(options2)); + { + threads.Threads.AddNew(); + // mtSem.Indexes[i] = -1; // actually we don't use these values + } for (i = 0; i < numThreads; i++) { CThreadInfo &threadInfo = threads.Threads[i]; + threadInfo.SetOptions(options2); ; #ifdef EXTERNAL_CODECS threadInfo.__externalCodecs = __externalCodecs; #endif RINOK(threadInfo.CreateEvents()); threadInfo.OutStreamSpec = new COutMemStream(&memManager); - RINOK(threadInfo.OutStreamSpec->CreateEvents()); + RINOK(threadInfo.OutStreamSpec->CreateEvents(SYNC_WFMO(&memManager.Synchro))); threadInfo.OutStream = threadInfo.OutStreamSpec; threadInfo.IsFree = true; threadInfo.ProgressSpec = new CMtCompressProgress(); threadInfo.Progress = threadInfo.ProgressSpec; - threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, (int)i); + threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i); threadInfo.InSeqMode = false; threadInfo.OutSeqMode = false; threadInfo.FileTime = 0; threadInfo.ExpectedDataSize = (UInt64)(Int64)-1; + threadInfo.ThreadIndex = i; + threadInfo.MtSem = &mtSem; RINOK(threadInfo.CreateThread()); } } @@ -890,7 +956,7 @@ static HRESULT Update2( } else { - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -958,10 +1024,9 @@ static HRESULT Update2( threadInfo.OutSeqMode = outSeqMode; threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode threadInfo.ExpectedDataSize = ui.Size; - + threadInfo.CompressEvent.Set(); - compressingCompletedEvents.Add(threadInfo.CompressionCompletedEvent); threadIndices.Add(k); } } @@ -982,7 +1047,7 @@ static HRESULT Update2( if (!ui.NewProps || !ui.NewData) { - itemEx = inputItems[ui.IndexInArc]; + itemEx = inputItems[(unsigned)ui.IndexInArc]; if (inArchive->ReadLocalItemAfterCdItemFull(itemEx) != S_OK) return E_NOTIMPL; (CItem &)item = itemEx; @@ -1004,7 +1069,7 @@ static HRESULT Update2( if (memRef.Finished) { if (lastRealStreamItemIndex < (int)itemIndex) - lastRealStreamItemIndex = itemIndex; + lastRealStreamItemIndex = (int)itemIndex; SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item); @@ -1030,7 +1095,7 @@ static HRESULT Update2( { // LocalHeader was not written for current itemIndex still - lastRealStreamItemIndex = itemIndex; + lastRealStreamItemIndex = (int)itemIndex; // thread was started before for that item already, and memRef.SeqMode was set @@ -1060,24 +1125,30 @@ static HRESULT Update2( } } - DWORD result = ::WaitForMultipleObjects(compressingCompletedEvents.Size(), - &compressingCompletedEvents.Front(), FALSE, INFINITE); - if (result == WAIT_FAILED) - { - DWORD lastError = GetLastError(); - return lastError != 0 ? lastError : E_FAIL; - } - - unsigned t = (unsigned)(result - WAIT_OBJECT_0); - if (t >= compressingCompletedEvents.Size()) + WRes wres = mtSem.Semaphore.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + int ti = mtSem.GetFreeItem(); + if (ti < 0) return E_FAIL; - CThreadInfo &threadInfo = threads.Threads[threadIndices[t]]; + CThreadInfo &threadInfo = threads.Threads[(unsigned)ti]; threadInfo.InStream.Release(); threadInfo.IsFree = true; RINOK(threadInfo.Result); + + unsigned t = 0; + + for (;;) + { + if (t == threadIndices.Size()) + return E_FAIL; + if (threadIndices[t] == (unsigned)ti) + break; + t++; + } threadIndices.Delete(t); - compressingCompletedEvents.Delete(t); if (t == 0) { @@ -1187,7 +1258,7 @@ HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *strea { RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos)); RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize)); - RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos)); + RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, &_virtPos)); } _phyPos = _virtPos; _phySize = _virtSize; @@ -1204,7 +1275,7 @@ HRESULT CCacheOutStream::MyWrite(size_t size) { if (!_stream) return E_FAIL; - RINOK(_stream->Seek(_cachedPos, STREAM_SEEK_SET, &_phyPos)); + RINOK(_stream->Seek((Int64)_cachedPos, STREAM_SEEK_SET, &_phyPos)); } size_t pos = (size_t)_cachedPos & kCacheMask; size_t curSize = MyMin(kCacheSize - pos, _cachedSize); @@ -1233,7 +1304,7 @@ CCacheOutStream::~CCacheOutStream() if (_virtSize != _phySize) _stream->SetSize(_virtSize); if (_virtPos != _phyPos) - _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL); + _stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, NULL); } ::MidFree(_cache); } @@ -1332,9 +1403,9 @@ STDMETHODIMP CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -1391,7 +1462,7 @@ HRESULT Update( { IInStream *baseStream = inArchive->GetBaseStream(); RINOK(baseStream->Seek(0, STREAM_SEEK_SET, NULL)); - RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, inArchive->ArcInfo.Base, NULL)); + RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, (UInt64)inArchive->ArcInfo.Base, NULL)); } } @@ -1412,7 +1483,7 @@ HRESULT Update( { IInStream *baseStream = inArchive->GetBaseStream(); RINOK(baseStream->Seek(inArchive->ArcInfo.Base, STREAM_SEEK_SET, NULL)); - UInt64 embStubSize = inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base; + const UInt64 embStubSize = (UInt64)((Int64)inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base); RINOK(NCompress::CopyStream_ExactSize(baseStream, outStream, embStubSize, NULL)); outArchive.MoveCurPos(embStubSize); } diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.h b/CPP/7zip/Archive/Zip/ZipUpdate.h index 8785ae60..95e72a47 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.h +++ b/CPP/7zip/Archive/Zip/ZipUpdate.h @@ -34,11 +34,12 @@ struct CUpdateItem bool IsUtf8; // bool IsAltStream; int IndexInArc; - int IndexInClient; + unsigned IndexInClient; UInt32 Attrib; UInt32 Time; UInt64 Size; AString Name; + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra CByteBuffer Comment; // bool Commented; // CUpdateRange CommentRange; @@ -54,6 +55,7 @@ struct CUpdateItem // IsAltStream = false; Size = 0; Name.Empty(); + Name_Utf.Free(); Comment.Free(); } diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index 96a0f054..145cf2ca 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gz /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -69,7 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gz /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gz /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # SUBTRACT CPP /WX # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" @@ -95,7 +95,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -122,7 +122,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c -# ADD CPP /nologo /Gz /MDd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gz /MDd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /D "WIN_LONG_PATH" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -350,10 +350,18 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha1Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha1Reg.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha256Reg.cpp # End Source File # Begin Source File @@ -390,10 +398,6 @@ SOURCE=..\..\..\Common\StringToInt.h # End Source File # Begin Source File -SOURCE=..\..\..\Common\Types.h -# End Source File -# Begin Source File - SOURCE=..\..\..\Common\UTFConvert.cpp # End Source File # Begin Source File @@ -526,6 +530,14 @@ SOURCE=..\..\..\Windows\PropVariantUtils.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Registry.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Registry.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -542,6 +554,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # Begin Source File @@ -999,14 +1019,6 @@ SOURCE=..\..\Compress\LzmaRegister.cpp # PROP Default_Filter "" # Begin Source File -SOURCE=..\..\Compress\PpmdContext.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdDecode.h -# End Source File -# Begin Source File - SOURCE=..\..\Compress\PpmdDecoder.cpp !IF "$(CFG)" == "Alone - Win32 Release" @@ -1032,10 +1044,6 @@ SOURCE=..\..\Compress\PpmdDecoder.h # End Source File # Begin Source File -SOURCE=..\..\Compress\PpmdEncode.h -# End Source File -# Begin Source File - SOURCE=..\..\Compress\PpmdEncoder.cpp !IF "$(CFG)" == "Alone - Win32 Release" @@ -1065,14 +1073,6 @@ SOURCE=..\..\Compress\PpmdRegister.cpp # End Source File # Begin Source File -SOURCE=..\..\Compress\PpmdSubAlloc.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdType.h -# End Source File -# Begin Source File - SOURCE=..\..\Compress\PpmdZip.cpp # End Source File # Begin Source File @@ -1457,10 +1457,6 @@ SOURCE=..\..\Archive\Zip\ZipItem.h # End Source File # Begin Source File -SOURCE=..\..\Archive\Zip\ZipItemEx.h -# End Source File -# Begin Source File - SOURCE=..\..\Archive\Zip\ZipOut.cpp # End Source File # Begin Source File @@ -2223,6 +2219,10 @@ SOURCE=..\..\..\..\C\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Aes.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2981,6 +2981,11 @@ SOURCE=..\..\..\..\C\Sha1.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha1Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sha256.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -3009,6 +3014,11 @@ SOURCE=..\..\..\..\C\Sha256.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -3063,10 +3073,6 @@ SOURCE=..\..\..\..\C\Threads.c SOURCE=..\..\..\..\C\Threads.h # End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Types.h -# End Source File # End Group # End Target # End Project diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index 27bdbb94..0fce341b 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -1,4 +1,6 @@ PROG = 7za.exe +# USE_C_AES = 1 +# USE_C_SHA = 1 COMMON_OBJS = \ $O\CommandLineParser.obj \ @@ -33,8 +35,10 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\PropVariantUtils.obj \ + $O\Registry.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ @@ -133,7 +137,7 @@ COMPRESS_OBJS = \ $O\BranchMisc.obj \ $O\BranchRegister.obj \ $O\ByteSwap.obj \ - $O\BZip2CRC.obj \ + $O\BZip2Crc.obj \ $O\BZip2Decoder.obj \ $O\BZip2Encoder.obj \ $O\BZip2Register.obj \ @@ -201,8 +205,6 @@ C_OBJS = \ $O\Ppmd8.obj \ $O\Ppmd8Dec.obj \ $O\Ppmd8Enc.obj \ - $O\Sha1.obj \ - $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ $O\Xz.obj \ @@ -216,5 +218,7 @@ C_OBJS = \ !include "../../Crc.mak" !include "../../Crc64.mak" !include "../../LzmaDec.mak" +!include "../../Sha1.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone/makefile.gcc b/CPP/7zip/Bundles/Alone/makefile.gcc new file mode 100644 index 00000000..e63fc332 --- /dev/null +++ b/CPP/7zip/Bundles/Alone/makefile.gcc @@ -0,0 +1,334 @@ +PROG = 7za + + + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef SystemDrive +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/LzFindMt.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/MemBlocks.o \ + $O/OutMemStream.o \ + $O/ProgressMt.o \ + $O/Threads.o \ + +endif + + + +LOCAL_FLAGS_SYS = + +ifdef SystemDrive + +LOCAL_FLAGS_SYS = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ + + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/ListFileUtils.o \ + $O/MyString.o \ + $O/NewHandler.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/Sha1Prepare.o \ + $O/Sha1Reg.o \ + $O/Sha256Prepare.o \ + $O/Sha256Reg.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/MyVector.o \ + $O/Wildcard.o \ + $O/XzCrc64Init.o \ + $O/XzCrc64Reg.o \ + + +WIN_OBJS = \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileLink.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + $O/PropVariantUtils.o \ + $O/System.o \ + $O/SystemInfo.o \ + $O/TimeUtils.o \ + + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +AR_OBJS = \ + $O/Bz2Handler.o \ + $O/GzHandler.o \ + $O/LzmaHandler.o \ + $O/SplitHandler.o \ + $O/XzHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/DummyOutStream.o \ + $O/HandlerOut.o \ + $O/InStreamWithCRC.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + $O/ParseProperties.o \ + +7Z_OBJS = \ + $O/7zCompressionMode.o \ + $O/7zDecode.o \ + $O/7zEncode.o \ + $O/7zExtract.o \ + $O/7zFolderInStream.o \ + $O/7zHandler.o \ + $O/7zHandlerOut.o \ + $O/7zHeader.o \ + $O/7zIn.o \ + $O/7zOut.o \ + $O/7zProperties.o \ + $O/7zRegister.o \ + $O/7zSpecStream.o \ + $O/7zUpdate.o \ + +CAB_OBJS = \ + $O/CabBlockInStream.o \ + $O/CabHandler.o \ + $O/CabHeader.o \ + $O/CabIn.o \ + $O/CabRegister.o \ + +TAR_OBJS = \ + $O/TarHandler.o \ + $O/TarHandlerOut.o \ + $O/TarHeader.o \ + $O/TarIn.o \ + $O/TarOut.o \ + $O/TarUpdate.o \ + $O/TarRegister.o \ + +ZIP_OBJS = \ + $O/ZipAddCommon.o \ + $O/ZipHandler.o \ + $O/ZipHandlerOut.o \ + $O/ZipIn.o \ + $O/ZipItem.o \ + $O/ZipOut.o \ + $O/ZipUpdate.o \ + $O/ZipRegister.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BitlDecoder.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/ByteSwap.o \ + $O/BZip2Crc.o \ + $O/BZip2Decoder.o \ + $O/BZip2Encoder.o \ + $O/BZip2Register.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/Deflate64Register.o \ + $O/DeflateDecoder.o \ + $O/DeflateEncoder.o \ + $O/DeflateRegister.o \ + $O/DeltaFilter.o \ + $O/ImplodeDecoder.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Encoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + $O/LzOutWindow.o \ + $O/LzxDecoder.o \ + $O/PpmdDecoder.o \ + $O/PpmdEncoder.o \ + $O/PpmdRegister.o \ + $O/PpmdZip.o \ + $O/QuantumDecoder.o \ + $O/ShrinkDecoder.o \ + $O/XzDecoder.o \ + $O/XzEncoder.o \ + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/HmacSha1.o \ + $O/MyAes.o \ + $O/MyAesReg.o \ + $O/Pbkdf2HmacSha1.o \ + $O/RandGen.o \ + $O/WzAes.o \ + $O/ZipCrypto.o \ + $O/ZipStrong.o \ + +C_OBJS = \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bcj2Enc.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/BwtSort.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/HuffEnc.o \ + $O/LzFind.o \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/Lzma2Enc.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/MtCoder.o \ + $O/MtDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/Ppmd7Enc.o \ + $O/Ppmd8.o \ + $O/Ppmd8Dec.o \ + $O/Ppmd8Enc.o \ + $O/Sort.o \ + $O/Xz.o \ + $O/XzDec.o \ + $O/XzEnc.o \ + $O/XzIn.o \ + $O/XzCrc64.o \ + $O/XzCrc64Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/Sha1.o \ + $O/Sha1Opt.o \ + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(CAB_OBJS) \ + $(TAR_OBJS) \ + $(ZIP_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/Alone2/StdAfx.cpp b/CPP/7zip/Bundles/Alone2/StdAfx.cpp new file mode 100644 index 00000000..d0feea85 --- /dev/null +++ b/CPP/7zip/Bundles/Alone2/StdAfx.cpp @@ -0,0 +1,3 @@ +// StdAfx.cpp + +#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Alone2/StdAfx.h b/CPP/7zip/Bundles/Alone2/StdAfx.h new file mode 100644 index 00000000..2854ff3e --- /dev/null +++ b/CPP/7zip/Bundles/Alone2/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/Common.h" + +#endif diff --git a/CPP/7zip/Bundles/Alone2/makefile b/CPP/7zip/Bundles/Alone2/makefile new file mode 100644 index 00000000..56de5b2e --- /dev/null +++ b/CPP/7zip/Bundles/Alone2/makefile @@ -0,0 +1,29 @@ +PROG = 7zz.exe +# USE_C_AES = 1 +# USE_C_SHA = 1 +CFLAGS = $(CFLAGS) -DPROG_VARIANT_Z + +!include "../Format7zF/Arc.mak" +!include "../../UI/Console/Console.mak" + +COMMON_OBJS = $(COMMON_OBJS) \ + $O\CommandLineParser.obj \ + $O\ListFileUtils.obj \ + $O\StdInStream.obj \ + $O\StdOutStream.obj \ + +WIN_OBJS = $(WIN_OBJS) \ + $O\DLL.obj \ + $O\ErrorMsg.obj \ + $O\FileLink.obj \ + $O\FileSystem.obj \ + $O\MemoryLock.obj \ + $O\PropVariantConv.obj \ + $O\Registry.obj \ + $O\SystemInfo.obj \ + +7ZIP_COMMON_OBJS = $(7ZIP_COMMON_OBJS) \ + $O\FilePathAutoRename.obj \ + $O\FileStreams.obj \ + +!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone2/makefile.gcc b/CPP/7zip/Bundles/Alone2/makefile.gcc new file mode 100644 index 00000000..1e7e17f1 --- /dev/null +++ b/CPP/7zip/Bundles/Alone2/makefile.gcc @@ -0,0 +1,100 @@ +PROG = 7zz + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +CONSOLE_VARIANT_FLAGS=-DPROG_VARIANT_Z + +include ../Format7zF/Arc_gcc.mak + +ifdef SystemDrive + +LOCAL_FLAGS_SYS = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_SYS) \ + $(LOCAL_FLAGS_ST) \ + + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +COMMON_OBJS_2 = \ + $O/CommandLineParser.o \ + $O/ListFileUtils.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + +WIN_OBJS_2 = \ + $O/ErrorMsg.o \ + $O/FileLink.o \ + $O/PropVariantConv.o \ + $O/SystemInfo.o \ + +7ZIP_COMMON_OBJS_2 = \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + +OBJS = \ + $(ARC_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS_2) \ + $(WIN_OBJS_2) \ + $(7ZIP_COMMON_OBJS_2) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/Alone2/resource.rc b/CPP/7zip/Bundles/Alone2/resource.rc new file mode 100644 index 00000000..af24c175 --- /dev/null +++ b/CPP/7zip/Bundles/Alone2/resource.rc @@ -0,0 +1,7 @@ +#include "../../MyVersionInfo.rc" + +MY_VERSION_INFO_APP("7-Zip Standalone 2 Console", "7zz") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index bb7ca4c9..a46526bf 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAc /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAc /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -69,7 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -342,6 +342,14 @@ SOURCE=..\..\..\Common\MyVector.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyWindows.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyWindows.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\NewHandler.cpp # End Source File # Begin Source File @@ -350,6 +358,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha256Reg.cpp # End Source File # Begin Source File @@ -518,6 +530,14 @@ SOURCE=..\..\..\Windows\PropVariantConv.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Registry.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Registry.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -534,6 +554,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # Begin Source File @@ -1233,34 +1261,6 @@ SOURCE=..\..\PropID.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\..\..\..\C\Sha256.c - -!IF "$(CFG)" == "Alone - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" - -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha256.h -# End Source File -# Begin Source File - SOURCE=..\..\..\..\C\Xz.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1457,6 +1457,10 @@ SOURCE=..\..\..\..\C\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Aes.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1862,6 +1866,39 @@ SOURCE=..\..\..\..\C\MtDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha256.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1899,6 +1936,10 @@ SOURCE=..\..\Crypto\MyAes.h # End Source File # Begin Source File +SOURCE=..\..\Crypto\MyAesReg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\Crypto\RandGen.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index 10be3cfb..803277ab 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -1,6 +1,8 @@ PROG = 7zr.exe -CFLAGS = $(CFLAGS) \ - -DPROG_VARIANT_R \ + +# USE_C_AES = 1 + +CFLAGS = $(CFLAGS) -DPROG_VARIANT_R COMMON_OBJS = \ $O\CommandLineParser.obj \ @@ -33,8 +35,10 @@ WIN_OBJS = \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ + $O\Registry.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ @@ -136,7 +140,6 @@ C_OBJS = \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ $O\MtDec.obj \ - $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ $O\Xz.obj \ @@ -150,5 +153,6 @@ C_OBJS = \ !include "../../Crc.mak" !include "../../Crc64.mak" !include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone7z/makefile.gcc b/CPP/7zip/Bundles/Alone7z/makefile.gcc new file mode 100644 index 00000000..b1f69734 --- /dev/null +++ b/CPP/7zip/Bundles/Alone7z/makefile.gcc @@ -0,0 +1,266 @@ +PROG = 7zr + +CONSOLE_VARIANT_FLAGS=-DPROG_VARIANT_R + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef SystemDrive +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/LzFindMt.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/Threads.o \ + + + +endif + + + +LOCAL_FLAGS_SYS = + +ifdef SystemDrive + +LOCAL_FLAGS_SYS = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ + +# -D_LZMA_PROB32 + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/ListFileUtils.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/Sha256Prepare.o \ + $O/Sha256Reg.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + $O/XzCrc64Init.o \ + $O/XzCrc64Reg.o \ + +WIN_OBJS = \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileLink.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + $O/System.o \ + $O/SystemInfo.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +AR_OBJS = \ + $O/LzmaHandler.o \ + $O/SplitHandler.o \ + $O/XzHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/DummyOutStream.o \ + $O/HandlerOut.o \ + $O/InStreamWithCRC.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + $O/ParseProperties.o \ + +7Z_OBJS = \ + $O/7zCompressionMode.o \ + $O/7zDecode.o \ + $O/7zEncode.o \ + $O/7zExtract.o \ + $O/7zFolderInStream.o \ + $O/7zHandler.o \ + $O/7zHandlerOut.o \ + $O/7zHeader.o \ + $O/7zIn.o \ + $O/7zOut.o \ + $O/7zProperties.o \ + $O/7zRegister.o \ + $O/7zSpecStream.o \ + $O/7zUpdate.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/ByteSwap.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/DeltaFilter.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Encoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + $O/XzDecoder.o \ + $O/XzEncoder.o \ + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/MyAes.o \ + $O/MyAesReg.o \ + $O/RandGen.o \ + +C_OBJS = \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bcj2Enc.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/LzFind.o \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/Lzma2Enc.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/MtCoder.o \ + $O/MtDec.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/Sort.o \ + $O/Xz.o \ + $O/XzDec.o \ + $O/XzEnc.o \ + $O/XzIn.o \ + $O/XzCrc64.o \ + $O/XzCrc64Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index 03fbb50e..cccf1be5 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -55,7 +55,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zFM.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "FM - Win32 Debug" @@ -82,7 +82,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zFM.exe" /pdbtype:sept !ELSEIF "$(CFG)" == "FM - Win32 ReleaseU" @@ -109,7 +109,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib /nologo /subsystem:windows /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zFM.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "FM - Win32 DebugU" @@ -137,7 +137,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib /nologo /subsystem:windows /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zFM.exe" /pdbtype:sept !ENDIF @@ -579,6 +579,10 @@ SOURCE=..\..\UI\FileManager\PanelSort.cpp SOURCE=..\..\UI\FileManager\PanelSplitFile.cpp # End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\VerCtrl.cpp +# End Source File # End Group # Begin Group "Dialog" @@ -1103,6 +1107,11 @@ SOURCE=..\..\..\..\C\Sha256.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1405,6 +1414,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # Begin Source File @@ -1537,7 +1554,7 @@ SOURCE=..\..\..\Common\Random.h # End Source File # Begin Source File -SOURCE=..\..\..\Common\Sha256Reg.cpp +SOURCE=..\..\..\Common\Sha256Prepare.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Fm/StdAfx.h b/CPP/7zip/Bundles/Fm/StdAfx.h index d2301e48..15fd136e 100644 --- a/CPP/7zip/Bundles/Fm/StdAfx.h +++ b/CPP/7zip/Bundles/Fm/StdAfx.h @@ -9,8 +9,8 @@ #include "../../../Common/Common.h" -#include <commctrl.h> +#include <CommCtrl.h> #include <ShlObj.h> -#include <shlwapi.h> +#include <Shlwapi.h> #endif diff --git a/CPP/7zip/Bundles/Fm/makefile b/CPP/7zip/Bundles/Fm/makefile index 8da34edb..2c80ca1d 100644 --- a/CPP/7zip/Bundles/Fm/makefile +++ b/CPP/7zip/Bundles/Fm/makefile @@ -23,6 +23,7 @@ WIN_OBJS = $(WIN_OBJS) \ $O\PropVariantConv.obj \ $O\Registry.obj \ $O\ResourceString.obj \ + $O\SystemInfo.obj \ $O\Shell.obj \ $O\Window.obj \ diff --git a/CPP/7zip/Bundles/Fm/resource.rc b/CPP/7zip/Bundles/Fm/resource.rc index dd2058a1..6c329588 100644 --- a/CPP/7zip/Bundles/Fm/resource.rc +++ b/CPP/7zip/Bundles/Fm/resource.rc @@ -3,5 +3,5 @@ STRINGTABLE BEGIN - 100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs" + 100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lzh lha rpm deb arj vhd wim swm esd fat ntfs dmg hfs xar squashfs" END diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 1233d4fc..ca66599a 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -132,12 +132,12 @@ C_OBJS = \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ - $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zExtract/makefile b/CPP/7zip/Bundles/Format7zExtract/makefile index e22faf36..9b4831a9 100644 --- a/CPP/7zip/Bundles/Format7zExtract/makefile +++ b/CPP/7zip/Bundles/Format7zExtract/makefile @@ -104,11 +104,11 @@ C_OBJS = \ $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ - $O\Sha256.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index fd24d5a8..fedce5e0 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -55,6 +55,7 @@ AR_OBJS = \ $O\ApmHandler.obj \ $O\ArHandler.obj \ $O\ArjHandler.obj \ + $O\Base64Handler.obj \ $O\Bz2Handler.obj \ $O\ComHandler.obj \ $O\CpioHandler.obj \ @@ -271,12 +272,11 @@ C_OBJS = \ $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ + $O\Ppmd7aDec.obj \ $O\Ppmd7Enc.obj \ $O\Ppmd8.obj \ $O\Ppmd8Dec.obj \ $O\Ppmd8Enc.obj \ - $O\Sha1.obj \ - $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ $O\Xz.obj \ @@ -288,3 +288,5 @@ C_OBJS = \ !include "../../Crc.mak" !include "../../Crc64.mak" !include "../../LzmaDec.mak" +!include "../../Sha1.mak" +!include "../../Sha256.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak new file mode 100644 index 00000000..c3dbf349 --- /dev/null +++ b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak @@ -0,0 +1,345 @@ +include ../../LzmaDec_gcc.mak + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef SystemDrive +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/LzFindMt.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/MemBlocks.o \ + $O/OutMemStream.o \ + $O/ProgressMt.o \ + $O/Threads.o \ + +endif + + + +COMMON_OBJS = \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/DynLimBuf.o \ + $O/IntToString.o \ + $O/MyMap.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/MyXml.o \ + $O/NewHandler.o \ + $O/Sha1Prepare.o \ + $O/Sha1Reg.o \ + $O/Sha256Prepare.o \ + $O/Sha256Reg.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + $O/XzCrc64Init.o \ + $O/XzCrc64Reg.o \ + +WIN_OBJS = \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantUtils.o \ + $O/System.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/LockedStream.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +AR_OBJS = \ + $O/ApmHandler.o \ + $O/ArHandler.o \ + $O/ArjHandler.o \ + $O/Base64Handler.o \ + $O/Bz2Handler.o \ + $O/ComHandler.o \ + $O/CpioHandler.o \ + $O/CramfsHandler.o \ + $O/DeflateProps.o \ + $O/DmgHandler.o \ + $O/ElfHandler.o \ + $O/ExtHandler.o \ + $O/FatHandler.o \ + $O/FlvHandler.o \ + $O/GzHandler.o \ + $O/GptHandler.o \ + $O/HandlerCont.o \ + $O/HfsHandler.o \ + $O/IhexHandler.o \ + $O/LzhHandler.o \ + $O/LzmaHandler.o \ + $O/MachoHandler.o \ + $O/MbrHandler.o \ + $O/MslzHandler.o \ + $O/MubHandler.o \ + $O/NtfsHandler.o \ + $O/PeHandler.o \ + $O/PpmdHandler.o \ + $O/QcowHandler.o \ + $O/RpmHandler.o \ + $O/SplitHandler.o \ + $O/SquashfsHandler.o \ + $O/SwfHandler.o \ + $O/UefiHandler.o \ + $O/VdiHandler.o \ + $O/VhdHandler.o \ + $O/VmdkHandler.o \ + $O/XarHandler.o \ + $O/XzHandler.o \ + $O/ZHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/DummyOutStream.o \ + $O/FindSignature.o \ + $O/InStreamWithCRC.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + $O/OutStreamWithSha1.o \ + $O/HandlerOut.o \ + $O/ParseProperties.o \ + + +7Z_OBJS = \ + $O/7zCompressionMode.o \ + $O/7zDecode.o \ + $O/7zEncode.o \ + $O/7zExtract.o \ + $O/7zFolderInStream.o \ + $O/7zHandler.o \ + $O/7zHandlerOut.o \ + $O/7zHeader.o \ + $O/7zIn.o \ + $O/7zOut.o \ + $O/7zProperties.o \ + $O/7zSpecStream.o \ + $O/7zUpdate.o \ + $O/7zRegister.o \ + +CAB_OBJS = \ + $O/CabBlockInStream.o \ + $O/CabHandler.o \ + $O/CabHeader.o \ + $O/CabIn.o \ + $O/CabRegister.o \ + +CHM_OBJS = \ + $O/ChmHandler.o \ + $O/ChmIn.o \ + +ISO_OBJS = \ + $O/IsoHandler.o \ + $O/IsoHeader.o \ + $O/IsoIn.o \ + $O/IsoRegister.o \ + +NSIS_OBJS = \ + $O/NsisDecode.o \ + $O/NsisHandler.o \ + $O/NsisIn.o \ + $O/NsisRegister.o \ + +RAR_OBJS = \ + $O/RarHandler.o \ + $O/Rar5Handler.o \ + +TAR_OBJS = \ + $O/TarHandler.o \ + $O/TarHandlerOut.o \ + $O/TarHeader.o \ + $O/TarIn.o \ + $O/TarOut.o \ + $O/TarUpdate.o \ + $O/TarRegister.o \ + +UDF_OBJS = \ + $O/UdfHandler.o \ + $O/UdfIn.o \ + +WIM_OBJS = \ + $O/WimHandler.o \ + $O/WimHandlerOut.o \ + $O/WimIn.o \ + $O/WimRegister.o \ + +ZIP_OBJS = \ + $O/ZipAddCommon.o \ + $O/ZipHandler.o \ + $O/ZipHandlerOut.o \ + $O/ZipIn.o \ + $O/ZipItem.o \ + $O/ZipOut.o \ + $O/ZipUpdate.o \ + $O/ZipRegister.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BitlDecoder.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/ByteSwap.o \ + $O/BZip2Crc.o \ + $O/BZip2Decoder.o \ + $O/BZip2Encoder.o \ + $O/BZip2Register.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/Deflate64Register.o \ + $O/DeflateDecoder.o \ + $O/DeflateEncoder.o \ + $O/DeflateRegister.o \ + $O/DeltaFilter.o \ + $O/ImplodeDecoder.o \ + $O/LzfseDecoder.o \ + $O/LzhDecoder.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Encoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + $O/LzmsDecoder.o \ + $O/LzOutWindow.o \ + $O/LzxDecoder.o \ + $O/PpmdDecoder.o \ + $O/PpmdEncoder.o \ + $O/PpmdRegister.o \ + $O/PpmdZip.o \ + $O/QuantumDecoder.o \ + $O/Rar1Decoder.o \ + $O/Rar2Decoder.o \ + $O/Rar3Decoder.o \ + $O/Rar3Vm.o \ + $O/Rar5Decoder.o \ + $O/RarCodecsRegister.o \ + $O/ShrinkDecoder.o \ + $O/XpressDecoder.o \ + $O/XzDecoder.o \ + $O/XzEncoder.o \ + $O/ZlibDecoder.o \ + $O/ZlibEncoder.o \ + $O/ZDecoder.o \ + + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/HmacSha1.o \ + $O/HmacSha256.o \ + $O/MyAes.o \ + $O/MyAesReg.o \ + $O/Pbkdf2HmacSha1.o \ + $O/RandGen.o \ + $O/Rar20Crypto.o \ + $O/Rar5Aes.o \ + $O/RarAes.o \ + $O/WzAes.o \ + $O/ZipCrypto.o \ + $O/ZipStrong.o \ + + +C_OBJS = \ + $O/7zBuf2.o \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bcj2Enc.o \ + $O/Blake2s.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/BwtSort.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/HuffEnc.o \ + $O/LzFind.o \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/Lzma2Enc.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/MtCoder.o \ + $O/MtDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/Ppmd7aDec.o \ + $O/Ppmd7Enc.o \ + $O/Ppmd8.o \ + $O/Ppmd8Dec.o \ + $O/Ppmd8Enc.o \ + $O/Sort.o \ + $O/Xz.o \ + $O/XzDec.o \ + $O/XzEnc.o \ + $O/XzIn.o \ + $O/XzCrc64.o \ + $O/XzCrc64Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/Sha1.o \ + $O/Sha1Opt.o \ + +ARC_OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(CAB_OBJS) \ + $(CHM_OBJS) \ + $(COM_OBJS) \ + $(ISO_OBJS) \ + $(NSIS_OBJS) \ + $(RAR_OBJS) \ + $(TAR_OBJS) \ + $(UDF_OBJS) \ + $(WIM_OBJS) \ + $(ZIP_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + diff --git a/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/CPP/7zip/Bundles/Format7zF/Format7z.dsp index 3e202fe7..67883ebc 100644 --- a/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -53,7 +53,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"C:\Program Files\7-Zip\7z.dll" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"C:\Util\7z.dll" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none /debug !ELSEIF "$(CFG)" == "7z - Win32 Debug" @@ -80,7 +80,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\7-Zip\7z.dll" /pdbtype:sept /ignore:4033 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"C:\Util\7z.dll" /pdbtype:sept /ignore:4033 # SUBTRACT LINK32 /pdb:none !ENDIF @@ -347,10 +347,18 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha1Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha1Reg.cpp # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha256Reg.cpp # End Source File # Begin Source File @@ -1060,6 +1068,16 @@ SOURCE=..\..\Compress\ZDecoder.h # Begin Source File SOURCE=..\..\Crypto\7zAes.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1068,6 +1086,16 @@ SOURCE=..\..\Crypto\7zAes.h # Begin Source File SOURCE=..\..\Crypto\7zAesRegister.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1090,6 +1118,16 @@ SOURCE=..\..\Crypto\HmacSha1.h # Begin Source File SOURCE=..\..\Crypto\HmacSha256.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1098,6 +1136,16 @@ SOURCE=..\..\Crypto\HmacSha256.h # Begin Source File SOURCE=..\..\Crypto\MyAes.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1106,6 +1154,16 @@ SOURCE=..\..\Crypto\MyAes.h # Begin Source File SOURCE=..\..\Crypto\MyAesReg.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1164,6 +1222,16 @@ SOURCE=..\..\Crypto\Rar20Crypto.h # Begin Source File SOURCE=..\..\Crypto\Rar5Aes.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1172,6 +1240,16 @@ SOURCE=..\..\Crypto\Rar5Aes.h # Begin Source File SOURCE=..\..\Crypto\RarAes.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1184,6 +1262,16 @@ SOURCE=..\..\Crypto\Sha1Cls.h # Begin Source File SOURCE=..\..\Crypto\WzAes.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + # End Source File # Begin Source File @@ -1938,6 +2026,22 @@ SOURCE=..\..\..\..\C\Ppmd7.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Ppmd7aDec.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7Dec.c !IF "$(CFG)" == "7z - Win32 Release" @@ -2651,6 +2755,10 @@ SOURCE=..\..\Archive\ArjHandler.cpp # End Source File # Begin Source File +SOURCE=..\..\Archive\Base64Handler.cpp +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Bz2Handler.cpp # End Source File # Begin Source File @@ -3003,6 +3111,68 @@ InputName=AesOpt !ENDIF # End Source File +# Begin Source File + +SOURCE=..\..\..\..\Asm\x86\Sha1Opt.asm + +!IF "$(CFG)" == "7z - Win32 Release" + +# Begin Custom Build +OutDir=.\Release +InputPath=..\..\..\..\Asm\x86\Sha1Opt.asm +InputName=Sha1Opt + +"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" + ml.exe -c -Fo$(OutDir)\$(InputName).obj $(InputPath) + +# End Custom Build + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# Begin Custom Build +OutDir=.\Debug +InputPath=..\..\..\..\Asm\x86\Sha1Opt.asm +InputName=Sha1Opt + +"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" + ml.exe -c -omf -Fo$(OutDir)\$(InputName).obj $(InputPath) + +# End Custom Build + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\Asm\x86\Sha256Opt.asm + +!IF "$(CFG)" == "7z - Win32 Release" + +# Begin Custom Build +OutDir=.\Release +InputPath=..\..\..\..\Asm\x86\Sha256Opt.asm +InputName=Sha256Opt + +"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" + ml.exe -c -Fo$(OutDir)\$(InputName).obj $(InputPath) + +# End Custom Build + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# Begin Custom Build +OutDir=.\Debug +InputPath=..\..\..\..\Asm\x86\Sha256Opt.asm +InputName=Sha256Opt + +"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" + ml.exe -c -omf -Fo$(OutDir)\$(InputName).obj $(InputPath) + +# End Custom Build + +!ENDIF + +# End Source File # End Group # End Target # End Project diff --git a/CPP/7zip/Bundles/Format7zF/makefile.gcc b/CPP/7zip/Bundles/Format7zF/makefile.gcc new file mode 100644 index 00000000..1953dd55 --- /dev/null +++ b/CPP/7zip/Bundles/Format7zF/makefile.gcc @@ -0,0 +1,45 @@ +PROG = 7z +DEF_FILE = ../../Archive/Archive2.def + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include Arc_gcc.mak + +ifdef SystemDrive + +LOCAL_FLAGS_WIN = \ + -D_7ZIP_LARGE_PAGES \ + $(LOCAL_FLAGS_ST) \ + +SYS_OBJS = \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + -DEXTERNAL_CODECS \ + $(LOCAL_FLAGS_WIN) \ + $(LOCAL_FLAGS_ST) \ + + +COMPRESS_OBJS_2 = \ + $O/CodecExports.o \ + +AR_OBJS_2 = \ + $O/ArchiveExports.o \ + $O/DllExports2.o \ + +OBJS = \ + $(ARC_OBJS) \ + $(AR_OBJS_2) \ + $(COMPRESS_OBJS_2) \ + $(SYS_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/Format7zF/resource.rc b/CPP/7zip/Bundles/Format7zF/resource.rc index 705814f5..5330a415 100644 --- a/CPP/7zip/Bundles/Format7zF/resource.rc +++ b/CPP/7zip/Bundles/Format7zF/resource.rc @@ -33,5 +33,5 @@ MY_VERSION_INFO_DLL("7z Plugin" , "7z") STRINGTABLE BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24" + 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 esd:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24" END diff --git a/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp b/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp index 62d3a84c..a08d9c09 100644 --- a/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp +++ b/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp @@ -2,7 +2,7 @@ #include "StdAfx.h" -#include <stdio.h> +// #include <stdio.h> #include "../../../../C/CpuArch.h" @@ -71,7 +71,7 @@ static const char * const kHelpString = " -so : write data to stdout\n"; -static const char * const kCantAllocate = "Can not allocate memory"; +static const char * const kCantAllocate = "Cannot allocate memory"; static const char * const kReadError = "Read error"; static const char * const kWriteError = "Write error"; @@ -99,24 +99,32 @@ enum Enum }; } +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL + +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_STRING SWFRM_1(NSwitchType::kString) + +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) + static const CSwitchForm kSwitchForms[] = { - { "?", NSwitchType::kSimple, false }, - { "H", NSwitchType::kSimple, false }, - { "MM", NSwitchType::kString, false, 1 }, - { "X", NSwitchType::kString, false, 1 }, - { "A", NSwitchType::kString, false, 1 }, - { "D", NSwitchType::kString, false, 1 }, - { "FB", NSwitchType::kString, false, 1 }, - { "MC", NSwitchType::kString, false, 1 }, - { "LC", NSwitchType::kString, false, 1 }, - { "LP", NSwitchType::kString, false, 1 }, - { "PB", NSwitchType::kString, false, 1 }, - { "MF", NSwitchType::kString, false, 1 }, - { "MT", NSwitchType::kString, false, 0 }, - { "EOS", NSwitchType::kSimple, false }, - { "SI", NSwitchType::kSimple, false }, - { "SO", NSwitchType::kSimple, false }, + { "?", SWFRM_SIMPLE }, + { "H", SWFRM_SIMPLE }, + { "MM", SWFRM_STRING_SINGL(1) }, + { "X", SWFRM_STRING_SINGL(1) }, + { "A", SWFRM_STRING_SINGL(1) }, + { "D", SWFRM_STRING_SINGL(1) }, + { "FB", SWFRM_STRING_SINGL(1) }, + { "MC", SWFRM_STRING_SINGL(1) }, + { "LC", SWFRM_STRING_SINGL(1) }, + { "LP", SWFRM_STRING_SINGL(1) }, + { "PB", SWFRM_STRING_SINGL(1) }, + { "MF", SWFRM_STRING_SINGL(1) }, + { "MT", SWFRM_STRING }, + { "EOS", SWFRM_SIMPLE }, + { "SI", SWFRM_SIMPLE }, + { "SO", SWFRM_SIMPLE }, { "F86", NSwitchType::kChar, false, 0, "+" } }; @@ -264,6 +272,7 @@ STDMETHODIMP CProgressPrint::SetRatioInfo(const UInt64 *inSize, const UInt64 *ou } +MY_ATTR_NORETURN static void IncorrectCommand() { throw "Incorrect command"; @@ -314,7 +323,9 @@ static int Error_HRESULT(const char *s, HRESULT res) return 1; } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION PrintError("Unsupported Windows version"); return 1; +#endif static void AddProp(CObjectVector<CProperty> &props2, const char *name, const wchar_t *val) { @@ -474,7 +485,7 @@ static int main2(int numArgs, const char *args[]) inStream = inStreamSpec; if (!inStreamSpec->Open(us2fs(inputName))) { - PrintError2("can not open input file", inputName); + PrintError2("Cannot open input file", inputName); return 1; } } @@ -494,7 +505,7 @@ static int main2(int numArgs, const char *args[]) outStream = outStreamSpec; if (!outStreamSpec->Create(us2fs(outputName), true)) { - PrintError2("can not open output file", outputName); + PrintError2("Cannot open output file", outputName); return 1; } } @@ -505,7 +516,7 @@ static int main2(int numArgs, const char *args[]) if (inStreamSpec) { if (!inStreamSpec->File.GetLength(fileSize)) - throw "Can not get file length"; + throw "Cannot get file length"; fileSizeDefined = true; if (!stdOutMode) Print_Size("Input size: ", fileSize); @@ -532,7 +543,7 @@ static int main2(int numArgs, const char *args[]) You can use xz format instead, if you want to use filters */ if (parser[NKey::kEOS].ThereIs || stdInMode) - throw "Can not use stdin in this mode"; + throw "Cannot use stdin in this mode"; size_t inSize = (size_t)fileSize; @@ -549,7 +560,7 @@ static int main2(int numArgs, const char *args[]) } if (ReadStream_FAIL(inStream, inBuffer, inSize) != S_OK) - throw "Can not read"; + throw "Cannot read"; Byte *outBuffer = NULL; size_t outSize; diff --git a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp index f8267a11..907b23e0 100644 --- a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp +++ b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp @@ -140,6 +140,14 @@ SOURCE=..\..\..\Windows\PropVariant.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Registry.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Registry.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -156,6 +164,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # End Group diff --git a/CPP/7zip/Bundles/LzmaCon/makefile b/CPP/7zip/Bundles/LzmaCon/makefile index 000ca446..e87becc4 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile +++ b/CPP/7zip/Bundles/LzmaCon/makefile @@ -15,23 +15,24 @@ COMMON_OBJS = \ $O\CrcReg.obj \ $O\IntToString.obj \ $O\MyString.obj \ + $O\MyVector.obj \ $O\NewHandler.obj \ $O\StringConvert.obj \ $O\StringToInt.obj \ - $O\MyVector.obj WIN_OBJS = \ $O\FileIO.obj \ $O\PropVariant.obj \ - $O\System.obj + $O\Registry.obj \ + $O\System.obj \ + $O\SystemInfo.obj \ 7ZIP_COMMON_OBJS = \ - $O\CWrappers.obj \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\FilterCoder.obj \ $O\MethodProps.obj \ - $O\OutBuffer.obj \ $O\StreamUtils.obj \ UI_COMMON_OBJS = \ diff --git a/CPP/7zip/Bundles/LzmaCon/makefile.gcc b/CPP/7zip/Bundles/LzmaCon/makefile.gcc index 97fe4508..f9ccfce4 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile.gcc +++ b/CPP/7zip/Bundles/LzmaCon/makefile.gcc @@ -1,195 +1,113 @@ PROG = lzma -CXX = g++ -O2 -# -Wall -Werror -Wno-delete-non-virtual-dtor -CXX_C = gcc -O2 -Wall -Werror -ifdef SystemDrive -IS_MINGW = 1 -endif +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include ../../LzmaDec_gcc.mak -ifdef IS_MINGW +LOCAL_FLAGS_ST = +MT_OBJS = -RM = del -CFLAGS = -c -LIB2 = -loleaut32 -luuid -LDFLAGS = -s -FILE_IO =FileIO -FILE_IO_2 =Windows/$(FILE_IO) +ifdef ST_MODE -MT_FILES = \ - LzFindMt.o \ - Threads.o \ +LOCAL_FLAGS_ST = -D_7ZIP_ST else -RM = rm -f -CFLAGS = -c -D_7ZIP_ST +MT_OBJS = \ + $O/LzFindMt.o \ + $O/Synchronization.o \ + $O/Threads.o \ -FILE_IO =C_FileIO -FILE_IO_2 =Common/$(FILE_IO) endif -OBJS = \ - $(MT_FILES) \ - $(FILE_IO).o \ - LzmaAlone.o \ - Bench.o \ - BenchCon.o \ - ConsoleClose.o \ - LzmaDecoder.o \ - LzmaEncoder.o \ - LzmaRegister.o \ - CreateCoder.o \ - CWrappers.o \ - FileStreams.o \ - FilterCoder.o \ - MethodProps.o \ - StreamUtils.o \ - CommandLineParser.o \ - CRC.o \ - CrcReg.o \ - IntToString.o \ - MyString.o \ - MyVector.o \ - MyWindows.o \ - StringConvert.o \ - StringToInt.o \ - PropVariant.o \ - System.o \ - 7zCrc.o \ - 7zCrcOpt.o \ - Alloc.o \ - Bra86.o \ - CpuArch.o \ - LzFind.o \ - LzmaDec.o \ - LzmaEnc.o \ - Lzma86Dec.o \ - Lzma86Enc.o \ - - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB2) - -LzmaAlone.o: LzmaAlone.cpp - $(CXX) $(CFLAGS) LzmaAlone.cpp - -Bench.o: ../../UI/Common/Bench.cpp - $(CXX) $(CFLAGS) ../../UI/Common/Bench.cpp - -BenchCon.o: ../../UI/Console/BenchCon.cpp - $(CXX) $(CFLAGS) ../../UI/Console/BenchCon.cpp - -ConsoleClose.o: ../../UI/Console/ConsoleClose.cpp - $(CXX) $(CFLAGS) ../../UI/Console/ConsoleClose.cpp - -LzmaDecoder.o: ../../Compress/LzmaDecoder.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaDecoder.cpp - -LzmaEncoder.o: ../../Compress/LzmaEncoder.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaEncoder.cpp - -LzmaRegister.o: ../../Compress/LzmaRegister.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaRegister.cpp - -CreateCoder.o: ../../Common/CreateCoder.cpp - $(CXX) $(CFLAGS) ../../Common/CreateCoder.cpp - -CWrappers.o: ../../Common/CWrappers.cpp - $(CXX) $(CFLAGS) ../../Common/CWrappers.cpp - -FileStreams.o: ../../Common/FileStreams.cpp - $(CXX) $(CFLAGS) ../../Common/FileStreams.cpp - -FilterCoder.o: ../../Common/FilterCoder.cpp - $(CXX) $(CFLAGS) ../../Common/FilterCoder.cpp - -MethodProps.o: ../../Common/MethodProps.cpp - $(CXX) $(CFLAGS) ../../Common/MethodProps.cpp - -StreamUtils.o: ../../Common/StreamUtils.cpp - $(CXX) $(CFLAGS) ../../Common/StreamUtils.cpp - -$(FILE_IO).o: ../../../$(FILE_IO_2).cpp - $(CXX) $(CFLAGS) ../../../$(FILE_IO_2).cpp +LOCAL_FLAGS_SYS = -CommandLineParser.o: ../../../Common/CommandLineParser.cpp - $(CXX) $(CFLAGS) ../../../Common/CommandLineParser.cpp - -CRC.o: ../../../Common/CRC.cpp - $(CXX) $(CFLAGS) ../../../Common/CRC.cpp - -CrcReg.o: ../../../Common/CrcReg.cpp - $(CXX) $(CFLAGS) ../../../Common/CrcReg.cpp - -IntToString.o: ../../../Common/IntToString.cpp - $(CXX) $(CFLAGS) ../../../Common/IntToString.cpp - -MyString.o: ../../../Common/MyString.cpp - $(CXX) $(CFLAGS) ../../../Common/MyString.cpp - -MyVector.o: ../../../Common/MyVector.cpp - $(CXX) $(CFLAGS) ../../../Common/MyVector.cpp - -MyWindows.o: ../../../Common/MyWindows.cpp - $(CXX) $(CFLAGS) ../../../Common/MyWindows.cpp - -StringConvert.o: ../../../Common/StringConvert.cpp - $(CXX) $(CFLAGS) ../../../Common/StringConvert.cpp - -StringToInt.o: ../../../Common/StringToInt.cpp - $(CXX) $(CFLAGS) ../../../Common/StringToInt.cpp - -PropVariant.o: ../../../Windows/PropVariant.cpp - $(CXX) $(CFLAGS) ../../../Windows/PropVariant.cpp - -System.o: ../../../Windows/System.cpp - $(CXX) $(CFLAGS) ../../../Windows/System.cpp - -7zCrc.o: ../../../../C/7zCrc.c - $(CXX_C) $(CFLAGS) ../../../../C/7zCrc.c - -7zCrcOpt.o: ../../../../C/7zCrcOpt.c - $(CXX_C) $(CFLAGS) ../../../../C/7zCrcOpt.c - -Alloc.o: ../../../../C/Alloc.c - $(CXX_C) $(CFLAGS) ../../../../C/Alloc.c - -Bra86.o: ../../../../C/Bra86.c - $(CXX_C) $(CFLAGS) ../../../../C/Bra86.c +ifdef SystemDrive -CpuArch.o: ../../../../C/CpuArch.c - $(CXX_C) $(CFLAGS) ../../../../C/CpuArch.c +SYS_OBJS = \ + $O/Registry.o \ + $O/resource.o \ -LzFind.o: ../../../../C/LzFind.c - $(CXX_C) $(CFLAGS) ../../../../C/LzFind.c +else -ifdef MT_FILES -LzFindMt.o: ../../../../C/LzFindMt.c - $(CXX_C) $(CFLAGS) ../../../../C/LzFindMt.c +SYS_OBJS = \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileName.o \ + $O/MyWindows.o \ + $O/TimeUtils.o \ -Threads.o: ../../../../C/Threads.c - $(CXX_C) $(CFLAGS) ../../../../C/Threads.c endif -LzmaDec.o: ../../../../C/LzmaDec.c - $(CXX_C) $(CFLAGS) ../../../../C/LzmaDec.c - -LzmaEnc.o: ../../../../C/LzmaEnc.c - $(CXX_C) $(CFLAGS) ../../../../C/LzmaEnc.c - -Lzma86Dec.o: ../../../../C/Lzma86Dec.c - $(CXX_C) $(CFLAGS) ../../../../C/Lzma86Dec.c +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + +WIN_OBJS = \ + $O/FileIO.o \ + $O/PropVariant.o \ + $O/System.o \ + $O/SystemInfo.o \ + +COMPRESS_OBJS = \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FileStreams.o \ + $O/FilterCoder.o \ + $O/MethodProps.o \ + $O/StreamUtils.o \ + +C_OBJS = \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Alloc.o \ + $O/Bra86.o \ + $O/CpuArch.o \ + $O/LzFind.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/Lzma86Dec.o \ + $O/Lzma86Enc.o \ -Lzma86Enc.o: ../../../../C/Lzma86Enc.c - $(CXX_C) $(CFLAGS) ../../../../C/Lzma86Enc.c - -clean: - -$(RM) $(PROG) $(OBJS) +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + $O/LzmaAlone.o \ + $O/Bench.o \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index 0f032d81..87e27c4f 100644 --- a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -481,6 +481,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\StdInStream.cpp # End Source File # Begin Source File @@ -888,6 +892,11 @@ SOURCE=..\..\..\..\C\Sha256.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXCon/SfxCon.cpp b/CPP/7zip/Bundles/SFXCon/SfxCon.cpp index 3209a01c..b149889b 100644 --- a/CPP/7zip/Bundles/SFXCon/SfxCon.cpp +++ b/CPP/7zip/Bundles/SFXCon/SfxCon.cpp @@ -13,8 +13,10 @@ #ifdef _WIN32 #include "../../../Windows/DLL.h" -#include "../../../Windows/FileDir.h" +#else +#include "../../../Common/StringConvert.h" #endif +#include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" #include "../../UI/Common/ExitCode.h" @@ -82,14 +84,20 @@ static const char kImmediateNameID = '!'; static const char kSomeCludePostStringMinSize = 2; // at least <@|!><N>ame must be static const char kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!><N>ame must be */ + +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) + static const CSwitchForm kSwitchForms[kNumSwitches] = { - { "?", NSwitchType::kSimple }, - { "H", NSwitchType::kSimple }, - { "BD", NSwitchType::kSimple }, - { "Y", NSwitchType::kSimple }, - { "P", NSwitchType::kString, false, 1 }, - { "O", NSwitchType::kString, false, 1 }, + { "?", SWFRM_SIMPLE }, + { "H", SWFRM_SIMPLE }, + { "BD", SWFRM_SIMPLE }, + { "Y", SWFRM_SIMPLE }, + { "P", SWFRM_STRING_SINGL(1) }, + { "O", SWFRM_STRING_SINGL(1) }, }; static const int kNumCommandForms = 3; @@ -103,7 +111,6 @@ static const NRecursedType::EEnum kCommandRecursedDefault[kNumCommandForms] = // static const bool kAddRecursedDefault = false; static const char * const kUniversalWildcard = "*"; -static const int kCommandIndex = 0; static const char * const kHelpString = "\nUsage: 7zSFX [<command>] [<switches>...] [<file_name>...]\n" @@ -152,7 +159,7 @@ struct CArchiveCommand NRecursedType::EEnum DefaultRecursedType() const; }; -bool ParseArchiveCommand(const UString &commandString, CArchiveCommand &command) +static bool ParseArchiveCommand(const UString &commandString, CArchiveCommand &command) { UString s = commandString; s.MakeLower_Ascii(); @@ -172,17 +179,19 @@ NRecursedType::EEnum CArchiveCommand::DefaultRecursedType() const return kCommandRecursedDefault[CommandType]; } -void PrintHelp(void) +static void PrintHelp(void) { g_StdOut << kHelpString; } +MY_ATTR_NORETURN static void ShowMessageAndThrowException(const char *message, NExitCode::EEnum code) { g_StdOut << message << endl; throw code; } +MY_ATTR_NORETURN static void PrintHelpAndExit() // yyy { PrintHelp(); @@ -218,7 +227,7 @@ static bool AddNameToCensor(NWildcard::CCensor &wildcardCensor, return true; } -void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, +static void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, const UString &name, bool include, NRecursedType::EEnum type) { if (!AddNameToCensor(wildcardCensor, name, include, type)) @@ -227,7 +236,7 @@ void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, #ifndef _WIN32 -static void GetArguments(int numArgs, const char *args[], UStringVector &parts) +static void GetArguments(int numArgs, char *args[], UStringVector &parts) { parts.Clear(); for (int i = 0; i < numArgs; i++) @@ -238,9 +247,15 @@ static void GetArguments(int numArgs, const char *args[], UStringVector &parts) } #endif + int Main2( #ifndef _WIN32 - int numArgs, const char *args[] + int numArgs, char *args[] + #endif +); +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] #endif ) { @@ -253,6 +268,10 @@ int Main2( SetFileApisToOEM(); #endif + #ifdef ENV_HAVE_LOCALE + MY_SetLocale(); + #endif + g_StdOut << kCopyrightString; UStringVector commandStrings; @@ -277,7 +296,10 @@ int Main2( #else - UString arcPath = commandStrings.Front(); + if (commandStrings.IsEmpty()) + return NExitCode::kFatalError; + + const FString arcPath = us2fs(commandStrings.Front()); #endif @@ -356,7 +378,7 @@ int Main2( if (passwordEnabled) password = parser[NKey::kPassword].PostStrings[0]; - if (!NFind::DoesFileExist(arcPath)) + if (!NFind::DoesFileExist_FollowLink(arcPath)) throw kCantFindSFX; FString outputDir; diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index 077dbc0a..633bd775 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -29,8 +29,8 @@ COMMON_OBJS = \ $O\StdInStream.obj \ $O\StdOutStream.obj \ $O\StringConvert.obj \ - $O\Wildcard.obj \ $O\UTFConvert.obj \ + $O\Wildcard.obj \ WIN_OBJS = \ $O\DLL.obj \ @@ -64,9 +64,9 @@ UI_COMMON_OBJS = \ $O\ArchiveExtractCallback.obj \ $O\ArchiveOpenCallback.obj \ $O\DefaultName.obj \ - $O\LoadCodecs.obj \ $O\Extract.obj \ $O\ExtractingFilePath.obj \ + $O\LoadCodecs.obj \ $O\OpenArchive.obj \ $O\PropIDUtils.obj \ @@ -79,7 +79,6 @@ AR_COMMON_OBJS = \ $O\MultiStream.obj \ $O\OutStreamWithCRC.obj \ - 7Z_OBJS = \ $O\7zDecode.obj \ $O\7zExtract.obj \ @@ -124,11 +123,11 @@ C_OBJS = \ $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ - $O\Sha256.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXCon/makefile.gcc b/CPP/7zip/Bundles/SFXCon/makefile.gcc new file mode 100644 index 00000000..889ec1f9 --- /dev/null +++ b/CPP/7zip/Bundles/SFXCon/makefile.gcc @@ -0,0 +1,204 @@ +PROG = 7zCon + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef SystemDrive +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/Threads.o \ + +endif + + + +LOCAL_FLAGS_SYS = + +ifdef SystemDrive + +LOCAL_FLAGS_SYS = \ + +SYS_OBJS = \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ + -DEXTRACT_ONLY \ + -DNO_READ_FROM_CODER \ + -D_SFX \ + + +CURRENT_OBJS = \ + $O/SfxCon.o \ + +CONSOLE_OBJS = \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/List.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UserInputUtils.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/IntToString.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/Sha256Prepare.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/StringConvert.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + \ + $O/System.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + \ + +UI_COMMON_OBJS = \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/DefaultName.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + +AR_OBJS = \ + $O/SplitHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + +7Z_OBJS = \ + $O/7zDecode.o \ + $O/7zExtract.o \ + $O/7zHandler.o \ + $O/7zIn.o \ + $O/7zRegister.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/DeltaFilter.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaRegister.o \ + $O/PpmdDecoder.o \ + $O/PpmdRegister.o \ + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/MyAes.o \ + +C_OBJS = \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/LzmaDec.o \ + $O/MtDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + $(CURRENT_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp b/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp index 00e3ba53..8eaeabe7 100644 --- a/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp +++ b/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp @@ -15,8 +15,8 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static LPCSTR const kCantDeleteFile = "Can not delete output file"; -static LPCSTR const kCantOpenFile = "Can not open output file"; +static LPCSTR const kCantDeleteFile = "Cannot delete output file"; +static LPCSTR const kCantOpenFile = "Cannot open output file"; static LPCSTR const kUnsupportedMethod = "Unsupported Method"; void CExtractCallbackImp::Init(IInArchive *archiveHandler, diff --git a/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp b/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp index e510e5c8..71b65c62 100644 --- a/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp +++ b/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp @@ -18,8 +18,8 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static LPCSTR const kCantFindArchive = "Can not find archive file"; -static LPCSTR const kCantOpenArchive = "Can not open the file as archive"; +static LPCSTR const kCantFindArchive = "Cannot find archive file"; +static LPCSTR const kCantOpenArchive = "Cannot open the file as archive"; struct CThreadExtracting { diff --git a/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp b/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp index 0c302332..86b4f0fb 100644 --- a/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp +++ b/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp @@ -44,32 +44,32 @@ static bool ReadDataString(CFSTR fileName, LPCSTR startID, NIO::CInFile inFile; if (!inFile.Open(fileName)) return false; - const int kBufferSize = (1 << 12); + const size_t kBufferSize = (1 << 12); Byte buffer[kBufferSize]; - int signatureStartSize = MyStringLen(startID); - int signatureEndSize = MyStringLen(endID); + const unsigned signatureStartSize = MyStringLen(startID); + const unsigned signatureEndSize = MyStringLen(endID); - UInt32 numBytesPrev = 0; + size_t numBytesPrev = 0; bool writeMode = false; UInt64 posTotal = 0; for (;;) { if (posTotal > (1 << 20)) return (stringResult.IsEmpty()); - UInt32 numReadBytes = kBufferSize - numBytesPrev; - UInt32 processedSize; - if (!inFile.Read(buffer + numBytesPrev, numReadBytes, processedSize)) + const size_t numReadBytes = kBufferSize - numBytesPrev; + size_t processedSize; + if (!inFile.ReadFull(buffer + numBytesPrev, numReadBytes, processedSize)) return false; if (processedSize == 0) return true; - UInt32 numBytesInBuffer = numBytesPrev + processedSize; + const size_t numBytesInBuffer = numBytesPrev + processedSize; UInt32 pos = 0; for (;;) { if (writeMode) { - if (pos > numBytesInBuffer - signatureEndSize) + if (pos + signatureEndSize > numBytesInBuffer) break; if (memcmp(buffer + pos, endID, signatureEndSize) == 0) return true; @@ -81,7 +81,7 @@ static bool ReadDataString(CFSTR fileName, LPCSTR startID, } else { - if (pos > numBytesInBuffer - signatureStartSize) + if (pos + signatureStartSize > numBytesInBuffer) break; if (memcmp(buffer + pos, startID, signatureStartSize) == 0) { @@ -111,7 +111,9 @@ struct CInstallIDInit } g_CInstallIDInit; +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ShowErrorMessage(L"Unsupported Windows version"); return 1; +#endif static void ShowErrorMessageSpec(const UString &name) { @@ -207,7 +209,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (!tempDir.Create(kTempDirPrefix)) { if (!assumeYes) - ShowErrorMessage(L"Can not create temp folder archive"); + ShowErrorMessage(L"Cannot create temp folder archive"); return 1; } @@ -217,7 +219,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, HRESULT result = codecs->Load(); if (result != S_OK) { - ShowErrorMessage(L"Can not load codecs"); + ShowErrorMessage(L"Cannot load codecs"); return 1; } } @@ -292,7 +294,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (result <= 32) { if (!assumeYes) - ShowErrorMessage(L"Can not open file"); + ShowErrorMessage(L"Cannot open file"); return 1; } hProcess = execInfo.hProcess; @@ -303,10 +305,10 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (appLaunched.IsEmpty()) { appLaunched = L"setup.exe"; - if (!NFind::DoesFileExist(us2fs(appLaunched))) + if (!NFind::DoesFileExist_FollowLink(us2fs(appLaunched))) { if (!assumeYes) - ShowErrorMessage(L"Can not find setup.exe"); + ShowErrorMessage(L"Cannot find setup.exe"); return 1; } } diff --git a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 14492ca1..c80736de 100644 --- a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -701,6 +701,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\StringConvert.cpp # End Source File # Begin Source File @@ -960,6 +964,11 @@ SOURCE=..\..\..\..\C\Sha256.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXWin/SfxWin.cpp b/CPP/7zip/Bundles/SFXWin/SfxWin.cpp index 3c77356e..2f809322 100644 --- a/CPP/7zip/Bundles/SFXWin/SfxWin.cpp +++ b/CPP/7zip/Bundles/SFXWin/SfxWin.cpp @@ -67,12 +67,12 @@ bool g_LVN_ITEMACTIVATE_Support = true; static const wchar_t * const kUnknownExceptionMessage = L"ERROR: Unknown Error!"; -void ErrorMessageForHRESULT(HRESULT res) +static void ErrorMessageForHRESULT(HRESULT res) { ShowErrorMessage(HResultToMessage(res)); } -int APIENTRY WinMain2() +static int APIENTRY WinMain2() { // OleInitialize is required for ProgressBar in TaskBar. #ifndef UNDER_CE @@ -206,7 +206,9 @@ int APIENTRY WinMain2() return NExitCode::kFatalError; } +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ShowErrorMessage(L"Unsupported Windows version"); return NExitCode::kFatalError; +#endif int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, #ifdef UNDER_CE diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index b3b4546f..57d922fb 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -143,11 +143,11 @@ C_OBJS = \ $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ - $O\Sha256.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/makefile b/CPP/7zip/Bundles/makefile index e3b4b88f..0651da54 100644 --- a/CPP/7zip/Bundles/makefile +++ b/CPP/7zip/Bundles/makefile @@ -1,5 +1,6 @@ DIRS = \ Alone\~ \ + Alone2\~ \ Alone7z\~ \ Fm\~ \ Format7z\~ \ diff --git a/CPP/7zip/Common/CWrappers.cpp b/CPP/7zip/Common/CWrappers.cpp index d6b04206..ee4c36a2 100644 --- a/CPP/7zip/Common/CWrappers.cpp +++ b/CPP/7zip/Common/CWrappers.cpp @@ -1,4 +1,4 @@ -// CWrappers.h +// CWrappers.c #include "StdAfx.h" @@ -158,27 +158,27 @@ void CSeekInStreamWrap::Init(IInStream *stream) throw() void CByteInBufWrap::Free() throw() { ::MidFree(Buf); - Buf = 0; + Buf = NULL; } bool CByteInBufWrap::Alloc(UInt32 size) throw() { - if (Buf == 0 || size != Size) + if (!Buf || size != Size) { Free(); Lim = Cur = Buf = (Byte *)::MidAlloc((size_t)size); Size = size; } - return (Buf != 0); + return (Buf != NULL); } Byte CByteInBufWrap::ReadByteFromNewBlock() throw() { - if (Res == S_OK) + if (!Extra && Res == S_OK) { UInt32 avail; - Processed += (Cur - Buf); Res = Stream->Read(Buf, Size, &avail); + Processed += (size_t)(Cur - Buf); Cur = Buf; Lim = Buf + avail; if (avail != 0) @@ -196,41 +196,88 @@ static Byte Wrap_ReadByte(const IByteIn *pp) throw() return p->ReadByteFromNewBlock(); } -CByteInBufWrap::CByteInBufWrap(): Buf(0) +CByteInBufWrap::CByteInBufWrap(): Buf(NULL) { vt.Read = Wrap_ReadByte; } + +/* ---------- CByteOutBufWrap ---------- */ + +/* +void CLookToSequentialWrap::Free() throw() +{ + ::MidFree(BufBase); + BufBase = NULL; +} + +bool CLookToSequentialWrap::Alloc(UInt32 size) throw() +{ + if (!BufBase || size != Size) + { + Free(); + BufBase = (Byte *)::MidAlloc((size_t)size); + Size = size; + } + return (BufBase != NULL); +} +*/ + +/* +EXTERN_C_BEGIN + +void CLookToSequentialWrap_Look(ILookInSeqStream *pp) +{ + CLookToSequentialWrap *p = (CLookToSequentialWrap *)pp->Obj; + + if (p->Extra || p->Res != S_OK) + return; + { + UInt32 avail; + p->Res = p->Stream->Read(p->BufBase, p->Size, &avail); + p->Processed += avail; + pp->Buf = p->BufBase; + pp->Limit = pp->Buf + avail; + if (avail == 0) + p->Extra = true; + } +} + +EXTERN_C_END +*/ + + /* ---------- CByteOutBufWrap ---------- */ void CByteOutBufWrap::Free() throw() { ::MidFree(Buf); - Buf = 0; + Buf = NULL; } bool CByteOutBufWrap::Alloc(size_t size) throw() { - if (Buf == 0 || size != Size) + if (!Buf || size != Size) { Free(); Buf = (Byte *)::MidAlloc(size); Size = size; } - return (Buf != 0); + return (Buf != NULL); } HRESULT CByteOutBufWrap::Flush() throw() { if (Res == S_OK) { - size_t size = (Cur - Buf); + const size_t size = (size_t)(Cur - Buf); Res = WriteStream(Stream, Buf, size); if (Res == S_OK) Processed += size; - Cur = Buf; + // else throw 11; } + Cur = Buf; // reset pointer for later Wrap_WriteByte() return Res; } @@ -244,7 +291,57 @@ static void Wrap_WriteByte(const IByteOut *pp, Byte b) throw() p->Flush(); } -CByteOutBufWrap::CByteOutBufWrap() throw(): Buf(0) +CByteOutBufWrap::CByteOutBufWrap() throw(): Buf(NULL), Size(0) { vt.Write = Wrap_WriteByte; } + + +/* ---------- CLookOutWrap ---------- */ + +/* +void CLookOutWrap::Free() throw() +{ + ::MidFree(Buf); + Buf = NULL; +} + +bool CLookOutWrap::Alloc(size_t size) throw() +{ + if (!Buf || size != Size) + { + Free(); + Buf = (Byte *)::MidAlloc(size); + Size = size; + } + return (Buf != NULL); +} + +static size_t LookOutWrap_GetOutBuf(const ILookOutStream *pp, void **buf) throw() +{ + CLookOutWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CLookOutWrap, vt); + *buf = p->Buf; + return p->Size; +} + +static size_t LookOutWrap_Write(const ILookOutStream *pp, size_t size) throw() +{ + CLookOutWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CLookOutWrap, vt); + if (p->Res == S_OK && size != 0) + { + p->Res = WriteStream(p->Stream, p->Buf, size); + if (p->Res == S_OK) + { + p->Processed += size; + return size; + } + } + return 0; +} + +CLookOutWrap::CLookOutWrap() throw(): Buf(NULL), Size(0) +{ + vt.GetOutBuf = LookOutWrap_GetOutBuf; + vt.Write = LookOutWrap_Write; +} +*/ diff --git a/CPP/7zip/Common/CWrappers.h b/CPP/7zip/Common/CWrappers.h index e5c890d8..e7196a5c 100644 --- a/CPP/7zip/Common/CWrappers.h +++ b/CPP/7zip/Common/CWrappers.h @@ -74,7 +74,7 @@ struct CByteInBufWrap Extra = false; Res = S_OK; } - UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + UInt64 GetProcessed() const { return Processed + (size_t)(Cur - Buf); } Byte ReadByteFromNewBlock() throw(); Byte ReadByte() { @@ -85,6 +85,45 @@ struct CByteInBufWrap }; +/* +struct CLookToSequentialWrap +{ + Byte *BufBase; + UInt32 Size; + ISequentialInStream *Stream; + UInt64 Processed; + bool Extra; + HRESULT Res; + + CLookToSequentialWrap(): BufBase(NULL) {} + ~CLookToSequentialWrap() { Free(); } + void Free() throw(); + bool Alloc(UInt32 size) throw(); + void Init() + { + // Lim = Cur = Buf; + Processed = 0; + Extra = false; + Res = S_OK; + } + // UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + + Byte ReadByteFromNewBlock() throw(); + Byte ReadByte() + { + if (Cur != Lim) + return *Cur++; + return ReadByteFromNewBlock(); + } +}; + +EXTERN_C_BEGIN +// void CLookToSequentialWrap_Look(ILookInSeqStream *pp); +EXTERN_C_END +*/ + + + struct CByteOutBufWrap { IByteOut vt; @@ -107,7 +146,7 @@ struct CByteOutBufWrap Processed = 0; Res = S_OK; } - UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + UInt64 GetProcessed() const { return Processed + (size_t)(Cur - Buf); } HRESULT Flush() throw(); void WriteByte(Byte b) { @@ -117,4 +156,27 @@ struct CByteOutBufWrap } }; + +/* +struct CLookOutWrap +{ + ILookOutStream vt; + Byte *Buf; + size_t Size; + ISequentialOutStream *Stream; + UInt64 Processed; + HRESULT Res; + + CLookOutWrap() throw(); + ~CLookOutWrap() { Free(); } + void Free() throw(); + bool Alloc(size_t size) throw(); + void Init() + { + Processed = 0; + Res = S_OK; + } +}; +*/ + #endif diff --git a/CPP/7zip/Common/CreateCoder.cpp b/CPP/7zip/Common/CreateCoder.cpp index bf525dc8..e62bb0b4 100644 --- a/CPP/7zip/Common/CreateCoder.cpp +++ b/CPP/7zip/Common/CreateCoder.cpp @@ -11,19 +11,23 @@ #include "RegisterCodec.h" static const unsigned kNumCodecsMax = 64; +extern +unsigned g_NumCodecs; unsigned g_NumCodecs = 0; +extern +const CCodecInfo *g_Codecs[]; const CCodecInfo *g_Codecs[kNumCodecsMax]; // We use g_ExternalCodecs in other stages. -/* #ifdef EXTERNAL_CODECS +/* extern CExternalCodecs g_ExternalCodecs; #define CHECK_GLOBAL_CODECS \ if (!__externalCodecs || !__externalCodecs->IsSet()) __externalCodecs = &g_ExternalCodecs; -#endif */ - #define CHECK_GLOBAL_CODECS +#endif + void RegisterCodec(const CCodecInfo *codecInfo) throw() { @@ -32,7 +36,11 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw() } static const unsigned kNumHashersMax = 16; +extern +unsigned g_NumHashers; unsigned g_NumHashers = 0; +extern +const CHasherInfo *g_Hashers[]; const CHasherInfo *g_Hashers[kNumHashersMax]; void RegisterHasher(const CHasherInfo *hashInfo) throw() @@ -164,7 +172,7 @@ int FindMethod_Index( { methodId = codec.Id; numStreams = codec.NumStreams; - return i; + return (int)i; } } @@ -181,7 +189,7 @@ int FindMethod_Index( { methodId = codec.Id; numStreams = codec.NumStreams; - return g_NumCodecs + i; + return (int)(g_NumCodecs + i); } } @@ -200,7 +208,7 @@ static int FindMethod_Index( { const CCodecInfo &codec = *g_Codecs[i]; if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder)) - return i; + return (int)i; } #ifdef EXTERNAL_CODECS @@ -212,7 +220,7 @@ static int FindMethod_Index( { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)) - return g_NumCodecs + i; + return (int)(g_NumCodecs + i); } #endif @@ -441,7 +449,7 @@ HRESULT CreateCoder_Id( int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode); if (index < 0) return S_OK; - return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS index, encode, filter, cod); + return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS (unsigned)index, encode, filter, cod); } diff --git a/CPP/7zip/Common/FilePathAutoRename.cpp b/CPP/7zip/Common/FilePathAutoRename.cpp index 0904e696..1ebfd72b 100644 --- a/CPP/7zip/Common/FilePathAutoRename.cpp +++ b/CPP/7zip/Common/FilePathAutoRename.cpp @@ -26,8 +26,8 @@ bool AutoRenamePath(FString &path) FString extension; if (dotPos > slashPos + 1) { - name.DeleteFrom(dotPos); - extension = path.Ptr(dotPos); + name.DeleteFrom((unsigned)dotPos); + extension = path.Ptr((unsigned)dotPos); } name += '_'; diff --git a/CPP/7zip/Common/FileStreams.cpp b/CPP/7zip/Common/FileStreams.cpp index f3a322fc..466f46e9 100644 --- a/CPP/7zip/Common/FileStreams.cpp +++ b/CPP/7zip/Common/FileStreams.cpp @@ -6,6 +6,7 @@ #include <fcntl.h> #include <unistd.h> #include <errno.h> +#include "../../Windows/FileFind.h" #endif #ifdef SUPPORT_DEVICE_FILE @@ -15,22 +16,26 @@ #include "FileStreams.h" -static inline HRESULT ConvertBoolToHRESULT(bool result) +static inline HRESULT GetLastError_HRESULT() { - #ifdef _WIN32 - if (result) - return S_OK; DWORD lastError = ::GetLastError(); if (lastError == 0) return E_FAIL; return HRESULT_FROM_WIN32(lastError); - #else - return result ? S_OK: E_FAIL; - #endif +} + +static inline HRESULT ConvertBoolToHRESULT(bool result) +{ + if (result) + return S_OK; + return GetLastError_HRESULT(); } +#ifdef SUPPORT_DEVICE_FILE static const UInt32 kClusterSize = 1 << 18; +#endif + CInFileStream::CInFileStream(): #ifdef SUPPORT_DEVICE_FILE VirtPos(0), @@ -111,7 +116,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (alignedPos != PhyPos) { UInt64 realNewPosition; - bool result = File.Seek(alignedPos, FILE_BEGIN, realNewPosition); + bool result = File.Seek((Int64)alignedPos, FILE_BEGIN, realNewPosition); if (!result) return ConvertBoolToHRESULT(result); PhyPos = realNewPosition; @@ -140,7 +145,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (VirtPos != PhyPos) { UInt64 realNewPosition; - bool result = File.Seek(VirtPos, FILE_BEGIN, realNewPosition); + bool result = File.Seek((Int64)VirtPos, FILE_BEGIN, realNewPosition); if (!result) return ConvertBoolToHRESULT(result); PhyPos = VirtPos = realNewPosition; @@ -149,7 +154,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) #endif UInt32 realProcessedSize; - bool result = File.ReadPart(data, size, realProcessedSize); + const bool result = File.ReadPart(data, size, realProcessedSize); if (processedSize) *processedSize = realProcessedSize; @@ -161,33 +166,27 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (result) return S_OK; + #else // USE_WIN_FILE + + if (processedSize) + *processedSize = 0; + const ssize_t res = File.read_part(data, (size_t)size); + if (res != -1) { - DWORD error = ::GetLastError(); + if (processedSize) + *processedSize = (UInt32)res; + return S_OK; + } + #endif // USE_WIN_FILE + { + const DWORD error = ::GetLastError(); if (Callback) return Callback->InFileStream_On_Error(CallbackRef, error); if (error == 0) return E_FAIL; - return HRESULT_FROM_WIN32(error); } - - #else - - if (processedSize) - *processedSize = 0; - ssize_t res = File.Read(data, (size_t)size); - if (res == -1) - { - if (Callback) - return Callback->InFileStream_On_Error(CallbackRef, E_FAIL); - return E_FAIL; - } - if (processedSize) - *processedSize = (UInt32)res; - return S_OK; - - #endif } #ifdef UNDER_CE @@ -228,7 +227,7 @@ STDMETHODIMP CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSi } while (res < 0 && (errno == EINTR)); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); if (processedSize) *processedSize = (UInt32)res; return S_OK; @@ -257,9 +256,9 @@ STDMETHODIMP CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - VirtPos = offset; + VirtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } #endif @@ -277,9 +276,9 @@ STDMETHODIMP CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos #else - off_t res = File.Seek((off_t)offset, seekOrigin); + off_t res = File.seek((off_t)offset, (int)seekOrigin); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); if (newPosition) *newPosition = (UInt64)res; return S_OK; @@ -306,7 +305,7 @@ STDMETHODIMP CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aT if (attrib) *attrib = info.dwFileAttributes; return S_OK; } - return GetLastError(); + return GetLastError_HRESULT(); } STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) @@ -325,7 +324,71 @@ STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) props->MTime = info.ftLastWriteTime; return S_OK; } - return GetLastError(); + return GetLastError_HRESULT(); +} + +#elif !defined(_WIN32) + +STDMETHODIMP CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib) +{ + struct stat st; + if (File.my_fstat(&st) != 0) + return GetLastError_HRESULT(); + + if (size) *size = (UInt64)st.st_size; + #ifdef __APPLE__ + if (cTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_ctimespec, *cTime); + if (aTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_atimespec, *aTime); + if (mTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_mtimespec, *mTime); + #else + if (cTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_ctim, *cTime); + if (aTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_atim, *aTime); + if (mTime) NWindows::NFile::NFind::timespec_To_FILETIME(st.st_mtim, *mTime); + #endif + if (attrib) *attrib = NWindows::NFile::NFind::Get_WinAttribPosix_From_PosixMode(st.st_mode); + + return S_OK; +} + +// #include <stdio.h> + +STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) +{ + struct stat st; + if (File.my_fstat(&st) != 0) + return GetLastError_HRESULT(); + + props->Size = (UInt64)st.st_size; + /* + dev_t stat::st_dev: + GCC:Linux long unsigned int : __dev_t + Mac: int + */ + props->VolID = (UInt64)(Int64)st.st_dev; + props->FileID_Low = st.st_ino; + props->FileID_High = 0; + props->NumLinks = (UInt32)st.st_nlink; // we reduce to UInt32 from (nlink_t) that is (unsigned long) + props->Attrib = NWindows::NFile::NFind::Get_WinAttribPosix_From_PosixMode(st.st_mode); + + #ifdef __APPLE__ + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_ctimespec, props->CTime); + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_atimespec, props->ATime); + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_mtimespec, props->MTime); + #else + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_ctim, props->CTime); + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_atim, props->ATime); + NWindows::NFile::NFind::timespec_To_FILETIME(st.st_mtim, props->MTime); + #endif + + /* + printf("\nGetProps2() NumLinks=%d = st_dev=%d st_ino = %d\n" + , (unsigned)(props->NumLinks) + , (unsigned)(st.st_dev) + , (unsigned)(st.st_ino) + ); + */ + + return S_OK; } #endif @@ -343,7 +406,7 @@ STDMETHODIMP COutFileStream::Write(const void *data, UInt32 size, UInt32 *proces #ifdef USE_WIN_FILE UInt32 realProcessedSize; - bool result = File.Write(data, size, realProcessedSize); + const bool result = File.Write(data, size, realProcessedSize); ProcessedSize += realProcessedSize; if (processedSize) *processedSize = realProcessedSize; @@ -353,12 +416,13 @@ STDMETHODIMP COutFileStream::Write(const void *data, UInt32 size, UInt32 *proces if (processedSize) *processedSize = 0; - ssize_t res = File.Write(data, (size_t)size); - if (res == -1) - return E_FAIL; + size_t realProcessedSize; + const ssize_t res = File.write_full(data, (size_t)size, realProcessedSize); + ProcessedSize += realProcessedSize; if (processedSize) - *processedSize = (UInt32)res; - ProcessedSize += res; + *processedSize = (UInt32)realProcessedSize; + if (res == -1) + return GetLastError_HRESULT(); return S_OK; #endif @@ -379,9 +443,9 @@ STDMETHODIMP COutFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo #else - off_t res = File.Seek((off_t)offset, seekOrigin); + off_t res = File.seek((off_t)offset, (int)seekOrigin); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); if (newPosition) *newPosition = (UInt64)res; return S_OK; @@ -403,8 +467,11 @@ STDMETHODIMP COutFileStream::SetSize(UInt64 newSize) #else - return E_FAIL; - + // SetLength() uses ftruncate() that doesn't change file offset + if (!File.SetLength(newSize)) + return GetLastError_HRESULT(); + return S_OK; + #endif } @@ -462,7 +529,7 @@ STDMETHODIMP CStdOutFileStream::Write(const void *data, UInt32 size, UInt32 *pro while (res < 0 && (errno == EINTR)); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); _size += (size_t)res; if (processedSize) diff --git a/CPP/7zip/Common/FileStreams.h b/CPP/7zip/Common/FileStreams.h index ef2986fd..fe9f4c19 100644 --- a/CPP/7zip/Common/FileStreams.h +++ b/CPP/7zip/Common/FileStreams.h @@ -7,42 +7,30 @@ #define USE_WIN_FILE #endif +#include "../../Common/MyCom.h" #include "../../Common/MyString.h" -#ifdef USE_WIN_FILE #include "../../Windows/FileIO.h" -#else -#include "../../Common/C_FileIO.h" -#endif - -#include "../../Common/MyCom.h" #include "../IStream.h" -#ifdef _WIN32 -typedef UINT_PTR My_UINT_PTR; -#else -typedef UINT My_UINT_PTR; -#endif - struct IInFileStream_Callback { - virtual HRESULT InFileStream_On_Error(My_UINT_PTR val, DWORD error) = 0; - virtual void InFileStream_On_Destroy(My_UINT_PTR val) = 0; + virtual HRESULT InFileStream_On_Error(UINT_PTR val, DWORD error) = 0; + virtual void InFileStream_On_Destroy(UINT_PTR val) = 0; }; class CInFileStream: public IInStream, public IStreamGetSize, - #ifdef USE_WIN_FILE public IStreamGetProps, public IStreamGetProps2, - #endif public CMyUnknownImp { public: - #ifdef USE_WIN_FILE NWindows::NFile::NIO::CInFile File; + + #ifdef USE_WIN_FILE #ifdef SUPPORT_DEVICE_FILE UInt64 VirtPos; @@ -52,14 +40,12 @@ public: UInt32 BufSize; #endif - #else - NC::NFile::NIO::CInFile File; #endif bool SupportHardLinks; IInFileStream_Callback *Callback; - My_UINT_PTR CallbackRef; + UINT_PTR CallbackRef; virtual ~CInFileStream(); @@ -77,10 +63,8 @@ public: MY_QUERYINTERFACE_BEGIN2(IInStream) MY_QUERYINTERFACE_ENTRY(IStreamGetSize) - #ifdef USE_WIN_FILE MY_QUERYINTERFACE_ENTRY(IStreamGetProps) MY_QUERYINTERFACE_ENTRY(IStreamGetProps2) - #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -88,10 +72,8 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); STDMETHOD(GetSize)(UInt64 *size); - #ifdef USE_WIN_FILE STDMETHOD(GetProps)(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib); STDMETHOD(GetProps2)(CStreamFileProps *props); - #endif }; class CStdInFileStream: @@ -110,11 +92,8 @@ class COutFileStream: public CMyUnknownImp { public: - #ifdef USE_WIN_FILE NWindows::NFile::NIO::COutFile File; - #else - NC::NFile::NIO::COutFile File; - #endif + virtual ~COutFileStream() {} bool Create(CFSTR fileName, bool createAlways) { @@ -131,14 +110,11 @@ public: UInt64 ProcessedSize; - #ifdef USE_WIN_FILE bool SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) { return File.SetTime(cTime, aTime, mTime); } bool SetMTime(const FILETIME *mTime) { return File.SetMTime(mTime); } - #endif - MY_UNKNOWN_IMP1(IOutStream) @@ -146,6 +122,15 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); STDMETHOD(SetSize)(UInt64 newSize); + bool SeekToBegin_bool() + { + #ifdef USE_WIN_FILE + return File.SeekToBegin(); + #else + return File.seekToBegin() == 0; + #endif + } + HRESULT GetSize(UInt64 *size); }; diff --git a/CPP/7zip/Common/FilterCoder.cpp b/CPP/7zip/Common/FilterCoder.cpp index f4c00273..fb99f610 100644 --- a/CPP/7zip/Common/FilterCoder.cpp +++ b/CPP/7zip/Common/FilterCoder.cpp @@ -86,47 +86,47 @@ STDMETHODIMP CFilterCoder::Code(ISequentialInStream *inStream, ISequentialOutStr { RINOK(Init_and_Alloc()); + UInt64 prev = 0; UInt64 nowPos64 = 0; bool inputFinished = false; UInt32 pos = 0; while (!outSize || nowPos64 < *outSize) { - UInt32 endPos = pos; - if (!inputFinished) { size_t processedSize = _bufSize - pos; RINOK(ReadStream(inStream, _buf + pos, &processedSize)); - endPos = pos + (UInt32)processedSize; - inputFinished = (endPos != _bufSize); + pos += (UInt32)processedSize; + inputFinished = (pos != _bufSize); } - pos = Filter->Filter(_buf, endPos); + if (pos == 0) + return S_OK; + + UInt32 filtered = Filter->Filter(_buf, pos); - if (pos > endPos) + if (filtered > pos) { // AES - if (!inputFinished || pos > _bufSize) + if (!inputFinished || filtered > _bufSize) return E_FAIL; if (!_encodeMode) return S_FALSE; + Byte *buf = _buf; do - _buf[endPos] = 0; - while (++endPos != pos); + buf[pos] = 0; + while (++pos != filtered); - if (pos != Filter->Filter(_buf, pos)) + if (filtered != Filter->Filter(buf, filtered)) return E_FAIL; } - if (endPos == 0) - return S_OK; - - UInt32 size = (pos != 0 ? pos : endPos); + UInt32 size = (filtered != 0 ? filtered : pos); if (outSize) { - UInt64 remSize = *outSize - nowPos64; + const UInt64 remSize = *outSize - nowPos64; if (size > remSize) size = (UInt32)remSize; } @@ -134,16 +134,17 @@ STDMETHODIMP CFilterCoder::Code(ISequentialInStream *inStream, ISequentialOutStr RINOK(WriteStream(outStream, _buf, size)); nowPos64 += size; - if (pos == 0) + if (filtered == 0) return S_OK; + pos -= filtered; + for (UInt32 i = 0; i < pos; i++) + _buf[i] = _buf[filtered++]; - if (progress) + if (progress && (nowPos64 - prev) >= (1 << 22)) + { + prev = nowPos64; RINOK(progress->SetRatioInfo(&nowPos64, &nowPos64)); - - UInt32 i = 0; - while (pos < endPos) - _buf[i++] = _buf[pos++]; - pos = i; + } } return S_OK; diff --git a/CPP/7zip/Common/InBuffer.cpp b/CPP/7zip/Common/InBuffer.cpp index d1bc9b9a..6f6eecad 100644 --- a/CPP/7zip/Common/InBuffer.cpp +++ b/CPP/7zip/Common/InBuffer.cpp @@ -56,7 +56,7 @@ bool CInBufferBase::ReadBlock() #endif if (_wasFinished) return false; - _processedSize += (_buf - _bufBase); + _processedSize += (size_t)(_buf - _bufBase); _buf = _bufBase; _bufLim = _bufBase; UInt32 processed; @@ -100,7 +100,7 @@ size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) size_t num = 0; for (;;) { - const size_t rem = _bufLim - _buf; + const size_t rem = (size_t)(_bufLim - _buf); if (size <= rem) { if (size != 0) @@ -148,7 +148,7 @@ size_t CInBufferBase::Skip(size_t size) size_t processed = 0; for (;;) { - size_t rem = (_bufLim - _buf); + const size_t rem = (size_t)(_bufLim - _buf); if (rem >= size) { _buf += size; diff --git a/CPP/7zip/Common/InBuffer.h b/CPP/7zip/Common/InBuffer.h index 4b9c2c32..fa063949 100644 --- a/CPP/7zip/Common/InBuffer.h +++ b/CPP/7zip/Common/InBuffer.h @@ -38,8 +38,16 @@ public: CInBufferBase() throw(); - UInt64 GetStreamSize() const { return _processedSize + (_buf - _bufBase); } - UInt64 GetProcessedSize() const { return _processedSize + NumExtraBytes + (_buf - _bufBase); } + // the size of portion of data in real stream that was already read from this object + // it doesn't include unused data in buffer + // it doesn't include virtual Extra bytes after the end of real stream data + UInt64 GetStreamSize() const { return _processedSize + (size_t)(_buf - _bufBase); } + + // the size of virtual data that was read from this object + // it doesn't include unused data in buffers + // it includes any virtual Extra bytes after the end of real data + UInt64 GetProcessedSize() const { return _processedSize + NumExtraBytes + (size_t)(_buf - _bufBase); } + bool WasFinished() const { return _wasFinished; } void SetStream(ISequentialInStream *stream) { _stream = stream; } @@ -68,6 +76,15 @@ public: b = *_buf++; return true; } + + MY_FORCE_INLINE + bool ReadByte_FromBuf(Byte &b) + { + if (_buf >= _bufLim) + return false; + b = *_buf++; + return true; + } MY_FORCE_INLINE Byte ReadByte() diff --git a/CPP/7zip/Common/InOutTempBuffer.cpp b/CPP/7zip/Common/InOutTempBuffer.cpp index 7523f3a3..cae6b803 100644 --- a/CPP/7zip/Common/InOutTempBuffer.cpp +++ b/CPP/7zip/Common/InOutTempBuffer.cpp @@ -2,13 +2,13 @@ #include "StdAfx.h" -#include "../../../C/7zCrc.h" - -#include "../../Common/Defs.h" - #include "InOutTempBuffer.h" #include "StreamUtils.h" +#ifdef USE_InOutTempBuffer_FILE + +#include "../../../C/7zCrc.h" + using namespace NWindows; using namespace NFile; using namespace NDir; @@ -16,50 +16,58 @@ using namespace NDir; static const size_t kTempBufSize = (1 << 20); #define kTempFilePrefixString FTEXT("7zt") +CInOutTempBuffer::~CInOutTempBuffer() +{ + delete []_buf; +} +#endif -CInOutTempBuffer::CInOutTempBuffer(): _buf(NULL) { } +CInOutTempBuffer::CInOutTempBuffer() + #ifdef USE_InOutTempBuffer_FILE + : _buf(NULL) + #endif +{ } void CInOutTempBuffer::Create() { + #ifdef USE_InOutTempBuffer_FILE if (!_buf) _buf = new Byte[kTempBufSize]; -} - -CInOutTempBuffer::~CInOutTempBuffer() -{ - delete []_buf; + #endif } void CInOutTempBuffer::InitWriting() { + #ifdef USE_InOutTempBuffer_FILE _bufPos = 0; + _crc = CRC_INIT_VAL; _tempFileCreated = false; + #endif _size = 0; - _crc = CRC_INIT_VAL; } -bool CInOutTempBuffer::WriteToFile(const void *data, UInt32 size) + +#ifdef USE_InOutTempBuffer_FILE + +static inline HRESULT Get_HRESULT_LastError() { - if (size == 0) - return true; - if (!_tempFileCreated) - { - if (!_tempFile.CreateRandomInTempFolder(kTempFilePrefixString, &_outFile)) - return false; - _tempFileCreated = true; - } - UInt32 processed; - if (!_outFile.Write(data, size, processed)) - return false; - _crc = CrcUpdate(_crc, data, processed); - _size += processed; - return (processed == size); + #ifdef _WIN32 + DWORD lastError = ::GetLastError(); + if (lastError != 0) + return HRESULT_FROM_WIN32(lastError); + #endif + return E_FAIL; } -bool CInOutTempBuffer::Write(const void *data, UInt32 size) +#endif + + +HRESULT CInOutTempBuffer::Write_HRESULT(const void *data, UInt32 size) { + #ifdef USE_InOutTempBuffer_FILE + if (size == 0) - return true; + return S_OK; size_t cur = kTempBufSize - _bufPos; if (cur != 0) { @@ -72,11 +80,42 @@ bool CInOutTempBuffer::Write(const void *data, UInt32 size) size -= (UInt32)cur; data = ((const Byte *)data) + cur; } - return WriteToFile(data, size); + + if (size == 0) + return S_OK; + + if (!_tempFileCreated) + { + if (!_tempFile.CreateRandomInTempFolder(kTempFilePrefixString, &_outFile)) + return Get_HRESULT_LastError(); + _tempFileCreated = true; + } + UInt32 processed; + if (!_outFile.Write(data, size, processed)) + return Get_HRESULT_LastError(); + _crc = CrcUpdate(_crc, data, processed); + _size += processed; + return (processed == size) ? S_OK : E_FAIL; + + #else + + const size_t newSize = _size + size; + if (newSize < _size) + return E_OUTOFMEMORY; + if (!_dynBuffer.EnsureCapacity(newSize)) + return E_OUTOFMEMORY; + memcpy(((Byte *)_dynBuffer) + _size, data, size); + _size = newSize; + return S_OK; + + #endif } + HRESULT CInOutTempBuffer::WriteToStream(ISequentialOutStream *stream) { + #ifdef USE_InOutTempBuffer_FILE + if (!_outFile.Close()) return E_FAIL; @@ -107,8 +146,13 @@ HRESULT CInOutTempBuffer::WriteToStream(ISequentialOutStream *stream) size += processed; } } - return (_crc == crc && size == _size) ? S_OK : E_FAIL; + + #else + + return WriteStream(stream, (const Byte *)_dynBuffer, _size); + + #endif } /* diff --git a/CPP/7zip/Common/InOutTempBuffer.h b/CPP/7zip/Common/InOutTempBuffer.h index 204a105f..755935ea 100644 --- a/CPP/7zip/Common/InOutTempBuffer.h +++ b/CPP/7zip/Common/InOutTempBuffer.h @@ -3,30 +3,48 @@ #ifndef __IN_OUT_TEMP_BUFFER_H #define __IN_OUT_TEMP_BUFFER_H -#include "../../Common/MyCom.h" +#ifdef _WIN32 +// #define USE_InOutTempBuffer_FILE +#endif + +#ifdef USE_InOutTempBuffer_FILE #include "../../Windows/FileDir.h" +#else +#include "StreamObjects.h" +#endif #include "../IStream.h" class CInOutTempBuffer { + #ifdef USE_InOutTempBuffer_FILE + NWindows::NFile::NDir::CTempFile _tempFile; NWindows::NFile::NIO::COutFile _outFile; + bool _tempFileCreated; Byte *_buf; size_t _bufPos; UInt64 _size; UInt32 _crc; - bool _tempFileCreated; - bool WriteToFile(const void *data, UInt32 size); + #else + + CByteDynBuffer _dynBuffer; + size_t _size; + + #endif + + CLASS_NO_COPY(CInOutTempBuffer); public: CInOutTempBuffer(); - ~CInOutTempBuffer(); void Create(); - void InitWriting(); - bool Write(const void *data, UInt32 size); + #ifdef USE_InOutTempBuffer_FILE + ~CInOutTempBuffer(); + #endif + void InitWriting(); + HRESULT Write_HRESULT(const void *data, UInt32 size); HRESULT WriteToStream(ISequentialOutStream *stream); UInt64 GetDataSize() const { return _size; } }; diff --git a/CPP/7zip/Common/LimitedStreams.cpp b/CPP/7zip/Common/LimitedStreams.cpp index 8e032561..add6636b 100644 --- a/CPP/7zip/Common/LimitedStreams.cpp +++ b/CPP/7zip/Common/LimitedStreams.cpp @@ -67,7 +67,7 @@ STDMETHODIMP CLimitedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *new } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -115,7 +115,7 @@ STDMETHODIMP CClusterInStream::Read(void *data, UInt32 size, UInt32 *processedSi _curRem = blockSize - offsetInBlock; - for (int i = 1; i < 64 && (virtBlock + i) < (UInt32)Vector.Size() && phyBlock + i == Vector[virtBlock + i]; i++) + for (unsigned i = 1; i < 64 && (virtBlock + i) < (UInt32)Vector.Size() && phyBlock + i == Vector[virtBlock + i]; i++) _curRem += (UInt32)1 << BlockSizeLog; } @@ -143,9 +143,9 @@ STDMETHODIMP CClusterInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *new return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; if (_virtPos != (UInt64)offset) _curRem = 0; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -203,7 +203,7 @@ STDMETHODIMP CExtentsStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -268,10 +268,10 @@ STDMETHODIMP CTailInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; - return Stream->Seek(Offset + _virtPos, STREAM_SEEK_SET, NULL); + return Stream->Seek((Int64)(Offset + _virtPos), STREAM_SEEK_SET, NULL); } STDMETHODIMP CLimitedCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSize) @@ -325,7 +325,7 @@ STDMETHODIMP CLimitedCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt6 } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -354,10 +354,10 @@ STDMETHODIMP CTailOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; - return Stream->Seek(Offset + _virtPos, STREAM_SEEK_SET, NULL); + return Stream->Seek((Int64)(Offset + _virtPos), STREAM_SEEK_SET, NULL); } STDMETHODIMP CTailOutStream::SetSize(UInt64 newSize) diff --git a/CPP/7zip/Common/LimitedStreams.h b/CPP/7zip/Common/LimitedStreams.h index fb1ac3cd..ade29937 100644 --- a/CPP/7zip/Common/LimitedStreams.h +++ b/CPP/7zip/Common/LimitedStreams.h @@ -44,7 +44,7 @@ class CLimitedInStream: UInt64 _size; UInt64 _startOffset; - HRESULT SeekToPhys() { return _stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return _stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } public: void SetStream(IInStream *stream) { _stream = stream; } HRESULT InitAndSeek(UInt64 startOffset, UInt64 size) @@ -80,7 +80,7 @@ public: CRecordVector<UInt32> Vector; UInt64 StartOffset; - HRESULT SeekToPhys() { return Stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return Stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } HRESULT InitAndSeek() { @@ -115,7 +115,7 @@ class CExtentsStream: UInt64 _virtPos; bool _needStartSeek; - HRESULT SeekToPhys() { return Stream->Seek(_phyPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return Stream->Seek((Int64)_phyPos, STREAM_SEEK_SET, NULL); } public: CMyComPtr<IInStream> Stream; @@ -177,7 +177,7 @@ public: STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); - HRESULT SeekToStart() { return Stream->Seek(Offset, STREAM_SEEK_SET, NULL); } + HRESULT SeekToStart() { return Stream->Seek((Int64)Offset, STREAM_SEEK_SET, NULL); } }; class CLimitedCachedInStream: @@ -195,7 +195,7 @@ class CLimitedCachedInStream: size_t _cachePhyPos; - HRESULT SeekToPhys() { return _stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return _stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } public: CByteBuffer Buffer; diff --git a/CPP/7zip/Common/MemBlocks.cpp b/CPP/7zip/Common/MemBlocks.cpp index 5aba046b..f351abbd 100644 --- a/CPP/7zip/Common/MemBlocks.cpp +++ b/CPP/7zip/Common/MemBlocks.cpp @@ -7,21 +7,26 @@ #include "MemBlocks.h" #include "StreamUtils.h" -bool CMemBlockManager::AllocateSpace(size_t numBlocks) +bool CMemBlockManager::AllocateSpace_bool(size_t numBlocks) { FreeSpace(); - if (_blockSize < sizeof(void *) || numBlocks < 1) + if (numBlocks == 0) + { + return true; + // return false; + } + if (_blockSize < sizeof(void *)) return false; - size_t totalSize = numBlocks * _blockSize; + const size_t totalSize = numBlocks * _blockSize; if (totalSize / _blockSize != numBlocks) return false; _data = ::MidAlloc(totalSize); - if (_data == 0) + if (!_data) return false; Byte *p = (Byte *)_data; for (size_t i = 0; i + 1 < numBlocks; i++, p += _blockSize) - *(Byte **)p = (p + _blockSize); - *(Byte **)p = 0; + *(Byte **)(void *)p = (p + _blockSize); + *(Byte **)(void *)p = NULL; _headFree = _data; return true; } @@ -35,41 +40,64 @@ void CMemBlockManager::FreeSpace() void *CMemBlockManager::AllocateBlock() { - if (_headFree == 0) - return 0; void *p = _headFree; - _headFree = *(void **)_headFree; + if (p) + _headFree = *(void **)p; return p; } void CMemBlockManager::FreeBlock(void *p) { - if (p == 0) + if (!p) return; *(void **)p = _headFree; _headFree = p; } +// #include <stdio.h> + HRes CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) { if (numNoLockBlocks > numBlocks) return E_INVALIDARG; - if (!CMemBlockManager::AllocateSpace(numBlocks)) + const size_t numLockBlocks = numBlocks - numNoLockBlocks; + UInt32 maxCount = (UInt32)numLockBlocks; + if (maxCount != numLockBlocks) + return E_OUTOFMEMORY; + if (!CMemBlockManager::AllocateSpace_bool(numBlocks)) return E_OUTOFMEMORY; - size_t numLockBlocks = numBlocks - numNoLockBlocks; Semaphore.Close(); - return Semaphore.Create((LONG)numLockBlocks, (LONG)numLockBlocks); + // we need (maxCount = 1), if we want to create non-use empty Semaphore + if (maxCount == 0) + maxCount = 1; + + // printf("\n Synchro.Create() \n"); + WRes wres; + #ifndef _WIN32 + wres = Synchro.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + wres = Semaphore.Create(&Synchro, (UInt32)numLockBlocks, maxCount); + #else + wres = Semaphore.Create((UInt32)numLockBlocks, maxCount); + #endif + + return HRESULT_FROM_WIN32(wres); } + HRes CMemBlockManagerMt::AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks) { + // desiredNumberOfBlocks = 0; // for debug if (numNoLockBlocks > desiredNumberOfBlocks) return E_INVALIDARG; for (;;) { - if (AllocateSpace(desiredNumberOfBlocks, numNoLockBlocks) == 0) - return 0; + // if (desiredNumberOfBlocks == 0) return E_OUTOFMEMORY; + HRes hres = AllocateSpace(desiredNumberOfBlocks, numNoLockBlocks); + if (hres != E_OUTOFMEMORY) + return hres; if (desiredNumberOfBlocks == numNoLockBlocks) return E_OUTOFMEMORY; desiredNumberOfBlocks = numNoLockBlocks + ((desiredNumberOfBlocks - numNoLockBlocks) >> 1); @@ -91,7 +119,7 @@ void *CMemBlockManagerMt::AllocateBlock() void CMemBlockManagerMt::FreeBlock(void *p, bool lockMode) { - if (p == 0) + if (!p) return; { NWindows::NSynchronization::CCriticalSectionLock lock(_criticalSection); @@ -101,6 +129,8 @@ void CMemBlockManagerMt::FreeBlock(void *p, bool lockMode) Semaphore.Release(); } + + void CMemBlocks::Free(CMemBlockManagerMt *manager) { while (Blocks.Size() > 0) @@ -122,9 +152,9 @@ HRESULT CMemBlocks::WriteToStream(size_t blockSize, ISequentialOutStream *outStr UInt64 totalSize = TotalSize; for (unsigned blockIndex = 0; totalSize > 0; blockIndex++) { - UInt32 curSize = (UInt32)blockSize; - if (totalSize < curSize) - curSize = (UInt32)totalSize; + size_t curSize = blockSize; + if (curSize > totalSize) + curSize = (size_t)totalSize; if (blockIndex >= Blocks.Size()) return E_FAIL; RINOK(WriteStream(outStream, Blocks[blockIndex], curSize)); @@ -134,10 +164,10 @@ HRESULT CMemBlocks::WriteToStream(size_t blockSize, ISequentialOutStream *outStr } -void CMemLockBlocks::FreeBlock(int index, CMemBlockManagerMt *memManager) +void CMemLockBlocks::FreeBlock(unsigned index, CMemBlockManagerMt *memManager) { memManager->FreeBlock(Blocks[index], LockMode); - Blocks[index] = 0; + Blocks[index] = NULL; } void CMemLockBlocks::Free(CMemBlockManagerMt *memManager) @@ -150,6 +180,7 @@ void CMemLockBlocks::Free(CMemBlockManagerMt *memManager) TotalSize = 0; } +/* HRes CMemLockBlocks::SwitchToNoLockMode(CMemBlockManagerMt *memManager) { if (LockMode) @@ -162,13 +193,14 @@ HRes CMemLockBlocks::SwitchToNoLockMode(CMemBlockManagerMt *memManager) } return 0; } +*/ void CMemLockBlocks::Detach(CMemLockBlocks &blocks, CMemBlockManagerMt *memManager) { blocks.Free(memManager); blocks.LockMode = LockMode; UInt64 totalSize = 0; - size_t blockSize = memManager->GetBlockSize(); + const size_t blockSize = memManager->GetBlockSize(); FOR_VECTOR (i, Blocks) { if (totalSize < TotalSize) diff --git a/CPP/7zip/Common/MemBlocks.h b/CPP/7zip/Common/MemBlocks.h index ec56c14d..3c9cefdb 100644 --- a/CPP/7zip/Common/MemBlocks.h +++ b/CPP/7zip/Common/MemBlocks.h @@ -15,10 +15,10 @@ class CMemBlockManager size_t _blockSize; void *_headFree; public: - CMemBlockManager(size_t blockSize = (1 << 20)): _data(0), _blockSize(blockSize), _headFree(0) {} + CMemBlockManager(size_t blockSize = (1 << 20)): _data(NULL), _blockSize(blockSize), _headFree(NULL) {} ~CMemBlockManager() { FreeSpace(); } - bool AllocateSpace(size_t numBlocks); + bool AllocateSpace_bool(size_t numBlocks); void FreeSpace(); size_t GetBlockSize() const { return _blockSize; } void *AllocateBlock(); @@ -30,17 +30,18 @@ class CMemBlockManagerMt: public CMemBlockManager { NWindows::NSynchronization::CCriticalSection _criticalSection; public: - NWindows::NSynchronization::CSemaphore Semaphore; + SYNC_OBJ_DECL(Synchro); + NWindows::NSynchronization::CSemaphore_WFMO Semaphore; CMemBlockManagerMt(size_t blockSize = (1 << 20)): CMemBlockManager(blockSize) {} ~CMemBlockManagerMt() { FreeSpace(); } - HRes AllocateSpace(size_t numBlocks, size_t numNoLockBlocks = 0); + HRes AllocateSpace(size_t numBlocks, size_t numNoLockBlocks); HRes AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks = 0); void FreeSpace(); void *AllocateBlock(); void FreeBlock(void *p, bool lockMode = true); - HRes ReleaseLockedBlocks(int number) { return Semaphore.Release(number); } + // WRes ReleaseLockedBlocks_WRes(unsigned number) { return Semaphore.Release(number); } }; @@ -63,8 +64,8 @@ struct CMemLockBlocks: public CMemBlocks CMemLockBlocks(): LockMode(true) {}; void Free(CMemBlockManagerMt *memManager); - void FreeBlock(int index, CMemBlockManagerMt *memManager); - HRes SwitchToNoLockMode(CMemBlockManagerMt *memManager); + void FreeBlock(unsigned index, CMemBlockManagerMt *memManager); + // HRes SwitchToNoLockMode(CMemBlockManagerMt *memManager); void Detach(CMemLockBlocks &blocks, CMemBlockManagerMt *memManager); }; diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index 8a6ebcaf..bea51db3 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -184,10 +184,12 @@ class CCoderProps unsigned _numProps; unsigned _numPropsMax; public: - CCoderProps(unsigned numPropsMax) + CCoderProps(unsigned numPropsMax): + _propIDs(NULL), + _props(NULL), + _numProps(0), + _numPropsMax(numPropsMax) { - _numPropsMax = numPropsMax; - _numProps = 0; _propIDs = new PROPID[numPropsMax]; _props = new NCOM::CPropVariant[numPropsMax]; } @@ -214,7 +216,15 @@ void CCoderProps::AddProp(const CProp &prop) HRESULT CProps::SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce) const { - CCoderProps coderProps(Props.Size() + (dataSizeReduce ? 1 : 0)); + return SetCoderProps_DSReduce_Aff(scp, dataSizeReduce, NULL); +} + +HRESULT CProps::SetCoderProps_DSReduce_Aff( + ICompressSetCoderProperties *scp, + const UInt64 *dataSizeReduce, + const UInt64 *affinity) const +{ + CCoderProps coderProps(Props.Size() + (dataSizeReduce ? 1 : 0) + (affinity ? 1 : 0) ); FOR_VECTOR (i, Props) coderProps.AddProp(Props[i]); if (dataSizeReduce) @@ -224,27 +234,34 @@ HRESULT CProps::SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *da prop.Value = *dataSizeReduce; coderProps.AddProp(prop); } + if (affinity) + { + CProp prop; + prop.Id = NCoderPropID::kAffinity; + prop.Value = *affinity; + coderProps.AddProp(prop); + } return coderProps.SetProps(scp); } int CMethodProps::FindProp(PROPID id) const { - for (int i = Props.Size() - 1; i >= 0; i--) - if (Props[i].Id == id) + for (int i = (int)Props.Size() - 1; i >= 0; i--) + if (Props[(unsigned)i].Id == id) return i; return -1; } -int CMethodProps::GetLevel() const +unsigned CMethodProps::GetLevel() const { int i = FindProp(NCoderPropID::kLevel); if (i < 0) return 5; - if (Props[i].Value.vt != VT_UI4) + if (Props[(unsigned)i].Value.vt != VT_UI4) return 9; - UInt32 level = Props[i].Value.ulVal; - return level > 9 ? 9 : (int)level; + UInt32 level = Props[(unsigned)i].Value.ulVal; + return level > 9 ? 9 : (unsigned)level; } struct CNameToPropID @@ -286,7 +303,7 @@ static int FindPropIdExact(const UString &name) { for (unsigned i = 0; i < ARRAY_SIZE(g_NameToPropID); i++) if (StringsAreEqualNoCase_Ascii(name, g_NameToPropID[i].Name)) - return i; + return (int)i; return -1; } @@ -346,8 +363,8 @@ static void SplitParam(const UString ¶m, UString &name, UString &value) int eqPos = param.Find(L'='); if (eqPos >= 0) { - name.SetFrom(param, eqPos); - value = param.Ptr(eqPos + 1); + name.SetFrom(param, (unsigned)eqPos); + value = param.Ptr((unsigned)(eqPos + 1)); return; } unsigned i; @@ -382,7 +399,7 @@ HRESULT CMethodProps::SetParam(const UString &name, const UString &value) return E_INVALIDARG; const CNameToPropID &nameToPropID = g_NameToPropID[(unsigned)index]; CProp prop; - prop.Id = index; + prop.Id = (unsigned)index; if (IsLogSizeProp(prop.Id)) { @@ -463,7 +480,7 @@ HRESULT CMethodProps::ParseParamsFromPROPVARIANT(const UString &realName, const return E_INVALIDARG; const CNameToPropID &nameToPropID = g_NameToPropID[(unsigned)index]; CProp prop; - prop.Id = index; + prop.Id = (unsigned)index; if (IsLogSizeProp(prop.Id)) { @@ -485,14 +502,14 @@ HRESULT COneMethodInfo::ParseMethodFromString(const UString &s) { UString temp = s; if (splitPos >= 0) - temp.DeleteFrom(splitPos); + temp.DeleteFrom((unsigned)splitPos); if (!temp.IsAscii()) return E_INVALIDARG; MethodName.SetFromWStr_if_Ascii(temp); } if (splitPos < 0) return S_OK; - PropsString = s.Ptr(splitPos + 1); + PropsString = s.Ptr((unsigned)(splitPos + 1)); return ParseParamsFromString(PropsString); } diff --git a/CPP/7zip/Common/MethodProps.h b/CPP/7zip/Common/MethodProps.h index 39580881..e0519b16 100644 --- a/CPP/7zip/Common/MethodProps.h +++ b/CPP/7zip/Common/MethodProps.h @@ -53,20 +53,21 @@ struct CProps prop.Value = s; } - HRESULT SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce) const; + HRESULT SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce = NULL) const; + HRESULT SetCoderProps_DSReduce_Aff(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce, const UInt64 *affinity) const; }; class CMethodProps: public CProps { HRESULT SetParam(const UString &name, const UString &value); public: - int GetLevel() const; + unsigned GetLevel() const; int Get_NumThreads() const { int i = FindProp(NCoderPropID::kNumThreads); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return (int)Props[i].Value.ulVal; + if (Props[(unsigned)i].Value.vt == VT_UI4) + return (int)Props[(unsigned)i].Value.ulVal; return -1; } @@ -75,9 +76,9 @@ public: res = 0; int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) + if (Props[(unsigned)i].Value.vt == VT_UI4) { - res = Props[i].Value.ulVal; + res = Props[(unsigned)i].Value.ulVal; return true; } return false; @@ -89,8 +90,8 @@ public: { int i = FindProp(NCoderPropID::kAlgorithm); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; + if (Props[(unsigned)i].Value.vt == VT_UI4) + return Props[(unsigned)i].Value.ulVal; return GetLevel() >= 5 ? 1 : 0; } @@ -98,10 +99,14 @@ public: { int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; - int level = GetLevel(); - return level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)); + if (Props[(unsigned)i].Value.vt == VT_UI4) + return Props[(unsigned)i].Value.ulVal; + unsigned level = GetLevel(); + return + ( level <= 3 ? (1 << (level * 2 + 16)) : + ( level <= 6 ? (1 << (level + 19)) : + ( level <= 7 ? (1 << 25) : (1 << 26) + ))); } bool Get_Lzma_Eos() const @@ -109,7 +114,7 @@ public: int i = FindProp(NCoderPropID::kEndMarker); if (i >= 0) { - const NWindows::NCOM::CPropVariant &val = Props[i].Value; + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; if (val.vt == VT_BOOL) return VARIANT_BOOLToBool(val.boolVal); } @@ -150,7 +155,7 @@ public: int i = FindProp(id); if (i >= 0) { - const NWindows::NCOM::CPropVariant &val = Props[i].Value; + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; if (val.vt == VT_UI4) { return val.ulVal; } if (val.vt == VT_UI8) { return val.uhVal.QuadPart; } } @@ -191,8 +196,8 @@ public: fixedNumber = true; if (numThreads < 1) return 1; const unsigned kNumBZip2ThreadsMax = 64; - if (numThreads > kNumBZip2ThreadsMax) return kNumBZip2ThreadsMax; - return numThreads; + if ((unsigned)numThreads > kNumBZip2ThreadsMax) return kNumBZip2ThreadsMax; + return (unsigned)numThreads; } return 1; } @@ -201,16 +206,16 @@ public: { int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) + if (Props[(unsigned)i].Value.vt == VT_UI4) { - UInt32 blockSize = Props[i].Value.ulVal; + UInt32 blockSize = Props[(unsigned)i].Value.ulVal; const UInt32 kDicSizeMin = 100000; const UInt32 kDicSizeMax = 900000; if (blockSize < kDicSizeMin) blockSize = kDicSizeMin; if (blockSize > kDicSizeMax) blockSize = kDicSizeMax; return blockSize; } - int level = GetLevel(); + unsigned level = GetLevel(); return 100000 * (level >= 5 ? 9 : (level >= 1 ? level * 2 - 1: 1)); } @@ -218,10 +223,10 @@ public: { int i = FindProp(NCoderPropID::kUsedMemorySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; - int level = GetLevel(); - return level >= 9 ? (192 << 20) : ((UInt32)1 << (level + 19)); + if (Props[(unsigned)i].Value.vt == VT_UI4) + return Props[(unsigned)i].Value.ulVal; + unsigned level = GetLevel(); + return ((UInt32)1 << (level + 19)); } void AddProp_Level(UInt32 level) diff --git a/CPP/7zip/Common/OffsetStream.cpp b/CPP/7zip/Common/OffsetStream.cpp index 368d39b6..b3e710f9 100644 --- a/CPP/7zip/Common/OffsetStream.cpp +++ b/CPP/7zip/Common/OffsetStream.cpp @@ -10,7 +10,7 @@ HRESULT COffsetOutStream::Init(IOutStream *stream, UInt64 offset) { _offset = offset; _stream = stream; - return _stream->Seek(offset, STREAM_SEEK_SET, NULL); + return _stream->Seek((Int64)offset, STREAM_SEEK_SET, NULL); } STDMETHODIMP COffsetOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) diff --git a/CPP/7zip/Common/OutMemStream.cpp b/CPP/7zip/Common/OutMemStream.cpp index 768c2d45..241589d2 100644 --- a/CPP/7zip/Common/OutMemStream.cpp +++ b/CPP/7zip/Common/OutMemStream.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +// #include <stdio.h> + #include "OutMemStream.h" void COutMemStream::Free() @@ -34,11 +36,12 @@ HRESULT COutMemStream::WriteToRealStream() return S_OK; } + STDMETHODIMP COutMemStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { if (_realStreamMode) return OutSeqStream->Write(data, size, processedSize); - if (processedSize != 0) + if (processedSize) *processedSize = 0; while (size != 0) { @@ -49,7 +52,7 @@ STDMETHODIMP COutMemStream::Write(const void *data, UInt32 size, UInt32 *process if (size < curSize) curSize = size; memcpy(p, data, curSize); - if (processedSize != 0) + if (processedSize) *processedSize += (UInt32)curSize; data = (const void *)((const Byte *)data + curSize); size -= (UInt32)curSize; @@ -65,8 +68,14 @@ STDMETHODIMP COutMemStream::Write(const void *data, UInt32 size, UInt32 *process } continue; } - HANDLE events[3] = { StopWritingEvent, WriteToRealStreamEvent, /* NoLockEvent, */ _memManager->Semaphore }; - DWORD waitResult = ::WaitForMultipleObjects((Blocks.LockMode ? 3 : 2), events, FALSE, INFINITE); + + const NWindows::NSynchronization::CHandle_WFMO events[3] = + { StopWritingEvent, WriteToRealStreamEvent, /* NoLockEvent, */ _memManager->Semaphore }; + const DWORD waitResult = NWindows::NSynchronization::WaitForMultiObj_Any_Infinite( + ((Blocks.LockMode /* && _memManager->Semaphore.IsCreated() */) ? 3 : 2), events); + + // printf("\n 1- outMemStream %d\n", waitResult - WAIT_OBJECT_0); + switch (waitResult) { case (WAIT_OBJECT_0 + 0): @@ -77,27 +86,34 @@ STDMETHODIMP COutMemStream::Write(const void *data, UInt32 size, UInt32 *process RINOK(WriteToRealStream()); UInt32 processedSize2; HRESULT res = OutSeqStream->Write(data, size, &processedSize2); - if (processedSize != 0) + if (processedSize) *processedSize += processedSize2; return res; } - /* case (WAIT_OBJECT_0 + 2): { // it has bug: no write. + /* if (!Blocks.SwitchToNoLockMode(_memManager)) return E_FAIL; + */ break; } - */ - case (WAIT_OBJECT_0 + 2): - break; default: + { + if (waitResult == WAIT_FAILED) + { + DWORD res = ::GetLastError(); + if (res != 0) + return HRESULT_FROM_WIN32(res); + } return E_FAIL; + } } - Blocks.Blocks.Add(_memManager->AllocateBlock()); - if (Blocks.Blocks.Back() == 0) + void *p = _memManager->AllocateBlock(); + if (!p) return E_FAIL; + Blocks.Blocks.Add(p); } return S_OK; } diff --git a/CPP/7zip/Common/OutMemStream.h b/CPP/7zip/Common/OutMemStream.h index 0a892c52..873742ed 100644 --- a/CPP/7zip/Common/OutMemStream.h +++ b/CPP/7zip/Common/OutMemStream.h @@ -17,8 +17,8 @@ class COutMemStream: bool _realStreamMode; bool _unlockEventWasSent; - NWindows::NSynchronization::CAutoResetEvent StopWritingEvent; - NWindows::NSynchronization::CAutoResetEvent WriteToRealStreamEvent; + NWindows::NSynchronization::CAutoResetEvent_WFMO StopWritingEvent; + NWindows::NSynchronization::CAutoResetEvent_WFMO WriteToRealStreamEvent; // NWindows::NSynchronization::CAutoResetEvent NoLockEvent; HRESULT StopWriteResult; @@ -31,10 +31,13 @@ class COutMemStream: public: - HRes CreateEvents() + + HRes CreateEvents(SYNC_PARAM_DECL(synchro)) { - RINOK(StopWritingEvent.CreateIfNotCreated()); - return WriteToRealStreamEvent.CreateIfNotCreated(); + WRes wres = StopWritingEvent.CreateIfNotCreated_Reset(SYNC_WFMO(synchro)); + if (wres == 0) + wres = WriteToRealStreamEvent.CreateIfNotCreated_Reset(SYNC_WFMO(synchro)); + return HRESULT_FROM_WIN32(wres); } void SetOutStream(IOutStream *outStream) @@ -55,7 +58,16 @@ public: OutSeqStream.Release(); } - COutMemStream(CMemBlockManagerMt *memManager): _memManager(memManager) { } + COutMemStream(CMemBlockManagerMt *memManager): + _memManager(memManager) + { + /* + #ifndef _WIN32 + StopWritingEvent._sync = + WriteToRealStreamEvent._sync = &memManager->Synchro; + #endif + */ + } ~COutMemStream() { Free(); } void Free(); diff --git a/CPP/7zip/Common/ProgressMt.cpp b/CPP/7zip/Common/ProgressMt.cpp index 319bd241..c2714a27 100644 --- a/CPP/7zip/Common/ProgressMt.cpp +++ b/CPP/7zip/Common/ProgressMt.cpp @@ -4,12 +4,12 @@ #include "ProgressMt.h" -void CMtCompressProgressMixer::Init(int numItems, ICompressProgressInfo *progress) +void CMtCompressProgressMixer::Init(unsigned numItems, ICompressProgressInfo *progress) { NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection); InSizes.Clear(); OutSizes.Clear(); - for (int i = 0; i < numItems; i++) + for (unsigned i = 0; i < numItems; i++) { InSizes.Add(0); OutSizes.Add(0); @@ -19,23 +19,23 @@ void CMtCompressProgressMixer::Init(int numItems, ICompressProgressInfo *progres _progress = progress; } -void CMtCompressProgressMixer::Reinit(int index) +void CMtCompressProgressMixer::Reinit(unsigned index) { NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection); InSizes[index] = 0; OutSizes[index] = 0; } -HRESULT CMtCompressProgressMixer::SetRatioInfo(int index, const UInt64 *inSize, const UInt64 *outSize) +HRESULT CMtCompressProgressMixer::SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize) { NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection); - if (inSize != 0) + if (inSize) { UInt64 diff = *inSize - InSizes[index]; InSizes[index] = *inSize; TotalInSize += diff; } - if (outSize != 0) + if (outSize) { UInt64 diff = *outSize - OutSizes[index]; OutSizes[index] = *outSize; diff --git a/CPP/7zip/Common/ProgressMt.h b/CPP/7zip/Common/ProgressMt.h index 26079d4e..32da976b 100644 --- a/CPP/7zip/Common/ProgressMt.h +++ b/CPP/7zip/Common/ProgressMt.h @@ -19,9 +19,9 @@ class CMtCompressProgressMixer UInt64 TotalOutSize; public: NWindows::NSynchronization::CCriticalSection CriticalSection; - void Init(int numItems, ICompressProgressInfo *progress); - void Reinit(int index); - HRESULT SetRatioInfo(int index, const UInt64 *inSize, const UInt64 *outSize); + void Init(unsigned numItems, ICompressProgressInfo *progress); + void Reinit(unsigned index); + HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize); }; class CMtCompressProgress: @@ -29,9 +29,9 @@ class CMtCompressProgress: public CMyUnknownImp { CMtCompressProgressMixer *_progress; - int _index; + unsigned _index; public: - void Init(CMtCompressProgressMixer *progress, int index) + void Init(CMtCompressProgressMixer *progress, unsigned index) { _progress = progress; _index = index; diff --git a/CPP/7zip/Common/RegisterCodec.h b/CPP/7zip/Common/RegisterCodec.h index 7ddb7604..a942da7a 100644 --- a/CPP/7zip/Common/RegisterCodec.h +++ b/CPP/7zip/Common/RegisterCodec.h @@ -26,11 +26,11 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); #define REGISTER_CODEC_CREATE(name, cls) REGISTER_CODEC_CREATE_2(name, cls, ICompressCoder) #define REGISTER_CODEC_NAME(x) CRegisterCodec ## x -#define REGISTER_CODEC_VAR static const CCodecInfo g_CodecInfo = +#define REGISTER_CODEC_VAR(x) static const CCodecInfo g_CodecInfo_ ## x = #define REGISTER_CODEC(x) struct REGISTER_CODEC_NAME(x) { \ - REGISTER_CODEC_NAME(x)() { RegisterCodec(&g_CodecInfo); }}; \ - static REGISTER_CODEC_NAME(x) g_RegisterCodec; + REGISTER_CODEC_NAME(x)() { RegisterCodec(&g_CodecInfo_ ## x); }}; \ + static REGISTER_CODEC_NAME(x) g_RegisterCodec_ ## x; #define REGISTER_CODECS_NAME(x) CRegisterCodecs ## x @@ -43,7 +43,7 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); #define REGISTER_CODEC_2(x, crDec, crEnc, id, name) \ - REGISTER_CODEC_VAR \ + REGISTER_CODEC_VAR(x) \ { crDec, crEnc, id, name, 1, false }; \ REGISTER_CODEC(x) @@ -67,19 +67,19 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); { crDec, crEnc, id, name, 1, true } #define REGISTER_FILTER(x, crDec, crEnc, id, name) \ - REGISTER_CODEC_VAR \ + REGISTER_CODEC_VAR(x) \ REGISTER_FILTER_ITEM(crDec, crEnc, id, name); \ REGISTER_CODEC(x) #ifdef EXTRACT_ONLY #define REGISTER_FILTER_E(x, clsDec, clsEnc, id, name) \ - REGISTER_FILTER_CREATE(CreateDec, clsDec) \ - REGISTER_FILTER(x, CreateDec, NULL, id, name) + REGISTER_FILTER_CREATE(x ## _CreateDec, clsDec) \ + REGISTER_FILTER(x, x ## _CreateDec, NULL, id, name) #else #define REGISTER_FILTER_E(x, clsDec, clsEnc, id, name) \ - REGISTER_FILTER_CREATE(CreateDec, clsDec) \ - REGISTER_FILTER_CREATE(CreateEnc, clsEnc) \ - REGISTER_FILTER(x, CreateDec, CreateEnc, id, name) + REGISTER_FILTER_CREATE(x ## _CreateDec, clsDec) \ + REGISTER_FILTER_CREATE(x ## _CreateEnc, clsEnc) \ + REGISTER_FILTER(x, x ## _CreateDec, x ## _CreateEnc, id, name) #endif diff --git a/CPP/7zip/Common/StreamBinder.cpp b/CPP/7zip/Common/StreamBinder.cpp index 435440c6..fbf2a0de 100644 --- a/CPP/7zip/Common/StreamBinder.cpp +++ b/CPP/7zip/Common/StreamBinder.cpp @@ -14,7 +14,7 @@ class CBinderInStream: public: MY_UNKNOWN_IMP1(ISequentialInStream) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); - ~CBinderInStream() { _binder->CloseRead(); } + ~CBinderInStream() { _binder->CloseRead_CallOnce(); } CBinderInStream(CStreamBinder *binder): _binder(binder) {} }; @@ -37,19 +37,24 @@ STDMETHODIMP CBinderOutStream::Write(const void *data, UInt32 size, UInt32 *proc { return _binder->Write(data, size, processedSize); } - -WRes CStreamBinder::CreateEvents() +static HRESULT Event__Create_or_Reset(NWindows::NSynchronization::CAutoResetEvent &event) { - RINOK(_canWrite_Event.Create()); - RINOK(_canRead_Event.Create()); - return _readingWasClosed_Event.Create(); + WRes wres; + if (event.IsCreated()) + wres = event.Reset(); + else + wres = event.Create(); + return HRESULT_FROM_WIN32(wres); } -void CStreamBinder::ReInit() +HRESULT CStreamBinder::Create_ReInit() { - _canWrite_Event.Reset(); - _canRead_Event.Reset(); - _readingWasClosed_Event.Reset(); + RINOK(Event__Create_or_Reset(_canRead_Event)); + // RINOK(Event__Create_or_Reset(_canWrite_Event)); + + _canWrite_Semaphore.Close(); + // we need at least 3 items of maxCount: 1 for normal unlock in Read(), 2 items for unlock in CloseRead_CallOnce() + _canWrite_Semaphore.Create(0, 3); // _readingWasClosed = false; _readingWasClosed2 = false; @@ -59,27 +64,14 @@ void CStreamBinder::ReInit() _buf = NULL; ProcessedSize = 0; // WritingWasCut = false; + return S_OK; } -void CStreamBinder::CreateStreams(ISequentialInStream **inStream, ISequentialOutStream **outStream) +void CStreamBinder::CreateStreams2(CMyComPtr<ISequentialInStream> &inStream, CMyComPtr<ISequentialOutStream> &outStream) { - // _readingWasClosed = false; - _readingWasClosed2 = false; - - _waitWrite = true; - _bufSize = 0; - _buf = NULL; - ProcessedSize = 0; - // WritingWasCut = false; - - CBinderInStream *inStreamSpec = new CBinderInStream(this); - CMyComPtr<ISequentialInStream> inStreamLoc(inStreamSpec); - *inStream = inStreamLoc.Detach(); - - CBinderOutStream *outStreamSpec = new CBinderOutStream(this); - CMyComPtr<ISequentialOutStream> outStreamLoc(outStreamSpec); - *outStream = outStreamLoc.Detach(); + inStream = new CBinderInStream(this); + outStream = new CBinderOutStream(this); } // (_canRead_Event && _bufSize == 0) means that stream is finished. @@ -92,7 +84,9 @@ HRESULT CStreamBinder::Read(void *data, UInt32 size, UInt32 *processedSize) { if (_waitWrite) { - RINOK(_canRead_Event.Lock()); + WRes wres = _canRead_Event.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); _waitWrite = false; } if (size > _bufSize) @@ -105,17 +99,25 @@ HRESULT CStreamBinder::Read(void *data, UInt32 size, UInt32 *processedSize) if (processedSize) *processedSize = size; _bufSize -= size; + + /* + if (_bufSize == 0), then we have read whole buffer + we have two ways here: + - if we check (_bufSize == 0) here, we unlock Write only after full data Reading - it reduces the number of syncs + - if we don't check (_bufSize == 0) here, we unlock Write after partial data Reading + */ if (_bufSize == 0) { _waitWrite = true; - _canRead_Event.Reset(); - _canWrite_Event.Set(); + // _canWrite_Event.Set(); + _canWrite_Semaphore.Release(); } } } return S_OK; } + HRESULT CStreamBinder::Write(const void *data, UInt32 size, UInt32 *processedSize) { if (processedSize) @@ -135,20 +137,20 @@ HRESULT CStreamBinder::Write(const void *data, UInt32 size, UInt32 *processedSiz _readingWasClosed2 = true; */ - HANDLE events[2] = { _canWrite_Event, _readingWasClosed_Event }; - DWORD waitResult = ::WaitForMultipleObjects(2, events, FALSE, INFINITE); - if (waitResult >= WAIT_OBJECT_0 + 2) - return E_FAIL; + _canWrite_Semaphore.Lock(); + // _bufSize : is remain size that was not read size -= _bufSize; + + // size : is size of data that was read if (size != 0) { + // if some data was read, then we report that size and return if (processedSize) *processedSize = size; return S_OK; } - // if (waitResult == WAIT_OBJECT_0 + 1) - _readingWasClosed2 = true; + _readingWasClosed2 = true; } // WritingWasCut = true; diff --git a/CPP/7zip/Common/StreamBinder.h b/CPP/7zip/Common/StreamBinder.h index 12088a94..16c872fb 100644 --- a/CPP/7zip/Common/StreamBinder.h +++ b/CPP/7zip/Common/StreamBinder.h @@ -8,45 +8,63 @@ #include "../IStream.h" /* -We don't use probably UNSAFE version: -reader thread: +We can use one from two code versions here: with Event or with Semaphore to unlock Writer thread +The difference for cases where Reading must be closed before Writing closing + +1) Event Version: _canWrite_Event + We call _canWrite_Event.Set() without waiting _canRead_Event in CloseRead() function. + The writer thread can get (_readingWasClosed) status in one from two iterations. + It's ambiguity of processing flow. But probably it's SAFE to use, if Event functions provide memory barriers. + reader thread: _canWrite_Event.Set(); - _readingWasClosed = true + _readingWasClosed = true; _canWrite_Event.Set(); -writer thread: + writer thread: _canWrite_Event.Wait() if (_readingWasClosed) -Can second call of _canWrite_Event.Set() be executed without memory barrier, if event is already set? + +2) Semaphore Version: _canWrite_Semaphore + writer thread always will detect closing of reading in latest iteration after all data processing iterations */ class CStreamBinder { - NWindows::NSynchronization::CAutoResetEvent _canWrite_Event; - NWindows::NSynchronization::CManualResetEvent _canRead_Event; - NWindows::NSynchronization::CManualResetEvent _readingWasClosed_Event; + NWindows::NSynchronization::CAutoResetEvent _canRead_Event; + // NWindows::NSynchronization::CAutoResetEvent _canWrite_Event; + NWindows::NSynchronization::CSemaphore _canWrite_Semaphore; - // bool _readingWasClosed; - bool _readingWasClosed2; + // bool _readingWasClosed; // set it in reader thread and check it in write thread + bool _readingWasClosed2; // use it in writer thread // bool WritingWasCut; - bool _waitWrite; + bool _waitWrite; // use it in reader thread UInt32 _bufSize; const void *_buf; public: - UInt64 ProcessedSize; + UInt64 ProcessedSize; // the size that was read by reader thread - WRes CreateEvents(); - void CreateStreams(ISequentialInStream **inStream, ISequentialOutStream **outStream); + void CreateStreams2(CMyComPtr<ISequentialInStream> &inStream, CMyComPtr<ISequentialOutStream> &outStream); - void ReInit(); + HRESULT Create_ReInit(); HRESULT Read(void *data, UInt32 size, UInt32 *processedSize); HRESULT Write(const void *data, UInt32 size, UInt32 *processedSize); - void CloseRead() + void CloseRead_CallOnce() { - _readingWasClosed_Event.Set(); - // _readingWasClosed = true; - // _canWrite_Event.Set(); + // call it only once: for example, in destructor + + /* + _readingWasClosed = true; + _canWrite_Event.Set(); + */ + + /* + We must relase Semaphore only once !!! + we must release at least 2 items of Semaphore: + one item to unlock partial Write(), if Read() have read some items + then additional item to stop writing (_bufSize will be 0) + */ + _canWrite_Semaphore.Release(2); } void CloseWrite() diff --git a/CPP/7zip/Common/StreamObjects.cpp b/CPP/7zip/Common/StreamObjects.cpp index 8136716d..2d941df6 100644 --- a/CPP/7zip/Common/StreamObjects.cpp +++ b/CPP/7zip/Common/StreamObjects.cpp @@ -37,9 +37,9 @@ STDMETHODIMP CBufferInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -72,9 +72,9 @@ STDMETHODIMP CBufInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosi } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -108,14 +108,10 @@ bool CByteDynBuffer::EnsureCapacity(size_t cap) throw() { if (cap <= _capacity) return true; - size_t delta; - if (_capacity > 64) - delta = _capacity / 4; - else if (_capacity > 8) - delta = 16; - else - delta = 4; - cap = MyMax(_capacity + delta, cap); + size_t delta = _capacity / 4; + size_t cap2 = _capacity + delta; + if (cap < cap2) + cap = cap2; Byte *buf = (Byte *)realloc(_buf, cap); if (!buf) return false; @@ -185,9 +181,9 @@ static const UInt64 kEmptyTag = (UInt64)(Int64)-1; void CCachedInStream::Free() throw() { MyFree(_tags); - _tags = 0; + _tags = NULL; MidFree(_data); - _data = 0; + _data = NULL; } bool CCachedInStream::Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw() @@ -196,19 +192,19 @@ bool CCachedInStream::Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw( if (sizeLog >= sizeof(size_t) * 8) return false; size_t dataSize = (size_t)1 << sizeLog; - if (_data == 0 || dataSize != _dataSize) + if (!_data || dataSize != _dataSize) { MidFree(_data); _data = (Byte *)MidAlloc(dataSize); - if (_data == 0) + if (!_data) return false; _dataSize = dataSize; } - if (_tags == 0 || numBlocksLog != _numBlocksLog) + if (!_tags || numBlocksLog != _numBlocksLog) { MyFree(_tags); _tags = (UInt64 *)MyAlloc(sizeof(UInt64) << numBlocksLog); - if (_tags == 0) + if (!_tags) return false; _numBlocksLog = numBlocksLog; } @@ -242,21 +238,32 @@ STDMETHODIMP CCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSiz while (size != 0) { - UInt64 cacheTag = _pos >> _blockSizeLog; - size_t cacheIndex = (size_t)cacheTag & (((size_t)1 << _numBlocksLog) - 1); + const UInt64 cacheTag = _pos >> _blockSizeLog; + const size_t cacheIndex = (size_t)cacheTag & (((size_t)1 << _numBlocksLog) - 1); Byte *p = _data + (cacheIndex << _blockSizeLog); + if (_tags[cacheIndex] != cacheTag) { + _tags[cacheIndex] = kEmptyTag; UInt64 remInBlock = _size - (cacheTag << _blockSizeLog); size_t blockSize = (size_t)1 << _blockSizeLog; if (blockSize > remInBlock) blockSize = (size_t)remInBlock; + RINOK(ReadBlock(cacheTag, p, blockSize)); + _tags[cacheIndex] = cacheTag; } - size_t offset = (size_t)_pos & (((size_t)1 << _blockSizeLog) - 1); - UInt32 cur = (UInt32)MyMin(((size_t)1 << _blockSizeLog) - offset, (size_t)size); + + const size_t kBlockSize = (size_t)1 << _blockSizeLog; + const size_t offset = (size_t)_pos & (kBlockSize - 1); + UInt32 cur = size; + const size_t rem = kBlockSize - offset; + if (cur > rem) + cur = (UInt32)rem; + memcpy(data, p + offset, cur); + if (processedSize) *processedSize += cur; data = (void *)((const Byte *)data + cur); @@ -266,6 +273,7 @@ STDMETHODIMP CCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSiz return S_OK; } + STDMETHODIMP CCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) { @@ -278,8 +286,8 @@ STDMETHODIMP CCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } diff --git a/CPP/7zip/Common/StreamObjects.h b/CPP/7zip/Common/StreamObjects.h index e20e9bd8..a8fb229c 100644 --- a/CPP/7zip/Common/StreamObjects.h +++ b/CPP/7zip/Common/StreamObjects.h @@ -24,6 +24,7 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); }; + struct CReferenceBuf: public IUnknown, public CMyUnknownImp @@ -32,6 +33,7 @@ struct CReferenceBuf: MY_UNKNOWN_IMP }; + class CBufInStream: public IInStream, public CMyUnknownImp @@ -41,7 +43,7 @@ class CBufInStream: size_t _size; CMyComPtr<IUnknown> _ref; public: - void Init(const Byte *data, size_t size, IUnknown *ref = 0) + void Init(const Byte *data, size_t size, IUnknown *ref = NULL) { _data = data; _size = size; @@ -55,17 +57,20 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); }; + void Create_BufInStream_WithReference(const void *data, size_t size, IUnknown *ref, ISequentialInStream **stream); void Create_BufInStream_WithNewBuffer(const void *data, size_t size, ISequentialInStream **stream); inline void Create_BufInStream_WithNewBuffer(const CByteBuffer &buf, ISequentialInStream **stream) { Create_BufInStream_WithNewBuffer(buf, buf.Size(), stream); } + class CByteDynBuffer { size_t _capacity; Byte *_buf; + CLASS_NO_COPY(CByteDynBuffer); public: - CByteDynBuffer(): _capacity(0), _buf(0) {}; + CByteDynBuffer(): _capacity(0), _buf(NULL) {}; // there is no copy constructor. So don't copy this object. ~CByteDynBuffer() { Free(); } void Free() throw(); @@ -75,6 +80,7 @@ public: bool EnsureCapacity(size_t capacity) throw(); }; + class CDynBufSeqOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -94,6 +100,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CBufPtrSeqOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -114,6 +121,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CSequentialOutStreamSizeCount: public ISequentialOutStream, public CMyUnknownImp @@ -129,6 +137,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CCachedInStream: public IInStream, public CMyUnknownImp @@ -143,7 +152,7 @@ class CCachedInStream: protected: virtual HRESULT ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) = 0; public: - CCachedInStream(): _tags(0), _data(0) {} + CCachedInStream(): _tags(NULL), _data(NULL) {} virtual ~CCachedInStream() { Free(); } // the destructor must be virtual (release calls it) !!! void Free() throw(); bool Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw(); diff --git a/CPP/7zip/Common/UniqBlocks.h b/CPP/7zip/Common/UniqBlocks.h index a376024e..d6cd3728 100644 --- a/CPP/7zip/Common/UniqBlocks.h +++ b/CPP/7zip/Common/UniqBlocks.h @@ -19,7 +19,7 @@ struct CUniqBlocks bool IsOnlyEmpty() const { - return (Bufs.Size() == 0 || Bufs.Size() == 1 && Bufs[0].Size() == 0); + return (Bufs.Size() == 0 || (Bufs.Size() == 1 && Bufs[0].Size() == 0)); } }; diff --git a/CPP/7zip/Common/VirtThread.cpp b/CPP/7zip/Common/VirtThread.cpp index 77e3c1ac..bf24bb1c 100644 --- a/CPP/7zip/Common/VirtThread.cpp +++ b/CPP/7zip/Common/VirtThread.cpp @@ -19,20 +19,20 @@ static THREAD_FUNC_DECL CoderThread(void *p) WRes CVirtThread::Create() { - RINOK(StartEvent.CreateIfNotCreated()); - RINOK(FinishedEvent.CreateIfNotCreated()); - StartEvent.Reset(); - FinishedEvent.Reset(); + RINOK_WRes(StartEvent.CreateIfNotCreated_Reset()); + RINOK_WRes(FinishedEvent.CreateIfNotCreated_Reset()); + // StartEvent.Reset(); + // FinishedEvent.Reset(); Exit = false; if (Thread.IsCreated()) return S_OK; return Thread.Create(CoderThread, this); } -void CVirtThread::Start() +WRes CVirtThread::Start() { Exit = false; - StartEvent.Set(); + return StartEvent.Set(); } void CVirtThread::WaitThreadFinish() @@ -42,7 +42,6 @@ void CVirtThread::WaitThreadFinish() StartEvent.Set(); if (Thread.IsCreated()) { - Thread.Wait(); - Thread.Close(); + Thread.Wait_Close(); } } diff --git a/CPP/7zip/Common/VirtThread.h b/CPP/7zip/Common/VirtThread.h index ebee158c..b4d8a5a9 100644 --- a/CPP/7zip/Common/VirtThread.h +++ b/CPP/7zip/Common/VirtThread.h @@ -16,9 +16,9 @@ struct CVirtThread ~CVirtThread() { WaitThreadFinish(); } void WaitThreadFinish(); // call it in destructor of child class ! WRes Create(); - void Start(); + WRes Start(); virtual void Execute() = 0; - void WaitExecuteFinish() { FinishedEvent.Lock(); } + WRes WaitExecuteFinish() { return FinishedEvent.Lock(); } }; #endif diff --git a/CPP/7zip/Compress/BZip2Const.h b/CPP/7zip/Compress/BZip2Const.h index 0efecba3..7927d018 100644 --- a/CPP/7zip/Compress/BZip2Const.h +++ b/CPP/7zip/Compress/BZip2Const.h @@ -55,10 +55,15 @@ The number of selectors stored in bzip2 block: (numSelectors <= 18001) - must work with any decoder. (numSelectors == 18002) - works with bzip2 1.0.6 decoder and all derived decoders. (numSelectors > 18002) - 7-Zip decoder doesn't support it. - bzip2 1.0.6 decoder can overflow selector[18002] arrays. But there are another + lbzip2 2.5: encoder can write up to (18001 + 7) selectors. + + 7-Zip before 19.03: decoder doesn't support it. + 7-Zip 19.03: decoder allows 8 additional selector records for lbzip2 compatibility. + + bzip2 1.0.6: decoder can overflow selector[18002] arrays. But there are another arrays after selector arrays. So the compiled code works. - lbzip2 2.5 encoder can write up to (18001 + 7) selectors. + bzip2 1.0.7: decoder doesn't support it. + bzip2 1.0.8: decoder allows additional selector records for lbzip2 compatibility. */ }} diff --git a/CPP/7zip/Compress/BZip2Crc.cpp b/CPP/7zip/Compress/BZip2Crc.cpp index bf8e540f..9639c7b7 100644 --- a/CPP/7zip/Compress/BZip2Crc.cpp +++ b/CPP/7zip/Compress/BZip2Crc.cpp @@ -19,6 +19,7 @@ void CBZip2Crc::InitTable() } } +static class CBZip2CrcTableInit { public: diff --git a/CPP/7zip/Compress/BZip2Decoder.cpp b/CPP/7zip/Compress/BZip2Decoder.cpp index b414d088..c09c160e 100644 --- a/CPP/7zip/Compress/BZip2Decoder.cpp +++ b/CPP/7zip/Compress/BZip2Decoder.cpp @@ -20,7 +20,6 @@ #define PRIN(s) #define PRIN_VAL(s, val) -#define PRIN_MT(s) PRIN(" " s) #include "../../../C/Alloc.h" @@ -183,10 +182,10 @@ SRes CBase::ReadStreamSignature2() unsigned b; READ_BITS_8(b, 8); - if ( state2 == 0 && b != kArSig0 - || state2 == 1 && b != kArSig1 - || state2 == 2 && b != kArSig2 - || state2 == 3 && (b <= kArSig3 || b > kArSig3 + kBlockSizeMultMax)) + if ( (state2 == 0 && b != kArSig0) + || (state2 == 1 && b != kArSig1) + || (state2 == 2 && b != kArSig2) + || (state2 == 3 && (b <= kArSig3 || b > kArSig3 + kBlockSizeMultMax))) return SZ_ERROR_DATA; state2++; @@ -342,7 +341,11 @@ SRes CBase::ReadBlock2() state2 = 0x543210; state3 = 0; state4 = 0; - if (numSelectors == 0 || numSelectors > kNumSelectorsMax) + // lbzip2 can write small number of additional selectors, + // 20.01: we allow big number of selectors here like bzip2-1.0.8 + if (numSelectors == 0 + // || numSelectors > kNumSelectorsMax_Decoder + ) return SZ_ERROR_DATA; } @@ -365,10 +368,19 @@ SRes CBase::ReadBlock2() UInt32 mask = ((UInt32)1 << ((state4 + 1) * kMtfBits)) - 1; state4 = 0; state2 = ((state2 << kMtfBits) & mask) | (state2 & ~mask) | tmp; - selectors[state3] = (Byte)tmp; + // 20.01: here we keep compatibility with bzip2-1.0.8 decoder: + if (state3 < kNumSelectorsMax) + selectors[state3] = (Byte)tmp; } while (++state3 < numSelectors); + // we allowed additional dummy selector records filled above to support lbzip2's archives. + // but we still don't allow to use these additional dummy selectors in the code bellow + // bzip2 1.0.8 decoder also has similar restriction. + + if (numSelectors > kNumSelectorsMax) + numSelectors = kNumSelectorsMax; + state = STATE_LEVELS; state2 = 0; state3 = 0; @@ -412,14 +424,15 @@ SRes CBase::ReadBlock2() state5 = 0; } + // 19.03: we use Build() instead of BuildFull() to support lbzip2 archives // lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen // BuildFull() returns error for such tree - /* for (unsigned i = state4; i < kMaxAlphaSize; i++) lens[i] = 0; if (!huffs[state2].Build(lens)) - */ + /* if (!huffs[state2].BuildFull(lens, state4)) + */ return SZ_ERROR_DATA; state3 = 0; } @@ -476,10 +489,11 @@ SRes CBase::ReadBlock2() val = VAL >> (32 - kMaxHuffmanLen); unsigned len; for (len = kNumTableBits + 1; val >= huff->_limits[len]; len++); - /* + + // 19.03: we use that check to support partial trees created Build() for lbzip2 archives if (len > kNumBitsMax) return SZ_ERROR_DATA; // that check is required, if NHuffman::Build() was used instead of BuildFull() - */ + if (_numBits < len) { SAVE_LOCAL @@ -769,7 +783,7 @@ Byte * CSpecState::Decode(Byte *data, size_t size) throw() continue; } - reps = b; + reps = (int)b; while (reps) { reps--; @@ -802,7 +816,7 @@ Byte * CSpecState::Decode(Byte *data, size_t size) throw() _randToGo--; } - reps = b; + reps = (int)b; } _tPos = tPos; @@ -857,7 +871,7 @@ HRESULT CDecoder::DecodeBlock(const CBlockProps &props) } TICKS_START - const size_t processed = block.Decode(data, size) - data; + const size_t processed = (size_t)(block.Decode(data, size) - data); TICKS_UPDATE(2) _outPosTotal += processed; @@ -879,11 +893,12 @@ HRESULT CDecoder::DecodeBlock(const CBlockProps &props) CDecoder::CDecoder(): - _inBuf(NULL), _outBuf(NULL), - _counters(NULL), FinishMode(false), - _outSizeDefined(false) + _outSizeDefined(false), + _counters(NULL), + _inBuf(NULL), + _inProcessed(0) { #ifndef _7ZIP_ST MtMode = false; @@ -909,9 +924,8 @@ CDecoder::~CDecoder() ScoutEvent.Set(); PRIN("\nThread.Wait()()"); - Thread.Wait(); + Thread.Wait_Close(); PRIN("\n after Thread.Wait()()"); - Thread.Close(); // if (ScoutRes != S_OK) throw ScoutRes; } @@ -929,7 +943,7 @@ HRESULT CDecoder::ReadInput() if (Base._buf != Base._lim || _inputFinished || _inputRes != S_OK) return _inputRes; - _inProcessed += (Base._buf - _inBuf); + _inProcessed += (size_t)(Base._buf - _inBuf); Base._buf = _inBuf; Base._lim = _inBuf; UInt32 size = 0; @@ -1138,7 +1152,11 @@ HRESULT CDecoder::DecodeStreams(ICompressProgressInfo *progress) if (useMt) { PRIN("DecoderEvent.Lock()"); - RINOK(DecoderEvent.Lock()); + { + WRes wres = DecoderEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } NeedWaitScout = false; PRIN("-- DecoderEvent.Lock()"); props = _block.Props; @@ -1186,7 +1204,11 @@ HRESULT CDecoder::DecodeStreams(ICompressProgressInfo *progress) */ PRIN("ScoutEvent.Set()"); - RINOK(ScoutEvent.Set()); + { + WRes wres = ScoutEvent.Set(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } NeedWaitScout = true; } #endif @@ -1219,14 +1241,17 @@ bool CDecoder::CreateInputBufer() _inBuf = (Byte *)MidAlloc(kInBufSize); if (!_inBuf) return false; + Base._buf = _inBuf; + Base._lim = _inBuf; } if (!_counters) { - _counters = (UInt32 *)::BigAlloc((256 + kBlockSizeMax) * sizeof(UInt32) + const size_t size = (256 + kBlockSizeMax) * sizeof(UInt32) #ifdef BZIP2_BYTE_MODE + kBlockSizeMax #endif - + 256); + + 256; + _counters = (UInt32 *)::BigAlloc(size); if (!_counters) return false; Base.Counters = _counters; @@ -1266,14 +1291,19 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream } */ - InitOutSize(outSize); - _inputFinished = false; _inputRes = S_OK; _writeRes = S_OK; try { + InitOutSize(outSize); + + // we can request data from InputBuffer after Code(). + // so we init InputBuffer before any function return. + + InitInputBuffer(); + if (!CreateInputBufer()) return E_OUTOFMEMORY; @@ -1286,7 +1316,7 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream Base.InStream = inStream; - InitInputBuffer(); + // InitInputBuffer(); _outStream = outStream; _outWritten = 0; @@ -1323,23 +1353,43 @@ STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) { - *value = GetInputProcessedSize(); + *value = GetInStreamSize(); + return S_OK; +} + + +STDMETHODIMP CDecoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize) +{ + Base.AlignToByte(); + UInt32 i; + for (i = 0; i < size; i++) + { + int b; + Base.ReadByte(b); + if (b < 0) + break; + ((Byte *)data)[i] = (Byte)b; + } + if (processedSize) + *processedSize = i; return S_OK; } #ifndef _7ZIP_ST -#define RINOK_THREAD(x) { WRes __result_ = (x); if (__result_ != 0) return __result_; } +#define PRIN_MT(s) PRIN(" " s) + +// #define RINOK_THREAD(x) { WRes __result_ = (x); if (__result_ != 0) return __result_; } static THREAD_FUNC_DECL RunScout2(void *p) { ((CDecoder *)p)->RunScout(); return 0; } HRESULT CDecoder::CreateThread() { - RINOK_THREAD(DecoderEvent.CreateIfNotCreated()); - RINOK_THREAD(ScoutEvent.CreateIfNotCreated()); - RINOK_THREAD(Thread.Create(RunScout2, this)); - return S_OK; + WRes wres = DecoderEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = ScoutEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = Thread.Create(RunScout2, this); }} + return HRESULT_FROM_WIN32(wres); } void CDecoder::RunScout() @@ -1512,10 +1562,12 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) { InitOutSize(outSize); + InitInputBuffer(); + if (!CreateInputBufer()) return E_OUTOFMEMORY; - InitInputBuffer(); + // InitInputBuffer(); StartNewStream(); diff --git a/CPP/7zip/Compress/BZip2Decoder.h b/CPP/7zip/Compress/BZip2Decoder.h index 68aa7094..8fe4ef1b 100644 --- a/CPP/7zip/Compress/BZip2Decoder.h +++ b/CPP/7zip/Compress/BZip2Decoder.h @@ -73,9 +73,19 @@ struct CBitDecoder */ SRes ReadByte(int &b); + + CBitDecoder(): + _buf(NULL), + _lim(NULL) + { + InitBitDecoder(); + } }; +// 19.03: we allow additional 8 selectors to support files created by lbzip2. +const UInt32 kNumSelectorsMax_Decoder = kNumSelectorsMax + 8; + struct CBase: public CBitDecoder { unsigned numInUse; @@ -100,7 +110,7 @@ struct CBase: public CBitDecoder private: CMtf8Decoder mtf; - Byte selectors[kNumSelectorsMax]; + Byte selectors[kNumSelectorsMax_Decoder]; CHuffmanDecoder huffs[kNumTablesMax]; Byte lens[kMaxAlphaSize]; @@ -188,6 +198,7 @@ class CDecoder : public ICompressCoder, public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, #ifndef NO_READ_FROM_CODER public ICompressSetInStream, @@ -293,6 +304,8 @@ public: void InitInputBuffer() { + // We use InitInputBuffer() before stream init. + // So don't read from stream here _inProcessed = 0; Base._buf = _inBuf; Base._lim = _inBuf; @@ -302,7 +315,12 @@ public: UInt64 GetInputProcessedSize() const { // for NSIS case : we need also look the number of bits in bitDecoder - return _inProcessed + (Base._buf - _inBuf); + return _inProcessed + (size_t)(Base._buf - _inBuf); + } + + UInt64 GetInStreamSize() const + { + return _inProcessed + (size_t)(Base._buf - _inBuf) - (Base._numBits >> 3); } UInt64 GetOutProcessedSize() const { return _outWritten + _outPos; } @@ -324,6 +342,7 @@ public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + MY_QUERYINTERFACE_ENTRY(ICompressReadUnusedFromInBuf) #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) @@ -344,6 +363,7 @@ public: STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + STDMETHOD(ReadUnusedFromInBuf)(void *data, UInt32 size, UInt32 *processedSize); UInt64 GetNumStreams() const { return Base.NumStreams; } UInt64 GetNumBlocks() const { return Base.NumBlocks; } diff --git a/CPP/7zip/Compress/BZip2Encoder.cpp b/CPP/7zip/Compress/BZip2Encoder.cpp index c9f9c664..25c3f045 100644 --- a/CPP/7zip/Compress/BZip2Encoder.cpp +++ b/CPP/7zip/Compress/BZip2Encoder.cpp @@ -20,17 +20,17 @@ static const unsigned kNumHuffPasses = 4; bool CThreadInfo::Alloc() { - if (m_BlockSorterIndex == 0) + if (!m_BlockSorterIndex) { m_BlockSorterIndex = (UInt32 *)::BigAlloc(BLOCK_SORT_BUF_SIZE(kBlockSizeMax) * sizeof(UInt32)); - if (m_BlockSorterIndex == 0) + if (!m_BlockSorterIndex) return false; } - if (m_Block == 0) + if (!m_Block) { m_Block = (Byte *)::MidAlloc(kBlockSizeMax * 5 + kBlockSizeMax / 10 + (20 << 10)); - if (m_Block == 0) + if (!m_Block) return false; m_MtfArray = m_Block + kBlockSizeMax; m_TempArray = m_MtfArray + kBlockSizeMax * 2 + 2; @@ -41,9 +41,9 @@ bool CThreadInfo::Alloc() void CThreadInfo::Free() { ::BigFree(m_BlockSorterIndex); - m_BlockSorterIndex = 0; + m_BlockSorterIndex = NULL; ::MidFree(m_Block); - m_Block = 0; + m_Block = NULL; } #ifndef _7ZIP_ST @@ -53,15 +53,19 @@ static THREAD_FUNC_DECL MFThread(void *threadCoderInfo) return ((CThreadInfo *)threadCoderInfo)->ThreadFunc(); } -#define RINOK_THREAD(x) { WRes __result_ = (x); if (__result_ != 0) return __result_; } - HRESULT CThreadInfo::Create() { - RINOK_THREAD(StreamWasFinishedEvent.Create()); - RINOK_THREAD(WaitingWasStartedEvent.Create()); - RINOK_THREAD(CanWriteEvent.Create()); - RINOK_THREAD(Thread.Create(MFThread, this)); - return S_OK; + WRes wres = StreamWasFinishedEvent.Create(); + if (wres == 0) { wres = WaitingWasStartedEvent.Create(); + if (wres == 0) { wres = CanWriteEvent.Create(); + if (wres == 0) + { + if (Encoder->_props.Affinity != 0) + wres = Thread.Create_With_Affinity(MFThread, this, (CAffinityMask)Encoder->_props.Affinity); + else + wres = Thread.Create(MFThread, this); + }}} + return HRESULT_FROM_WIN32(wres); } void CThreadInfo::FinishStream(bool needLeave) @@ -74,7 +78,7 @@ void CThreadInfo::FinishStream(bool needLeave) WaitingWasStartedEvent.Set(); } -DWORD CThreadInfo::ThreadFunc() +THREAD_FUNC_RET_TYPE CThreadInfo::ThreadFunc() { for (;;) { @@ -133,7 +137,7 @@ void CEncProps::Normalize(int level) if (NumPasses > kNumPassesMax) NumPasses = kNumPassesMax; if (BlockSizeMult == (UInt32)(Int32)-1) - BlockSizeMult = (level >= 5 ? 9 : (level >= 1 ? level * 2 - 1: 1)); + BlockSizeMult = (level >= 5 ? 9 : (level >= 1 ? (unsigned)level * 2 - 1: 1)); if (BlockSizeMult < kBlockSizeMultMin) BlockSizeMult = kBlockSizeMultMin; if (BlockSizeMult > kBlockSizeMultMax) BlockSizeMult = kBlockSizeMultMax; } @@ -143,7 +147,7 @@ CEncoder::CEncoder() _props.Normalize(-1); #ifndef _7ZIP_ST - ThreadsInfo = 0; + ThreadsInfo = NULL; m_NumThreadsPrev = 0; NumThreads = 1; #endif @@ -157,9 +161,14 @@ CEncoder::~CEncoder() HRESULT CEncoder::Create() { - RINOK_THREAD(CanProcessEvent.CreateIfNotCreated()); - RINOK_THREAD(CanStartWaitingEvent.CreateIfNotCreated()); - if (ThreadsInfo != 0 && m_NumThreadsPrev == NumThreads) + { + WRes wres = CanProcessEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = CanStartWaitingEvent.CreateIfNotCreated_Reset(); } + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + + if (ThreadsInfo && m_NumThreadsPrev == NumThreads) return S_OK; try { @@ -167,7 +176,7 @@ HRESULT CEncoder::Create() MtMode = (NumThreads > 1); m_NumThreadsPrev = NumThreads; ThreadsInfo = new CThreadInfo[NumThreads]; - if (ThreadsInfo == 0) + if (!ThreadsInfo) return E_OUTOFMEMORY; } catch(...) { return E_OUTOFMEMORY; } @@ -199,11 +208,11 @@ void CEncoder::Free() { CThreadInfo &ti = ThreadsInfo[t]; if (MtMode) - ti.Thread.Wait(); + ti.Thread.Wait_Close(); ti.Free(); } delete []ThreadsInfo; - ThreadsInfo = 0; + ThreadsInfo = NULL; } #endif @@ -745,9 +754,11 @@ HRESULT CEncoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * CThreadInfo &ti = ThreadsInfo[t]; if (MtMode) { - RINOK(ti.StreamWasFinishedEvent.Reset()); - RINOK(ti.WaitingWasStartedEvent.Reset()); - RINOK(ti.CanWriteEvent.Reset()); + WRes wres = ti.StreamWasFinishedEvent.Reset(); + if (wres == 0) { wres = ti.WaitingWasStartedEvent.Reset(); + if (wres == 0) { wres = ti.CanWriteEvent.Reset(); }} + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } #else CThreadInfo &ti = ThreadsInfo; @@ -854,6 +865,16 @@ HRESULT CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *c { const PROPVARIANT &prop = coderProps[i]; PROPID propID = propIDs[i]; + + if (propID == NCoderPropID::kAffinity) + { + if (prop.vt == VT_UI8) + props.Affinity = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + continue; + } + if (propID >= NCoderPropID::kReduceSize) continue; if (prop.vt != VT_UI4) @@ -863,7 +884,7 @@ HRESULT CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *c { case NCoderPropID::kNumPasses: props.NumPasses = v; break; case NCoderPropID::kDictionarySize: props.BlockSizeMult = v / kBlockSizeStep; break; - case NCoderPropID::kLevel: level = v; break; + case NCoderPropID::kLevel: level = (int)v; break; case NCoderPropID::kNumThreads: { #ifndef _7ZIP_ST diff --git a/CPP/7zip/Compress/BZip2Encoder.h b/CPP/7zip/Compress/BZip2Encoder.h index 05d6fa16..5e63a730 100644 --- a/CPP/7zip/Compress/BZip2Encoder.h +++ b/CPP/7zip/Compress/BZip2Encoder.h @@ -53,7 +53,7 @@ public: unsigned numNewBits = MyMin(numBits, _bitPos); numBits -= numNewBits; - _curByte <<= numNewBits; + _curByte = (Byte)(_curByte << numNewBits); UInt32 newBits = value >> numBits; _curByte |= Byte(newBits); value -= (newBits << numBits); @@ -134,10 +134,10 @@ public: Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size. HRESULT Create(); void FinishStream(bool needLeave); - DWORD ThreadFunc(); + THREAD_FUNC_RET_TYPE ThreadFunc(); #endif - CThreadInfo(): m_BlockSorterIndex(0), m_Block(0) {} + CThreadInfo(): m_Block(NULL), m_BlockSorterIndex(NULL) {} ~CThreadInfo() { Free(); } bool Alloc(); void Free(); @@ -149,11 +149,13 @@ struct CEncProps { UInt32 BlockSizeMult; UInt32 NumPasses; + UInt64 Affinity; CEncProps() { BlockSizeMult = (UInt32)(Int32)-1; NumPasses = (UInt32)(Int32)-1; + Affinity = 0; } void Normalize(int level); bool DoOptimizeNumTables() const { return NumPasses > 1; } diff --git a/CPP/7zip/Compress/Bcj2Coder.cpp b/CPP/7zip/Compress/Bcj2Coder.cpp index 4906e78c..561fd08b 100644 --- a/CPP/7zip/Compress/Bcj2Coder.cpp +++ b/CPP/7zip/Compress/Bcj2Coder.cpp @@ -234,14 +234,14 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 Bcj2Enc_Encode(&enc); - currentInPos = totalStreamRead - numBytes_in_ReadBuf + (enc.src - _bufs[BCJ2_NUM_STREAMS]) - enc.tempPos; + currentInPos = totalStreamRead - numBytes_in_ReadBuf + (size_t)(enc.src - _bufs[BCJ2_NUM_STREAMS]) - enc.tempPos; if (Bcj2Enc_IsFinished(&enc)) break; if (enc.state < BCJ2_NUM_STREAMS) { - size_t curSize = enc.bufs[enc.state] - _bufs[enc.state]; + const size_t curSize = (size_t)(enc.bufs[enc.state] - _bufs[enc.state]); // printf("Write stream = %2d %6d\n", enc.state, curSize); RINOK(WriteStream(outStreams[enc.state], _bufs[enc.state], curSize)); if (enc.state == BCJ2_STREAM_RC) @@ -286,7 +286,7 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 if (progress && currentInPos - prevProgress >= (1 << 20)) { - UInt64 outSize2 = currentInPos + outSizeRc + enc.bufs[BCJ2_STREAM_RC] - enc.bufs[BCJ2_STREAM_RC]; + const UInt64 outSize2 = currentInPos + outSizeRc + (size_t)(enc.bufs[BCJ2_STREAM_RC] - enc.bufs[BCJ2_STREAM_RC]); prevProgress = currentInPos; // printf("progress %8d, %8d\n", (int)inSize2, (int)outSize2); RINOK(progress->SetRatioInfo(¤tInPos, &outSize2)); @@ -295,7 +295,7 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 for (int i = 0; i < BCJ2_NUM_STREAMS; i++) { - RINOK(WriteStream(outStreams[i], _bufs[i], enc.bufs[i] - _bufs[i])); + RINOK(WriteStream(outStreams[i], _bufs[i], (size_t)(enc.bufs[i] - _bufs[i]))); } // if (currentInPos != subStreamStartPos + subStreamSize) return E_FAIL; @@ -440,7 +440,7 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co } else // if (dec.state <= BCJ2_STATE_ORIG) { - size_t curSize = dec.dest - _bufs[BCJ2_NUM_STREAMS]; + const size_t curSize = (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (curSize != 0) { outSizeProcessed += curSize; @@ -463,17 +463,17 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co if (progress) { - const UInt64 outSize2 = outSizeProcessed + (dec.dest - _bufs[BCJ2_NUM_STREAMS]); + const UInt64 outSize2 = outSizeProcessed + (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (outSize2 - prevProgress >= (1 << 22)) { - const UInt64 inSize2 = outSize2 + _inStreamsProcessed[BCJ2_STREAM_RC] - (dec.lims[BCJ2_STREAM_RC] - dec.bufs[BCJ2_STREAM_RC]); + const UInt64 inSize2 = outSize2 + _inStreamsProcessed[BCJ2_STREAM_RC] - (size_t)(dec.lims[BCJ2_STREAM_RC] - dec.bufs[BCJ2_STREAM_RC]); RINOK(progress->SetRatioInfo(&inSize2, &outSize2)); prevProgress = outSize2; } } } - size_t curSize = dec.dest - _bufs[BCJ2_NUM_STREAMS]; + const size_t curSize = (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (curSize != 0) { outSizeProcessed += curSize; @@ -498,7 +498,7 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co { for (int i = 0; i < BCJ2_NUM_STREAMS; i++) { - size_t rem = dec.lims[i] - dec.bufs[i] + _extraReadSizes[i]; + const size_t rem = (size_t)(dec.lims[i] - dec.bufs[i]) + _extraReadSizes[i]; /* if (rem != 0) return S_FALSE; @@ -658,7 +658,7 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) STDMETHODIMP CDecoder::GetInStreamProcessedSize2(UInt32 streamIndex, UInt64 *value) { - const size_t rem = dec.lims[streamIndex] - dec.bufs[streamIndex] + _extraReadSizes[streamIndex]; + const size_t rem = (size_t)(dec.lims[streamIndex] - dec.bufs[streamIndex]) + _extraReadSizes[streamIndex]; *value = _inStreamsProcessed[streamIndex] - rem; return S_OK; } diff --git a/CPP/7zip/Compress/Bcj2Register.cpp b/CPP/7zip/Compress/Bcj2Register.cpp index 7a48f91c..2868f1c9 100644 --- a/CPP/7zip/Compress/Bcj2Register.cpp +++ b/CPP/7zip/Compress/Bcj2Register.cpp @@ -16,7 +16,7 @@ REGISTER_CODEC_CREATE_2(CreateCodecOut, CEncoder(), ICompressCoder2) #define CreateCodecOut NULL #endif -REGISTER_CODEC_VAR +REGISTER_CODEC_VAR(BCJ2) { CreateCodec, CreateCodecOut, 0x303011B, "BCJ2", 4, false }; REGISTER_CODEC(BCJ2) diff --git a/CPP/7zip/Compress/BitlDecoder.cpp b/CPP/7zip/Compress/BitlDecoder.cpp index 516b0932..876e6497 100644 --- a/CPP/7zip/Compress/BitlDecoder.cpp +++ b/CPP/7zip/Compress/BitlDecoder.cpp @@ -8,6 +8,7 @@ namespace NBitl { Byte kInvertTable[256]; +static struct CInverterTableInitializer { CInverterTableInitializer() diff --git a/CPP/7zip/Compress/BitlDecoder.h b/CPP/7zip/Compress/BitlDecoder.h index f3b8248f..e85942cf 100644 --- a/CPP/7zip/Compress/BitlDecoder.h +++ b/CPP/7zip/Compress/BitlDecoder.h @@ -38,7 +38,18 @@ public: _value = 0; } - UInt64 GetStreamSize() const { return _stream.GetStreamSize(); } + // the size of portion data in real stream that was already read from this object. + // it doesn't include unused data in BitStream object buffer (up to 4 bytes) + // it doesn't include unused data in TInByte buffers + // it doesn't include virtual Extra bytes after the end of real stream data + UInt64 GetStreamSize() const + { + return ExtraBitsWereRead() ? + _stream.GetStreamSize(): + GetProcessedSize(); + } + + // the size of virtual data that was read from this object. UInt64 GetProcessedSize() const { return _stream.GetProcessedSize() - ((kNumBigValueBits - _bitPos) >> 3); } bool ThereAreDataInBitsBuffer() const { return this->_bitPos != kNumBigValueBits; } @@ -139,6 +150,17 @@ public: MovePos(8); return b; } + + // call it only if the object is aligned for byte. + MY_FORCE_INLINE + bool ReadAlignedByte_FromBuf(Byte &b) + { + if (this->_bitPos == kNumBigValueBits) + return this->_stream.ReadByte_FromBuf(b); + b = (Byte)(_normalValue & 0xFF); + MovePos(8); + return true; + } }; } diff --git a/CPP/7zip/Compress/BitlEncoder.h b/CPP/7zip/Compress/BitlEncoder.h index 22b83545..9a4612fc 100644 --- a/CPP/7zip/Compress/BitlEncoder.h +++ b/CPP/7zip/Compress/BitlEncoder.h @@ -39,7 +39,7 @@ public: { if (numBits < _bitPos) { - _curByte |= (value & ((1 << numBits) - 1)) << (8 - _bitPos); + _curByte |= (Byte)((value & ((1 << numBits) - 1)) << (8 - _bitPos)); _bitPos -= numBits; return; } diff --git a/CPP/7zip/Compress/BitmEncoder.h b/CPP/7zip/Compress/BitmEncoder.h index 05079ace..4499c79d 100644 --- a/CPP/7zip/Compress/BitmEncoder.h +++ b/CPP/7zip/Compress/BitmEncoder.h @@ -33,7 +33,7 @@ public: { if (numBits < _bitPos) { - _curByte |= ((Byte)value << (_bitPos -= numBits)); + _curByte = (Byte)(_curByte | (value << (_bitPos -= numBits))); return; } numBits -= _bitPos; diff --git a/CPP/7zip/Compress/CodecExports.cpp b/CPP/7zip/Compress/CodecExports.cpp index 99085040..2aea7b33 100644 --- a/CPP/7zip/Compress/CodecExports.cpp +++ b/CPP/7zip/Compress/CodecExports.cpp @@ -82,6 +82,14 @@ static HRESULT FindCodecClassId(const GUID *clsid, bool isCoder2, bool isFilter, return S_OK; } +/* +#ifdef __GNUC__ +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wduplicated-branches" +#endif +#endif +*/ + static HRESULT CreateCoderMain(unsigned index, bool encode, void **coder) { COM_TRY_BEGIN @@ -97,12 +105,15 @@ static HRESULT CreateCoderMain(unsigned index, bool encode, void **coder) if (c) { IUnknown *unk; + unk = (IUnknown *)c; + /* if (codec.IsFilter) unk = (IUnknown *)(ICompressFilter *)c; else if (codec.NumStreams != 1) unk = (IUnknown *)(ICompressCoder2 *)c; else unk = (IUnknown *)(ICompressCoder *)c; + */ unk->AddRef(); *coder = c; } @@ -136,16 +147,22 @@ static HRESULT CreateCoder2(bool encode, UInt32 index, const GUID *iid, void **o return CreateCoderMain(index, encode, outObject); } + +STDAPI CreateDecoder(UInt32 index, const GUID *iid, void **outObject); STDAPI CreateDecoder(UInt32 index, const GUID *iid, void **outObject) { return CreateCoder2(false, index, iid, outObject); } + +STDAPI CreateEncoder(UInt32 index, const GUID *iid, void **outObject); STDAPI CreateEncoder(UInt32 index, const GUID *iid, void **outObject) { return CreateCoder2(true, index, iid, outObject); } + +STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject) { *outObject = NULL; @@ -175,6 +192,8 @@ STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject) return CreateCoderMain(codecIndex, encode, outObject); } + +STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value); STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) { ::VariantClear((VARIANTARG *)value); @@ -238,6 +257,8 @@ STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) return S_OK; } + +STDAPI GetNumberOfMethods(UINT32 *numCodecs); STDAPI GetNumberOfMethods(UINT32 *numCodecs) { *numCodecs = g_NumCodecs; @@ -270,6 +291,7 @@ static HRESULT CreateHasher2(UInt32 index, IHasher **hasher) COM_TRY_END } +STDAPI CreateHasher(const GUID *clsid, IHasher **outObject); STDAPI CreateHasher(const GUID *clsid, IHasher **outObject) { COM_TRY_BEGIN @@ -281,6 +303,7 @@ STDAPI CreateHasher(const GUID *clsid, IHasher **outObject) COM_TRY_END } +STDAPI GetHasherProp(UInt32 codecIndex, PROPID propID, PROPVARIANT *value); STDAPI GetHasherProp(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) { ::VariantClear((VARIANTARG *)value); @@ -318,6 +341,7 @@ public: STDMETHOD(CreateHasher)(UInt32 index, IHasher **hasher); }; +STDAPI GetHashers(IHashers **hashers); STDAPI GetHashers(IHashers **hashers) { COM_TRY_BEGIN diff --git a/CPP/7zip/Compress/DeflateDecoder.cpp b/CPP/7zip/Compress/DeflateDecoder.cpp index fceef857..e34c2c0c 100644 --- a/CPP/7zip/Compress/DeflateDecoder.cpp +++ b/CPP/7zip/Compress/DeflateDecoder.cpp @@ -9,8 +9,8 @@ namespace NDeflate { namespace NDecoder { CCoder::CCoder(bool deflate64Mode): - _deflate64Mode(deflate64Mode), _deflateNSIS(false), + _deflate64Mode(deflate64Mode), _keepHistory(false), _needFinishInput(false), _needInitInStream(true), @@ -408,9 +408,25 @@ STDMETHODIMP CCoder::SetFinishMode(UInt32 finishMode) STDMETHODIMP CCoder::GetInStreamProcessedSize(UInt64 *value) { - if (!value) - return E_INVALIDARG; - *value = m_InBitStream.GetProcessedSize(); + *value = m_InBitStream.GetStreamSize(); + return S_OK; +} + + +STDMETHODIMP CCoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize) +{ + AlignToByte(); + UInt32 i = 0; + if (!m_InBitStream.ExtraBitsWereRead()) + { + for (i = 0; i < size; i++) + { + if (!m_InBitStream.ReadAlignedByte_FromBuf(((Byte *)data)[i])) + break; + } + } + if (processedSize) + *processedSize = i; return S_OK; } diff --git a/CPP/7zip/Compress/DeflateDecoder.h b/CPP/7zip/Compress/DeflateDecoder.h index 0a724247..141184ef 100644 --- a/CPP/7zip/Compress/DeflateDecoder.h +++ b/CPP/7zip/Compress/DeflateDecoder.h @@ -25,6 +25,7 @@ class CCoder: public ICompressCoder, public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, @@ -103,6 +104,7 @@ public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + MY_QUERYINTERFACE_ENTRY(ICompressReadUnusedFromInBuf) #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) @@ -119,6 +121,7 @@ public: STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + STDMETHOD(ReadUnusedFromInBuf)(void *data, UInt32 size, UInt32 *processedSize); STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); @@ -141,7 +144,10 @@ public: } bool InputEofError() const { return m_InBitStream.ExtraBitsWereRead(); } + // size of used real data from input stream UInt64 GetStreamSize() const { return m_InBitStream.GetStreamSize(); } + + // size of virtual input stream processed UInt64 GetInputProcessedSize() const { return m_InBitStream.GetProcessedSize(); } }; diff --git a/CPP/7zip/Compress/DeflateEncoder.cpp b/CPP/7zip/Compress/DeflateEncoder.cpp index 233edb5f..fb24c6b0 100644 --- a/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/CPP/7zip/Compress/DeflateEncoder.cpp @@ -36,7 +36,7 @@ static const UInt32 kMatchArrayLimit = kMatchArraySize - kMatchMaxLen * 4 * size static const UInt32 kBlockUncompressedSizeThreshold = kMaxUncompressedBlockSize - kMatchMaxLen - kNumOpts; -static const unsigned kMaxCodeBitLength = 11; +// static const unsigned kMaxCodeBitLength = 11; static const unsigned kMaxLevelBitLength = 7; static const Byte kNoLiteralStatPrice = 11; @@ -89,7 +89,7 @@ void CEncProps::Normalize() if (algo < 0) algo = (level < 5 ? 0 : 1); if (fb < 0) fb = (level < 7 ? 32 : (level < 9 ? 64 : 128)); if (btMode < 0) btMode = (algo == 0 ? 0 : 1); - if (mc == 0) mc = (16 + (fb >> 1)); + if (mc == 0) mc = (16 + ((unsigned)fb >> 1)); if (numPasses == (UInt32)(Int32)-1) numPasses = (level < 7 ? 1 : (level < 9 ? 3 : 10)); } @@ -100,7 +100,7 @@ void CCoder::SetProps(const CEncProps *props2) m_MatchFinderCycles = props.mc; { - unsigned fb = props.fb; + unsigned fb = (unsigned)props.fb; if (fb < kMatchMinLen) fb = kMatchMinLen; if (fb > m_MatchMaxLen) @@ -125,12 +125,12 @@ void CCoder::SetProps(const CEncProps *props2) } CCoder::CCoder(bool deflate64Mode): - m_Deflate64Mode(deflate64Mode), - m_OnePosMatchesMemory(0), - m_DistanceMemory(0), + m_Values(NULL), + m_OnePosMatchesMemory(NULL), + m_DistanceMemory(NULL), m_Created(false), - m_Values(0), - m_Tables(0) + m_Deflate64Mode(deflate64Mode), + m_Tables(NULL) { m_MatchMaxLen = deflate64Mode ? kMatchMaxLen64 : kMatchMaxLen32; m_NumLenCombinations = deflate64Mode ? kNumLenSymbols64 : kNumLenSymbols32; @@ -213,10 +213,10 @@ HRESULT CCoder::BaseSetEncoderProperties2(const PROPID *propIDs, const PROPVARIA switch (propID) { case NCoderPropID::kNumPasses: props.numPasses = v; break; - case NCoderPropID::kNumFastBytes: props.fb = v; break; + case NCoderPropID::kNumFastBytes: props.fb = (int)v; break; case NCoderPropID::kMatchFinderCycles: props.mc = v; break; - case NCoderPropID::kAlgorithm: props.algo = v; break; - case NCoderPropID::kLevel: props.Level = v; break; + case NCoderPropID::kAlgorithm: props.algo = (int)v; break; + case NCoderPropID::kLevel: props.Level = (int)v; break; case NCoderPropID::kNumThreads: break; default: return E_INVALIDARG; } @@ -595,7 +595,7 @@ NO_INLINE void CCoder::MakeTables(unsigned maxHuffLen) Huffman_Generate(distFreqs, distCodes, m_NewLevels.distLevels, kDistTableSize64, maxHuffLen); } -NO_INLINE UInt32 Huffman_GetPrice(const UInt32 *freqs, const Byte *lens, UInt32 num) +static NO_INLINE UInt32 Huffman_GetPrice(const UInt32 *freqs, const Byte *lens, UInt32 num) { UInt32 price = 0; UInt32 i; @@ -604,7 +604,7 @@ NO_INLINE UInt32 Huffman_GetPrice(const UInt32 *freqs, const Byte *lens, UInt32 return price; } -NO_INLINE UInt32 Huffman_GetPrice_Spec(const UInt32 *freqs, const Byte *lens, UInt32 num, const Byte *extraBits, UInt32 extraBase) +static NO_INLINE UInt32 Huffman_GetPrice_Spec(const UInt32 *freqs, const Byte *lens, UInt32 num, const Byte *extraBits, UInt32 extraBase) { return Huffman_GetPrice(freqs, lens, num) + Huffman_GetPrice(freqs + extraBase, extraBits, num - extraBase); @@ -629,8 +629,9 @@ NO_INLINE void CCoder::TryBlock() { if (m_OptimumCurrentIndex == m_OptimumEndIndex) { - if (m_Pos >= kMatchArrayLimit || BlockSizeRes >= blockSize || !m_SecondPass && - ((Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) == 0) || m_ValueIndex >= m_ValueBlockSize)) + if (m_Pos >= kMatchArrayLimit + || BlockSizeRes >= blockSize + || (!m_SecondPass && ((Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) == 0) || m_ValueIndex >= m_ValueBlockSize))) break; } UInt32 pos; @@ -688,7 +689,7 @@ NO_INLINE void CCoder::SetPrices(const CLevels &levels) } } -NO_INLINE void Huffman_ReverseBits(UInt32 *codes, const Byte *lens, UInt32 num) +static NO_INLINE void Huffman_ReverseBits(UInt32 *codes, const Byte *lens, UInt32 num) { for (UInt32 i = 0; i < num; i++) { diff --git a/CPP/7zip/Compress/DllExportsCompress.cpp b/CPP/7zip/Compress/DllExportsCompress.cpp index c58d2d5e..24749d27 100644 --- a/CPP/7zip/Compress/DllExportsCompress.cpp +++ b/CPP/7zip/Compress/DllExportsCompress.cpp @@ -27,6 +27,16 @@ void RegisterHasher(const CHasherInfo *hashInfo) throw() } #ifdef _WIN32 + +extern "C" +BOOL WINAPI DllMain( + #ifdef UNDER_CE + HANDLE + #else + HINSTANCE + #endif + , DWORD /* dwReason */, LPVOID /*lpReserved*/); + extern "C" BOOL WINAPI DllMain( #ifdef UNDER_CE @@ -42,6 +52,7 @@ BOOL WINAPI DllMain( STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject); +STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject) { return CreateCoder(clsid, iid, outObject); diff --git a/CPP/7zip/Compress/ImplodeDecoder.cpp b/CPP/7zip/Compress/ImplodeDecoder.cpp index 736c832e..97a3cdf7 100644 --- a/CPP/7zip/Compress/ImplodeDecoder.cpp +++ b/CPP/7zip/Compress/ImplodeDecoder.cpp @@ -83,8 +83,8 @@ static const UInt32 kHistorySize = (1 << kNumDistDirectBitsBig) * kDistTableSize CCoder::CCoder(): - _fullStreamMode(false), - _flags(0) + _flags(0), + _fullStreamMode(false) {} diff --git a/CPP/7zip/Compress/Lzma2Encoder.cpp b/CPP/7zip/Compress/Lzma2Encoder.cpp index caecbc09..6a5d6797 100644 --- a/CPP/7zip/Compress/Lzma2Encoder.cpp +++ b/CPP/7zip/Compress/Lzma2Encoder.cpp @@ -34,6 +34,7 @@ CEncoder::~CEncoder() } +HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props); HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props) { switch (propID) @@ -49,7 +50,10 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm break; } case NCoderPropID::kNumThreads: - if (prop.vt != VT_UI4) return E_INVALIDARG; lzma2Props.numTotalThreads = (int)(prop.ulVal); break; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + lzma2Props.numTotalThreads = (int)(prop.ulVal); + break; default: RINOK(NLzma::SetLzmaProp(propID, prop, lzma2Props.lzmaProps)); } diff --git a/CPP/7zip/Compress/LzmaDecoder.cpp b/CPP/7zip/Compress/LzmaDecoder.cpp index 83c24f1c..a25d36d1 100644 --- a/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/CPP/7zip/Compress/LzmaDecoder.cpp @@ -153,7 +153,7 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream *inStream, ISequentialOutStream * SizeT inProcessed = _inLim - _inPos; ELzmaStatus status; - + SRes res = LzmaDec_DecodeToDic(&_state, dicPos + size, _inBuf + _inPos, &inProcessed, finishMode, &status); _lzmaStatus = status; @@ -183,7 +183,10 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream *inStream, ISequentialOutStream * if (needStop) { if (res != 0) + { + // return SResToHRESULT(res); return S_FALSE; + } if (status == LZMA_STATUS_FINISHED_WITH_MARK) { diff --git a/CPP/7zip/Compress/LzmaEncoder.cpp b/CPP/7zip/Compress/LzmaEncoder.cpp index e47f776b..5a6c6831 100644 --- a/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/CPP/7zip/Compress/LzmaEncoder.cpp @@ -9,6 +9,20 @@ #include "LzmaEncoder.h" +#include "../../Common/IntToString.h" +#include "../../Windows/TimeUtils.h" + +// #define LOG_LZMA_THREADS + +#ifdef LOG_LZMA_THREADS +#include <stdio.h> + +EXTERN_C_BEGIN +void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]); +EXTERN_C_END + +#endif + namespace NCompress { namespace NLzma { @@ -26,46 +40,48 @@ CEncoder::~CEncoder() LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc); } -static inline wchar_t GetUpperChar(wchar_t c) +static inline wchar_t GetLowCharFast(wchar_t c) { - if (c >= 'a' && c <= 'z') - c -= 0x20; - return c; + return c |= 0x20; } static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes) { - wchar_t c = GetUpperChar(*s++); - if (c == L'H') + wchar_t c = GetLowCharFast(*s++); + if (c == 'h') { - if (GetUpperChar(*s++) != L'C') + if (GetLowCharFast(*s++) != 'c') return 0; - int numHashBytesLoc = (int)(*s++ - L'0'); - if (numHashBytesLoc < 4 || numHashBytesLoc > 4) + int num = (int)(*s++ - L'0'); + if (num < 4 || num > 5) return 0; if (*s != 0) return 0; *btMode = 0; - *numHashBytes = numHashBytesLoc; + *numHashBytes = num; return 1; } - if (c != L'B') - return 0; - if (GetUpperChar(*s++) != L'T') - return 0; - int numHashBytesLoc = (int)(*s++ - L'0'); - if (numHashBytesLoc < 2 || numHashBytesLoc > 4) - return 0; - if (*s != 0) + if (c != 'b') return 0; - *btMode = 1; - *numHashBytes = numHashBytesLoc; - return 1; + { + if (GetLowCharFast(*s++) != 't') + return 0; + int num = (int)(*s++ - L'0'); + if (num < 2 || num > 5) + return 0; + if (*s != 0) + return 0; + *btMode = 1; + *numHashBytes = num; + return 1; + } } -#define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break; +#define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break; +#define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break; +HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep); HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) { if (propID == NCoderPropID::kMatchFinder) @@ -74,7 +90,16 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) return E_INVALIDARG; return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG; } - + + if (propID == NCoderPropID::kAffinity) + { + if (prop.vt == VT_UI8) + ep.affinity = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } + if (propID > NCoderPropID::kReduceSize) return S_OK; @@ -95,9 +120,9 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) case NCoderPropID::kDefaultProp: if (v > 31) return E_INVALIDARG; ep.dictSize = (UInt32)1 << (unsigned)v; break; SET_PROP_32(kLevel, level) SET_PROP_32(kNumFastBytes, fb) - SET_PROP_32(kMatchFinderCycles, mc) + SET_PROP_32U(kMatchFinderCycles, mc) SET_PROP_32(kAlgorithm, algo) - SET_PROP_32(kDictionarySize, dictSize) + SET_PROP_32U(kDictionarySize, dictSize) SET_PROP_32(kPosStateBits, pb) SET_PROP_32(kLitPosBits, lp) SET_PROP_32(kLitContextBits, lc) @@ -120,7 +145,10 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, switch (propID) { case NCoderPropID::kEndMarker: - if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal != VARIANT_FALSE); break; + if (prop.vt != VT_BOOL) + return E_INVALIDARG; + props.writeEndMark = (prop.boolVal != VARIANT_FALSE); + break; default: RINOK(SetLzmaProp(propID, prop, props)); } @@ -156,6 +184,92 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) #define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + + +#ifdef LOG_LZMA_THREADS + +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') +{ + char temp[64]; + char *p = temp + 32; + ConvertUInt64ToString(val, p); + unsigned len = (unsigned)strlen(p); + for (; len < numDigits; len++) + *--p = c; + printf("%s", p); +} + +static void PrintTime(const char *s, UInt64 val, UInt64 total) +{ + printf(" %s :", s); + const UInt32 kFreq = 10000000; + UInt64 sec = val / kFreq; + PrintNum(sec, 6); + printf(" ."); + UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000); + PrintNum(ms, 3, '0'); + + while (val > ((UInt64)1 << 56)) + { + val >>= 1; + total >>= 1; + } + + UInt64 percent = 0; + if (total != 0) + percent = val * 100 / total; + printf(" ="); + PrintNum(percent, 4); + printf("%%"); +} + + +struct CBaseStat +{ + UInt64 kernelTime, userTime; + + BOOL Get(HANDLE thread, const CBaseStat *prevStat) + { + FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT; + BOOL res = GetThreadTimes(thread + , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT); + if (res) + { + kernelTime = GetTime64(kernelTimeFT); + userTime = GetTime64(userTimeFT); + if (prevStat) + { + kernelTime -= prevStat->kernelTime; + userTime -= prevStat->userTime; + } + } + return res; + } +}; + + +static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat) +{ + CBaseStat newStat; + if (!newStat.Get(thread, prevStat)) + return; + + PrintTime("K", newStat.kernelTime, totalTime); + + const UInt64 processTime = newStat.kernelTime + newStat.userTime; + + PrintTime("U", newStat.userTime, totalTime); + PrintTime("S", processTime, totalTime); + printf("\n"); + // PrintTime("G ", totalTime, totalTime); +} + +#endif + + + STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) { @@ -167,6 +281,18 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream outWrap.Init(outStream); progressWrap.Init(progress); + #ifdef LOG_LZMA_THREADS + + FILETIME startTimeFT; + NWindows::NTime::GetCurUtcFileTime(startTimeFT); + UInt64 totalTime = GetTime64(startTimeFT); + CBaseStat oldStat; + if (!oldStat.Get(GetCurrentThread(), NULL)) + return E_FAIL; + + #endif + + SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc); @@ -175,7 +301,23 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + + + #ifdef LOG_LZMA_THREADS + NWindows::NTime::GetCurUtcFileTime(startTimeFT); + totalTime = GetTime64(startTimeFT) - totalTime; + HANDLE lz_threads[2]; + LzmaEnc_GetLzThreads(_encoder, lz_threads); + printf("\n"); + printf("Main: "); PrintStat(GetCurrentThread(), totalTime, &oldStat); + printf("Hash: "); PrintStat(lz_threads[0], totalTime, NULL); + printf("BinT: "); PrintStat(lz_threads[1], totalTime, NULL); + // PrintTime("Total: ", totalTime, totalTime); + printf("\n"); + + #endif + return SResToHRESULT(res); } diff --git a/CPP/7zip/Compress/LzmsDecoder.cpp b/CPP/7zip/Compress/LzmsDecoder.cpp index 5189ce16..e27afa3c 100644 --- a/CPP/7zip/Compress/LzmsDecoder.cpp +++ b/CPP/7zip/Compress/LzmsDecoder.cpp @@ -123,7 +123,7 @@ static void x86_Filter(Byte *data, UInt32 size, Int32 *history) for (;;) { - const Byte *p = data + (UInt32)i; + Byte *p = data + (UInt32)i; for (;;) { @@ -198,7 +198,7 @@ static void x86_Filter(Byte *data, UInt32 size, Int32 *history) Int32 *target; { - const Byte *p2 = p + codeLen; + Byte *p2 = p + codeLen; UInt32 n = GetUi32(p2); if (i - last_x86_pos <= maxTransOffset) { @@ -208,7 +208,7 @@ static void x86_Filter(Byte *data, UInt32 size, Int32 *history) target = history + (((UInt32)i + n) & 0xFFFF); } - i += codeLen + sizeof(UInt32) - 1; + i += (Int32)(codeLen + sizeof(UInt32) - 1); if (i - *target <= k_x86_WindowSize) last_x86_pos = i; @@ -220,7 +220,7 @@ static void x86_Filter(Byte *data, UInt32 size, Int32 *history) -static const int kLenIdNeedInit = -2; +// static const int kLenIdNeedInit = -2; CDecoder::CDecoder(): _x86_history(NULL) @@ -232,7 +232,7 @@ CDecoder::~CDecoder() ::MidFree(_x86_history); } -#define RIF(x) { if (!(x)) return false; } +// #define RIF(x) { if (!(x)) return false; } #define LIMIT_CHECK if (_bs._buf < _rc.cur) return S_FALSE; // #define LIMIT_CHECK @@ -539,8 +539,8 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out _rc.Normalize(); if (_rc.code != 0) return S_FALSE; - if (_rc.cur > _bs._buf || - _rc.cur == _bs._buf && _bs._bitPos != 0) + if (_rc.cur > _bs._buf + || (_rc.cur == _bs._buf && _bs._bitPos != 0)) return S_FALSE; /* diff --git a/CPP/7zip/Compress/LzmsDecoder.h b/CPP/7zip/Compress/LzmsDecoder.h index 510d3389..f0909a1e 100644 --- a/CPP/7zip/Compress/LzmsDecoder.h +++ b/CPP/7zip/Compress/LzmsDecoder.h @@ -263,7 +263,7 @@ public: ~CDecoder(); HRESULT Code(const Byte *in, size_t inSize, Byte *out, size_t outSize); - const size_t GetUnpackSize() const { return _pos; } + size_t GetUnpackSize() const { return _pos; } }; }} diff --git a/CPP/7zip/Compress/LzxDecoder.cpp b/CPP/7zip/Compress/LzxDecoder.cpp index a7bba903..e59cd400 100644 --- a/CPP/7zip/Compress/LzxDecoder.cpp +++ b/CPP/7zip/Compress/LzxDecoder.cpp @@ -32,7 +32,7 @@ static void x86_Filter(Byte *data, UInt32 size, UInt32 processedSize, UInt32 tra for (UInt32 i = 0;;) { - const Byte *p = data + i; + Byte *p = data + i; for (;;) { if (*p++ == 0xE8) break; @@ -46,13 +46,13 @@ static void x86_Filter(Byte *data, UInt32 size, UInt32 processedSize, UInt32 tra if (i > size) break; { - Int32 v = GetUi32(p); + Int32 v = (Int32)GetUi32(p); Int32 pos = (Int32)((Int32)1 - (Int32)(processedSize + i)); i += 4; if (v >= pos && v < (Int32)translationSize) { - v += (v >= 0 ? pos : translationSize); - SetUi32(p, v); + v += (v >= 0 ? pos : (Int32)translationSize); + SetUi32(p, (UInt32)v); } } } @@ -63,15 +63,15 @@ static void x86_Filter(Byte *data, UInt32 size, UInt32 processedSize, UInt32 tra CDecoder::CDecoder(bool wimMode): _win(NULL), - _keepHistory(false), _skipByte(false), + _unpackBlockSize(0), + KeepHistoryForNext(true), + NeedAlloc(true), + _keepHistory(false), _wimMode(wimMode), _numDictBits(15), - _unpackBlockSize(0), _x86_buf(NULL), _x86_translationSize(0), - KeepHistoryForNext(true), - NeedAlloc(true), _unpackedData(NULL) { } diff --git a/CPP/7zip/Compress/LzxDecoder.h b/CPP/7zip/Compress/LzxDecoder.h index a5c6f12c..4d70b272 100644 --- a/CPP/7zip/Compress/LzxDecoder.h +++ b/CPP/7zip/Compress/LzxDecoder.h @@ -30,7 +30,7 @@ public: _extraSize = 0; } - size_t GetRem() const { return _bufLim + 1 - _buf; } + size_t GetRem() const { return (size_t)(_bufLim + 1 - _buf); } bool WasExtraReadError_Fast() const { return _extraSize > 4; } bool WasFinishedOK() const @@ -238,7 +238,7 @@ public: bool WasBlockFinished() const { return _unpackBlockSize == 0; } const Byte *GetUnpackData() const { return _unpackedData; } - const UInt32 GetUnpackSize() const { return _pos - _writePos; } + UInt32 GetUnpackSize() const { return _pos - _writePos; } }; }} diff --git a/CPP/7zip/Compress/PpmdDecoder.cpp b/CPP/7zip/Compress/PpmdDecoder.cpp index ed0e9e29..7f54ec3b 100644 --- a/CPP/7zip/Compress/PpmdDecoder.cpp +++ b/CPP/7zip/Compress/PpmdDecoder.cpp @@ -1,5 +1,5 @@ // PpmdDecoder.cpp -// 2009-03-11 : Igor Pavlov : Public domain +// 2020-07-03 : Igor Pavlov : Public domain #include "StdAfx.h" @@ -13,13 +13,13 @@ namespace NCompress { namespace NPpmd { -static const UInt32 kBufSize = (1 << 20); +static const UInt32 kBufSize = (1 << 16); enum { kStatus_NeedInit, kStatus_Normal, - kStatus_Finished, + kStatus_Finished_With_Mark, kStatus_Error }; @@ -35,7 +35,8 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *props, UInt32 size) return E_INVALIDARG; _order = props[0]; UInt32 memSize = GetUi32(props + 1); - if (_order < PPMD7_MIN_ORDER || + if ( + // _order < PPMD7_MIN_ORDER || _order > PPMD7_MAX_ORDER || memSize < PPMD7_MIN_MEM_SIZE || memSize > PPMD7_MAX_MEM_SIZE) @@ -47,23 +48,36 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *props, UInt32 size) return S_OK; } +#define _rangeDec _ppmd.rc.dec + +#define CHECK_EXTRA_ERROR \ + if (_inStream.Extra) { \ + _status = kStatus_Error; \ + return (_res = (_inStream.Res != SZ_OK ? _inStream.Res: S_FALSE)); } + + HRESULT CDecoder::CodeSpec(Byte *memStream, UInt32 size) { + if (_res != S_OK) + return _res; + switch (_status) { - case kStatus_Finished: return S_OK; + case kStatus_Finished_With_Mark: return S_OK; case kStatus_Error: return S_FALSE; case kStatus_NeedInit: _inStream.Init(); if (!Ppmd7z_RangeDec_Init(&_rangeDec)) { _status = kStatus_Error; - return S_FALSE; + return (_res = S_FALSE); } + CHECK_EXTRA_ERROR _status = kStatus_Normal; Ppmd7_Init(&_ppmd, _order); break; } + if (_outSizeDefined) { const UInt64 rem = _outSize - _processedSize; @@ -71,29 +85,54 @@ HRESULT CDecoder::CodeSpec(Byte *memStream, UInt32 size) size = (UInt32)rem; } - UInt32 i; int sym = 0; - for (i = 0; i != size; i++) { - sym = Ppmd7_DecodeSymbol(&_ppmd, &_rangeDec.vt); - if (_inStream.Extra || sym < 0) - break; - memStream[i] = (Byte)sym; + Byte *buf = memStream; + const Byte *lim = buf + size; + for (; buf != lim; buf++) + { + sym = Ppmd7z_DecodeSymbol(&_ppmd); + if (_inStream.Extra || sym < 0) + break; + *buf = (Byte)sym; + } + /* + buf = Ppmd7z_DecodeSymbols(&_ppmd, buf, lim); + sym = _ppmd.LastSymbol; + */ + _processedSize += (size_t)(buf - memStream); } - _processedSize += i; - if (_inStream.Extra) + CHECK_EXTRA_ERROR + + if (sym >= 0) + { + if (!FinishStream + || !_outSizeDefined + || _outSize != _processedSize + || _rangeDec.Code == 0) + return S_OK; + /* + // We can decode additional End Marker here: + sym = Ppmd7z_DecodeSymbol(&_ppmd); + CHECK_EXTRA_ERROR + */ + } + + if (sym != PPMD7_SYM_END || _rangeDec.Code != 0) { _status = kStatus_Error; - return _inStream.Res; + return (_res = S_FALSE); } - if (sym < 0) - _status = (sym < -1) ? kStatus_Error : kStatus_Finished; + + _status = kStatus_Finished_With_Mark; return S_OK; } + + STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { if (!_outBuf) { @@ -112,18 +151,23 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream size_t processed = (size_t)(_processedSize - startPos); RINOK(WriteStream(outStream, _outBuf, processed)); RINOK(res); - if (_status == kStatus_Finished) + if (_status == kStatus_Finished_With_Mark) break; if (progress) { - UInt64 inSize = _inStream.GetProcessed(); - RINOK(progress->SetRatioInfo(&inSize, &_processedSize)); + const UInt64 inProcessed = _inStream.GetProcessed(); + RINOK(progress->SetRatioInfo(&inProcessed, &_processedSize)); } } while (!_outSizeDefined || _processedSize < _outSize); + + if (FinishStream && inSize && *inSize != _inStream.GetProcessed()) + return S_FALSE; + return S_OK; } + STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) { _outSizeDefined = (outSize != NULL); @@ -131,9 +175,15 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) _outSize = *outSize; _processedSize = 0; _status = kStatus_NeedInit; + _res = SZ_OK; return S_OK; } +STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) +{ + FinishStream = (finishMode != 0); + return S_OK; +} STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) { diff --git a/CPP/7zip/Compress/PpmdDecoder.h b/CPP/7zip/Compress/PpmdDecoder.h index 2b6213c0..f1a8ee21 100644 --- a/CPP/7zip/Compress/PpmdDecoder.h +++ b/CPP/7zip/Compress/PpmdDecoder.h @@ -1,5 +1,5 @@ // PpmdDecoder.h -// 2009-03-11 : Igor Pavlov : Public domain +// 2020-07-03 : Igor Pavlov : Public domain #ifndef __COMPRESS_PPMD_DECODER_H #define __COMPRESS_PPMD_DECODER_H @@ -8,16 +8,17 @@ #include "../../Common/MyCom.h" -#include "../Common/CWrappers.h" - #include "../ICoder.h" +#include "../Common/CWrappers.h" + namespace NCompress { namespace NPpmd { class CDecoder : public ICompressCoder, public ICompressSetDecoderProperties2, + public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, #ifndef NO_READ_FROM_CODER public ICompressSetInStream, @@ -27,12 +28,13 @@ class CDecoder : public CMyUnknownImp { Byte *_outBuf; - CPpmd7z_RangeDec _rangeDec; CByteInBufWrap _inStream; CPpmd7 _ppmd; Byte _order; + bool FinishStream; bool _outSizeDefined; + HRESULT _res; int _status; UInt64 _outSize; UInt64 _processedSize; @@ -47,7 +49,7 @@ public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) - // MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) @@ -61,6 +63,7 @@ public: STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); + STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); @@ -71,11 +74,13 @@ public: STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); #endif - CDecoder(): _outBuf(NULL), _outSizeDefined(false) + CDecoder(): + _outBuf(NULL), + FinishStream(false), + _outSizeDefined(false) { - Ppmd7z_RangeDec_CreateVTable(&_rangeDec); - _rangeDec.Stream = &_inStream.vt; Ppmd7_Construct(&_ppmd); + _ppmd.rc.dec.Stream = &_inStream.vt; } ~CDecoder(); diff --git a/CPP/7zip/Compress/PpmdEncoder.cpp b/CPP/7zip/Compress/PpmdEncoder.cpp index 34e6d361..e2754772 100644 --- a/CPP/7zip/Compress/PpmdEncoder.cpp +++ b/CPP/7zip/Compress/PpmdEncoder.cpp @@ -3,7 +3,6 @@ #include "StdAfx.h" #include "../../../C/Alloc.h" -#include "../../../C/CpuArch.h" #include "../Common/StreamUtils.h" @@ -21,7 +20,7 @@ void CEncProps::Normalize(int level) if (level < 0) level = 5; if (level > 9) level = 9; if (MemSize == (UInt32)(Int32)-1) - MemSize = level >= 9 ? ((UInt32)192 << 20) : ((UInt32)1 << (level + 19)); + MemSize = (UInt32)1 << (level + 19); const unsigned kMult = 16; if (MemSize / kMult > ReduceSize) { @@ -43,8 +42,8 @@ CEncoder::CEncoder(): _inBuf(NULL) { _props.Normalize(-1); - _rangeEnc.Stream = &_outStream.vt; Ppmd7_Construct(&_ppmd); + _ppmd.rc.enc.Stream = &_outStream.vt; } CEncoder::~CEncoder() @@ -120,8 +119,8 @@ HRESULT CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outS _outStream.Stream = outStream; _outStream.Init(); - Ppmd7z_RangeEnc_Init(&_rangeEnc); - Ppmd7_Init(&_ppmd, _props.Order); + Ppmd7z_Init_RangeEnc(&_ppmd); + Ppmd7_Init(&_ppmd, (unsigned)_props.Order); UInt64 processed = 0; for (;;) @@ -131,19 +130,27 @@ HRESULT CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outS if (size == 0) { // We don't write EndMark in PPMD-7z. - // Ppmd7_EncodeSymbol(&_ppmd, &_rangeEnc, -1); - Ppmd7z_RangeEnc_FlushData(&_rangeEnc); + // Ppmd7z_EncodeSymbol(&_ppmd, -1); + Ppmd7z_Flush_RangeEnc(&_ppmd); return _outStream.Flush(); } - for (UInt32 i = 0; i < size; i++) + const Byte *buf = _inBuf; + const Byte *lim = buf + size; + /* + for (; buf < lim; buf++) { - Ppmd7_EncodeSymbol(&_ppmd, &_rangeEnc, _inBuf[i]); + Ppmd7z_EncodeSymbol(&_ppmd, *buf); RINOK(_outStream.Res); } + */ + + Ppmd7z_EncodeSymbols(&_ppmd, buf, lim); + RINOK(_outStream.Res); + processed += size; if (progress) { - UInt64 outSize = _outStream.GetProcessed(); + const UInt64 outSize = _outStream.GetProcessed(); RINOK(progress->SetRatioInfo(&processed, &outSize)); } } diff --git a/CPP/7zip/Compress/PpmdEncoder.h b/CPP/7zip/Compress/PpmdEncoder.h index 671a5353..0104663f 100644 --- a/CPP/7zip/Compress/PpmdEncoder.h +++ b/CPP/7zip/Compress/PpmdEncoder.h @@ -37,7 +37,6 @@ class CEncoder : { Byte *_inBuf; CByteOutBufWrap _outStream; - CPpmd7z_RangeEnc _rangeEnc; CPpmd7 _ppmd; CEncProps _props; public: diff --git a/CPP/7zip/Compress/PpmdZip.cpp b/CPP/7zip/Compress/PpmdZip.cpp index d9ce218a..434e143b 100644 --- a/CPP/7zip/Compress/PpmdZip.cpp +++ b/CPP/7zip/Compress/PpmdZip.cpp @@ -15,8 +15,8 @@ namespace NPpmdZip { CDecoder::CDecoder(bool fullFileMode): _fullFileMode(fullFileMode) { - _ppmd.Stream.In = &_inStream.vt; Ppmd8_Construct(&_ppmd); + _ppmd.Stream.In = &_inStream.vt; } CDecoder::~CDecoder() @@ -27,6 +27,8 @@ CDecoder::~CDecoder() STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { + // try { + if (!_outStream.Alloc()) return E_OUTOFMEMORY; if (!_inStream.Alloc(1 << 20)) @@ -43,9 +45,9 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream return S_FALSE; UInt32 val = GetUi16(buf); - UInt32 order = (val & 0xF) + 1; + unsigned order = (val & 0xF) + 1; UInt32 mem = ((val >> 4) & 0xFF) + 1; - UInt32 restor = (val >> 12); + unsigned restor = (val >> 12); if (order < 2 || restor > 2) return S_FALSE; @@ -57,7 +59,7 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream if (!Ppmd8_Alloc(&_ppmd, mem << 20, &g_BigAlloc)) return E_OUTOFMEMORY; - if (!Ppmd8_RangeDec_Init(&_ppmd)) + if (!Ppmd8_Init_RangeDec(&_ppmd)) return S_FALSE; Ppmd8_Init(&_ppmd, order, restor); } @@ -79,21 +81,23 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream } } - Byte *data = _outStream.Buf; - size_t i = 0; - int sym = 0; + int sym; + Byte *buf = _outStream.Buf; + const Byte *lim = buf + size; + do { sym = Ppmd8_DecodeSymbol(&_ppmd); if (_inStream.Extra || sym < 0) break; - data[i] = (Byte)sym; + *buf++ = (Byte)sym; } - while (++i != size); + while (buf != lim); - processedSize += i; + size_t cur = (size_t)(buf - _outStream.Buf); + processedSize += cur; - RINOK(WriteStream(outStream, _outStream.Buf, i)); + RINOK(WriteStream(outStream, _outStream.Buf, cur)); RINOK(_inStream.Res); if (_inStream.Extra) @@ -133,6 +137,8 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream } return S_OK; + + // } catch (...) { return E_FAIL; } } @@ -158,21 +164,14 @@ void CEncProps::Normalize(int level) if (level == 0) level = 1; if (level > 9) level = 9; if (MemSizeMB == (UInt32)(Int32)-1) - MemSizeMB = (1 << ((level > 8 ? 8 : level) - 1)); + MemSizeMB = 1 << (level - 1); const unsigned kMult = 16; - if ((MemSizeMB << 20) / kMult > ReduceSize) - { - for (UInt32 m = (1 << 20); m <= (1 << 28); m <<= 1) + for (UInt32 m = 1; m < MemSizeMB; m <<= 1) + if (ReduceSize <= (m << 20) / kMult) { - if (ReduceSize <= m / kMult) - { - m >>= 20; - if (MemSizeMB > m) - MemSizeMB = m; - break; - } + MemSizeMB = m; + break; } - } if (Order == -1) Order = 3 + level; if (Restor == -1) Restor = level < 7 ? @@ -197,6 +196,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA continue; if (propID == NCoderPropID::kReduceSize) { + props.ReduceSize = (UInt32)(Int32)-1; if (prop.vt == VT_UI8 && prop.uhVal.QuadPart < (UInt32)(Int32)-1) props.ReduceSize = (UInt32)prop.uhVal.QuadPart; continue; @@ -219,9 +219,9 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA case NCoderPropID::kNumThreads: break; case NCoderPropID::kLevel: level = (int)v; break; case NCoderPropID::kAlgorithm: - if (v > 1) + if (v >= PPMD8_RESTORE_METHOD_UNSUPPPORTED) return E_INVALIDARG; - props.Restor = v; + props.Restor = (int)v; break; default: return E_INVALIDARG; } @@ -251,12 +251,14 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream _outStream.Stream = outStream; _outStream.Init(); - Ppmd8_RangeEnc_Init(&_ppmd); - Ppmd8_Init(&_ppmd, _props.Order, _props.Restor); + Ppmd8_Init_RangeEnc(&_ppmd); + Ppmd8_Init(&_ppmd, (unsigned)_props.Order, (unsigned)_props.Restor); - UInt32 val = (UInt32)((_props.Order - 1) + ((_props.MemSizeMB - 1) << 4) + (_props.Restor << 12)); - _outStream.WriteByte((Byte)(val & 0xFF)); - _outStream.WriteByte((Byte)(val >> 8)); + { + UInt32 val = (UInt32)(((unsigned)_props.Order - 1) + ((_props.MemSizeMB - 1) << 4) + ((unsigned)_props.Restor << 12)); + _outStream.WriteByte((Byte)(val & 0xFF)); + _outStream.WriteByte((Byte)(val >> 8)); + } RINOK(_outStream.Res); UInt64 processed = 0; @@ -267,15 +269,23 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream if (size == 0) { Ppmd8_EncodeSymbol(&_ppmd, -1); - Ppmd8_RangeEnc_FlushData(&_ppmd); + Ppmd8_Flush_RangeEnc(&_ppmd); return _outStream.Flush(); } - for (UInt32 i = 0; i < size; i++) + + processed += size; + const Byte *buf = _inStream.Buf; + const Byte *lim = buf + size; + do { - Ppmd8_EncodeSymbol(&_ppmd, _inStream.Buf[i]); - RINOK(_outStream.Res); + Ppmd8_EncodeSymbol(&_ppmd, *buf); + if (_outStream.Res != S_OK) + break; } - processed += size; + while (++buf != lim); + + RINOK(_outStream.Res); + if (progress) { const UInt64 outProccessed = _outStream.GetProcessed(); @@ -284,4 +294,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream } } + + + }} diff --git a/CPP/7zip/Compress/PpmdZip.h b/CPP/7zip/Compress/PpmdZip.h index b8c7aae0..9d1fc4bd 100644 --- a/CPP/7zip/Compress/PpmdZip.h +++ b/CPP/7zip/Compress/PpmdZip.h @@ -8,10 +8,10 @@ #include "../../Common/MyCom.h" -#include "../Common/CWrappers.h" - #include "../ICoder.h" +#include "../Common/CWrappers.h" + namespace NCompress { namespace NPpmdZip { @@ -21,7 +21,7 @@ struct CBuf { Byte *Buf; - CBuf(): Buf(0) {} + CBuf(): Buf(NULL) {} ~CBuf() { ::MidFree(Buf); } bool Alloc() { @@ -52,7 +52,7 @@ public: STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); - CDecoder(bool fullFileMode); + CDecoder(bool fullFileMode = true); ~CDecoder(); }; diff --git a/CPP/7zip/Compress/QuantumDecoder.cpp b/CPP/7zip/Compress/QuantumDecoder.cpp index 64e35bf2..8c650581 100644 --- a/CPP/7zip/Compress/QuantumDecoder.cpp +++ b/CPP/7zip/Compress/QuantumDecoder.cpp @@ -41,7 +41,7 @@ unsigned CModelDecoder::Decode(CRangeDecoder *rc) unsigned res = Vals[--i]; do - Freqs[i] += kUpdateStep; + Freqs[i] = (UInt16)(Freqs[i] + kUpdateStep); while (i--); if (Freqs[0] > kFreqSumMax) @@ -72,7 +72,7 @@ unsigned CModelDecoder::Decode(CRangeDecoder *rc) i = NumItems - 1; do { - Freqs[i] >>= 1; + Freqs[i] = (UInt16)(Freqs[i] >> 1); if (Freqs[i] <= Freqs[(size_t)i + 1]) Freqs[i] = (UInt16)(Freqs[(size_t)i + 1] + 1); } diff --git a/CPP/7zip/Compress/Rar1Decoder.cpp b/CPP/7zip/Compress/Rar1Decoder.cpp index 6b55705b..510bbd17 100644 --- a/CPP/7zip/Compress/Rar1Decoder.cpp +++ b/CPP/7zip/Compress/Rar1Decoder.cpp @@ -239,6 +239,7 @@ HRESULT CDecoder::LongLZ() oldAvr3 = AvrLn3; if (len != 1 && len != 4) + { if (len == 0 && dist <= MaxDist3) { AvrLn3++; @@ -246,6 +247,7 @@ HRESULT CDecoder::LongLZ() } else if (AvrLn3 > 0) AvrLn3--; + } len += 3; @@ -254,7 +256,7 @@ HRESULT CDecoder::LongLZ() if (dist <= 256) len += 8; - if (oldAvr3 > 0xb0 || AvrPlc >= 0x2a00 && oldAvr2 < 0x40) + if (oldAvr3 > 0xb0 || (AvrPlc >= 0x2a00 && oldAvr2 < 0x40)) MaxDist3 = 0x7f00; else MaxDist3 = 0x2001; @@ -412,7 +414,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * { // InitStructures - for (int i = 0; i < kNumRepDists; i++) + for (unsigned i = 0; i < kNumRepDists; i++) m_RepDists[i] = 0; m_RepDistPtr = 0; LastLength = 0; diff --git a/CPP/7zip/Compress/Rar2Decoder.cpp b/CPP/7zip/Compress/Rar2Decoder.cpp index fe458c85..b1c8c2d4 100644 --- a/CPP/7zip/Compress/Rar2Decoder.cpp +++ b/CPP/7zip/Compress/Rar2Decoder.cpp @@ -4,6 +4,8 @@ #include "StdAfx.h" +#include <stdlib.h> + #include "Rar2Decoder.h" namespace NCompress { @@ -77,7 +79,7 @@ Byte CFilter::Decode(int &channelDelta, Byte deltaByte) static const UInt32 kHistorySize = 1 << 20; -static const UInt32 kWindowReservSize = (1 << 22) + 256; +// static const UInt32 kWindowReservSize = (1 << 22) + 256; CDecoder::CDecoder(): _isSolid(false), @@ -209,6 +211,7 @@ bool CDecoder::ReadLastTables() // + 2 works for: return 0xFF; in CInBuffer::ReadByte. if (m_InBitStream.GetProcessedSize() + 7 <= m_PackSize) // test it: probably incorrect; // if (m_InBitStream.GetProcessedSize() + 2 <= m_PackSize) // test it: probably incorrect; + { if (m_AudioMode) { UInt32 symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); @@ -225,6 +228,7 @@ bool CDecoder::ReadLastTables() if (sym >= kMainTableSize) return false; } + } return true; } diff --git a/CPP/7zip/Compress/Rar3Decoder.cpp b/CPP/7zip/Compress/Rar3Decoder.cpp index 4456ba63..2271dcc8 100644 --- a/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/CPP/7zip/Compress/Rar3Decoder.cpp @@ -19,20 +19,14 @@ static const UInt32 kNumAlignReps = 15; static const UInt32 kSymbolReadTable = 256; static const UInt32 kSymbolRep = 259; -static const UInt32 kSymbolLen2 = kSymbolRep + kNumReps; - -static const Byte kLenStart [kLenTableSize] = {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224}; -static const Byte kLenDirectBits[kLenTableSize] = {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; static const Byte kDistDirectBits[kDistTableSize] = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15, 16,16,16,16,16,16,16,16,16,16,16,16,16,16, 18,18,18,18,18,18,18,18,18,18,18,18}; -static const Byte kLevelDirectBits[kLevelTableSize] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7}; - -static const Byte kLen2DistStarts[kNumLen2Symbols]={0,4,8,16,32,64,128,192}; -static const Byte kLen2DistDirectBits[kNumLen2Symbols]={2,2,3, 4, 5, 6, 6, 6}; +static const Byte kLen2DistStarts[kNumLen2Symbols] = {0,4,8,16,32,64,128,192}; +static const Byte kLen2DistDirectBits[kNumLen2Symbols] = {2,2,3, 4, 5, 6, 6, 6}; static const UInt32 kDistLimit3 = 0x2000 - 2; static const UInt32 kDistLimit4 = 0x40000 - 2; @@ -44,47 +38,20 @@ static const UInt32 kVmCodeSizeMax = 1 << 16; extern "C" { -#define GET_RangeDecoder CRangeDecoder *p = CONTAINER_FROM_VTBL_CLS(pp, CRangeDecoder, vt); - -static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) +static Byte Wrap_ReadByte(const IByteIn *pp) throw() { - GET_RangeDecoder; - return p->Code / (p->Range /= total); + CByteIn *p = CONTAINER_FROM_VTBL_CLS(pp, CByteIn, IByteIn_obj); + return p->BitDecoder.Stream.ReadByte(); } -static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) +static Byte Wrap_ReadBits8(const IByteIn *pp) throw() { - GET_RangeDecoder; - start *= p->Range; - p->Low += start; - p->Code -= start; - p->Range *= size; - p->Normalize(); + CByteIn *p = CONTAINER_FROM_VTBL_CLS(pp, CByteIn, IByteIn_obj); + return (Byte)p->BitDecoder.ReadByteFromAligned(); } -static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) -{ - GET_RangeDecoder; - if (p->Code / (p->Range >>= 14) < size0) - { - Range_Decode(&p->vt, 0, size0); - return 0; - } - else - { - Range_Decode(&p->vt, size0, (1 << 14) - size0); - return 1; - } } -} - -CRangeDecoder::CRangeDecoder() throw() -{ - vt.GetThreshold = Range_GetThreshold; - vt.Decode = Range_Decode; - vt.DecodeBit = Range_DecodeBit; -} CDecoder::CDecoder(): _window(0), @@ -98,6 +65,13 @@ CDecoder::CDecoder(): _solidAllowed(false) { Ppmd7_Construct(&_ppmd); + + UInt32 start = 0; + for (UInt32 i = 0; i < kDistTableSize; i++) + { + kDistStart[i] = start; + start += ((UInt32)1 << kDistDirectBits[i]); + } } CDecoder::~CDecoder() @@ -360,32 +334,37 @@ bool CDecoder::AddVmCode(UInt32 firstByte, UInt32 codeSize) bool CDecoder::ReadVmCodeLZ() { UInt32 firstByte = ReadBits(8); - UInt32 length = (firstByte & 7) + 1; - if (length == 7) - length = ReadBits(8) + 7; - else if (length == 8) - length = ReadBits(16); - if (length > kVmDataSizeMax) + UInt32 len = (firstByte & 7) + 1; + if (len == 7) + len = ReadBits(8) + 7; + else if (len == 8) + len = ReadBits(16); + if (len > kVmDataSizeMax) return false; - for (UInt32 i = 0; i < length; i++) + for (UInt32 i = 0; i < len; i++) _vmData[i] = (Byte)ReadBits(8); - return AddVmCode(firstByte, length); + return AddVmCode(firstByte, len); } + +// int CDecoder::DecodePpmSymbol() { return Ppmd7a_DecodeSymbol(&_ppmd); } +#define DecodePpmSymbol() Ppmd7a_DecodeSymbol(&_ppmd) + + bool CDecoder::ReadVmCodePPM() { int firstByte = DecodePpmSymbol(); if (firstByte < 0) return false; - UInt32 length = (firstByte & 7) + 1; - if (length == 7) + UInt32 len = (firstByte & 7) + 1; + if (len == 7) { int b1 = DecodePpmSymbol(); if (b1 < 0) return false; - length = b1 + 7; + len = b1 + 7; } - else if (length == 8) + else if (len == 8) { int b1 = DecodePpmSymbol(); if (b1 < 0) @@ -393,20 +372,20 @@ bool CDecoder::ReadVmCodePPM() int b2 = DecodePpmSymbol(); if (b2 < 0) return false; - length = b1 * 256 + b2; + len = b1 * 256 + b2; } - if (length > kVmDataSizeMax) + if (len > kVmDataSizeMax) return false; if (InputEofError_Fast()) return false; - for (UInt32 i = 0; i < length; i++) + for (UInt32 i = 0; i < len; i++) { int b = DecodePpmSymbol(); if (b < 0) return false; _vmData[i] = (Byte)b; } - return AddVmCode(firstByte, length); + return AddVmCode(firstByte, len); } #define RIF(x) { if (!(x)) return S_FALSE; } @@ -422,19 +401,22 @@ HRESULT CDecoder::InitPPM() bool reset = ((maxOrder & 0x20) != 0); UInt32 maxMB = 0; if (reset) - maxMB = (Byte)ReadBits(8); + maxMB = (Byte)Wrap_ReadBits8(&m_InBitStream.IByteIn_obj); else { if (PpmError || !Ppmd7_WasAllocated(&_ppmd)) return S_FALSE; } if (maxOrder & 0x40) - PpmEscChar = (Byte)ReadBits(8); - m_InBitStream.InitRangeCoder(); - /* - if (m_InBitStream.m_BitPos != 0) - return S_FALSE; - */ + PpmEscChar = (Byte)Wrap_ReadBits8(&m_InBitStream.IByteIn_obj); + + _ppmd.rc.dec.Stream = &m_InBitStream.IByteIn_obj; + m_InBitStream.IByteIn_obj.Read = Wrap_ReadBits8; + + Ppmd7a_RangeDec_Init(&_ppmd.rc.dec); + + m_InBitStream.IByteIn_obj.Read = Wrap_ReadByte; + if (reset) { PpmError = true; @@ -454,7 +436,6 @@ HRESULT CDecoder::InitPPM() return S_OK; } -int CDecoder::DecodePpmSymbol() { return Ppmd7_DecodeSymbol(&_ppmd, &m_InBitStream.vt); } HRESULT CDecoder::DecodePPM(Int32 num, bool &keepDecompressing) { @@ -503,8 +484,8 @@ HRESULT CDecoder::DecodePPM(Int32 num, bool &keepDecompressing) } if (nextCh == 4 || nextCh == 5) { - UInt32 distance = 0; - UInt32 length = 4; + UInt32 dist = 0; + UInt32 len = 4; if (nextCh == 4) { for (int i = 0; i < 3; i++) @@ -515,10 +496,10 @@ HRESULT CDecoder::DecodePPM(Int32 num, bool &keepDecompressing) PpmError = true; return S_FALSE; } - distance = (distance << 8) + (Byte)c2; + dist = (dist << 8) + (Byte)c2; } - distance++; - length += 28; + dist++; + len += 28; } int c2 = DecodePpmSymbol(); if (c2 < 0) @@ -526,11 +507,11 @@ HRESULT CDecoder::DecodePPM(Int32 num, bool &keepDecompressing) PpmError = true; return S_FALSE; } - length += c2; - if (distance >= _lzSize) + len += c2; + if (dist >= _lzSize) return S_FALSE; - CopyBlock(distance, length); - num -= (Int32)length; + CopyBlock(dist, len); + num -= (Int32)len; continue; } } @@ -571,8 +552,8 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) for (i = 0; i < kLevelTableSize; i++) { - UInt32 length = ReadBits(4); - if (length == 15) + UInt32 len = ReadBits(4); + if (len == 15) { UInt32 zeroCount = ReadBits(4); if (zeroCount != 0) @@ -584,7 +565,7 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) continue; } } - levelLevels[i] = (Byte)length; + levelLevels[i] = (Byte)len; } RIF(m_LevelDecoder.Build(levelLevels)); @@ -674,22 +655,6 @@ HRESULT CDecoder::ReadEndOfBlock(bool &keepDecompressing) return ReadTables(keepDecompressing); } -UInt32 kDistStart[kDistTableSize]; - -class CDistInit -{ -public: - CDistInit() { Init(); } - void Init() - { - UInt32 start = 0; - for (UInt32 i = 0; i < kDistTableSize; i++) - { - kDistStart[i] = start; - start += (1 << kDistDirectBits[i]); - } - } -} g_DistInit; HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) { @@ -697,7 +662,7 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) UInt32 rep1 = _reps[1]; UInt32 rep2 = _reps[2]; UInt32 rep3 = _reps[3]; - UInt32 length = _lastLength; + UInt32 len = _lastLength; for (;;) { if (((_wrPtr - _winPos) & kWindowMask) < 260 && _wrPtr != _winPos) @@ -732,35 +697,40 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) } else if (sym == 258) { - if (length == 0) + if (len == 0) return S_FALSE; } else if (sym < kSymbolRep + 4) { if (sym != kSymbolRep) { - UInt32 distance; + UInt32 dist; if (sym == kSymbolRep + 1) - distance = rep1; + dist = rep1; else { if (sym == kSymbolRep + 2) - distance = rep2; + dist = rep2; else { - distance = rep3; + dist = rep3; rep3 = rep2; } rep2 = rep1; } rep1 = rep0; - rep0 = distance; + rep0 = dist; } const UInt32 sym2 = m_LenDecoder.Decode(&m_InBitStream.BitDecoder); if (sym2 >= kLenTableSize) return S_FALSE; - length = 2 + kLenStart[sym2] + m_InBitStream.BitDecoder.ReadBits(kLenDirectBits[sym2]); + len = 2 + sym2; + if (sym2 >= 8) + { + unsigned num = (sym2 >> 2) - 1; + len = 2 + ((4 + (sym2 & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); + } } else { @@ -770,13 +740,18 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) if (sym < 271) { sym -= 263; - rep0 = kLen2DistStarts[sym] + m_InBitStream.BitDecoder.ReadBits(kLen2DistDirectBits[sym]); - length = 2; + rep0 = kLen2DistStarts[sym] + m_InBitStream.BitDecoder.ReadBits_upto8(kLen2DistDirectBits[sym]); + len = 2; } else if (sym < 299) { sym -= 271; - length = kNormalMatchMinLen + (UInt32)kLenStart[sym] + m_InBitStream.BitDecoder.ReadBits(kLenDirectBits[sym]); + len = kNormalMatchMinLen + sym; + if (sym >= 8) + { + unsigned num = (sym >> 2) - 1; + len = kNormalMatchMinLen + ((4 + (sym & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); + } const UInt32 sym2 = m_DistDecoder.Decode(&m_InBitStream.BitDecoder); if (sym2 >= kDistTableSize) return S_FALSE; @@ -809,21 +784,21 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) } } else - rep0 += m_InBitStream.BitDecoder.ReadBits(numBits); - length += ((kDistLimit4 - rep0) >> 31) + ((kDistLimit3 - rep0) >> 31); + rep0 += m_InBitStream.BitDecoder.ReadBits_upto8(numBits); + len += ((UInt32)(kDistLimit4 - rep0) >> 31) + ((UInt32)(kDistLimit3 - rep0) >> 31); } else return S_FALSE; } if (rep0 >= _lzSize) return S_FALSE; - CopyBlock(rep0, length); + CopyBlock(rep0, len); } _reps[0] = rep0; _reps[1] = rep1; _reps[2] = rep2; _reps[3] = rep3; - _lastLength = length; + _lastLength = len; return S_OK; } @@ -839,7 +814,7 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) _lzSize = 0; _winPos = 0; _wrPtr = 0; - for (int i = 0; i < kNumReps; i++) + for (unsigned i = 0; i < kNumReps; i++) _reps[i] = 0; _lastLength = 0; memset(m_LastLevels, 0, kTablesSizesSum); diff --git a/CPP/7zip/Compress/Rar3Decoder.h b/CPP/7zip/Compress/Rar3Decoder.h index 3d319ded..fdecc55f 100644 --- a/CPP/7zip/Compress/Rar3Decoder.h +++ b/CPP/7zip/Compress/Rar3Decoder.h @@ -95,44 +95,40 @@ public: MovePos(numBits); return res; } -}; - -const UInt32 kTopValue = (1 << 24); -const UInt32 kBot = (1 << 15); - -struct CRangeDecoder -{ - IPpmd7_RangeDec vt; - UInt32 Range; - UInt32 Code; - UInt32 Low; - CBitDecoder BitDecoder; - SRes Res; -public: - void InitRangeCoder() + UInt32 ReadBits_upto8(unsigned numBits) { - Code = 0; - Low = 0; - Range = 0xFFFFFFFF; - for (int i = 0; i < 4; i++) - Code = (Code << 8) | BitDecoder.ReadBits(8); + if (_bitPos < numBits) + { + _bitPos += 8; + _value = (_value << 8) | Stream.ReadByte(); + } + _bitPos -= numBits; + UInt32 res = _value >> _bitPos; + _value = _value & ((1 << _bitPos) - 1); + return res; } - void Normalize() + Byte ReadByteFromAligned() { - while ((Low ^ (Low + Range)) < kTopValue || - Range < kBot && ((Range = (0 - Low) & (kBot - 1)), 1)) - { - Code = (Code << 8) | BitDecoder.Stream.ReadByte(); - Range <<= 8; - Low <<= 8; - } + if (_bitPos == 0) + return Stream.ReadByte(); + unsigned bitsPos = _bitPos - 8; + Byte b = (Byte)(_value >> bitsPos); + _value = _value & ((1 << bitsPos) - 1); + _bitPos = bitsPos; + return b; } +}; + - CRangeDecoder() throw(); +struct CByteIn +{ + IByteIn IByteIn_obj; + CBitDecoder BitDecoder; }; + struct CFilter: public NVm::CProgram { CRecordVector<Byte> GlobalData; @@ -165,7 +161,7 @@ class CDecoder: public ICompressSetDecoderProperties2, public CMyUnknownImp { - CRangeDecoder m_InBitStream; + CByteIn m_InBitStream; Byte *_window; UInt32 _winPos; UInt32 _wrPtr; @@ -174,6 +170,7 @@ class CDecoder: UInt64 _writtenFileSize; // if it's > _unpackSize, then _unpackSize only written ISequentialOutStream *_outStream; NHuffman::CDecoder<kNumHuffmanBits, kMainTableSize> m_MainDecoder; + UInt32 kDistStart[kDistTableSize]; NHuffman::CDecoder<kNumHuffmanBits, kDistTableSize> m_DistDecoder; NHuffman::CDecoder<kNumHuffmanBits, kAlignTableSize> m_AlignDecoder; NHuffman::CDecoder<kNumHuffmanBits, kLenTableSize> m_LenDecoder; @@ -223,7 +220,7 @@ class CDecoder: UInt32 ReadBits(unsigned numBits); HRESULT InitPPM(); - int DecodePpmSymbol(); + // int DecodePpmSymbol(); HRESULT DecodePPM(Int32 num, bool &keepDecompressing); HRESULT ReadTables(bool &keepDecompressing); @@ -245,10 +242,10 @@ public: STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); - void CopyBlock(UInt32 distance, UInt32 len) + void CopyBlock(UInt32 dist, UInt32 len) { _lzSize += len; - UInt32 pos = (_winPos - distance - 1) & kWindowMask; + UInt32 pos = (_winPos - dist - 1) & kWindowMask; Byte *window = _window; UInt32 winPos = _winPos; if (kWindowSize - winPos > len && kWindowSize - pos > len) @@ -273,12 +270,11 @@ public: void PutByte(Byte b) { - _window[_winPos] = b; - _winPos = (_winPos + 1) & kWindowMask; + UInt32 wp = _winPos; + _window[wp] = b; + _winPos = (wp + 1) & kWindowMask; _lzSize++; } - - }; }} diff --git a/CPP/7zip/Compress/Rar3Vm.cpp b/CPP/7zip/Compress/Rar3Vm.cpp index 61cf657c..6accd08c 100644 --- a/CPP/7zip/Compress/Rar3Vm.cpp +++ b/CPP/7zip/Compress/Rar3Vm.cpp @@ -34,7 +34,7 @@ UInt32 CMemBitDecoder::ReadBits(unsigned numBits) if (numBits <= avail) { _bitPos += numBits; - return res | (b >> (avail - numBits)) & ((1 << numBits) - 1); + return res | ((b >> (avail - numBits)) & ((1 << numBits) - 1)); } numBits -= avail; res |= (UInt32)(b & ((1 << avail) - 1)) << numBits; diff --git a/CPP/7zip/Compress/Rar5Decoder.cpp b/CPP/7zip/Compress/Rar5Decoder.cpp index d7c68e21..caabb316 100644 --- a/CPP/7zip/Compress/Rar5Decoder.cpp +++ b/CPP/7zip/Compress/Rar5Decoder.cpp @@ -64,16 +64,16 @@ enum FilterType static const size_t kWriteStep = (size_t)1 << 22; CDecoder::CDecoder(): + _isSolid(false), + _solidAllowed(false), + _wasInit(false), + _dictSizeLog(0), _window(NULL), _winPos(0), - _winSizeAllocated(0), _lzSize(0), _lzEnd(0), _writtenFileSize(0), - _dictSizeLog(0), - _isSolid(false), - _solidAllowed(false), - _wasInit(false), + _winSizeAllocated(0), _inputBuf(NULL) { } diff --git a/CPP/7zip/Compress/ShrinkDecoder.cpp b/CPP/7zip/Compress/ShrinkDecoder.cpp index 41f0e352..22f3844c 100644 --- a/CPP/7zip/Compress/ShrinkDecoder.cpp +++ b/CPP/7zip/Compress/ShrinkDecoder.cpp @@ -160,7 +160,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * } } - lastSym = sym; + lastSym = (int)sym; unsigned cur = sym; unsigned i = 0; diff --git a/CPP/7zip/Compress/XpressDecoder.cpp b/CPP/7zip/Compress/XpressDecoder.cpp index 864c6f8f..a8bbd008 100644 --- a/CPP/7zip/Compress/XpressDecoder.cpp +++ b/CPP/7zip/Compress/XpressDecoder.cpp @@ -7,6 +7,7 @@ #include "../../../C/CpuArch.h" #include "HuffmanDecoder.h" +#include "XpressDecoder.h" namespace NCompress { namespace NXpress { diff --git a/CPP/7zip/Compress/XzDecoder.cpp b/CPP/7zip/Compress/XzDecoder.cpp index 7a974311..0371173c 100644 --- a/CPP/7zip/Compress/XzDecoder.cpp +++ b/CPP/7zip/Compress/XzDecoder.cpp @@ -34,7 +34,7 @@ static HRESULT SResToHRESULT_Code(SRes res) throw() HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) { - MainDecodeSRes = S_OK; + MainDecodeSRes = SZ_OK; MainDecodeSRes_wasUsed = false; XzStatInfo_Clear(&Stat); @@ -95,7 +95,7 @@ HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) - // return E_OUTOFMEMORY; + // return E_OUTOFMEMORY; // for debug check MainDecodeSRes_wasUsed = true; diff --git a/CPP/7zip/Compress/XzDecoder.h b/CPP/7zip/Compress/XzDecoder.h index b65b46b8..7ad81f49 100644 --- a/CPP/7zip/Compress/XzDecoder.h +++ b/CPP/7zip/Compress/XzDecoder.h @@ -38,8 +38,9 @@ struct CDecoder XzDecMt_Destroy(xz); } - /* Decode() can return ERROR code only if there is progress or stream error. - Decode() returns S_OK in case of xz decoding error, but DecodeRes and CStatInfo contain error information */ + /* Decode() can return S_OK, if there is data after good xz streams, and that data is not new xz stream. + check also (Stat.DataAfterEnd) flag */ + HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *compressProgress); }; diff --git a/CPP/7zip/Compress/XzEncoder.cpp b/CPP/7zip/Compress/XzEncoder.cpp index 7dd4d860..d6f42caf 100644 --- a/CPP/7zip/Compress/XzEncoder.cpp +++ b/CPP/7zip/Compress/XzEncoder.cpp @@ -159,7 +159,7 @@ HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop) int filterId = FilterIdFromName(prop.bstrVal); if (filterId < 0 /* || filterId == XZ_ID_LZMA2 */) return E_INVALIDARG; - id32 = filterId; + id32 = (unsigned)filterId; } } diff --git a/CPP/7zip/Compress/ZlibDecoder.cpp b/CPP/7zip/Compress/ZlibDecoder.cpp index 8f3a63cb..2356c7c5 100644 --- a/CPP/7zip/Compress/ZlibDecoder.cpp +++ b/CPP/7zip/Compress/ZlibDecoder.cpp @@ -15,6 +15,7 @@ namespace NZlib { #define ADLER_MOD 65521 #define ADLER_LOOP_MAX 5550 +UInt32 Adler32_Update(UInt32 adler, const Byte *buf, size_t size); UInt32 Adler32_Update(UInt32 adler, const Byte *buf, size_t size) { UInt32 a = adler & 0xFFFF; diff --git a/CPP/7zip/Crypto/7zAes.cpp b/CPP/7zip/Crypto/7zAes.cpp index 2ed69bad..7f34e910 100644 --- a/CPP/7zip/Crypto/7zAes.cpp +++ b/CPP/7zip/Crypto/7zAes.cpp @@ -2,9 +2,11 @@ #include "StdAfx.h" +#include "../../../C/CpuArch.h" #include "../../../C/Sha256.h" #include "../../Common/ComTry.h" +#include "../../Common/MyBuffer2.h" #ifndef _7ZIP_ST #include "../../Windows/Synchronization.h" @@ -48,31 +50,65 @@ void CKeyInfo::CalcKey() } else { - size_t bufSize = 8 + SaltSize + Password.Size(); - CObjArray<Byte> buf(bufSize); + const unsigned kUnrPow = 6; + const UInt32 numUnroll = (UInt32)1 << (NumCyclesPower <= kUnrPow ? (unsigned)NumCyclesPower : kUnrPow); + + const size_t bufSize = 8 + SaltSize + Password.Size(); + const size_t unrollSize = bufSize * numUnroll; + + // MY_ALIGN (16) + // CSha256 sha; + CAlignedBuffer sha(sizeof(CSha256) + unrollSize + bufSize * 2); + Byte *buf = sha + sizeof(CSha256); + memcpy(buf, Salt, SaltSize); memcpy(buf + SaltSize, Password, Password.Size()); + memset(buf + bufSize - 8, 0, 8); - CSha256 sha; - Sha256_Init(&sha); - - Byte *ctr = buf + SaltSize + Password.Size(); - - for (unsigned i = 0; i < 8; i++) - ctr[i] = 0; + Sha256_Init((CSha256 *)(void *)(Byte *)sha); + { + { + Byte *dest = buf; + for (UInt32 i = 1; i < numUnroll; i++) + { + dest += bufSize; + memcpy(dest, buf, bufSize); + } + } + + const UInt32 numRounds = (UInt32)1 << NumCyclesPower; + UInt32 r = 0; + do + { + Byte *dest = buf + bufSize - 8; + UInt32 i = r; + r += numUnroll; + do + { + SetUi32(dest, i); i++; dest += bufSize; + // SetUi32(dest, i); i++; dest += bufSize; + } + while (i < r); + Sha256_Update((CSha256 *)(void *)(Byte *)sha, buf, unrollSize); + } + while (r < numRounds); + } + /* UInt64 numRounds = (UInt64)1 << NumCyclesPower; do { - Sha256_Update(&sha, buf, bufSize); + Sha256_Update((CSha256 *)(Byte *)sha, buf, bufSize); for (unsigned i = 0; i < 8; i++) if (++(ctr[i]) != 0) break; } while (--numRounds != 0); + */ - Sha256_Final(&sha, Key); + Sha256_Final((CSha256 *)(void *)(Byte *)sha, Key); + memset(sha, 0, sha.Size()); } } @@ -250,6 +286,7 @@ STDMETHODIMP CBaseCoder::CryptoSetPassword(const Byte *data, UInt32 size) { COM_TRY_BEGIN + _key.Password.Wipe(); _key.Password.CopyFrom(data, (size_t)size); return S_OK; diff --git a/CPP/7zip/Crypto/7zAes.h b/CPP/7zip/Crypto/7zAes.h index 84e07ac9..a67ac0b6 100644 --- a/CPP/7zip/Crypto/7zAes.h +++ b/CPP/7zip/Crypto/7zAes.h @@ -37,6 +37,17 @@ public: for (unsigned i = 0; i < sizeof(Salt); i++) Salt[i] = 0; } + + void Wipe() + { + Password.Wipe(); + NumCyclesPower = 0; + SaltSize = 0; + MY_memset_0_ARRAY(Salt); + MY_memset_0_ARRAY(Key); + } + + ~CKeyInfo() { Wipe(); } }; class CKeyInfoCache diff --git a/CPP/7zip/Crypto/7zAesRegister.cpp b/CPP/7zip/Crypto/7zAesRegister.cpp index f9d59699..35605f43 100644 --- a/CPP/7zip/Crypto/7zAesRegister.cpp +++ b/CPP/7zip/Crypto/7zAesRegister.cpp @@ -9,9 +9,9 @@ namespace NCrypto { namespace N7z { -REGISTER_FILTER_E(7zAES, - CDecoder(), - CEncoder(), +REGISTER_FILTER_E(_7zAES, + CDecoder, + CEncoder, 0x6F10701, "7zAES") }} diff --git a/CPP/7zip/Crypto/HmacSha1.cpp b/CPP/7zip/Crypto/HmacSha1.cpp index 359b65bc..d085bb05 100644 --- a/CPP/7zip/Crypto/HmacSha1.cpp +++ b/CPP/7zip/Crypto/HmacSha1.cpp @@ -11,110 +11,82 @@ namespace NSha1 { void CHmac::SetKey(const Byte *key, size_t keySize) { - Byte keyTemp[kBlockSize]; + MY_ALIGN (16) + UInt32 temp[SHA1_NUM_BLOCK_WORDS]; size_t i; - for (i = 0; i < kBlockSize; i++) - keyTemp[i] = 0; + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] = 0; if (keySize > kBlockSize) { _sha.Init(); _sha.Update(key, keySize); - _sha.Final(keyTemp); + _sha.Final((Byte *)temp); } else - for (i = 0; i < keySize; i++) - keyTemp[i] = key[i]; + memcpy(temp, key, keySize); - for (i = 0; i < kBlockSize; i++) - keyTemp[i] ^= 0x36; + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636; _sha.Init(); - _sha.Update(keyTemp, kBlockSize); + _sha.Update((const Byte *)temp, kBlockSize); - for (i = 0; i < kBlockSize; i++) - keyTemp[i] ^= 0x36 ^ 0x5C; + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636 ^ 0x5C5C5C5C; _sha2.Init(); - _sha2.Update(keyTemp, kBlockSize); + _sha2.Update((const Byte *)temp, kBlockSize); } -void CHmac::Final(Byte *mac, size_t macSize) + +void CHmac::Final(Byte *mac) { - Byte digest[kDigestSize]; - _sha.Final(digest); - _sha2.Update(digest, kDigestSize); - _sha2.Final(digest); - for (size_t i = 0; i < macSize; i++) - mac[i] = digest[i]; + _sha.Final(mac); + _sha2.Update(mac, kDigestSize); + _sha2.Final(mac); } -void CHmac32::SetKey(const Byte *key, size_t keySize) +void CHmac::GetLoopXorDigest1(void *mac, UInt32 numIteration) { - UInt32 keyTemp[kNumBlockWords]; - size_t i; - - for (i = 0; i < kNumBlockWords; i++) - keyTemp[i] = 0; - - if (keySize > kBlockSize) - { - CContext sha; - sha.Init(); - sha.Update(key, keySize); - Byte digest[kDigestSize]; - sha.Final(digest); - - for (i = 0 ; i < kNumDigestWords; i++) - keyTemp[i] = GetBe32(digest + i * 4 + 0); - } - else - for (i = 0; i < keySize; i++) - keyTemp[i / 4] |= (key[i] << (24 - 8 * (i & 3))); - - for (i = 0; i < kNumBlockWords; i++) - keyTemp[i] ^= 0x36363636; - - _sha.Init(); - _sha.Update(keyTemp, kNumBlockWords); + MY_ALIGN (16) UInt32 block [SHA1_NUM_BLOCK_WORDS]; + MY_ALIGN (16) UInt32 block2[SHA1_NUM_BLOCK_WORDS]; + MY_ALIGN (16) UInt32 mac2 [SHA1_NUM_BLOCK_WORDS]; - for (i = 0; i < kNumBlockWords; i++) - keyTemp[i] ^= 0x36363636 ^ 0x5C5C5C5C; - - _sha2.Init(); - _sha2.Update(keyTemp, kNumBlockWords); -} + _sha. PrepareBlock((Byte *)block, SHA1_DIGEST_SIZE); + _sha2.PrepareBlock((Byte *)block2, SHA1_DIGEST_SIZE); -void CHmac32::Final(UInt32 *mac, size_t macSize) -{ - UInt32 digest[kNumDigestWords]; - _sha.Final(digest); - _sha2.Update(digest, kNumDigestWords); - _sha2.Final(digest); - for (size_t i = 0; i < macSize; i++) - mac[i] = digest[i]; -} + block[0] = ((const UInt32 *)mac)[0]; + block[1] = ((const UInt32 *)mac)[1]; + block[2] = ((const UInt32 *)mac)[2]; + block[3] = ((const UInt32 *)mac)[3]; + block[4] = ((const UInt32 *)mac)[4]; -void CHmac32::GetLoopXorDigest(UInt32 *mac, UInt32 numIteration) -{ - UInt32 block[kNumBlockWords]; - UInt32 block2[kNumBlockWords]; - - _sha.PrepareBlock(block, kNumDigestWords); - _sha2.PrepareBlock(block2, kNumDigestWords); + mac2[0] = block[0]; + mac2[1] = block[1]; + mac2[2] = block[2]; + mac2[3] = block[3]; + mac2[4] = block[4]; - for (unsigned s = 0; s < kNumDigestWords; s++) - block[s] = mac[s]; - for (UInt32 i = 0; i < numIteration; i++) { - _sha.GetBlockDigest(block, block2); - _sha2.GetBlockDigest(block2, block); - for (unsigned s = 0; s < kNumDigestWords; s++) - mac[s] ^= block[s]; + _sha. GetBlockDigest((const Byte *)block, (Byte *)block2); + _sha2.GetBlockDigest((const Byte *)block2, (Byte *)block); + + mac2[0] ^= block[0]; + mac2[1] ^= block[1]; + mac2[2] ^= block[2]; + mac2[3] ^= block[3]; + mac2[4] ^= block[4]; } + + ((UInt32 *)mac)[0] = mac2[0]; + ((UInt32 *)mac)[1] = mac2[1]; + ((UInt32 *)mac)[2] = mac2[2]; + ((UInt32 *)mac)[3] = mac2[3]; + ((UInt32 *)mac)[4] = mac2[4]; } }} diff --git a/CPP/7zip/Crypto/HmacSha1.h b/CPP/7zip/Crypto/HmacSha1.h index 6ba015e8..d4b21b36 100644 --- a/CPP/7zip/Crypto/HmacSha1.h +++ b/CPP/7zip/Crypto/HmacSha1.h @@ -9,7 +9,7 @@ namespace NCrypto { namespace NSha1 { -// Use: SetKey(key, keySize); for () Update(data, size); Final(mac, macSize); +// Use: SetKey(key, keySize); for () Update(data, size); FinalFull(mac); class CHmac { @@ -18,20 +18,12 @@ class CHmac public: void SetKey(const Byte *key, size_t keySize); void Update(const Byte *data, size_t dataSize) { _sha.Update(data, dataSize); } - void Final(Byte *mac, size_t macSize = kDigestSize); -}; - -class CHmac32 -{ - CContext32 _sha; - CContext32 _sha2; -public: - void SetKey(const Byte *key, size_t keySize); - void Update(const UInt32 *data, size_t dataSize) { _sha.Update(data, dataSize); } - void Final(UInt32 *mac, size_t macSize = kNumDigestWords); - // It'sa for hmac function. in,out: mac[kNumDigestWords]. - void GetLoopXorDigest(UInt32 *mac, UInt32 numIteration); + // Final() : mac is recommended to be aligned for 4 bytes + // GetLoopXorDigest1() : mac is required to be aligned for 4 bytes + // The caller can use: UInt32 mac[NSha1::kNumDigestWords] and typecast to (Byte *) and (void *); + void Final(Byte *mac); + void GetLoopXorDigest1(void *mac, UInt32 numIteration); }; }} diff --git a/CPP/7zip/Crypto/HmacSha256.cpp b/CPP/7zip/Crypto/HmacSha256.cpp index 2e1efb48..cec5e752 100644 --- a/CPP/7zip/Crypto/HmacSha256.cpp +++ b/CPP/7zip/Crypto/HmacSha256.cpp @@ -9,39 +9,38 @@ namespace NCrypto { namespace NSha256 { -static const unsigned kBlockSize = 64; - void CHmac::SetKey(const Byte *key, size_t keySize) { - Byte temp[kBlockSize]; + MY_ALIGN (16) + UInt32 temp[SHA256_NUM_BLOCK_WORDS]; size_t i; - for (i = 0; i < kBlockSize; i++) + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) temp[i] = 0; if (keySize > kBlockSize) { Sha256_Init(&_sha); Sha256_Update(&_sha, key, keySize); - Sha256_Final(&_sha, temp); + Sha256_Final(&_sha, (Byte *)temp); } else - for (i = 0; i < keySize; i++) - temp[i] = key[i]; + memcpy(temp, key, keySize); - for (i = 0; i < kBlockSize; i++) - temp[i] ^= 0x36; + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636; Sha256_Init(&_sha); - Sha256_Update(&_sha, temp, kBlockSize); + Sha256_Update(&_sha, (const Byte *)temp, kBlockSize); - for (i = 0; i < kBlockSize; i++) - temp[i] ^= 0x36 ^ 0x5C; + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636 ^ 0x5C5C5C5C; Sha256_Init(&_sha2); - Sha256_Update(&_sha2, temp, kBlockSize); + Sha256_Update(&_sha2, (const Byte *)temp, kBlockSize); } + void CHmac::Final(Byte *mac) { Sha256_Final(&_sha, mac); @@ -49,14 +48,4 @@ void CHmac::Final(Byte *mac) Sha256_Final(&_sha2, mac); } -/* -void CHmac::Final(Byte *mac, size_t macSize) -{ - Byte digest[SHA256_DIGEST_SIZE]; - Final(digest); - for (size_t i = 0; i < macSize; i++) - mac[i] = digest[i]; -} -*/ - }} diff --git a/CPP/7zip/Crypto/HmacSha256.h b/CPP/7zip/Crypto/HmacSha256.h index 233424a0..d709f43d 100644 --- a/CPP/7zip/Crypto/HmacSha256.h +++ b/CPP/7zip/Crypto/HmacSha256.h @@ -9,6 +9,7 @@ namespace NCrypto { namespace NSha256 { +const unsigned kBlockSize = SHA256_BLOCK_SIZE; const unsigned kDigestSize = SHA256_DIGEST_SIZE; class CHmac @@ -19,7 +20,6 @@ public: void SetKey(const Byte *key, size_t keySize); void Update(const Byte *data, size_t dataSize) { Sha256_Update(&_sha, data, dataSize); } void Final(Byte *mac); - // void Final(Byte *mac, size_t macSize); }; }} diff --git a/CPP/7zip/Crypto/MyAes.cpp b/CPP/7zip/Crypto/MyAes.cpp index 52eaab7a..5cb7f463 100644 --- a/CPP/7zip/Crypto/MyAes.cpp +++ b/CPP/7zip/Crypto/MyAes.cpp @@ -10,91 +10,179 @@ namespace NCrypto { static struct CAesTabInit { CAesTabInit() { AesGenTables();} } g_AesTabInit; -CAesCbcCoder::CAesCbcCoder(bool encodeMode, unsigned keySize): +CAesCoder::CAesCoder(bool encodeMode, unsigned keySize, bool ctrMode): _keySize(keySize), _keyIsSet(false), - _encodeMode(encodeMode) + _encodeMode(encodeMode), + _ctrMode(ctrMode), + _aes(AES_NUM_IVMRK_WORDS * 4 + AES_BLOCK_SIZE * 2) { - _offset = ((0 - (unsigned)(ptrdiff_t)_aes) & 0xF) / sizeof(UInt32); + // _offset = ((0 - (unsigned)(ptrdiff_t)_aes) & 0xF) / sizeof(UInt32); memset(_iv, 0, AES_BLOCK_SIZE); + /* + // we can use the following code to test 32-bit overflow case for AES-CTR + for (unsigned i = 0; i < 16; i++) _iv[i] = (Byte)(i + 1); + _iv[0] = 0xFE; _iv[1] = _iv[2] = _iv[3] = 0xFF; + */ SetFunctions(0); } -STDMETHODIMP CAesCbcCoder::Init() +STDMETHODIMP CAesCoder::Init() { - AesCbc_Init(_aes + _offset, _iv); - return _keyIsSet ? S_OK : E_FAIL; + AesCbc_Init(Aes(), _iv); + return _keyIsSet ? S_OK : E_NOTIMPL; // E_FAIL } -STDMETHODIMP_(UInt32) CAesCbcCoder::Filter(Byte *data, UInt32 size) +STDMETHODIMP_(UInt32) CAesCoder::Filter(Byte *data, UInt32 size) { if (!_keyIsSet) return 0; if (size == 0) return 0; if (size < AES_BLOCK_SIZE) + { + #ifndef _SFX + if (_ctrMode) + { + // use that code only for last block !!! + Byte *ctr = (Byte *)(Aes() + AES_NUM_IVMRK_WORDS); + memset(ctr, 0, AES_BLOCK_SIZE); + memcpy(ctr, data, size); + _codeFunc(Aes(), ctr, 1); + memcpy(data, ctr, size); + return size; + } + #endif return AES_BLOCK_SIZE; + } size >>= 4; - _codeFunc(_aes + _offset, data, size); + _codeFunc(Aes(), data, size); return size << 4; } -STDMETHODIMP CAesCbcCoder::SetKey(const Byte *data, UInt32 size) +STDMETHODIMP CAesCoder::SetKey(const Byte *data, UInt32 size) { if ((size & 0x7) != 0 || size < 16 || size > 32) return E_INVALIDARG; if (_keySize != 0 && size != _keySize) return E_INVALIDARG; - AES_SET_KEY_FUNC setKeyFunc = _encodeMode ? Aes_SetKey_Enc : Aes_SetKey_Dec; - setKeyFunc(_aes + _offset + 4, data, size); + AES_SET_KEY_FUNC setKeyFunc = (_ctrMode | _encodeMode) ? Aes_SetKey_Enc : Aes_SetKey_Dec; + setKeyFunc(Aes() + 4, data, size); _keyIsSet = true; return S_OK; } -STDMETHODIMP CAesCbcCoder::SetInitVector(const Byte *data, UInt32 size) +STDMETHODIMP CAesCoder::SetInitVector(const Byte *data, UInt32 size) { if (size != AES_BLOCK_SIZE) return E_INVALIDARG; memcpy(_iv, data, size); - CAesCbcCoder::Init(); // don't call virtual function here !!! + CAesCoder::Init(); // don't call virtual function here !!! return S_OK; } -EXTERN_C_BEGIN +#ifndef _SFX -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); +#ifdef MY_CPU_X86_OR_AMD64 + #define USE_HW_AES +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif +#endif -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); +#endif -EXTERN_C_END -bool CAesCbcCoder::SetFunctions(UInt32 algo) +bool CAesCoder::SetFunctions(UInt32 + #ifndef _SFX + algo + #endif + ) { - _codeFunc = _encodeMode ? - g_AesCbc_Encode : - g_AesCbc_Decode; + _codeFunc = g_AesCbc_Decode; + + #ifdef _SFX + + return true; + + #else + + if (_ctrMode) + _codeFunc = g_AesCtr_Code; + else if (_encodeMode) + _codeFunc = g_AesCbc_Encode; + + if (algo < 1) + return true; + if (algo == 1) { - _codeFunc = _encodeMode ? - AesCbc_Encode: - AesCbc_Decode; + _codeFunc = AesCbc_Decode; + + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code; + else if (_encodeMode) + _codeFunc = AesCbc_Encode; + #endif + return true; } - if (algo == 2) + + #ifdef USE_HW_AES + // if (CPU_IsSupported_AES()) { - #ifdef MY_CPU_X86_OR_AMD64 - if (g_AesCbc_Encode != AesCbc_Encode_Intel) + if (algo == 2) + if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW) + { + _codeFunc = AesCbc_Decode_HW; + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code_HW; + else if (_encodeMode) + _codeFunc = AesCbc_Encode_HW; + #endif + return true; + } + + #if defined(MY_CPU_X86_OR_AMD64) + if (algo == 3) + if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW_256) + { + _codeFunc = AesCbc_Decode_HW_256; + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code_HW_256; + else if (_encodeMode) + _codeFunc = AesCbc_Encode_HW; + #endif + return true; + } #endif - return false; } - return true; + #endif + + return false; + + #endif } -STDMETHODIMP CAesCbcCoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) + +#ifndef _SFX + +STDMETHODIMP CAesCoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) { + UInt32 algo = 0; for (UInt32 i = 0; i < numProps; i++) { const PROPVARIANT &prop = coderProps[i]; @@ -102,11 +190,16 @@ STDMETHODIMP CAesCbcCoder::SetCoderProperties(const PROPID *propIDs, const PROPV { if (prop.vt != VT_UI4) return E_INVALIDARG; - if (!SetFunctions(prop.ulVal)) + if (prop.ulVal > 3) return E_NOTIMPL; + algo = prop.ulVal; } } + if (!SetFunctions(algo)) + return E_NOTIMPL; return S_OK; } +#endif + } diff --git a/CPP/7zip/Crypto/MyAes.h b/CPP/7zip/Crypto/MyAes.h index 182411db..a60042be 100644 --- a/CPP/7zip/Crypto/MyAes.h +++ b/CPP/7zip/Crypto/MyAes.h @@ -5,51 +5,73 @@ #include "../../../C/Aes.h" +#include "../../Common/MyBuffer2.h" #include "../../Common/MyCom.h" #include "../ICoder.h" namespace NCrypto { -class CAesCbcCoder: +class CAesCoder: public ICompressFilter, public ICryptoProperties, + #ifndef _SFX public ICompressSetCoderProperties, + #endif public CMyUnknownImp { AES_CODE_FUNC _codeFunc; - unsigned _offset; + // unsigned _offset; unsigned _keySize; bool _keyIsSet; bool _encodeMode; - UInt32 _aes[AES_NUM_IVMRK_WORDS + 3]; + bool _ctrMode; + + // UInt32 _aes[AES_NUM_IVMRK_WORDS + 3]; + CAlignedBuffer _aes; + Byte _iv[AES_BLOCK_SIZE]; + // UInt32 *Aes() { return _aes + _offset; } + UInt32 *Aes() { return (UInt32 *)(void *)(Byte *)_aes; } + bool SetFunctions(UInt32 algo); public: - CAesCbcCoder(bool encodeMode, unsigned keySize); + CAesCoder(bool encodeMode, unsigned keySize, bool ctrMode); - virtual ~CAesCbcCoder() {}; // we need virtual destructor for derived classes + virtual ~CAesCoder() {}; // we need virtual destructor for derived classes - MY_UNKNOWN_IMP3(ICompressFilter, ICryptoProperties, ICompressSetCoderProperties) + MY_QUERYINTERFACE_BEGIN2(ICompressFilter) + MY_QUERYINTERFACE_ENTRY(ICryptoProperties) + #ifndef _SFX + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE INTERFACE_ICompressFilter(;) + + void SetKeySize(unsigned size) { _keySize = size; } STDMETHOD(SetKey)(const Byte *data, UInt32 size); STDMETHOD(SetInitVector)(const Byte *data, UInt32 size); + #ifndef _SFX STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + #endif }; -struct CAesCbcEncoder: public CAesCbcCoder +#ifndef _SFX +struct CAesCbcEncoder: public CAesCoder { - CAesCbcEncoder(unsigned keySize = 0): CAesCbcCoder(true, keySize) {} + CAesCbcEncoder(unsigned keySize = 0): CAesCoder(true, keySize, false) {} }; +#endif -struct CAesCbcDecoder: public CAesCbcCoder +struct CAesCbcDecoder: public CAesCoder { - CAesCbcDecoder(unsigned keySize = 0): CAesCbcCoder(false, keySize) {} + CAesCbcDecoder(unsigned keySize = 0): CAesCoder(false, keySize, false) {} }; } diff --git a/CPP/7zip/Crypto/MyAesReg.cpp b/CPP/7zip/Crypto/MyAesReg.cpp index 28006835..86bd2547 100644 --- a/CPP/7zip/Crypto/MyAesReg.cpp +++ b/CPP/7zip/Crypto/MyAesReg.cpp @@ -8,9 +8,23 @@ namespace NCrypto { -REGISTER_FILTER_E(AES256CBC, - CAesCbcDecoder(32), - CAesCbcEncoder(32), - 0x6F00181, "AES256CBC") +#ifndef _SFX + +#define REGISTER_AES_2(name, nameString, keySize, isCtr) \ + REGISTER_FILTER_E(name, \ + CAesCoder(false, keySize, isCtr), \ + CAesCoder(true , keySize, isCtr), \ + 0x6F00100 | ((keySize - 16) * 8) | (isCtr ? 4 : 1), \ + nameString) \ + +#define REGISTER_AES(name, nameString, isCtr) \ + /* REGISTER_AES_2(AES128 ## name, "AES128" nameString, 16, isCtr) */ \ + /* REGISTER_AES_2(AES192 ## name, "AES192" nameString, 24, isCtr) */ \ + REGISTER_AES_2(AES256 ## name, "AES256" nameString, 32, isCtr) \ + +REGISTER_AES(CBC, "CBC", false) +// REGISTER_AES(CTR, "CTR", true) + +#endif } diff --git a/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp b/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp index a7fcb728..6b504ee9 100644 --- a/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp +++ b/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp @@ -5,6 +5,7 @@ #include "../../../C/CpuArch.h" #include "HmacSha1.h" +#include "Pbkdf2HmacSha1.h" namespace NCrypto { namespace NSha1 { @@ -14,81 +15,29 @@ void Pbkdf2Hmac(const Byte *pwd, size_t pwdSize, UInt32 numIterations, Byte *key, size_t keySize) { + MY_ALIGN (16) CHmac baseCtx; baseCtx.SetKey(pwd, pwdSize); for (UInt32 i = 1; keySize != 0; i++) { - CHmac ctx = baseCtx; + MY_ALIGN (16) + CHmac ctx; + ctx = baseCtx; ctx.Update(salt, saltSize); - Byte u[kDigestSize]; - SetBe32(u, i); - - ctx.Update(u, 4); - ctx.Final(u, kDigestSize); - - const unsigned curSize = (keySize < kDigestSize) ? (unsigned)keySize : kDigestSize; - unsigned s; - for (s = 0; s < curSize; s++) - key[s] = u[s]; - - for (UInt32 j = numIterations; j > 1; j--) - { - ctx = baseCtx; - ctx.Update(u, kDigestSize); - ctx.Final(u, kDigestSize); - for (s = 0; s < curSize; s++) - key[s] ^= u[s]; - } - - key += curSize; - keySize -= curSize; - } -} - -void Pbkdf2Hmac32(const Byte *pwd, size_t pwdSize, - const UInt32 *salt, size_t saltSize, - UInt32 numIterations, - UInt32 *key, size_t keySize) -{ - CHmac32 baseCtx; - baseCtx.SetKey(pwd, pwdSize); - - for (UInt32 i = 1; keySize != 0; i++) - { - CHmac32 ctx = baseCtx; - ctx.Update(salt, saltSize); - + MY_ALIGN (16) UInt32 u[kNumDigestWords]; - u[0] = i; + SetBe32(u, i); - ctx.Update(u, 1); - ctx.Final(u, kNumDigestWords); + ctx.Update((const Byte *)u, 4); + ctx.Final((Byte *)u); - // Speed-optimized code start ctx = baseCtx; - ctx.GetLoopXorDigest(u, numIterations - 1); - // Speed-optimized code end - - const unsigned curSize = (keySize < kNumDigestWords) ? (unsigned)keySize : kNumDigestWords; - unsigned s; - for (s = 0; s < curSize; s++) - key[s] = u[s]; - - /* - // Default code start - for (UInt32 j = numIterations; j > 1; j--) - { - ctx = baseCtx; - ctx.Update(u, kNumDigestWords); - ctx.Final(u, kNumDigestWords); - for (s = 0; s < curSize; s++) - key[s] ^= u[s]; - } - // Default code end - */ + ctx.GetLoopXorDigest1((void *)u, numIterations - 1); + const unsigned curSize = (keySize < kDigestSize) ? (unsigned)keySize : kDigestSize;; + memcpy(key, (const Byte *)u, curSize); key += curSize; keySize -= curSize; } diff --git a/CPP/7zip/Crypto/Pbkdf2HmacSha1.h b/CPP/7zip/Crypto/Pbkdf2HmacSha1.h index 6560b8d1..e9462e47 100644 --- a/CPP/7zip/Crypto/Pbkdf2HmacSha1.h +++ b/CPP/7zip/Crypto/Pbkdf2HmacSha1.h @@ -14,9 +14,6 @@ namespace NSha1 { void Pbkdf2Hmac(const Byte *pwd, size_t pwdSize, const Byte *salt, size_t saltSize, UInt32 numIterations, Byte *key, size_t keySize); -void Pbkdf2Hmac32(const Byte *pwd, size_t pwdSize, const UInt32 *salt, size_t saltSize, - UInt32 numIterations, UInt32 *key, size_t keySize); - }} #endif diff --git a/CPP/7zip/Crypto/RandGen.cpp b/CPP/7zip/Crypto/RandGen.cpp index f98878f5..65633fd8 100644 --- a/CPP/7zip/Crypto/RandGen.cpp +++ b/CPP/7zip/Crypto/RandGen.cpp @@ -19,7 +19,7 @@ #ifdef USE_STATIC_RtlGenRandom -#include <ntsecapi.h> +// #include <NTSecAPI.h> EXTERN_C_BEGIN #ifndef RtlGenRandom @@ -59,6 +59,7 @@ EXTERN_C_END void CRandomGenerator::Init() { + MY_ALIGN (16) CSha256 hash; Sha256_Init(&hash); @@ -68,6 +69,7 @@ void CRandomGenerator::Init() #ifndef UNDER_CE const unsigned kNumIterations_Small = 100; const unsigned kBufSize = 32; + MY_ALIGN (16) Byte buf[kBufSize]; #endif @@ -98,7 +100,7 @@ void CRandomGenerator::Init() if (hModule) { // SystemFunction036() is real name of RtlGenRandom() function - Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)GetProcAddress(hModule, "SystemFunction036"); + Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)(void *)GetProcAddress(hModule, "SystemFunction036"); if (my_RtlGenRandom) { if (my_RtlGenRandom(buf, kBufSize)) @@ -126,11 +128,11 @@ void CRandomGenerator::Init() { do { - int n = read(f, buf, numBytes); + ssize_t n = read(f, buf, numBytes); if (n <= 0) break; - Sha256_Update(&hash, buf, n); - numBytes -= n; + Sha256_Update(&hash, buf, (size_t)n); + numBytes -= (unsigned)n; } while (numBytes); close(f); @@ -211,6 +213,7 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) Init(); while (size != 0) { + MY_ALIGN (16) CSha256 hash; Sha256_Init(&hash); @@ -221,6 +224,7 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) UInt32 salt = 0xF672ABD1; HASH_UPD(salt); Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); + MY_ALIGN (16) Byte buff[SHA256_DIGEST_SIZE]; Sha256_Final(&hash, buff); for (unsigned i = 0; i < SHA256_DIGEST_SIZE && size != 0; i++, size--) diff --git a/CPP/7zip/Crypto/Rar20Crypto.cpp b/CPP/7zip/Crypto/Rar20Crypto.cpp index 878dfc5c..2bbaa931 100644 --- a/CPP/7zip/Crypto/Rar20Crypto.cpp +++ b/CPP/7zip/Crypto/Rar20Crypto.cpp @@ -54,7 +54,7 @@ void CData::SetPassword(const Byte *data, unsigned size) Keys[3] = 0xA4E7F123L; Byte psw[128]; - memset(psw, 0, sizeof(psw)); + MY_memset_0_ARRAY(psw); if (size != 0) { if (size >= sizeof(psw)) diff --git a/CPP/7zip/Crypto/Rar20Crypto.h b/CPP/7zip/Crypto/Rar20Crypto.h index 6d1d0f5d..596619ce 100644 --- a/CPP/7zip/Crypto/Rar20Crypto.h +++ b/CPP/7zip/Crypto/Rar20Crypto.h @@ -28,6 +28,13 @@ class CData void UpdateKeys(const Byte *data); void CryptBlock(Byte *buf, bool encrypt); public: + ~CData() { Wipe(); } + void Wipe() + { + MY_memset_0_ARRAY(SubstTable); + MY_memset_0_ARRAY(Keys); + } + void EncryptBlock(Byte *buf) { CryptBlock(buf, true); } void DecryptBlock(Byte *buf) { CryptBlock(buf, false); } void SetPassword(const Byte *password, unsigned passwordLen); diff --git a/CPP/7zip/Crypto/Rar5Aes.cpp b/CPP/7zip/Crypto/Rar5Aes.cpp index 005fa6bc..5834bbb7 100644 --- a/CPP/7zip/Crypto/Rar5Aes.cpp +++ b/CPP/7zip/Crypto/Rar5Aes.cpp @@ -9,6 +9,7 @@ #endif #include "Rar5Aes.h" +#include "HmacSha256.h" namespace NCrypto { namespace NRar5 { @@ -122,6 +123,7 @@ void CDecoder::SetPassword(const Byte *data, size_t size) if (size != _password.Size() || memcmp(data, _password, size) != 0) { _needCalc = true; + _password.Wipe(); _password.CopyFrom(data, size); } } @@ -132,28 +134,31 @@ STDMETHODIMP CDecoder::Init() CalcKey_and_CheckPassword(); RINOK(SetKey(_key, kAesKeySize)); RINOK(SetInitVector(_iv, AES_BLOCK_SIZE)); - return CAesCbcCoder::Init(); + return CAesCoder::Init(); } UInt32 CDecoder::Hmac_Convert_Crc32(UInt32 crc) const { + MY_ALIGN (16) NSha256::CHmac ctx; ctx.SetKey(_hashKey, NSha256::kDigestSize); - Byte v[4]; - SetUi32(v, crc); - ctx.Update(v, 4); - Byte h[NSha256::kDigestSize]; - ctx.Final(h); + UInt32 v; + SetUi32(&v, crc); + ctx.Update((const Byte *)&v, 4); + MY_ALIGN (16) + UInt32 h[SHA256_NUM_DIGEST_WORDS]; + ctx.Final((Byte *)h); crc = 0; - for (unsigned i = 0; i < NSha256::kDigestSize; i++) - crc ^= (UInt32)h[i] << ((i & 3) * 8); + for (unsigned i = 0; i < SHA256_NUM_DIGEST_WORDS; i++) + crc ^= (UInt32)GetUi32(h + i); return crc; }; void CDecoder::Hmac_Convert_32Bytes(Byte *data) const { + MY_ALIGN (16) NSha256::CHmac ctx; ctx.SetKey(_hashKey, NSha256::kDigestSize); ctx.Update(data, NSha256::kDigestSize); @@ -190,13 +195,16 @@ bool CDecoder::CalcKey_and_CheckPassword() { // Pbkdf HMAC-SHA-256 + MY_ALIGN (16) NSha256::CHmac baseCtx; baseCtx.SetKey(_password, _password.Size()); NSha256::CHmac ctx = baseCtx; ctx.Update(_salt, sizeof(_salt)); + MY_ALIGN (16) Byte u[NSha256::kDigestSize]; + MY_ALIGN (16) Byte key[NSha256::kDigestSize]; u[0] = 0; diff --git a/CPP/7zip/Crypto/Rar5Aes.h b/CPP/7zip/Crypto/Rar5Aes.h index c69b24e4..e779c17b 100644 --- a/CPP/7zip/Crypto/Rar5Aes.h +++ b/CPP/7zip/Crypto/Rar5Aes.h @@ -3,11 +3,10 @@ #ifndef __CRYPTO_RAR5_AES_H #define __CRYPTO_RAR5_AES_H -#include "../../../C/Aes.h" +#include "../../../C/Sha256.h" #include "../../Common/MyBuffer.h" -#include "HmacSha256.h" #include "MyAes.h" namespace NCrypto { @@ -50,6 +49,17 @@ struct CKey } CKey(); + + void Wipe() + { + _password.Wipe(); + MY_memset_0_ARRAY(_salt); + MY_memset_0_ARRAY(_key); + MY_memset_0_ARRAY(_check_Calced); + MY_memset_0_ARRAY(_hashKey); + } + + ~CKey() { Wipe(); } }; diff --git a/CPP/7zip/Crypto/RarAes.cpp b/CPP/7zip/Crypto/RarAes.cpp index 98e9f62d..aa9444cf 100644 --- a/CPP/7zip/Crypto/RarAes.cpp +++ b/CPP/7zip/Crypto/RarAes.cpp @@ -2,6 +2,9 @@ #include "StdAfx.h" +#include "../../../C/CpuArch.h" +#include "../../../C/RotateDefs.h" + #include "RarAes.h" #include "Sha1Cls.h" @@ -71,6 +74,7 @@ void CDecoder::SetPassword(const Byte *data, unsigned size) } if (!_needCalc && !same) _needCalc = true; + _password.Wipe(); _password.CopyFrom(data, (size_t)size); } @@ -79,9 +83,44 @@ STDMETHODIMP CDecoder::Init() CalcKey(); RINOK(SetKey(_key, kAesKeySize)); RINOK(SetInitVector(_iv, AES_BLOCK_SIZE)); - return CAesCbcCoder::Init(); + return CAesCoder::Init(); } + +// if (password_size_in_bytes + SaltSize > 64), +// the original rar code updates password_with_salt buffer +// with some generated data from SHA1 code. + +// #define RAR_SHA1_REDUCE + +#ifdef RAR_SHA1_REDUCE + #define kNumW 16 + #define WW(i) W[(i)&15] +#else + #define kNumW 80 + #define WW(i) W[i] +#endif + +static void UpdatePswDataSha1(Byte *data) +{ + UInt32 W[kNumW]; + size_t i; + + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + W[i] = GetBe32(data + i * 4); + + for (i = 16; i < 80; i++) + { + WW(i) = rotlFixed(WW((i)-3) ^ WW((i)-8) ^ WW((i)-14) ^ WW((i)-16), 1); + } + + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + { + SetUi32(data + i * 4, W[kNumW - SHA1_NUM_BLOCK_WORDS + i]); + } +} + + void CDecoder::CalcKey() { if (!_needCalc) @@ -102,20 +141,42 @@ void CDecoder::CalcKey() rawSize += kSaltSize; } + MY_ALIGN (16) NSha1::CContext sha; sha.Init(); + MY_ALIGN (16) Byte digest[NSha1::kDigestSize]; // rar reverts hash for sha. const UInt32 kNumRounds = ((UInt32)1 << 18); + UInt32 pos = 0; UInt32 i; for (i = 0; i < kNumRounds; i++) { - sha.UpdateRar(buf, rawSize /* , _rar350Mode */); + sha.Update(buf, rawSize); + // if (_rar350Mode) + { + const UInt32 kBlockSize = 64; + const UInt32 endPos = (pos + (UInt32)rawSize) & ~(kBlockSize - 1); + if (endPos > pos + kBlockSize) + { + UInt32 curPos = pos & ~(kBlockSize - 1); + curPos += kBlockSize; + do + { + UpdatePswDataSha1(buf + (curPos - pos)); + curPos += kBlockSize; + } + while (curPos != endPos); + } + } + pos += (UInt32)rawSize; Byte pswNum[3] = { (Byte)i, (Byte)(i >> 8), (Byte)(i >> 16) }; - sha.UpdateRar(pswNum, 3 /* , _rar350Mode */); + sha.Update(pswNum, 3); + pos += 3; if (i % (kNumRounds / 16) == 0) { + MY_ALIGN (16) NSha1::CContext shaTemp = sha; shaTemp.Final(digest); _iv[i / (kNumRounds / 16)] = (Byte)digest[4 * 4 + 3]; diff --git a/CPP/7zip/Crypto/RarAes.h b/CPP/7zip/Crypto/RarAes.h index 645da1af..2bb68667 100644 --- a/CPP/7zip/Crypto/RarAes.h +++ b/CPP/7zip/Crypto/RarAes.h @@ -44,6 +44,15 @@ public: HRESULT SetDecoderProperties2(const Byte *data, UInt32 size); CDecoder(); + + ~CDecoder() { Wipe(); } + void Wipe() + { + _password.Wipe(); + MY_memset_0_ARRAY(_salt); + MY_memset_0_ARRAY(_key); + MY_memset_0_ARRAY(_iv); + } // void SetRar350Mode(bool rar350Mode) { _rar350Mode = rar350Mode; } }; diff --git a/CPP/7zip/Crypto/Sha1Cls.h b/CPP/7zip/Crypto/Sha1Cls.h index 71acbdec..34616782 100644 --- a/CPP/7zip/Crypto/Sha1Cls.h +++ b/CPP/7zip/Crypto/Sha1Cls.h @@ -1,7 +1,7 @@ -// Crypto/Sha1.h +// Crypto/Sha1Cls.h -#ifndef __CRYPTO_SHA1_H -#define __CRYPTO_SHA1_H +#ifndef __CRYPTO_SHA1_CLS_H +#define __CRYPTO_SHA1_CLS_H #include "../../../C/Sha1.h" @@ -14,35 +14,21 @@ const unsigned kNumDigestWords = SHA1_NUM_DIGEST_WORDS; const unsigned kBlockSize = SHA1_BLOCK_SIZE; const unsigned kDigestSize = SHA1_DIGEST_SIZE; -class CContextBase +class CContext { -protected: CSha1 _s; public: void Init() throw() { Sha1_Init(&_s); } - void GetBlockDigest(const UInt32 *blockData, UInt32 *destDigest) throw() { Sha1_GetBlockDigest(&_s, blockData, destDigest); } -}; - -class CContext: public CContextBase -{ -public: void Update(const Byte *data, size_t size) throw() { Sha1_Update(&_s, data, size); } - void UpdateRar(Byte *data, size_t size /* , bool rar350Mode */) throw() { Sha1_Update_Rar(&_s, data, size /* , rar350Mode ? 1 : 0 */); } void Final(Byte *digest) throw() { Sha1_Final(&_s, digest); } -}; - -class CContext32: public CContextBase -{ -public: - void Update(const UInt32 *data, size_t size) throw() { Sha1_32_Update(&_s, data, size); } - void Final(UInt32 *digest) throw() { Sha1_32_Final(&_s, digest); } - - /* PrepareBlock can be used only when size <= 13. size in Words - _buffer must be empty (_count & 0xF) == 0) */ - void PrepareBlock(UInt32 *block, unsigned size) const throw() + void PrepareBlock(Byte *block, unsigned size) const throw() + { + Sha1_PrepareBlock(&_s, block, size); + } + void GetBlockDigest(const Byte *blockData, Byte *destDigest) const throw() { - Sha1_32_PrepareBlock(&_s, block, size); + Sha1_GetBlockDigest(&_s, blockData, destDigest); } }; diff --git a/CPP/7zip/Crypto/WzAes.cpp b/CPP/7zip/Crypto/WzAes.cpp index d415ab84..b422b501 100644 --- a/CPP/7zip/Crypto/WzAes.cpp +++ b/CPP/7zip/Crypto/WzAes.cpp @@ -16,9 +16,6 @@ Note: you must include MyAes.cpp to project to initialize AES tables #include "RandGen.h" #include "WzAes.h" -// define it if you don't want to use speed-optimized version of NSha1::Pbkdf2Hmac -// #define _NO_WZAES_OPTIMIZATIONS - namespace NCrypto { namespace NWzAes { @@ -30,6 +27,7 @@ STDMETHODIMP CBaseCoder::CryptoSetPassword(const Byte *data, UInt32 size) { if (size > kPasswordSizeMax) return E_INVALIDARG; + _key.Password.Wipe(); _key.Password.CopyFrom(data, (size_t)size); return S_OK; } @@ -40,52 +38,25 @@ void CBaseCoder::Init2() Byte dk[dkSizeMax32 * 4]; const unsigned keySize = _key.GetKeySize(); - const unsigned dkSize = 2 * keySize + kPwdVerifSize; + const unsigned dkSize = 2 * keySize + ((kPwdVerifSize + 3) & ~(unsigned)3); // for (unsigned ii = 0; ii < 1000; ii++) { - #ifdef _NO_WZAES_OPTIMIZATIONS - NSha1::Pbkdf2Hmac( _key.Password, _key.Password.Size(), _key.Salt, _key.GetSaltSize(), kNumKeyGenIterations, dk, dkSize); - - #else - - UInt32 dk32[dkSizeMax32]; - const unsigned dkSize32 = (dkSize + 3) / 4; - UInt32 salt[kSaltSizeMax / 4]; - unsigned numSaltWords = _key.GetNumSaltWords(); - - for (unsigned i = 0; i < numSaltWords; i++) - { - const Byte *src = _key.Salt + i * 4; - salt[i] = GetBe32(src); - } - - NSha1::Pbkdf2Hmac32( - _key.Password, _key.Password.Size(), - salt, numSaltWords, - kNumKeyGenIterations, - dk32, dkSize32); - - /* - for (unsigned j = 0; j < dkSize; j++) - dk[j] = (Byte)(dk32[j / 4] >> (24 - 8 * (j & 3))); - */ - for (unsigned j = 0; j < dkSize32; j++) - SetBe32(dk + j * 4, dk32[j]); - - #endif } - _hmac.SetKey(dk + keySize, keySize); + Hmac()->SetKey(dk + keySize, keySize); memcpy(_key.PwdVerifComputed, dk + 2 * keySize, kPwdVerifSize); - Aes_SetKey_Enc(_aes.aes + _aes.offset + 8, dk, keySize); - AesCtr2_Init(&_aes); + // Aes_SetKey_Enc(_aes.Aes() + 8, dk, keySize); + // AesCtr2_Init(&_aes); + _aesCoderSpec->SetKeySize(keySize); + if (_aesCoderSpec->SetKey(dk, keySize) != S_OK) throw 2; + if (_aesCoderSpec->Init() != S_OK) throw 3; } STDMETHODIMP CBaseCoder::Init() @@ -104,8 +75,9 @@ HRESULT CEncoder::WriteHeader(ISequentialOutStream *outStream) HRESULT CEncoder::WriteFooter(ISequentialOutStream *outStream) { - Byte mac[kMacSize]; - _hmac.Final(mac, kMacSize); + MY_ALIGN (16) + UInt32 mac[NSha1::kNumDigestWords]; + Hmac()->Final((Byte *)mac); return WriteStream(outStream, mac, kMacSize); } @@ -150,34 +122,41 @@ bool CDecoder::Init_and_CheckPassword() HRESULT CDecoder::CheckMac(ISequentialInStream *inStream, bool &isOK) { isOK = false; + MY_ALIGN (16) Byte mac1[kMacSize]; RINOK(ReadStream_FAIL(inStream, mac1, kMacSize)); - Byte mac2[kMacSize]; - _hmac.Final(mac2, kMacSize); - isOK = CompareArrays(mac1, mac2, kMacSize); + MY_ALIGN (16) + UInt32 mac2[NSha1::kNumDigestWords]; + Hmac()->Final((Byte *)mac2); + isOK = CompareArrays(mac1, (const Byte *)mac2, kMacSize); return S_OK; } -CAesCtr2::CAesCtr2() +/* + +CAesCtr2::CAesCtr2(): + aes((4 + AES_NUM_IVMRK_WORDS) * 4) { - offset = ((0 - (unsigned)(ptrdiff_t)aes) & 0xF) / sizeof(UInt32); + // offset = ((0 - (unsigned)(ptrdiff_t)aes) & 0xF) / sizeof(UInt32); + // first 16 bytes are buffer for last block data. + // so the ivAES is aligned for (Align + 16). } void AesCtr2_Init(CAesCtr2 *p) { - UInt32 *ctr = p->aes + p->offset + 4; + UInt32 *ctr = p->Aes() + 4; unsigned i; for (i = 0; i < 4; i++) ctr[i] = 0; p->pos = AES_BLOCK_SIZE; } -/* (size != 16 * N) is allowed only for last call */ +// (size != 16 * N) is allowed only for last call void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size) { unsigned pos = p->pos; - UInt32 *buf32 = p->aes + p->offset; + UInt32 *buf32 = p->Aes(); if (size == 0) return; @@ -188,6 +167,8 @@ void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size) *data++ ^= buf[pos++]; while (--size != 0 && pos != AES_BLOCK_SIZE); } + + // (size == 0 || pos == AES_BLOCK_SIZE) if (size >= 16) { @@ -196,8 +177,10 @@ void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size) size2 <<= 4; data += size2; size -= size2; - pos = AES_BLOCK_SIZE; + // (pos == AES_BLOCK_SIZE) } + + // (size < 16) if (size != 0) { @@ -215,20 +198,26 @@ void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size) p->pos = pos; } +*/ /* (size != 16 * N) is allowed only for last Filter() call */ STDMETHODIMP_(UInt32) CEncoder::Filter(Byte *data, UInt32 size) { - AesCtr2_Code(&_aes, data, size); - _hmac.Update(data, size); + // AesCtr2_Code(&_aes, data, size); + size = _aesCoder->Filter(data, size); + Hmac()->Update(data, size); return size; } STDMETHODIMP_(UInt32) CDecoder::Filter(Byte *data, UInt32 size) { - _hmac.Update(data, size); - AesCtr2_Code(&_aes, data, size); + if (size >= 16) + size &= ~(UInt32)15; + + Hmac()->Update(data, size); + // AesCtr2_Code(&_aes, data, size); + size = _aesCoder->Filter(data, size); return size; } diff --git a/CPP/7zip/Crypto/WzAes.h b/CPP/7zip/Crypto/WzAes.h index 3b22bc16..fa6221c5 100644 --- a/CPP/7zip/Crypto/WzAes.h +++ b/CPP/7zip/Crypto/WzAes.h @@ -12,15 +12,12 @@ specified in "A Password Based File Encryption Utility": #ifndef __CRYPTO_WZ_AES_H #define __CRYPTO_WZ_AES_H -#include "../../../C/Aes.h" - #include "../../Common/MyBuffer.h" -#include "../../Common/MyCom.h" -#include "../ICoder.h" #include "../IPassword.h" #include "HmacSha1.h" +#include "MyAes.h" namespace NCrypto { namespace NWzAes { @@ -64,18 +61,33 @@ struct CKeyInfo unsigned GetNumSaltWords() const { return (KeySizeMode + 1); } CKeyInfo(): KeySizeMode(kKeySizeMode_AES256) {} + + void Wipe() + { + Password.Wipe(); + MY_memset_0_ARRAY(Salt); + MY_memset_0_ARRAY(PwdVerifComputed); + } + + ~CKeyInfo() { Wipe(); } }; +/* struct CAesCtr2 { unsigned pos; - unsigned offset; - UInt32 aes[4 + AES_NUM_IVMRK_WORDS + 3]; + CAlignedBuffer aes; + UInt32 *Aes() { return (UInt32 *)(Byte *)aes; } + + // unsigned offset; + // UInt32 aes[4 + AES_NUM_IVMRK_WORDS + 3]; + // UInt32 *Aes() { return aes + offset; } CAesCtr2(); }; void AesCtr2_Init(CAesCtr2 *p); void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size); +*/ class CBaseCoder: public ICompressFilter, @@ -84,8 +96,21 @@ class CBaseCoder: { protected: CKeyInfo _key; - NSha1::CHmac _hmac; - CAesCtr2 _aes; + + // NSha1::CHmac _hmac; + // NSha1::CHmac *Hmac() { return &_hmac; } + CAlignedBuffer _hmacBuf; + NSha1::CHmac *Hmac() { return (NSha1::CHmac *)(void *)(Byte *)_hmacBuf; } + + // CAesCtr2 _aes; + CAesCoder *_aesCoderSpec; + CMyComPtr<ICompressFilter> _aesCoder; + CBaseCoder(): + _hmacBuf(sizeof(NSha1::CHmac)) + { + _aesCoderSpec = new CAesCoder(true, 32, true); + _aesCoder = _aesCoderSpec; + } void Init2(); public: diff --git a/CPP/7zip/Crypto/ZipCrypto.h b/CPP/7zip/Crypto/ZipCrypto.h index acc0b031..d2fe4c4f 100644 --- a/CPP/7zip/Crypto/ZipCrypto.h +++ b/CPP/7zip/Crypto/ZipCrypto.h @@ -51,7 +51,12 @@ public: STDMETHOD(Init)(); STDMETHOD(CryptoSetPassword)(const Byte *data, UInt32 size); - virtual ~CCipher() {} + virtual ~CCipher() + { + Key0 = KeyMem0 = + Key1 = KeyMem1 = + Key2 = KeyMem2 = 0; + } }; class CEncoder: public CCipher diff --git a/CPP/7zip/Crypto/ZipStrong.cpp b/CPP/7zip/Crypto/ZipStrong.cpp index 22a905cb..895ce949 100644 --- a/CPP/7zip/Crypto/ZipStrong.cpp +++ b/CPP/7zip/Crypto/ZipStrong.cpp @@ -26,10 +26,12 @@ static const UInt16 kAES128 = 0x660E; static void DeriveKey2(const Byte *digest, Byte c, Byte *dest) { + MY_ALIGN (16) Byte buf[64]; memset(buf, c, 64); for (unsigned i = 0; i < NSha1::kDigestSize; i++) buf[i] ^= digest[i]; + MY_ALIGN (16) NSha1::CContext sha; sha.Init(); sha.Update(buf, 64); @@ -38,8 +40,10 @@ static void DeriveKey2(const Byte *digest, Byte c, Byte *dest) static void DeriveKey(NSha1::CContext &sha, Byte *key) { + MY_ALIGN (16) Byte digest[NSha1::kDigestSize]; sha.Final(digest); + MY_ALIGN (16) Byte temp[NSha1::kDigestSize * 2]; DeriveKey2(digest, 0x36, temp); DeriveKey2(digest, 0x5C, temp + NSha1::kDigestSize); @@ -48,6 +52,7 @@ static void DeriveKey(NSha1::CContext &sha, Byte *key) void CKeyInfo::SetPassword(const Byte *data, UInt32 size) { + MY_ALIGN (16) NSha1::CContext sha; sha.Init(); sha.Update(data, size); @@ -103,21 +108,21 @@ HRESULT CDecoder::Init_and_CheckPassword(bool &passwOK) if (_remSize < 16) return E_NOTIMPL; Byte *p = _bufAligned; - UInt16 format = GetUi16(p); + const unsigned format = GetUi16(p); if (format != 3) return E_NOTIMPL; - UInt16 algId = GetUi16(p + 2); + unsigned algId = GetUi16(p + 2); if (algId < kAES128) return E_NOTIMPL; algId -= kAES128; if (algId > 2) return E_NOTIMPL; - UInt16 bitLen = GetUi16(p + 4); - UInt16 flags = GetUi16(p + 6); + const unsigned bitLen = GetUi16(p + 4); + const unsigned flags = GetUi16(p + 6); if (algId * 64 + 128 != bitLen) return E_NOTIMPL; _key.KeySize = 16 + algId * 8; - bool cert = ((flags & 2) != 0); + const bool cert = ((flags & 2) != 0); if ((flags & 0x4000) != 0) { @@ -198,7 +203,7 @@ HRESULT CDecoder::Init_and_CheckPassword(bool &passwOK) UInt32 validSize = GetUi16(p2); p2 += 2; - const size_t validOffset = p2 - p; + const size_t validOffset = (size_t)(p2 - p); if ((validSize & 0xF) != 0 || validOffset + validSize != _remSize) return E_NOTIMPL; @@ -214,7 +219,9 @@ HRESULT CDecoder::Init_and_CheckPassword(bool &passwOK) return S_OK; // passwOK = false; } + MY_ALIGN (16) Byte fileKey[32]; + MY_ALIGN (16) NSha1::CContext sha; sha.Init(); sha.Update(_iv, _ivSize); diff --git a/CPP/7zip/Crypto/ZipStrong.h b/CPP/7zip/Crypto/ZipStrong.h index 03a0f2c7..2b58a5cb 100644 --- a/CPP/7zip/Crypto/ZipStrong.h +++ b/CPP/7zip/Crypto/ZipStrong.h @@ -27,6 +27,12 @@ struct CKeyInfo UInt32 KeySize; void SetPassword(const Byte *data, UInt32 size); + + ~CKeyInfo() { Wipe(); } + void Wipe() + { + MY_memset_0_ARRAY(MasterKey); + } }; class CBaseCoder: @@ -58,6 +64,12 @@ public: // Change it, if is not AES return kAesPadAllign - (packSize32 & (kAesPadAllign - 1)); } + + ~CDecoder() { Wipe(); } + void Wipe() + { + MY_memset_0_ARRAY(_iv); + } }; }} diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index c9da7ed7..f28afcd1 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -50,6 +50,7 @@ 26 ICompressSetFinishMode 27 ICompressGetInStreamProcessedSize2 28 ICompressSetMemLimit + 29 ICompressReadUnusedFromInBuf 30 ICompressGetSubStreamSize 31 ICompressSetInStream @@ -166,6 +167,7 @@ Handler GUIDs: 0C xz 0D ppmd + C5 Base64 C6 COFF C7 Ext C8 VMDK diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index 677d8cfb..314d33d4 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -131,7 +131,8 @@ namespace NCoderPropID kBlockSize2, // VT_UI4 or VT_UI8 kCheckSize, // VT_UI4 : size of digest in bytes kFilter, // VT_BSTR - kMemUse // VT_UI8 + kMemUse, // VT_UI8 + kAffinity // VT_UI8 }; } @@ -198,6 +199,22 @@ CODER_INTERFACE(ICompressSetMemLimit, 0x28) }; +/* + ICompressReadUnusedFromInBuf is supported by ICoder object + call ReadUnusedFromInBuf() after ICoder::Code(inStream, ...). + ICoder::Code(inStream, ...) decodes data, and the ICoder object is allowed + to read from inStream to internal buffers more data than minimal data required for decoding. + So we can call ReadUnusedFromInBuf() from same ICoder object to read unused input + data from the internal buffer. + in ReadUnusedFromInBuf(): the Coder is not allowed to use (ISequentialInStream *inStream) object, that was sent to ICoder::Code(). +*/ + +CODER_INTERFACE(ICompressReadUnusedFromInBuf, 0x29) +{ + STDMETHOD(ReadUnusedFromInBuf)(void *data, UInt32 size, UInt32 *processedSize) PURE; +}; + + CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) { @@ -284,8 +301,24 @@ CODER_INTERFACE(ICompressSetInStreamSize2, 0x39) /* ICompressFilter - Filter() converts as most as possible bytes + Filter() converts as most as possible bytes required for fast processing. + Some filters have (smallest_fast_block). + For example, (smallest_fast_block == 16) for AES CBC/CTR filters. + If data stream is not finished, caller must call Filter() for larger block: + where (size >= smallest_fast_block). + if (size >= smallest_fast_block) + { + The filter can leave some bytes at the end of data without conversion: + if there are data alignment reasons or speed reasons. + The caller must read additional data from stream and call Filter() again. + } + If data stream was finished, caller can call Filter() for (size < smallest_fast_block) + + data : must be aligned for at least 16 bytes for some filters (AES) + returns: (outSize): + if (outSize == 0) : Filter have not converted anything. + So the caller can stop processing, if data stream was finished. if (outSize <= size) : Filter have converted outSize bytes if (outSize > size) : Filter have not converted anything. and it needs at least outSize bytes to convert one block diff --git a/CPP/7zip/IPassword.h b/CPP/7zip/IPassword.h index 7ea45537..4ccc9ac7 100644 --- a/CPP/7zip/IPassword.h +++ b/CPP/7zip/IPassword.h @@ -10,11 +10,41 @@ #define PASSWORD_INTERFACE(i, x) DECL_INTERFACE(i, 5, x) +/* +How to use output parameter (BSTR *password): + +in: The caller is required to set BSTR value as NULL (no string). + The callee (in 7-Zip code) ignores the input value stored in BSTR variable, + +out: The callee rewrites BSTR variable (*password) with new allocated string pointer. + The caller must free BSTR string with function SysFreeString(); +*/ + PASSWORD_INTERFACE(ICryptoGetTextPassword, 0x10) { STDMETHOD(CryptoGetTextPassword)(BSTR *password) PURE; }; + +/* +CryptoGetTextPassword2() +in: + The caller is required to set BSTR value as NULL (no string). + The caller is not required to set (*passwordIsDefined) value. + +out: + Return code: != S_OK : error code + Return code: S_OK : success + + if (*passwordIsDefined == 1), the variable (*password) contains password string + + if (*passwordIsDefined == 0), the password is not defined, + but the callee still could set (*password) to some allocated string, for example, as empty string. + + The caller must free BSTR string with function SysFreeString() +*/ + + PASSWORD_INTERFACE(ICryptoGetTextPassword2, 0x11) { STDMETHOD(CryptoGetTextPassword2)(Int32 *passwordIsDefined, BSTR *password) PURE; diff --git a/CPP/7zip/IStream.h b/CPP/7zip/IStream.h index 9a0bcbf3..48c67c1a 100644 --- a/CPP/7zip/IStream.h +++ b/CPP/7zip/IStream.h @@ -62,12 +62,21 @@ STREAM_INTERFACE(ISequentialOutStream, 0x02) */ }; +#ifdef _WIN32 + #ifdef __HRESULT_FROM_WIN32 #define HRESULT_WIN32_ERROR_NEGATIVE_SEEK __HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK) #else #define HRESULT_WIN32_ERROR_NEGATIVE_SEEK HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK) #endif +#else + +#define HRESULT_WIN32_ERROR_NEGATIVE_SEEK MY__E_ERROR_NEGATIVE_SEEK + +#endif + + /* Seek() Function If you seek before the beginning of the stream, Seek() function returns error code: Recommended error code is __HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK). diff --git a/CPP/7zip/LzmaDec.mak b/CPP/7zip/LzmaDec.mak index 8d7f6f98..bed4700a 100644 --- a/CPP/7zip/LzmaDec.mak +++ b/CPP/7zip/LzmaDec.mak @@ -1,5 +1,7 @@ !IF "$(PLATFORM)" == "x64" +!IFNDEF NO_ASM CFLAGS_C_SPEC = -D_LZMA_DEC_OPT ASM_OBJS = $(ASM_OBJS) \ $O\LzmaDecOpt.obj !ENDIF +!ENDIF diff --git a/CPP/7zip/LzmaDec_gcc.mak b/CPP/7zip/LzmaDec_gcc.mak new file mode 100644 index 00000000..51924f50 --- /dev/null +++ b/CPP/7zip/LzmaDec_gcc.mak @@ -0,0 +1,14 @@ +ifdef USE_ASM +ifdef IS_X64 +USE_LZMA_DEC_ASM=1 +endif +ifdef IS_ARM64 +USE_LZMA_DEC_ASM=1 +endif +endif + +ifdef USE_LZMA_DEC_ASM + +LZMA_DEC_OPT_OBJS= $O/LzmaDecOpt.o + +endif diff --git a/CPP/7zip/Sha1.mak b/CPP/7zip/Sha1.mak new file mode 100644 index 00000000..1b5f605f --- /dev/null +++ b/CPP/7zip/Sha1.mak @@ -0,0 +1,13 @@ +COMMON_OBJS = $(COMMON_OBJS) \ + $O\Sha1Prepare.obj + +C_OBJS = $(C_OBJS) \ + $O\Sha1.obj + +!IF defined(USE_C_SHA) || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +C_OBJS = $(C_OBJS) \ + $O\Sha1Opt.obj +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" +ASM_OBJS = $(ASM_OBJS) \ + $O\Sha1Opt.obj +!ENDIF diff --git a/CPP/7zip/Sha256.mak b/CPP/7zip/Sha256.mak new file mode 100644 index 00000000..0bdbcb60 --- /dev/null +++ b/CPP/7zip/Sha256.mak @@ -0,0 +1,13 @@ +COMMON_OBJS = $(COMMON_OBJS) \ + $O\Sha256Prepare.obj + +C_OBJS = $(C_OBJS) \ + $O\Sha256.obj + +!IF defined(USE_C_SHA) || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +C_OBJS = $(C_OBJS) \ + $O\Sha256Opt.obj +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" +ASM_OBJS = $(ASM_OBJS) \ + $O\Sha256Opt.obj +!ENDIF diff --git a/CPP/7zip/UI/Agent/Agent.cpp b/CPP/7zip/UI/Agent/Agent.cpp index 1c2e3397..76b243f2 100644 --- a/CPP/7zip/UI/Agent/Agent.cpp +++ b/CPP/7zip/UI/Agent/Agent.cpp @@ -27,8 +27,10 @@ CCodecs *g_CodecsObj; #ifdef EXTERNAL_CODECS CExternalCodecs g_ExternalCodecs; - CCodecs::CReleaser g_CodecsReleaser; + static CCodecs::CReleaser g_CodecsReleaser; #else + extern + CMyComPtr<IUnknown> g_CodecsRef; CMyComPtr<IUnknown> g_CodecsRef; #endif @@ -1245,7 +1247,7 @@ STDMETHODIMP CAgentFolder::GetFolderProperty(PROPID propID, PROPVARIANT *value) // case kpidName: prop = dir.Name; break; // case kpidPath: prop = _proxy2->GetFullPathPrefix(_proxyDirIndex); break; case kpidType: prop = UString("7-Zip.") + _agentSpec->ArchiveType; break; - case kpidCRC: if (dir.CrcIsDefined) prop = dir.Crc; break; + case kpidCRC: if (dir.CrcIsDefined) { prop = dir.Crc; } break; } } @@ -1533,8 +1535,8 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices, CAgent::CAgent(): _proxy(NULL), _proxy2(NULL), - _isDeviceFile(false), - _updatePathPrefix_is_AltFolder(false) + _updatePathPrefix_is_AltFolder(false), + _isDeviceFile(false) { } @@ -1611,24 +1613,30 @@ STDMETHODIMP CAgent::Open( options.filePath = _archiveFilePath; options.callback = openArchiveCallback; - RINOK(_archiveLink.Open(options)); + HRESULT res = _archiveLink.Open(options); - CArc &arc = _archiveLink.Arcs.Back(); - if (!inStream) + if (!_archiveLink.Arcs.IsEmpty()) { - arc.MTimeDefined = !fi.IsDevice; - arc.MTime = fi.MTime; + CArc &arc = _archiveLink.Arcs.Back(); + if (!inStream) + { + arc.MTimeDefined = !fi.IsDevice; + arc.MTime = fi.MTime; + } + + ArchiveType = GetTypeOfArc(arc); + if (archiveType) + { + RINOK(StringToBstr(ArchiveType, archiveType)); + } } - ArchiveType = GetTypeOfArc(arc); - if (archiveType) - { - RINOK(StringToBstr(ArchiveType, archiveType)); - } - return S_OK; + return res; + COM_TRY_END } + STDMETHODIMP CAgent::ReOpen(IArchiveOpenCallback *openArchiveCallback) { COM_TRY_BEGIN @@ -1717,7 +1725,10 @@ HRESULT CAgent::ReadItems() STDMETHODIMP CAgent::BindToRootFolder(IFolderFolder **resultFolder) { COM_TRY_BEGIN - RINOK(ReadItems()); + if (!_archiveLink.Arcs.IsEmpty()) + { + RINOK(ReadItems()); + } CAgentFolder *folderSpec = new CAgentFolder; CMyComPtr<IFolderFolder> rootFolder = folderSpec; folderSpec->Init(_proxy, _proxy2, k_Proxy_RootDirIndex, /* NULL, */ this); @@ -1813,6 +1824,20 @@ STDMETHODIMP CAgent::GetArcProp(UInt32 level, PROPID propID, PROPVARIANT *value) if (_archiveLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) prop = g_CodecsObj->Formats[_archiveLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name; break; + case kpidErrorFlags: + { + UInt32 flags = _archiveLink.NonOpen_ErrorInfo.GetErrorFlags(); + if (flags != 0) + prop = flags; + break; + } + case kpidWarningFlags: + { + UInt32 flags = _archiveLink.NonOpen_ErrorInfo.GetWarningFlags(); + if (flags != 0) + prop = flags; + break; + } } } else @@ -1822,7 +1847,10 @@ STDMETHODIMP CAgent::GetArcProp(UInt32 level, PROPID propID, PROPVARIANT *value) { case kpidType: prop = GetTypeOfArc(arc); break; case kpidPath: prop = arc.Path; break; - case kpidErrorType: if (arc.ErrorInfo.ErrorFormatIndex >= 0) prop = g_CodecsObj->Formats[arc.ErrorInfo.ErrorFormatIndex].Name; break; + case kpidErrorType: + if (arc.ErrorInfo.ErrorFormatIndex >= 0) + prop = g_CodecsObj->Formats[arc.ErrorInfo.ErrorFormatIndex].Name; + break; case kpidErrorFlags: { UInt32 flags = arc.ErrorInfo.GetErrorFlags(); diff --git a/CPP/7zip/UI/Agent/Agent.h b/CPP/7zip/UI/Agent/Agent.h index f7d7b5e4..498e7d0c 100644 --- a/CPP/7zip/UI/Agent/Agent.h +++ b/CPP/7zip/UI/Agent/Agent.h @@ -292,7 +292,7 @@ public: s2 += "Warning: The archive is open with offset"; else { - s2 += "Can not open the file as ["; + s2 += "Cannot open the file as ["; s2 += g_CodecsObj->GetFormatNamePtr(arc.ErrorInfo.ErrorFormatIndex); s2 += "] archive"; } diff --git a/CPP/7zip/UI/Agent/AgentOut.cpp b/CPP/7zip/UI/Agent/AgentOut.cpp index 40876d76..8dfb63a2 100644 --- a/CPP/7zip/UI/Agent/AgentOut.cpp +++ b/CPP/7zip/UI/Agent/AgentOut.cpp @@ -166,7 +166,7 @@ struct CDirItemsCallback_AgentOut: public IDirItemsCallback IFolderArchiveUpdateCallback *FolderArchiveUpdateCallback; HRESULT ErrorCode; - CDirItemsCallback_AgentOut(): ErrorCode(S_OK), FolderArchiveUpdateCallback(NULL) {} + CDirItemsCallback_AgentOut(): FolderArchiveUpdateCallback(NULL), ErrorCode(S_OK) {} HRESULT ScanError(const FString &name, DWORD systemError) { diff --git a/CPP/7zip/UI/Agent/AgentProxy.cpp b/CPP/7zip/UI/Agent/AgentProxy.cpp index 7f550f75..f0acd20b 100644 --- a/CPP/7zip/UI/Agent/AgentProxy.cpp +++ b/CPP/7zip/UI/Agent/AgentProxy.cpp @@ -51,6 +51,18 @@ int CProxyArc::FindSubDir(unsigned dirIndex, const wchar_t *name) const return FindSubDir(dirIndex, name, insertPos); } +static const wchar_t *AllocStringAndCopy(const wchar_t *s, size_t len) +{ + wchar_t *p = new wchar_t[len + 1]; + MyStringCopy(p, s); + return p; +} + +static const wchar_t *AllocStringAndCopy(const UString &s) +{ + return AllocStringAndCopy(s, s.Len()); +} + unsigned CProxyArc::AddDir(unsigned dirIndex, int arcIndex, const UString &name) { unsigned insertPos; @@ -70,8 +82,7 @@ unsigned CProxyArc::AddDir(unsigned dirIndex, int arcIndex, const UString &name) CProxyDir &item = Dirs.AddNew(); item.NameLen = name.Len(); - item.Name = new wchar_t[item.NameLen + 1]; - MyStringCopy((wchar_t *)item.Name, name); + item.Name = AllocStringAndCopy(name); item.ArcIndex = arcIndex; item.ParentDir = dirIndex; @@ -248,6 +259,10 @@ HRESULT CProxyArc::Load(const CArc &arc, IProgress *progress) unsigned len = 0; bool isPtrName = false; + #if WCHAR_PATH_SEPARATOR != L'/' + wchar_t replaceFromChar = 0; + #endif + #if defined(MY_CPU_LE) && defined(_WIN32) // it works only if (sizeof(wchar_t) == 2) if (arc.GetRawProps) @@ -263,6 +278,9 @@ HRESULT CProxyArc::Load(const CArc &arc, IProgress *progress) len = size / 2 - 1; s = (const wchar_t *)p; isPtrName = true; + #if WCHAR_PATH_SEPARATOR != L'/' + replaceFromChar = L'\\'; + #endif } } if (!s) @@ -309,9 +327,17 @@ HRESULT CProxyArc::Load(const CArc &arc, IProgress *progress) for (unsigned j = 0; j < len; j++) { - wchar_t c = s[j]; + const wchar_t c = s[j]; if (c == WCHAR_PATH_SEPARATOR || c == L'/') { + #if WCHAR_PATH_SEPARATOR != L'/' + if (c == replaceFromChar) + { + // s.ReplaceOneCharAtPos(j, WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT); + continue; + } + #endif + const unsigned kLevelLimit = 1 << 10; if (numLevels <= kLevelLimit) { @@ -351,9 +377,8 @@ HRESULT CProxyArc::Load(const CArc &arc, IProgress *progress) f.Name = s; else { - f.Name = new wchar_t[f.NameLen + 1]; + f.Name = AllocStringAndCopy(s, f.NameLen); f.NeedDeleteName = true; - MyStringCopy((wchar_t *)f.Name, s); } if (isDir) @@ -382,15 +407,15 @@ void CProxyArc2::GetDirPathParts(int dirIndex, UStringVector &pathParts, bool &i isAltStreamDir = false; - if (dirIndex == k_Proxy2_RootDirIndex) + if (dirIndex == (int)k_Proxy2_RootDirIndex) return; - if (dirIndex == k_Proxy2_AltRootDirIndex) + if (dirIndex == (int)k_Proxy2_AltRootDirIndex) { isAltStreamDir = true; return; } - while (dirIndex >= k_Proxy2_NumRootDirs) + while (dirIndex >= (int)k_Proxy2_NumRootDirs) { const CProxyDir2 &dir = Dirs[dirIndex]; const CProxyFile2 &file = Files[dir.ArcIndex]; @@ -608,9 +633,8 @@ HRESULT CProxyArc2::Load(const CArc &arc, IProgress *progress) tempAString = (const char *)p; ConvertUTF8ToUnicode(tempAString, tempUString); file.NameLen = tempUString.Len(); - file.Name = new wchar_t[file.NameLen + 1]; + file.Name = AllocStringAndCopy(tempUString); file.NeedDeleteName = true; - wmemcpy((wchar_t *)file.Name, tempUString.Ptr(), file.NameLen + 1); } else { @@ -624,9 +648,8 @@ HRESULT CProxyArc2::Load(const CArc &arc, IProgress *progress) else return E_FAIL; file.NameLen = MyStringLen(s); - file.Name = new wchar_t[file.NameLen + 1]; + file.Name = AllocStringAndCopy(s, file.NameLen); file.NeedDeleteName = true; - wmemcpy((wchar_t *)file.Name, s, file.NameLen + 1); } UInt32 parent = (UInt32)(Int32)-1; diff --git a/CPP/7zip/UI/Agent/AgentProxy.h b/CPP/7zip/UI/Agent/AgentProxy.h index 49012676..f2cb3d7a 100644 --- a/CPP/7zip/UI/Agent/AgentProxy.h +++ b/CPP/7zip/UI/Agent/AgentProxy.h @@ -12,7 +12,7 @@ struct CProxyFile bool NeedDeleteName; CProxyFile(): Name(NULL), NameLen(0), NeedDeleteName(false) {} - ~CProxyFile() { if (NeedDeleteName) delete [](wchar_t *)Name; } + ~CProxyFile() { if (NeedDeleteName) delete [](wchar_t *)(void *)Name; } // delete [](wchar_t *)Name; }; const unsigned k_Proxy_RootDirIndex = 0; @@ -35,7 +35,7 @@ struct CProxyDir bool CrcIsDefined; CProxyDir(): Name(NULL), NameLen(0), ParentDir(-1) {}; - ~CProxyDir() { delete [](wchar_t *)Name; } + ~CProxyDir() { delete [](wchar_t *)(void *)Name; } void Clear(); bool IsLeaf() const { return ArcIndex >= 0; } @@ -93,7 +93,7 @@ struct CProxyFile2 ~CProxyFile2() { if (NeedDeleteName) - delete [](wchar_t *)Name; + delete [](wchar_t *)(void *)Name; } }; diff --git a/CPP/7zip/UI/Agent/ArchiveFolderOpen.cpp b/CPP/7zip/UI/Agent/ArchiveFolderOpen.cpp index 62d5b1f8..55457f45 100644 --- a/CPP/7zip/UI/Agent/ArchiveFolderOpen.cpp +++ b/CPP/7zip/UI/Agent/ArchiveFolderOpen.cpp @@ -31,8 +31,21 @@ STDMETHODIMP CArchiveFolderManager::OpenFolderFile(IInStream *inStream, } CAgent *agent = new CAgent(); CMyComPtr<IInFolderArchive> archive = agent; - RINOK(agent->Open(inStream, filePath, arcFormat, NULL, openArchiveCallback)); - return agent->BindToRootFolder(resultFolder); + + HRESULT res = agent->Open(inStream, filePath, arcFormat, NULL, openArchiveCallback); + + if (res != S_OK) + { + if (res != S_FALSE) + return res; + /* 20.01: we create folder even for Non-Open cases, if there is NonOpen_ErrorInfo information. + So we can get error information from that IFolderFolder later. */ + if (!agent->_archiveLink.NonOpen_ErrorInfo.IsThereErrorOrWarning()) + return res; + } + + RINOK(agent->BindToRootFolder(resultFolder)); + return res; } /* diff --git a/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp b/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp index ecbfe58b..ba52f4e7 100644 --- a/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp +++ b/CPP/7zip/UI/Agent/ArchiveFolderOut.cpp @@ -27,30 +27,41 @@ void CAgentFolder::GetPathParts(UStringVector &pathParts, bool &isAltStreamFolde _proxy->GetDirPathParts(_proxyDirIndex, pathParts); } -static bool DeleteEmptyFolderAndEmptySubFolders(const FString &path) +static bool Delete_EmptyFolder_And_EmptySubFolders(const FString &path) { - NFind::CFileInfo fileInfo; - FString pathPrefix = path; - pathPrefix.Add_PathSepar(); { - NFind::CEnumerator enumerator; - enumerator.SetDirPrefix(pathPrefix); - while (enumerator.Next(fileInfo)) + const FString pathPrefix = path + FCHAR_PATH_SEPARATOR; + CObjectVector<FString> names; { - if (fileInfo.IsDir()) - if (!DeleteEmptyFolderAndEmptySubFolders(pathPrefix + fileInfo.Name)) + NFind::CDirEntry fileInfo; + NFind::CEnumerator enumerator; + enumerator.SetDirPrefix(pathPrefix); + for (;;) + { + bool found; + if (!enumerator.Next(fileInfo, found)) return false; + if (!found) + break; + if (fileInfo.IsDir()) + names.Add(fileInfo.Name); + } + } + bool res = true; + FOR_VECTOR (i, names) + { + if (!Delete_EmptyFolder_And_EmptySubFolders(pathPrefix + names[i])) + res = false; } + if (!res) + return false; } - /* - // we don't need clear readonly for folders + // we clear read-only attrib to remove read-only dir if (!SetFileAttrib(path, 0)) return false; - */ return RemoveDir(path); } - HRESULT CAgentFolder::CommonUpdateOperation( AGENT_OP operation, bool moveMode, @@ -123,7 +134,7 @@ HRESULT CAgentFolder::CommonUpdateOperation( case AGENT_OP_Uni: { Byte actionSetByte[NUpdateArchive::NPairState::kNumValues]; - for (int i = 0; i < NUpdateArchive::NPairState::kNumValues; i++) + for (unsigned i = 0; i < NUpdateArchive::NPairState::kNumValues; i++) actionSetByte[i] = (Byte)actionSet->StateActions[i]; result = _agentSpec->DoOperation2( moveMode ? &requestedPaths : NULL, @@ -162,7 +173,7 @@ HRESULT CAgentFolder::CommonUpdateOperation( { const FString &fs = requestedPaths[i]; if (NFind::DoesDirExist(fs)) - DeleteEmptyFolderAndEmptySubFolders(fs); + Delete_EmptyFolder_And_EmptySubFolders(fs); } } } diff --git a/CPP/7zip/UI/Client7z/Client7z.cpp b/CPP/7zip/UI/Client7z/Client7z.cpp index 4ebfb2dc..0c69bdc0 100644 --- a/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/CPP/7zip/UI/Client7z/Client7z.cpp @@ -28,10 +28,12 @@ #include "../../../../C/7zVersion.h" #ifdef _WIN32 +extern +HINSTANCE g_hInstance; HINSTANCE g_hInstance = 0; #endif -// Tou can find the list of all GUIDs in Guid.txt file. +// You can find the list of all GUIDs in Guid.txt file. // use another CLSIDs, if you want to support other formats (zip, rar, ...). // {23170F69-40C1-278A-1000-000110070000} @@ -47,7 +49,11 @@ using namespace NWindows; using namespace NFile; using namespace NDir; +#ifdef _WIN32 #define kDllName "7z.dll" +#else +#define kDllName "7z.so" +#endif static const char * const kCopyrightString = "\n" @@ -383,7 +389,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, { if (!DeleteFileAlways(fullProcessedPath)) { - PrintError("Can not delete output file", fullProcessedPath); + PrintError("Cannot delete output file", fullProcessedPath); return E_ABORT; } } @@ -392,7 +398,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, CMyComPtr<ISequentialOutStream> outStreamLoc(_outFileStreamSpec); if (!_outFileStreamSpec->Open(fullProcessedPath, CREATE_ALWAYS)) { - PrintError("Can not open output file", fullProcessedPath); + PrintError("Cannot open output file", fullProcessedPath); return E_ABORT; } _outFileStream = outStreamLoc; @@ -556,7 +562,11 @@ public: FStringVector FailedFiles; CRecordVector<HRESULT> FailedCodes; - CArchiveUpdateCallback(): PasswordIsDefined(false), AskPassword(false), DirItems(0) {}; + CArchiveUpdateCallback(): + DirItems(NULL), + PasswordIsDefined(false), + AskPassword(false) + {} ~CArchiveUpdateCallback() { Finilize(); } HRESULT Finilize(); @@ -726,12 +736,18 @@ STDMETHODIMP CArchiveUpdateCallback::CryptoGetTextPassword2(Int32 *passwordIsDef // Main function +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION PrintError("Unsupported Windows version"); return 1; +#endif int MY_CDECL main(int numArgs, const char *args[]) { NT_CHECK + #ifdef ENV_HAVE_LOCALE + MY_SetLocale(); + #endif + PrintStringLn(kCopyrightString); if (numArgs < 2) @@ -740,55 +756,96 @@ int MY_CDECL main(int numArgs, const char *args[]) return 0; } - if (numArgs < 3) + FString dllPrefix; + + #ifdef _WIN32 + dllPrefix = NDLL::GetModuleDirPrefix(); + #else { - PrintError(kIncorrectCommand); - return 1; + AString s (args[0]); + int sep = s.ReverseFind_PathSepar(); + s.DeleteFrom(sep + 1); + dllPrefix = s; } + #endif - NDLL::CLibrary lib; - if (!lib.Load(NDLL::GetModuleDirPrefix() + FTEXT(kDllName))) + if (!lib.Load(dllPrefix + FTEXT(kDllName))) { - PrintError("Can not load 7-zip library"); + PrintError("Cannot load 7-zip library"); return 1; } Func_CreateObject createObjectFunc = (Func_CreateObject)lib.GetProc("CreateObject"); if (!createObjectFunc) { - PrintError("Can not get CreateObject"); + PrintError("Cannot get CreateObject"); return 1; } - char c; + char c = 0; + UString password; + bool passwordIsDefined = false; + CObjectVector<FString> params; + + for (int curCmd = 1; curCmd < numArgs; curCmd++) { - AString command (args[1]); - if (command.Len() != 1) + AString a(args[curCmd]); + + if (!a.IsEmpty()) + { + if (a[0] == '-') + { + if (!passwordIsDefined && a[1] == 'p') + { + password = GetUnicodeString(a.Ptr(2)); + passwordIsDefined = true; + continue; + } + } + else + { + if (c) + { + params.Add(CmdStringToFString(a)); + continue; + } + if (a.Len() == 1) + { + c = (char)MyCharLower_Ascii(a[0]); + continue; + } + } + } { PrintError(kIncorrectCommand); return 1; } - c = (char)MyCharLower_Ascii(command[0]); } - FString archiveName = CmdStringToFString(args[2]); + if (!c || params.Size() < 1) + { + PrintError(kIncorrectCommand); + return 1; + } + + const FString &archiveName = params[0]; if (c == 'a') { // create archive command - if (numArgs < 4) + if (params.Size() < 2) { PrintError(kIncorrectCommand); return 1; } CObjectVector<CDirItem> dirItems; { - int i; - for (i = 3; i < numArgs; i++) + unsigned i; + for (i = 1; i < params.Size(); i++) { CDirItem di; - FString name = CmdStringToFString(args[i]); + const FString &name = params[i]; NFind::CFileInfo fi; if (!fi.Find(name)) @@ -819,15 +876,15 @@ int MY_CDECL main(int numArgs, const char *args[]) CMyComPtr<IOutArchive> outArchive; if (createObjectFunc(&CLSID_Format, &IID_IOutArchive, (void **)&outArchive) != S_OK) { - PrintError("Can not get class object"); + PrintError("Cannot get class object"); return 1; } CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; CMyComPtr<IArchiveUpdateCallback2> updateCallback(updateCallbackSpec); updateCallbackSpec->Init(&dirItems); - // updateCallbackSpec->PasswordIsDefined = true; - // updateCallbackSpec->Password = L"1"; + updateCallbackSpec->PasswordIsDefined = passwordIsDefined; + updateCallbackSpec->Password = password; /* { @@ -874,7 +931,7 @@ int MY_CDECL main(int numArgs, const char *args[]) } else { - if (numArgs != 3) + if (params.Size() != 1) { PrintError(kIncorrectCommand); return 1; @@ -895,7 +952,7 @@ int MY_CDECL main(int numArgs, const char *args[]) CMyComPtr<IInArchive> archive; if (createObjectFunc(&CLSID_Format, &IID_IInArchive, (void **)&archive) != S_OK) { - PrintError("Can not get class object"); + PrintError("Cannot get class object"); return 1; } @@ -904,21 +961,20 @@ int MY_CDECL main(int numArgs, const char *args[]) if (!fileSpec->Open(archiveName)) { - PrintError("Can not open archive file", archiveName); + PrintError("Cannot open archive file", archiveName); return 1; } { CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; CMyComPtr<IArchiveOpenCallback> openCallback(openCallbackSpec); - openCallbackSpec->PasswordIsDefined = false; - // openCallbackSpec->PasswordIsDefined = true; - // openCallbackSpec->Password = L"1"; + openCallbackSpec->PasswordIsDefined = passwordIsDefined; + openCallbackSpec->Password = password; const UInt64 scanSize = 1 << 23; if (archive->Open(file, &scanSize, openCallback) != S_OK) { - PrintError("Can not open file as archive", archiveName); + PrintError("Cannot open file as archive", archiveName); return 1; } } @@ -957,9 +1013,8 @@ int MY_CDECL main(int numArgs, const char *args[]) CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback; CMyComPtr<IArchiveExtractCallback> extractCallback(extractCallbackSpec); extractCallbackSpec->Init(archive, FString()); // second parameter is output folder path - extractCallbackSpec->PasswordIsDefined = false; - // extractCallbackSpec->PasswordIsDefined = true; - // extractCallbackSpec->Password = "1"; + extractCallbackSpec->PasswordIsDefined = passwordIsDefined; + extractCallbackSpec->Password = password; /* const wchar_t *names[] = diff --git a/CPP/7zip/UI/Client7z/makefile b/CPP/7zip/UI/Client7z/makefile index 99a6d494..988701ef 100644 --- a/CPP/7zip/UI/Client7z/makefile +++ b/CPP/7zip/UI/Client7z/makefile @@ -8,9 +8,9 @@ COMMON_OBJS = \ $O\IntToString.obj \ $O\NewHandler.obj \ $O\MyString.obj \ + $O\MyVector.obj \ $O\StringConvert.obj \ $O\StringToInt.obj \ - $O\MyVector.obj \ $O\Wildcard.obj \ WIN_OBJS = \ diff --git a/CPP/7zip/UI/Client7z/makefile.gcc b/CPP/7zip/UI/Client7z/makefile.gcc new file mode 100644 index 00000000..b65095bf --- /dev/null +++ b/CPP/7zip/UI/Client7z/makefile.gcc @@ -0,0 +1,59 @@ +PROG = 7zcl +IS_NOT_STANDALONE = 1 + +# IS_X64 = 1 + + + +ifdef SystemDrive + +SYS_OBJS = \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + $O/TimeUtils.o \ + +endif + + +LOCAL_FLAGS = \ + + +CURRENT_OBJS = \ + $O/Client7z.o \ + +COMMON_OBJS = \ + $O/IntToString.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + $O/DLL.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + +7ZIP_COMMON_OBJS = \ + $O/FileStreams.o \ + + +OBJS = \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(SYS_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(CURRENT_OBJS) \ + + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index 0e2a4700..ef7eb1a7 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -19,6 +19,7 @@ #include "../../../../C/Alloc.h" #endif +#include "../../../Common/IntToString.h" #include "../../../Common/ListFileUtils.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -26,6 +27,7 @@ #include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" +#include "../../../Windows/System.h" #ifdef _WIN32 #include "../../../Windows/FileMapping.h" #include "../../../Windows/MemoryLock.h" @@ -41,22 +43,37 @@ extern bool g_CaseSensitive; extern bool g_PathTrailReplaceMode; #ifdef _7ZIP_LARGE_PAGES +extern +bool g_LargePagesMode; bool g_LargePagesMode = false; #endif +/* +#ifdef ENV_HAVE_LSTAT +EXTERN_C_BEGIN +extern int global_use_lstat; +EXTERN_C_END +#endif +*/ + #ifdef UNDER_CE #define MY_IS_TERMINAL(x) false; #else -#if _MSC_VER >= 1400 -#define MY_isatty_fileno(x) _isatty(_fileno(x)) -#else -#define MY_isatty_fileno(x) isatty(fileno(x)) -#endif - -#define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0); +// #define MY_isatty_fileno(x) (isatty(fileno(x))) +// #define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0); +static inline bool MY_IS_TERMINAL(FILE *x) +{ + return ( + #if defined(_MSC_VER) && (_MSC_VER >= 1400) + _isatty(_fileno(x)) + #else + isatty(fileno(x)) + #endif + != 0); +} #endif @@ -74,8 +91,6 @@ static bool StringToUInt32(const wchar_t *s, UInt32 &v) } -int g_CodePage = -1; - namespace NKey { enum Enum { @@ -127,6 +142,7 @@ enum Enum kConsoleCharSet, kTechMode, + kPreserveATime, kShareForWrite, kStopAfterOpenError, kCaseSensitive, @@ -137,6 +153,7 @@ enum Enum kFullPathMode, kHardLinks, + kSymLinks_AllowDangerous, kSymLinks, kNtSecurity, @@ -164,7 +181,7 @@ static const char * const k_ArcNameMode_PostCharSet = "sea"; static const char * const k_Stream_PostCharSet = "012"; -static inline const EArcNameMode ParseArcNameMode(int postCharIndex) +static inline EArcNameMode ParseArcNameMode(int postCharIndex) { switch (postCharIndex) { @@ -182,12 +199,15 @@ namespace NRecursedPostCharIndex { }; } -static const char kImmediateNameID = '!'; -static const char kMapNameID = '#'; -static const char kFileListID = '@'; +// static const char +#define kImmediateNameID '!' +#ifdef _WIN32 +#define kMapNameID '#' +#endif +#define kFileListID '@' -static const char kSomeCludePostStringMinSize = 2; // at least <@|!><N>ame must be -static const char kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!><N>ame must be +static const Byte kSomeCludePostStringMinSize = 2; // at least <@|!><N>ame must be +static const Byte kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!><N>ame must be static const char * const kOverwritePostCharSet = "asut"; @@ -199,80 +219,95 @@ static const NExtract::NOverwriteMode::EEnum k_OverwriteModes[] = NExtract::NOverwriteMode::kRenameExisting }; + + +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL + +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_MINUS SWFRM_1(NSwitchType::kMinus) +#define SWFRM_STRING SWFRM_1(NSwitchType::kString) + +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) +#define SWFRM_STRING_MULT(mi) SWFRM_3(NSwitchType::kString, true, mi) + + static const CSwitchForm kSwitchForms[] = { - { "?" }, - { "h" }, - { "-help" }, + { "?", SWFRM_SIMPLE }, + { "h", SWFRM_SIMPLE }, + { "-help", SWFRM_SIMPLE }, - { "ba" }, - { "bd" }, - { "bt" }, - { "bb", NSwitchType::kString, false, 0 }, + { "ba", SWFRM_SIMPLE }, + { "bd", SWFRM_SIMPLE }, + { "bt", SWFRM_SIMPLE }, + { "bb", SWFRM_STRING_SINGL(0) }, { "bso", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, { "bse", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, { "bsp", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, - { "y" }, + { "y", SWFRM_SIMPLE }, - { "ad" }, + { "ad", SWFRM_SIMPLE }, { "ao", NSwitchType::kChar, false, 1, kOverwritePostCharSet}, - { "t", NSwitchType::kString, false, 1 }, - { "stx", NSwitchType::kString, true, 1 }, + { "t", SWFRM_STRING_SINGL(1) }, + { "stx", SWFRM_STRING_MULT(1) }, - { "m", NSwitchType::kString, true, 1 }, - { "o", NSwitchType::kString, false, 1 }, - { "w", NSwitchType::kString }, - - { "i", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "x", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "ai", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "ax", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "an" }, + { "m", SWFRM_STRING_MULT(1) }, + { "o", SWFRM_STRING_SINGL(1) }, + { "w", SWFRM_STRING }, + + { "i", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "x", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ai", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ax", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "an", SWFRM_SIMPLE }, - { "u", NSwitchType::kString, true, 1}, - { "v", NSwitchType::kString, true, 1}, + { "u", SWFRM_STRING_MULT(1) }, + { "v", SWFRM_STRING_MULT(1) }, { "r", NSwitchType::kChar, false, 0, kRecursedPostCharSet }, - { "stm", NSwitchType::kString }, - { "sfx", NSwitchType::kString }, - { "seml", NSwitchType::kString, false, 0}, - { "scrc", NSwitchType::kString, true, 0 }, + { "stm", SWFRM_STRING }, + { "sfx", SWFRM_STRING }, + { "seml", SWFRM_STRING_SINGL(0) }, + { "scrc", SWFRM_STRING_MULT(0) }, - { "si", NSwitchType::kString }, - { "so" }, - - { "slp", NSwitchType::kString }, - { "scs", NSwitchType::kString }, - { "scc", NSwitchType::kString }, - { "slt" }, - - { "ssw" }, - { "sse" }, - { "ssc", NSwitchType::kMinus }, + { "si", SWFRM_STRING }, + { "so", SWFRM_SIMPLE }, + + { "slp", SWFRM_STRING }, + { "scs", SWFRM_STRING }, + { "scc", SWFRM_STRING }, + { "slt", SWFRM_SIMPLE }, + + { "ssp", SWFRM_SIMPLE }, + { "ssw", SWFRM_SIMPLE }, + { "sse", SWFRM_SIMPLE }, + { "ssc", SWFRM_MINUS }, { "sa", NSwitchType::kChar, false, 1, k_ArcNameMode_PostCharSet }, - { "spd" }, - { "spe", NSwitchType::kMinus }, - { "spf", NSwitchType::kString, false, 0 }, + { "spd", SWFRM_SIMPLE }, + { "spe", SWFRM_MINUS }, + { "spf", SWFRM_STRING_SINGL(0) }, - { "snh", NSwitchType::kMinus }, - { "snl", NSwitchType::kMinus }, - { "sni" }, + { "snh", SWFRM_MINUS }, + { "snld", SWFRM_MINUS }, + { "snl", SWFRM_MINUS }, + { "sni", SWFRM_SIMPLE }, - { "sns", NSwitchType::kMinus }, - { "snr" }, - { "snc" }, + { "sns", SWFRM_MINUS }, + { "snr", SWFRM_SIMPLE }, + { "snc", SWFRM_SIMPLE }, - { "snt", NSwitchType::kMinus }, + { "snt", SWFRM_MINUS }, - { "sdel" }, - { "stl" } + { "sdel", SWFRM_SIMPLE }, + { "stl", SWFRM_SIMPLE } #ifndef _NO_CRYPTO - , { "p", NSwitchType::kString } + , { "p", SWFRM_STRING } #endif }; @@ -281,7 +316,7 @@ static const unsigned kMinNonSwitchWords = 1; static const unsigned kCommandIndex = 0; // static const char * const kUserErrorMessage = "Incorrect command line"; -static const char * const kCannotFindListFile = "Cannot find listfile"; +// static const char * const kCannotFindListFile = "Cannot find listfile"; static const char * const kIncorrectListFile = "Incorrect item in listfile.\nCheck charset encoding and -scs switch."; static const char * const kTerminalOutError = "I won't write compressed data to a terminal"; static const char * const kSameTerminalError = "I won't write data and program's messages to same stream"; @@ -295,8 +330,9 @@ bool CArcCommand::IsFromExtractGroup() const case NCommandType::kExtract: case NCommandType::kExtractFull: return true; + default: + return false; } - return false; } NExtract::NPathMode::EEnum CArcCommand::GetPathMode() const @@ -306,8 +342,9 @@ NExtract::NPathMode::EEnum CArcCommand::GetPathMode() const case NCommandType::kTest: case NCommandType::kExtractFull: return NExtract::NPathMode::kFullPaths; + default: + return NExtract::NPathMode::kNoPaths; } - return NExtract::NPathMode::kNoPaths; } bool CArcCommand::IsFromUpdateGroup() const @@ -319,8 +356,9 @@ bool CArcCommand::IsFromUpdateGroup() const case NCommandType::kDelete: case NCommandType::kRename: return true; + default: + return false; } - return false; } static NRecursedType::EEnum GetRecursedTypeFromIndex(int index) @@ -376,6 +414,8 @@ static void AddNameToCensor(NWildcard::CCensor &censor, case NRecursedType::kRecursed: recursed = true; break; + default: + break; } censor.AddPreItem(include, name, recursed, wildcardMatching); } @@ -408,11 +448,13 @@ static void AddRenamePair(CObjectVector<CRenamePair> *renamePairs, static void AddToCensorFromListFile( CObjectVector<CRenamePair> *renamePairs, NWildcard::CCensor &censor, - LPCWSTR fileName, bool include, NRecursedType::EEnum type, bool wildcardMatching, Int32 codePage) + LPCWSTR fileName, bool include, NRecursedType::EEnum type, bool wildcardMatching, UInt32 codePage) { UStringVector names; - if (!NFind::DoesFileExist(us2fs(fileName))) + /* + if (!NFind::DoesFileExist_FollowLink(us2fs(fileName))) throw CArcCmdLineException(kCannotFindListFile, fileName); + */ DWORD lastError = 0; if (!ReadNamesFromListFile2(us2fs(fileName), names, codePage, lastError)) { @@ -449,7 +491,7 @@ static void AddToCensorFromNonSwitchesStrings( int stopSwitchIndex, NRecursedType::EEnum type, bool wildcardMatching, - bool thereAreSwitchIncludes, Int32 codePage) + bool thereAreSwitchIncludes, UInt32 codePage) { if ((renamePairs || nonSwitchStrings.Size() == startIndex) && !thereAreSwitchIncludes) AddNameToCensor(censor, UString(kUniversalWildcard), true, type, @@ -459,7 +501,7 @@ static void AddToCensorFromNonSwitchesStrings( int oldIndex = -1; if (stopSwitchIndex < 0) - stopSwitchIndex = nonSwitchStrings.Size(); + stopSwitchIndex = (int)nonSwitchStrings.Size(); for (unsigned i = startIndex; i < nonSwitchStrings.Size(); i++) { @@ -471,11 +513,11 @@ static void AddToCensorFromNonSwitchesStrings( else if (renamePairs) { if (oldIndex == -1) - oldIndex = i; + oldIndex = (int)i; else { // NRecursedType::EEnum type is used for global wildcard (-i! switches) - AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, NRecursedType::kNonRecursed, wildcardMatching); + AddRenamePair(renamePairs, nonSwitchStrings[(unsigned)oldIndex], s, NRecursedType::kNonRecursed, wildcardMatching); // AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, type); oldIndex = -1; } @@ -486,7 +528,7 @@ static void AddToCensorFromNonSwitchesStrings( if (oldIndex != -1) { - throw CArcCmdLineException("There is no second file name for rename pair:", nonSwitchStrings[oldIndex]); + throw CArcCmdLineException("There is no second file name for rename pair:", nonSwitchStrings[(unsigned)oldIndex]); } } @@ -517,23 +559,23 @@ static const char *ParseMapWithPaths( int pos = s.Find(L':'); if (pos < 0) return k_IncorrectMapCommand; - int pos2 = s.Find(L':', pos + 1); + int pos2 = s.Find(L':', (unsigned)(pos + 1)); if (pos2 < 0) return k_IncorrectMapCommand; - CEventSetEnd eventSetEnd((const wchar_t *)s + ((unsigned)pos2 + 1)); - s.DeleteFrom(pos2); + CEventSetEnd eventSetEnd((const wchar_t *)s + (unsigned)(pos2 + 1)); + s.DeleteFrom((unsigned)pos2); UInt32 size; - if (!StringToUInt32(s.Ptr(pos + 1), size) + if (!StringToUInt32(s.Ptr((unsigned)(pos + 1)), size) || size < sizeof(wchar_t) || size > ((UInt32)1 << 31) || size % sizeof(wchar_t) != 0) return "Unsupported Map data size"; - s.DeleteFrom(pos); + s.DeleteFrom((unsigned)pos); CFileMapping map; if (map.Open(FILE_MAP_READ, GetSystemString(s)) != 0) - return "Can not open mapping"; + return "Cannot open mapping"; LPVOID data = map.Map(FILE_MAP_READ, 0, size); if (!data) return "MapViewOfFile error"; @@ -569,7 +611,7 @@ static void AddSwitchWildcardsToCensor( const UStringVector &strings, bool include, NRecursedType::EEnum commonRecursedType, bool wildcardMatching, - Int32 codePage) + UInt32 codePage) { const char *errorMessage = NULL; unsigned i; @@ -667,9 +709,9 @@ static bool ParseUpdateCommandString2(const UString &command, if (i >= command.Len()) return false; c = command[i]; - if (c < '0' || c >= '0' + kNumUpdatePairActions) + if (c < '0' || c >= (wchar_t)('0' + kNumUpdatePairActions)) return false; - unsigned actionPos = c - '0'; + unsigned actionPos = (unsigned)(c - '0'); actionSet.StateActions[(unsigned)statePos] = (NUpdateArchive::NPairAction::EEnum)(actionPos); if (kUpdatePairStateNotSupportedActions[(unsigned)statePos] == (int)actionPos) return false; @@ -791,8 +833,8 @@ static void SetMethodOptions(const CParser &parser, CObjectVector<CProperty> &pr int index = prop.Name.Find(L'='); if (index >= 0) { - prop.Value = prop.Name.Ptr(index + 1); - prop.Name.DeleteFrom(index); + prop.Value = prop.Name.Ptr((unsigned)(index + 1)); + prop.Name.DeleteFrom((unsigned)index); } properties.Add(prop); } @@ -803,13 +845,24 @@ static void SetMethodOptions(const CParser &parser, CObjectVector<CProperty> &pr static inline void SetStreamMode(const CSwitchResult &sw, unsigned &res) { if (sw.ThereIs) - res = sw.PostCharIndex; + res = (unsigned)sw.PostCharIndex; } +#if defined(_WIN32) && !defined(UNDER_CE) +static void PrintHex(UString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + + void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, CArcCmdLineOptions &options) { + Parse1Log.Empty(); if (!parser.ParseStrings(kSwitchForms, ARRAY_SIZE(kSwitchForms), commandStrings)) throw CArcCmdLineException(parser.ErrorMessage, parser.ErrorLine); @@ -879,14 +932,16 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, #ifdef _7ZIP_LARGE_PAGES if (slp > - #ifndef UNDER_CE + #if defined(_WIN32) && !defined(UNDER_CE) (unsigned)NSecurity::Get_LargePages_RiskLevel() #else 0 #endif ) { + #ifdef _WIN32 // change it ! SetLargePageSize(); + #endif // note: this process also can inherit that Privilege from parent process g_LargePagesMode = #if defined(_WIN32) && !defined(UNDER_CE) @@ -906,32 +961,83 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, const UString &s = parser[NKey::kAffinity].PostStrings[0]; if (!s.IsEmpty()) { - UInt32 v = 0; AString a; a.SetFromWStr_if_Ascii(s); - if (!a.IsEmpty()) + Parse1Log += "Set process affinity mask: "; + + #ifdef _WIN32 + + UInt64 v = 0; { const char *end; - v = ConvertHexStringToUInt32(a, &end); + v = ConvertHexStringToUInt64(a, &end); if (*end != 0) a.Empty(); } if (a.IsEmpty()) throw CArcCmdLineException("Unsupported switch postfix -stm", s); + + { + #ifndef _WIN64 + if (v >= ((UInt64)1 << 32)) + throw CArcCmdLineException("unsupported value -stm", s); + #endif + { + PrintHex(Parse1Log, v); + if (!SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)v)) + { + DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } + } - #ifdef _WIN32 - SetProcessAffinityMask(GetCurrentProcess(), v); - #endif + #else // _WIN32 + + { + Parse1Log += a; + NSystem::CProcessAffinity aff; + aff.CpuZero(); + for (unsigned i = 0; i < a.Len(); i++) + { + char c = a[i]; + unsigned v; + if (c >= '0' && c <= '9') v = (unsigned)(c - '0'); + else if (c >= 'A' && c <= 'F') v = 10 + (unsigned)(c - 'A'); + else if (c >= 'a' && c <= 'f') v = 10 + (unsigned)(c - 'a'); + else + throw CArcCmdLineException("Unsupported switch postfix -stm", s); + for (unsigned k = 0; k < 4; k++) + { + const unsigned cpu = (a.Len() - 1 - i) * 4 + k; + if (v & ((unsigned)1 << k)) + aff.CpuSet(cpu); + } + } + + if (!aff.SetProcAffinity()) + { + DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } + #endif // _WIN32 + + Parse1Log.Add_LF(); } } #endif } + + struct CCodePagePair { const char *Name; - Int32 CodePage; + UInt32 CodePage; }; static const unsigned kNumByteOnlyCodePages = 3; @@ -964,7 +1070,7 @@ static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned key throw CArcCmdLineException("Unsupported charset:", name); const CCodePagePair &pair = g_CodePagePairs[i]; if (name.IsEqualTo(pair.Name)) - return pair.CodePage; + return (Int32)pair.CodePage; } } @@ -1023,8 +1129,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kDisableWildcardParsing].ThereIs) wildcardMatching = false; - g_CodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); - Int32 codePage = FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); + options.ConsoleCodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); + + UInt32 codePage = (UInt32)FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); bool thereAreSwitchIncludes = false; @@ -1097,6 +1204,30 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) SetBoolPair(parser, NKey::kHardLinks, options.HardLinks); SetBoolPair(parser, NKey::kSymLinks, options.SymLinks); + CBoolPair symLinks_AllowDangerous; + SetBoolPair(parser, NKey::kSymLinks_AllowDangerous, symLinks_AllowDangerous); + + + /* + bool supportSymLink = options.SymLinks.Val; + + if (!options.SymLinks.Def) + { + if (isExtractOrList) + supportSymLink = true; + else + supportSymLink = false; + } + + #ifdef ENV_HAVE_LSTAT + if (supportSymLink) + global_use_lstat = 1; + else + global_use_lstat = 0; + #endif + */ + + if (isExtractOrList) { CExtractOptionsBase &eo = options.ExtractOptions; @@ -1117,6 +1248,8 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (!options.SymLinks.Def) nt.SymLinks.Val = true; + nt.SymLinks_AllowDangerous = symLinks_AllowDangerous; + nt.ReplaceColonForAltStream = parser[NKey::kReplaceColonForAltStream].ThereIs; nt.WriteToAltStreamIfColon = parser[NKey::kWriteToAltStreamIfColon].ThereIs; } @@ -1174,6 +1307,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kOutputDir].ThereIs) { eo.OutputDir = us2fs(parser[NKey::kOutputDir].PostStrings[0]); + #ifdef _WIN32 + NFile::NName::NormalizeDirSeparators(eo.OutputDir); + #endif NFile::NName::NormalizeDirPathPrefix(eo.OutputDir); } @@ -1213,6 +1349,8 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) updateOptions.MethodMode.Properties = options.Properties; + if (parser[NKey::kPreserveATime].ThereIs) + updateOptions.PreserveATime = true; if (parser[NKey::kShareForWrite].ThereIs) updateOptions.OpenShareForWrite = true; if (parser[NKey::kStopAfterOpenError].ThereIs) @@ -1270,7 +1408,7 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (curCommandIndex < numNonSwitchStrings) { if (!StringToUInt32(nonSwitchStrings[curCommandIndex], options.NumIterations)) - throw CArcCmdLineException("Incorrect Number of benmchmark iterations", nonSwitchStrings[curCommandIndex]); + throw CArcCmdLineException("Incorrect number of benchmark iterations", nonSwitchStrings[curCommandIndex]); curCommandIndex++; } } @@ -1282,10 +1420,13 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) CHashOptions &hashOptions = options.HashOptions; hashOptions.PathMode = censorPathMode; hashOptions.Methods = options.HashMethods; + if (parser[NKey::kPreserveATime].ThereIs) + hashOptions.PreserveATime = true; if (parser[NKey::kShareForWrite].ThereIs) hashOptions.OpenShareForWrite = true; hashOptions.StdInMode = options.StdInMode; hashOptions.AltStreamsMode = options.AltStreams.Val; + hashOptions.SymLinks = options.SymLinks; } else if (options.Command.CommandType == NCommandType::kInfo) { @@ -1293,3 +1434,45 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) else throw 20150919; } + + + +#ifndef _WIN32 + +static AString g_ModuleDirPrefix; + +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +void Set_ModuleDirPrefix_From_ProgArg0(const char *s) +{ + AString a (s); + int sep = a.ReverseFind_PathSepar(); + a.DeleteFrom((unsigned)(sep + 1)); + g_ModuleDirPrefix = a; +} + +namespace NWindows { +namespace NDLL { + +FString GetModuleDirPrefix(); +FString GetModuleDirPrefix() +{ + FString s; + + s = g_ModuleDirPrefix; + if (s.IsEmpty()) + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + /* + setenv("_7ZIP_HOME_DIR", "/test/", 0); + const char *home = getenv("_7ZIP_HOME_DIR"); + if (home) + s = home; + else + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + */ +} + +}} + +#endif // ! _WIN32 diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.h b/CPP/7zip/UI/Common/ArchiveCommandLine.h index 9ed0825f..150541e6 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -63,6 +63,11 @@ struct CArcCmdLineOptions bool YesToAll; bool ShowDialog; + bool TechMode; + bool ShowTime; + + int ConsoleCodePage; + NWildcard::CCensor Censor; CArcCommand Command; @@ -73,9 +78,6 @@ struct CArcCmdLineOptions UString Password; #endif - bool TechMode; - bool ShowTime; - UStringVector HashMethods; bool AppendName; @@ -109,13 +111,27 @@ struct CArcCmdLineOptions UInt32 NumIterations; CArcCmdLineOptions(): + HelpMode(false), // LargePages(false), CaseSensitiveChange(false), CaseSensitive(false), + IsInTerminal(false), + IsStdOutTerminal(false), + IsStdErrTerminal(false), + StdInMode(false), StdOutMode(false), + EnableHeaders(false), + + YesToAll(false), + ShowDialog(false), + TechMode(false), + ShowTime(false), + + ConsoleCodePage(-1), + Number_for_Out(k_OutStream_stdout), Number_for_Errors(k_OutStream_stderr), Number_for_Percents(k_OutStream_stdout), @@ -129,6 +145,7 @@ class CArcCmdLineParser { NCommandLineParser::CParser parser; public: + UString Parse1Log; void Parse1(const UStringVector &commandStrings, CArcCmdLineOptions &options); void Parse2(CArcCmdLineOptions &options); }; diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index de2f2e00..40e43d2a 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -15,6 +15,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../../Windows/ErrorMsg.h" @@ -30,7 +31,7 @@ #endif #include "../../Common/FilePathAutoRename.h" -// #include "../../Common/StreamUtils.h" +#include "../../Common/StreamUtils.h" #include "../Common/ExtractingFilePath.h" #include "../Common/PropIDUtils.h" @@ -41,15 +42,16 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static const char * const kCantAutoRename = "Can not create file with auto name"; -static const char * const kCantRenameFile = "Can not rename existing file"; -static const char * const kCantDeleteOutputFile = "Can not delete output file"; -static const char * const kCantDeleteOutputDir = "Can not delete output folder"; -static const char * const kCantCreateHardLink = "Can not create hard link"; -static const char * const kCantCreateSymLink = "Can not create symbolic link"; -static const char * const kCantOpenOutFile = "Can not open output file"; -static const char * const kCantSetFileLen = "Can not set length for output file"; - +static const char * const kCantAutoRename = "Cannot create file with auto name"; +static const char * const kCantRenameFile = "Cannot rename existing file"; +static const char * const kCantDeleteOutputFile = "Cannot delete output file"; +static const char * const kCantDeleteOutputDir = "Cannot delete output folder"; +static const char * const kCantOpenOutFile = "Cannot open output file"; +static const char * const kCantSetFileLen = "Cannot set length for output file"; +#ifdef SUPPORT_LINKS +static const char * const kCantCreateHardLink = "Cannot create hard link"; +static const char * const kCantCreateSymLink = "Cannot create symbolic link"; +#endif #ifndef _SFX @@ -66,9 +68,11 @@ STDMETHODIMP COutStreamWithHash::Write(const void *data, UInt32 size, UInt32 *pr return result; } -#endif +#endif // _SFX + #ifdef _USE_SECURITY_CODE +bool InitLocalPrivileges(); bool InitLocalPrivileges() { NSecurity::CAccessToken token; @@ -87,7 +91,8 @@ bool InitLocalPrivileges() return false; return (GetLastError() == ERROR_SUCCESS); } -#endif +#endif // _USE_SECURITY_CODE + #ifdef SUPPORT_LINKS @@ -158,7 +163,7 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector<UInt32> *r hardIDs.Sort2(); { - // wee keep only items that have 2 or more items + // we keep only items that have 2 or more items unsigned k = 0; unsigned numSame = 1; for (unsigned i = 1; i < hardIDs.Size(); i++) @@ -179,7 +184,8 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector<UInt32> *r return S_OK; } -#endif +#endif // SUPPORT_LINKS + CArchiveExtractCallback::CArchiveExtractCallback(): _arc(NULL), @@ -196,6 +202,7 @@ CArchiveExtractCallback::CArchiveExtractCallback(): #endif } + void CArchiveExtractCallback::Init( const CExtractNtOptions &ntOptions, const NWildcard::CCensorNode *wildcardCensor, @@ -208,6 +215,7 @@ void CArchiveExtractCallback::Init( { ClearExtractedDirsInfo(); _outFileStream.Release(); + _bufPtrSeqOutStream.Release(); #ifdef SUPPORT_LINKS _hardLinks.Clear(); @@ -274,6 +282,7 @@ void CArchiveExtractCallback::Init( } } + STDMETHODIMP CArchiveExtractCallback::SetTotal(UInt64 size) { COM_TRY_BEGIN @@ -285,6 +294,7 @@ STDMETHODIMP CArchiveExtractCallback::SetTotal(UInt64 size) COM_TRY_END } + static void NormalizeVals(UInt64 &v1, UInt64 &v2) { const UInt64 kMax = (UInt64)1 << 31; @@ -295,6 +305,7 @@ static void NormalizeVals(UInt64 &v1, UInt64 &v2) } } + static UInt64 MyMultDiv64(UInt64 unpCur, UInt64 unpTotal, UInt64 packTotal) { NormalizeVals(packTotal, unpTotal); @@ -304,6 +315,7 @@ static UInt64 MyMultDiv64(UInt64 unpCur, UInt64 unpTotal, UInt64 packTotal) return unpCur * packTotal / unpTotal; } + STDMETHODIMP CArchiveExtractCallback::SetCompleted(const UInt64 *completeValue) { COM_TRY_BEGIN @@ -324,6 +336,7 @@ STDMETHODIMP CArchiveExtractCallback::SetCompleted(const UInt64 *completeValue) COM_TRY_END } + STDMETHODIMP CArchiveExtractCallback::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) { COM_TRY_BEGIN @@ -331,6 +344,7 @@ STDMETHODIMP CArchiveExtractCallback::SetRatioInfo(const UInt64 *inSize, const U COM_TRY_END } + void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath) { bool isAbsPath = false; @@ -369,6 +383,7 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat } } + HRESULT CArchiveExtractCallback::GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined) { filetimeIsDefined = false; @@ -417,9 +432,14 @@ HRESULT CArchiveExtractCallback::SendMessageError_with_LastError(const char *mes return _extractCallback2->MessageError(s); } -HRESULT CArchiveExtractCallback::SendMessageError2(const char *message, const FString &path1, const FString &path2) +HRESULT CArchiveExtractCallback::SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2) { UString s (message); + if (errorCode != 0) + { + s += " : "; + s += NError::MyFormatMessage(errorCode); + } AddPathToMessage(s, path1); AddPathToMessage(s, path2); return _extractCallback2->MessageError(s); @@ -442,7 +462,7 @@ STDMETHODIMP CGetProp::GetProp(PROPID propID, PROPVARIANT *value) return Arc->Archive->GetProperty(IndexInArc, propID, value); } -#endif +#endif // _SFX #ifdef SUPPORT_LINKS @@ -455,22 +475,32 @@ static UString GetDirPrefixOf(const UString &src) if (IsPathSepar(s.Back())) s.DeleteBack(); int pos = s.ReverseFind_PathSepar(); - s.DeleteFrom(pos + 1); + s.DeleteFrom((unsigned)(pos + 1)); } return s; } -#endif +#endif // SUPPORT_LINKS +struct CLinkLevelsInfo +{ + bool IsAbsolute; + int LowLevel; + int FinalLevel; -bool IsSafePath(const UString &path) + void Parse(const UString &path); +}; + +void CLinkLevelsInfo::Parse(const UString &path) { - if (NName::IsAbsolutePath(path)) - return false; + IsAbsolute = NName::IsAbsolutePath(path); + + LowLevel = 0; + FinalLevel = 0; UStringVector parts; SplitPathToParts(path, parts); - unsigned level = 0; + int level = 0; FOR_VECTOR (i, parts) { @@ -478,29 +508,42 @@ bool IsSafePath(const UString &path) if (s.IsEmpty()) { if (i == 0) - return false; + IsAbsolute = true; continue; } if (s == L".") continue; if (s == L"..") { - if (level == 0) - return false; level--; + if (LowLevel > level) + LowLevel = level; } else level++; } - return level > 0; + FinalLevel = level; +} + + +bool IsSafePath(const UString &path); +bool IsSafePath(const UString &path) +{ + CLinkLevelsInfo levelsInfo; + levelsInfo.Parse(path); + return !levelsInfo.IsAbsolute + && levelsInfo.LowLevel >= 0 + && levelsInfo.FinalLevel > 0; } +bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include); bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include) { bool found = false; - + + // CheckPathVect() doesn't check path to Parent nodes if (node.CheckPathVect(item.PathParts, !item.MainIsDir, include)) { if (!include) @@ -535,11 +578,12 @@ bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcIte return true; } - #endif + #endif // SUPPORT_ALT_STREAMS return found; } + bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item) { bool include; @@ -548,6 +592,7 @@ bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem return false; } + static FString MakePath_from_2_Parts(const FString &prefix, const FString &path) { FString s (prefix); @@ -563,9 +608,10 @@ static FString MakePath_from_2_Parts(const FString &prefix, const FString &path) } -/* + #ifdef SUPPORT_LINKS +/* struct CTempMidBuffer { void *Buf; @@ -601,90 +647,23 @@ HRESULT CArchiveExtractCallback::MyCopyFile(ISequentialOutStream *outStream) RINOK(WriteStream(outStream, buf.Buf, num)); } } - -#endif */ -STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStream **outStream, Int32 askExtractMode) +HRESULT CArchiveExtractCallback::ReadLink() { - COM_TRY_BEGIN - - *outStream = NULL; - - #ifndef _SFX - if (_hashStream) - _hashStreamSpec->ReleaseStream(); - _hashStreamWasUsed = false; - #endif - - _outFileStream.Release(); - - _encrypted = false; - _position = 0; - _isSplit = false; - - _curSize = 0; - _curSizeDefined = false; - _fileLengthWasSet = false; - _index = index; - - _diskFilePath.Empty(); - - // _fi.Clear(); - - #ifdef SUPPORT_LINKS - // _CopyFile_Path.Empty(); - linkPath.Empty(); - #endif - IInArchive *archive = _arc->Archive; - - #ifndef _SFX - _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; - if (_use_baseParentFolder_mode) - { - _item._baseParentFolder = _baseParentFolder; - if (_pathMode == NExtract::NPathMode::kFullPaths || - _pathMode == NExtract::NPathMode::kAbsPaths) - _item._baseParentFolder = -1; - } - #endif - - #ifdef SUPPORT_ALT_STREAMS - _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; - #endif - - RINOK(_arc->GetItem(index, _item)); - - { - NCOM::CPropVariant prop; - RINOK(archive->GetProperty(index, kpidPosition, &prop)); - if (prop.vt != VT_EMPTY) - { - if (prop.vt != VT_UI8) - return E_FAIL; - _position = prop.uhVal.QuadPart; - _isSplit = true; - } - } - - #ifdef SUPPORT_LINKS - - // bool isCopyLink = false; - bool isHardLink = false; - bool isJunction = false; - bool isRelative = false; + const UInt32 index = _index; { NCOM::CPropVariant prop; RINOK(archive->GetProperty(index, kpidHardLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = true; - // isCopyLink = false; - isRelative = false; // RAR5, TAR: hard links are from root folder of archive - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = true; + // _link.isCopyLink = false; + _link.isRelative = false; // RAR5, TAR: hard links are from root folder of archive + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; @@ -696,10 +675,10 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre RINOK(archive->GetProperty(index, kpidCopyLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = false; - isCopyLink = true; - isRelative = false; // RAR5: copy links are from root folder of archive - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = false; + _link.isCopyLink = true; + _link.isRelative = false; // RAR5: copy links are from root folder of archive + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; @@ -711,19 +690,19 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre RINOK(archive->GetProperty(index, kpidSymLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = false; - // isCopyLink = false; - isRelative = true; // RAR5, TAR: symbolic links can be relative - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = false; + // _link.isCopyLink = false; + _link.isRelative = true; // RAR5, TAR: symbolic links can be relative + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; } + NtReparse_Data = NULL; + NtReparse_Size = 0; - bool isOkReparse = false; - - if (linkPath.IsEmpty() && _arc->GetRawProps) + if (_link.linkPath.IsEmpty() && _arc->GetRawProps) { const void *data; UInt32 dataSize; @@ -731,56 +710,75 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre _arc->GetRawProps->GetRawProp(_index, kpidNtReparse, &data, &dataSize, &propType); + // if (dataSize == 1234567) // for debug: unpacking without reparse if (dataSize != 0) { if (propType != NPropDataType::kRaw) return E_FAIL; - UString s; + + #ifdef _WIN32 + + NtReparse_Data = data; + NtReparse_Size = dataSize; + CReparseAttr reparse; - DWORD errorCode = 0; - isOkReparse = reparse.Parse((const Byte *)data, dataSize, errorCode); + bool isOkReparse = reparse.Parse((const Byte *)data, dataSize); if (isOkReparse) { - isHardLink = false; - // isCopyLink = false; - linkPath = reparse.GetPath(); - isJunction = reparse.IsMountPoint(); - isRelative = reparse.IsRelative(); + _link.isHardLink = false; + // _link.isCopyLink = false; + _link.linkPath = reparse.GetPath(); + _link.isJunction = reparse.IsMountPoint(); + + if (reparse.IsSymLink_WSL()) + { + _link.isWSL = true; + _link.isRelative = reparse.IsRelative_WSL(); + } + else + _link.isRelative = reparse.IsRelative_Win(); + #ifndef _WIN32 - linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); + _link.linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); #endif } + #endif } } - if (!linkPath.IsEmpty()) + if (_link.linkPath.IsEmpty()) + return S_OK; + { #ifdef _WIN32 - linkPath.Replace(L'/', WCHAR_PATH_SEPARATOR); + _link.linkPath.Replace(L'/', WCHAR_PATH_SEPARATOR); #endif // rar5 uses "\??\" prefix for absolute links - if (linkPath.IsPrefixedBy(WSTRING_PATH_SEPARATOR L"??" WSTRING_PATH_SEPARATOR)) + if (_link.linkPath.IsPrefixedBy(WSTRING_PATH_SEPARATOR L"??" WSTRING_PATH_SEPARATOR)) { - isRelative = false; - linkPath.DeleteFrontal(4); + _link.isRelative = false; + _link.linkPath.DeleteFrontal(4); } for (;;) // while (NName::IsAbsolutePath(linkPath)) { - unsigned n = NName::GetRootPrefixSize(linkPath); + unsigned n = NName::GetRootPrefixSize(_link.linkPath); if (n == 0) break; - isRelative = false; - linkPath.DeleteFrontal(n); + _link.isRelative = false; + _link.linkPath.DeleteFrontal(n); } } - if (!linkPath.IsEmpty() && !isRelative && _removePathParts.Size() != 0) + if (_link.linkPath.IsEmpty()) + return S_OK; + + if (!_link.isRelative && _removePathParts.Size() != 0) { UStringVector pathParts; - SplitPathToParts(linkPath, pathParts); + SplitPathToParts(_link.linkPath, pathParts); bool badPrefix = false; FOR_VECTOR (i, _removePathParts) { @@ -792,22 +790,597 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre } if (!badPrefix) pathParts.DeleteFrontal(_removePathParts.Size()); - linkPath = MakePathFromParts(pathParts); + _link.linkPath = MakePathFromParts(pathParts); + } + + /* + if (!_link.linkPath.IsEmpty()) + { + printf("\n_link %s to -> %s\n", GetOemString(_item.Path).Ptr(), GetOemString(_link.linkPath).Ptr()); + } + */ + + return S_OK; +} + +#endif // SUPPORT_LINKS + + + +HRESULT CArchiveExtractCallback::Read_fi_Props() +{ + IInArchive *archive = _arc->Archive; + const UInt32 index = _index; + + _fi.AttribDefined = false; + + { + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidPosixAttrib, &prop)); + if (prop.vt == VT_UI4) + { + _fi.SetFromPosixAttrib(prop.ulVal); + } + else if (prop.vt != VT_EMPTY) + return E_FAIL; } + + { + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidAttrib, &prop)); + if (prop.vt == VT_UI4) + { + _fi.Attrib = prop.ulVal; + _fi.AttribDefined = true; + } + else if (prop.vt != VT_EMPTY) + return E_FAIL; + } + + RINOK(GetTime(index, kpidCTime, _fi.CTime, _fi.CTimeDefined)); + RINOK(GetTime(index, kpidATime, _fi.ATime, _fi.ATimeDefined)); + RINOK(GetTime(index, kpidMTime, _fi.MTime, _fi.MTimeDefined)); + return S_OK; +} + + +void CArchiveExtractCallback::CorrectPathParts() +{ + UStringVector &pathParts = _item.PathParts; + + #ifdef SUPPORT_ALT_STREAMS + if (!_item.IsAltStream + || !pathParts.IsEmpty() + || !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt)) #endif + Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, _keepAndReplaceEmptyDirPrefixes, pathParts, _item.MainIsDir); - RINOK(Archive_GetItemBoolProp(archive, index, kpidEncrypted, _encrypted)); + #ifdef SUPPORT_ALT_STREAMS + + if (_item.IsAltStream) + { + UString s (_item.AltStreamName); + Correct_AltStream_Name(s); + bool needColon = true; + + if (pathParts.IsEmpty()) + { + pathParts.AddNew(); + if (_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt) + needColon = false; + } + #ifdef _WIN32 + else if (_pathMode == NExtract::NPathMode::kAbsPaths && + NWildcard::GetNumPrefixParts_if_DrivePath(pathParts) == pathParts.Size()) + pathParts.AddNew(); + #endif + + UString &name = pathParts.Back(); + if (needColon) + name += (char)(_ntOptions.ReplaceColonForAltStream ? '_' : ':'); + name += s; + } + + #endif // SUPPORT_ALT_STREAMS +} + - RINOK(GetUnpackSize()); + +void CArchiveExtractCallback::CreateFolders() +{ + UStringVector &pathParts = _item.PathParts; + + if (!_item.IsDir) + { + if (!pathParts.IsEmpty()) + pathParts.DeleteBack(); + } + + if (pathParts.IsEmpty()) + return; + + FString fullPathNew; + CreateComplexDirectory(pathParts, fullPathNew); + + if (!_item.IsDir) + return; + + CDirPathTime &pt = _extractedFolders.AddNew(); + + pt.CTime = _fi.CTime; + pt.CTimeDefined = (WriteCTime && _fi.CTimeDefined); + + pt.ATime = _fi.ATime; + pt.ATimeDefined = (WriteATime && _fi.ATimeDefined); + + pt.MTimeDefined = false; + + if (WriteMTime) + { + if (_fi.MTimeDefined) + { + pt.MTime = _fi.MTime; + pt.MTimeDefined = true; + } + else if (_arc->MTimeDefined) + { + pt.MTime = _arc->MTime; + pt.MTimeDefined = true; + } + } + + pt.Path = fullPathNew; + pt.SetDirTime(); +} + + + +/* + CheckExistFile(fullProcessedPath) + it can change: fullProcessedPath, _isRenamed, _overwriteMode + (needExit = true) means that we must exit GetStream() even for S_OK result. +*/ + +HRESULT CArchiveExtractCallback::CheckExistFile(FString &fullProcessedPath, bool &needExit) +{ + needExit = true; // it was set already before + + NFind::CFileInfo fileInfo; + + if (fileInfo.Find(fullProcessedPath)) + { + if (_overwriteMode == NExtract::NOverwriteMode::kSkip) + return S_OK; + + if (_overwriteMode == NExtract::NOverwriteMode::kAsk) + { + int slashPos = fullProcessedPath.ReverseFind_PathSepar(); + FString realFullProcessedPath (fullProcessedPath.Left((unsigned)(slashPos + 1)) + fileInfo.Name); + + /* (fileInfo) can be symbolic link. + we can show final file properties here. */ + + Int32 overwriteResult; + RINOK(_extractCallback2->AskOverwrite( + fs2us(realFullProcessedPath), &fileInfo.MTime, &fileInfo.Size, _item.Path, + _fi.MTimeDefined ? &_fi.MTime : NULL, + _curSizeDefined ? &_curSize : NULL, + &overwriteResult)) + + switch (overwriteResult) + { + case NOverwriteAnswer::kCancel: + return E_ABORT; + case NOverwriteAnswer::kNo: + return S_OK; + case NOverwriteAnswer::kNoToAll: + _overwriteMode = NExtract::NOverwriteMode::kSkip; + return S_OK; + + case NOverwriteAnswer::kYes: + break; + case NOverwriteAnswer::kYesToAll: + _overwriteMode = NExtract::NOverwriteMode::kOverwrite; + break; + case NOverwriteAnswer::kAutoRename: + _overwriteMode = NExtract::NOverwriteMode::kRename; + break; + default: + return E_FAIL; + } + } // NExtract::NOverwriteMode::kAsk + + if (_overwriteMode == NExtract::NOverwriteMode::kRename) + { + if (!AutoRenamePath(fullProcessedPath)) + { + RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); + return E_FAIL; + } + _isRenamed = true; + } + else if (_overwriteMode == NExtract::NOverwriteMode::kRenameExisting) + { + FString existPath (fullProcessedPath); + if (!AutoRenamePath(existPath)) + { + RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); + return E_FAIL; + } + // MyMoveFile can rename folders. So it's OK to use it for folders too + if (!MyMoveFile(fullProcessedPath, existPath)) + { + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantRenameFile, existPath, fullProcessedPath)); + return E_FAIL; + } + } + else // not Rename* + { + if (fileInfo.IsDir()) + { + // do we need to delete all files in folder? + if (!RemoveDir(fullProcessedPath)) + { + RINOK(SendMessageError_with_LastError(kCantDeleteOutputDir, fullProcessedPath)); + return S_OK; + } + } + else // fileInfo is not Dir + { + if (NFind::DoesFileExist_Raw(fullProcessedPath)) + if (!DeleteFileAlways(fullProcessedPath)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) // check it in linux + { + RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); + return S_OK; + // return E_FAIL; + } + } // fileInfo is not Dir + } // not Rename* + } + else // not Find(fullProcessedPath) + { + #if defined(_WIN32) && !defined(UNDER_CE) + // we need to clear READ-ONLY of parent before creating alt stream + int colonPos = NName::FindAltStreamColon(fullProcessedPath); + if (colonPos >= 0 && fullProcessedPath[(unsigned)colonPos + 1] != 0) + { + FString parentFsPath (fullProcessedPath); + parentFsPath.DeleteFrom((unsigned)colonPos); + NFind::CFileInfo parentFi; + if (parentFi.Find(parentFsPath)) + { + if (parentFi.IsReadOnly()) + SetFileAttrib(parentFsPath, parentFi.Attrib & ~(DWORD)FILE_ATTRIBUTE_READONLY); + } + } + #endif // defined(_WIN32) && !defined(UNDER_CE) + } + + needExit = false; + return S_OK; +} + + + + + + +HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr<ISequentialOutStream> &outStreamLoc, bool &needExit) +{ + needExit = true; + + RINOK(Read_fi_Props()); + + #ifdef SUPPORT_LINKS + IInArchive *archive = _arc->Archive; + #endif + + const UStringVector &pathParts = _item.PathParts; + const UInt32 index = _index; + + bool isAnti = false; + RINOK(_arc->IsItemAnti(index, isAnti)); + + CorrectPathParts(); + + UString processedPath (MakePathFromParts(pathParts)); + + if (!isAnti) + CreateFolders(); + + FString fullProcessedPath (us2fs(processedPath)); + if (_pathMode != NExtract::NPathMode::kAbsPaths + || !NName::IsAbsolutePath(processedPath)) + { + fullProcessedPath = MakePath_from_2_Parts(_dirPathPrefix, fullProcessedPath); + } #ifdef SUPPORT_ALT_STREAMS + if (_item.IsAltStream && _item.ParentIndex != (UInt32)(Int32)-1) + { + int renIndex = _renamedFiles.FindInSorted(CIndexToPathPair(_item.ParentIndex)); + if (renIndex >= 0) + { + const CIndexToPathPair &pair = _renamedFiles[(unsigned)renIndex]; + fullProcessedPath = pair.Path; + fullProcessedPath += ':'; + UString s (_item.AltStreamName); + Correct_AltStream_Name(s); + fullProcessedPath += us2fs(s); + } + } + #endif // SUPPORT_ALT_STREAMS + + if (_item.IsDir) + { + _diskFilePath = fullProcessedPath; + if (isAnti) + RemoveDir(_diskFilePath); + #ifdef SUPPORT_LINKS + if (_link.linkPath.IsEmpty()) + #endif + return S_OK; + } + else if (!_isSplit) + { + RINOK(CheckExistFile(fullProcessedPath, needExit)); + if (needExit) + return S_OK; + needExit = true; + } - if (!_ntOptions.AltStreams.Val && _item.IsAltStream) + _diskFilePath = fullProcessedPath; + + + if (isAnti) + { + needExit = false; return S_OK; + } + + // not anti + + #ifdef SUPPORT_LINKS + + if (!_link.linkPath.IsEmpty()) + { + #ifndef UNDER_CE + { + bool linkWasSet = false; + RINOK(SetFromLinkPath(fullProcessedPath, _link, linkWasSet)); + } + #endif // UNDER_CE + + // if (_CopyFile_Path.IsEmpty()) + { + needExit = false; + return S_OK; + } + } + + if (!_hardLinks.IDs.IsEmpty() && !_item.IsAltStream) + { + CHardLinkNode h; + bool defined; + RINOK(Archive_Get_HardLinkNode(archive, index, h, defined)); + if (defined) + { + int linkIndex = _hardLinks.IDs.FindInSorted2(h); + if (linkIndex >= 0) + { + FString &hl = _hardLinks.Links[(unsigned)linkIndex]; + if (hl.IsEmpty()) + hl = fullProcessedPath; + else + { + if (!MyCreateHardLink(fullProcessedPath, hl)) + { + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantCreateHardLink, fullProcessedPath, hl)); + return S_OK; + } + + needExit = false; + return S_OK; + } + } + } + } + + #endif // SUPPORT_LINKS + + + // ---------- CREATE WRITE FILE ----- + + _outFileStreamSpec = new COutFileStream; + CMyComPtr<ISequentialOutStream> outFileStream_Loc(_outFileStreamSpec); + + if (!_outFileStreamSpec->Open(fullProcessedPath, _isSplit ? OPEN_ALWAYS: CREATE_ALWAYS)) + { + // if (::GetLastError() != ERROR_FILE_EXISTS || !isSplit) + { + RINOK(SendMessageError_with_LastError(kCantOpenOutFile, fullProcessedPath)); + return S_OK; + } + } + + _fileWasExtracted = true; + + if (_curSizeDefined && _curSize > 0 && _curSize < (1 << 12)) + { + if (_fi.IsLinuxSymLink()) + { + _is_SymLink_in_Data = true; + _is_SymLink_in_Data_Linux = true; + } + else if (_fi.IsReparse()) + { + _is_SymLink_in_Data = true; + _is_SymLink_in_Data_Linux = false; + } + } + if (_is_SymLink_in_Data) + { + _outMemBuf.Alloc((size_t)_curSize); + _bufPtrSeqOutStream_Spec = new CBufPtrSeqOutStream; + _bufPtrSeqOutStream = _bufPtrSeqOutStream_Spec; + _bufPtrSeqOutStream_Spec->Init(_outMemBuf, _outMemBuf.Size()); + outStreamLoc = _bufPtrSeqOutStream; + } + else // not reprase + { + if (_ntOptions.PreAllocateOutFile && !_isSplit && _curSizeDefined && _curSize > (1 << 12)) + { + // UInt64 ticks = GetCpuTicks(); + _fileLength_that_WasSet = _curSize; + bool res = _outFileStreamSpec->File.SetLength(_curSize); + _fileLengthWasSet = res; + + // ticks = GetCpuTicks() - ticks; + // printf("\nticks = %10d\n", (unsigned)ticks); + if (!res) + { + RINOK(SendMessageError_with_LastError(kCantSetFileLen, fullProcessedPath)); + } + + /* + _outFileStreamSpec->File.Close(); + ticks = GetCpuTicks() - ticks; + printf("\nticks = %10d\n", (unsigned)ticks); + return S_FALSE; + */ + + /* + File.SetLength() on FAT (xp64): is fast, but then File.Close() can be slow, + if we don't write any data. + File.SetLength() for remote share file (exFAT) can be slow in some cases, + and the Windows can return "network error" after 1 minute, + while remote file still can grow. + We need some way to detect such bad cases and disable PreAllocateOutFile mode. + */ + + res = _outFileStreamSpec->SeekToBegin_bool(); + if (!res) + { + RINOK(SendMessageError_with_LastError("Cannot seek to begin of file", fullProcessedPath)); + } + } // PreAllocateOutFile + + #ifdef SUPPORT_ALT_STREAMS + if (_isRenamed && !_item.IsAltStream) + { + CIndexToPathPair pair(index, fullProcessedPath); + unsigned oldSize = _renamedFiles.Size(); + unsigned insertIndex = _renamedFiles.AddToUniqueSorted(pair); + if (oldSize == _renamedFiles.Size()) + _renamedFiles[insertIndex].Path = fullProcessedPath; + } + #endif // SUPPORT_ALT_STREAMS + + if (_isSplit) + { + RINOK(_outFileStreamSpec->Seek((Int64)_position, STREAM_SEEK_SET, NULL)); + } + outStreamLoc = outFileStream_Loc; + } // if not reprase + + _outFileStream = outFileStream_Loc; + + needExit = false; + return S_OK; +} + + + + + +STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStream **outStream, Int32 askExtractMode) +{ + COM_TRY_BEGIN + + *outStream = NULL; + + #ifndef _SFX + if (_hashStream) + _hashStreamSpec->ReleaseStream(); + _hashStreamWasUsed = false; #endif + _outFileStream.Release(); + _bufPtrSeqOutStream.Release(); + + _encrypted = false; + _position = 0; + _isSplit = false; + + _curSize = 0; + _curSizeDefined = false; + _fileLengthWasSet = false; + _fileLength_that_WasSet = 0; + _index = index; + + _diskFilePath.Empty(); + + _isRenamed = false; + // _fi.Clear(); + _is_SymLink_in_Data = false; + _is_SymLink_in_Data_Linux = false; + + _fileWasExtracted = false; + + #ifdef SUPPORT_LINKS + // _CopyFile_Path.Empty(); + _link.Clear(); + #endif + + IInArchive *archive = _arc->Archive; + + #ifndef _SFX + _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; + if (_use_baseParentFolder_mode) + { + _item._baseParentFolder = (int)_baseParentFolder; + if (_pathMode == NExtract::NPathMode::kFullPaths || + _pathMode == NExtract::NPathMode::kAbsPaths) + _item._baseParentFolder = -1; + } + #endif // _SFX + + #ifdef SUPPORT_ALT_STREAMS + _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; + #endif + + RINOK(_arc->GetItem(index, _item)); + + { + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidPosition, &prop)); + if (prop.vt != VT_EMPTY) + { + if (prop.vt != VT_UI8) + return E_FAIL; + _position = prop.uhVal.QuadPart; + _isSplit = true; + } + } + + #ifdef SUPPORT_LINKS + RINOK(ReadLink()); + #endif // SUPPORT_LINKS + + + RINOK(Archive_GetItemBoolProp(archive, index, kpidEncrypted, _encrypted)); + + RINOK(GetUnpackSize()); + + #ifdef SUPPORT_ALT_STREAMS + if (!_ntOptions.AltStreams.Val && _item.IsAltStream) + return S_OK; + #endif // SUPPORT_ALT_STREAMS UStringVector &pathParts = _item.PathParts; @@ -836,7 +1409,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre } } else - #endif + #endif // _SFX { if (pathParts.IsEmpty()) { @@ -924,11 +1497,14 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre case NExtract::NPathMode::kAbsPaths: break; */ + default: + break; } pathParts.DeleteFrontal(numRemovePathParts); } + #ifndef _SFX if (ExtractToStreamCallback) @@ -954,480 +1530,25 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre return ExtractToStreamCallback->GetStream7(name, BoolToInt(_item.IsDir), outStream, askExtractMode, GetProp); } - #endif + #endif // _SFX + CMyComPtr<ISequentialOutStream> outStreamLoc; -if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) -{ - if (_stdOutMode) + if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) { - outStreamLoc = new CStdOutFileStream; - } - else - { - { - NCOM::CPropVariant prop; - RINOK(archive->GetProperty(index, kpidAttrib, &prop)); - if (prop.vt == VT_UI4) - { - _fi.Attrib = prop.ulVal; - _fi.AttribDefined = true; - } - else if (prop.vt == VT_EMPTY) - _fi.AttribDefined = false; - else - return E_FAIL; - } - - RINOK(GetTime(index, kpidCTime, _fi.CTime, _fi.CTimeDefined)); - RINOK(GetTime(index, kpidATime, _fi.ATime, _fi.ATimeDefined)); - RINOK(GetTime(index, kpidMTime, _fi.MTime, _fi.MTimeDefined)); - - bool isAnti = false; - RINOK(_arc->IsItemAnti(index, isAnti)); - - #ifdef SUPPORT_ALT_STREAMS - if (!_item.IsAltStream - || !pathParts.IsEmpty() - || !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt)) - #endif - Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, _keepAndReplaceEmptyDirPrefixes, pathParts, _item.MainIsDir); - - #ifdef SUPPORT_ALT_STREAMS - - if (_item.IsAltStream) - { - UString s (_item.AltStreamName); - Correct_AltStream_Name(s); - bool needColon = true; - - if (pathParts.IsEmpty()) - { - pathParts.AddNew(); - if (_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt) - needColon = false; - } - else if (_pathMode == NExtract::NPathMode::kAbsPaths && - NWildcard::GetNumPrefixParts_if_DrivePath(pathParts) == pathParts.Size()) - pathParts.AddNew(); - - UString &name = pathParts.Back(); - if (needColon) - name += (char)(_ntOptions.ReplaceColonForAltStream ? '_' : ':'); - name += s; - } - - #endif - - UString processedPath (MakePathFromParts(pathParts)); - - if (!isAnti) - { - if (!_item.IsDir) - { - if (!pathParts.IsEmpty()) - pathParts.DeleteBack(); - } - - if (!pathParts.IsEmpty()) - { - FString fullPathNew; - CreateComplexDirectory(pathParts, fullPathNew); - - if (_item.IsDir) - { - CDirPathTime &pt = _extractedFolders.AddNew(); - - pt.CTime = _fi.CTime; - pt.CTimeDefined = (WriteCTime && _fi.CTimeDefined); - - pt.ATime = _fi.ATime; - pt.ATimeDefined = (WriteATime && _fi.ATimeDefined); - - pt.MTimeDefined = false; - - if (WriteMTime) - { - if (_fi.MTimeDefined) - { - pt.MTime = _fi.MTime; - pt.MTimeDefined = true; - } - else if (_arc->MTimeDefined) - { - pt.MTime = _arc->MTime; - pt.MTimeDefined = true; - } - } - - pt.Path = fullPathNew; - - pt.SetDirTime(); - } - } - } - - - FString fullProcessedPath (us2fs(processedPath)); - if (_pathMode != NExtract::NPathMode::kAbsPaths - || !NName::IsAbsolutePath(processedPath)) - { - fullProcessedPath = MakePath_from_2_Parts(_dirPathPrefix, fullProcessedPath); - } - - #ifdef SUPPORT_ALT_STREAMS - - if (_item.IsAltStream && _item.ParentIndex != (UInt32)(Int32)-1) - { - int renIndex = _renamedFiles.FindInSorted(CIndexToPathPair(_item.ParentIndex)); - if (renIndex >= 0) - { - const CIndexToPathPair &pair = _renamedFiles[renIndex]; - fullProcessedPath = pair.Path; - fullProcessedPath += ':'; - UString s (_item.AltStreamName); - Correct_AltStream_Name(s); - fullProcessedPath += us2fs(s); - } - } - - #endif - - bool isRenamed = false; - - if (_item.IsDir) + if (_stdOutMode) + outStreamLoc = new CStdOutFileStream; + else { - _diskFilePath = fullProcessedPath; - if (isAnti) - RemoveDir(_diskFilePath); - #ifdef SUPPORT_LINKS - if (linkPath.IsEmpty()) - #endif + bool needExit = true; + RINOK(GetExtractStream(outStreamLoc, needExit)); + if (needExit) return S_OK; } - else if (!_isSplit) - { - - // ----- Is file (not split) ----- - NFind::CFileInfo fileInfo; - if (fileInfo.Find(fullProcessedPath)) - { - switch (_overwriteMode) - { - case NExtract::NOverwriteMode::kSkip: - return S_OK; - case NExtract::NOverwriteMode::kAsk: - { - int slashPos = fullProcessedPath.ReverseFind_PathSepar(); - FString realFullProcessedPath (fullProcessedPath.Left(slashPos + 1) + fileInfo.Name); - - Int32 overwriteResult; - RINOK(_extractCallback2->AskOverwrite( - fs2us(realFullProcessedPath), &fileInfo.MTime, &fileInfo.Size, _item.Path, - _fi.MTimeDefined ? &_fi.MTime : NULL, - _curSizeDefined ? &_curSize : NULL, - &overwriteResult)) - - switch (overwriteResult) - { - case NOverwriteAnswer::kCancel: return E_ABORT; - case NOverwriteAnswer::kNo: return S_OK; - case NOverwriteAnswer::kNoToAll: _overwriteMode = NExtract::NOverwriteMode::kSkip; return S_OK; - case NOverwriteAnswer::kYes: break; - case NOverwriteAnswer::kYesToAll: _overwriteMode = NExtract::NOverwriteMode::kOverwrite; break; - case NOverwriteAnswer::kAutoRename: _overwriteMode = NExtract::NOverwriteMode::kRename; break; - default: - return E_FAIL; - } - } - } - if (_overwriteMode == NExtract::NOverwriteMode::kRename) - { - if (!AutoRenamePath(fullProcessedPath)) - { - RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); - return E_FAIL; - } - isRenamed = true; - } - else if (_overwriteMode == NExtract::NOverwriteMode::kRenameExisting) - { - FString existPath (fullProcessedPath); - if (!AutoRenamePath(existPath)) - { - RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); - return E_FAIL; - } - // MyMoveFile can raname folders. So it's OK to use it for folders too - if (!MyMoveFile(fullProcessedPath, existPath)) - { - RINOK(SendMessageError2(kCantRenameFile, existPath, fullProcessedPath)); - return E_FAIL; - } - } - else - { - if (fileInfo.IsDir()) - { - // do we need to delete all files in folder? - if (!RemoveDir(fullProcessedPath)) - { - RINOK(SendMessageError_with_LastError(kCantDeleteOutputDir, fullProcessedPath)); - return S_OK; - } - } - else - { - bool needDelete = true; - if (needDelete) - { - if (NFind::DoesFileExist(fullProcessedPath)) - if (!DeleteFileAlways(fullProcessedPath)) - if (GetLastError() != ERROR_FILE_NOT_FOUND) - { - RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); - return S_OK; - // return E_FAIL; - } - } - } - } - } - else // not Find(fullProcessedPath) - { - // we need to clear READ-ONLY of parent before creating alt stream - #if defined(_WIN32) && !defined(UNDER_CE) - int colonPos = NName::FindAltStreamColon(fullProcessedPath); - if (colonPos >= 0 && fullProcessedPath[(unsigned)colonPos + 1] != 0) - { - FString parentFsPath (fullProcessedPath); - parentFsPath.DeleteFrom(colonPos); - NFind::CFileInfo parentFi; - if (parentFi.Find(parentFsPath)) - { - if (parentFi.IsReadOnly()) - SetFileAttrib(parentFsPath, parentFi.Attrib & ~FILE_ATTRIBUTE_READONLY); - } - } - #endif - } - // ----- END of code for Is file (not split) ----- - - } - _diskFilePath = fullProcessedPath; - - - if (!isAnti) - { - #ifdef SUPPORT_LINKS - - if (!linkPath.IsEmpty()) - { - #ifndef UNDER_CE - - UString relatPath; - if (isRelative) - relatPath = GetDirPrefixOf(_item.Path); - relatPath += linkPath; - - if (!IsSafePath(relatPath)) - { - RINOK(SendMessageError("Dangerous link path was ignored", us2fs(relatPath))); - } - else - { - FString existPath; - if (isHardLink /* || isCopyLink */ || !isRelative) - { - if (!NName::GetFullPath(_dirPathPrefix_Full, us2fs(relatPath), existPath)) - { - RINOK(SendMessageError("Incorrect path", us2fs(relatPath))); - } - } - else - { - existPath = us2fs(linkPath); - } - - if (!existPath.IsEmpty()) - { - if (isHardLink /* || isCopyLink */) - { - // if (isHardLink) - { - if (!MyCreateHardLink(fullProcessedPath, existPath)) - { - RINOK(SendMessageError2(kCantCreateHardLink, fullProcessedPath, existPath)); - // return S_OK; - } - } - /* - else - { - NFind::CFileInfo fi; - if (!fi.Find(existPath)) - { - RINOK(SendMessageError2("Can not find the file for copying", existPath, fullProcessedPath)); - } - else - { - if (_curSizeDefined && _curSize == fi.Size) - _CopyFile_Path = existPath; - else - { - RINOK(SendMessageError2("File size collision for file copying", existPath, fullProcessedPath)); - } - - // RINOK(MyCopyFile(existPath, fullProcessedPath)); - } - } - */ - } - else if (_ntOptions.SymLinks.Val) - { - // bool isSymLink = true; // = false for junction - if (_item.IsDir && !isRelative) - { - // if it's before Vista we use Junction Point - // isJunction = true; - // convertToAbs = true; - } - - CByteBuffer data; - if (FillLinkData(data, fs2us(existPath), !isJunction)) - { - CReparseAttr attr; - DWORD errorCode = 0; - if (!attr.Parse(data, data.Size(), errorCode)) - { - RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path))); - // return E_FAIL; - } - else - if (!NFile::NIO::SetReparseData(fullProcessedPath, _item.IsDir, data, (DWORD)data.Size())) - { - RINOK(SendMessageError_with_LastError(kCantCreateSymLink, fullProcessedPath)); - } - } - } - } - } - - #endif - } - - if (linkPath.IsEmpty() /* || !_CopyFile_Path.IsEmpty() */) - #endif // SUPPORT_LINKS - { - bool needWriteFile = true; - - #ifdef SUPPORT_LINKS - if (!_hardLinks.IDs.IsEmpty() && !_item.IsAltStream) - { - CHardLinkNode h; - bool defined; - RINOK(Archive_Get_HardLinkNode(archive, index, h, defined)); - if (defined) - { - { - int linkIndex = _hardLinks.IDs.FindInSorted2(h); - if (linkIndex >= 0) - { - FString &hl = _hardLinks.Links[linkIndex]; - if (hl.IsEmpty()) - hl = fullProcessedPath; - else - { - if (!MyCreateHardLink(fullProcessedPath, hl)) - { - RINOK(SendMessageError2(kCantCreateHardLink, fullProcessedPath, hl)); - return S_OK; - } - needWriteFile = false; - } - } - } - } - } - #endif - - if (needWriteFile) - { - _outFileStreamSpec = new COutFileStream; - CMyComPtr<ISequentialOutStream> outStreamLoc2(_outFileStreamSpec); - if (!_outFileStreamSpec->Open(fullProcessedPath, _isSplit ? OPEN_ALWAYS: CREATE_ALWAYS)) - { - // if (::GetLastError() != ERROR_FILE_EXISTS || !isSplit) - { - RINOK(SendMessageError_with_LastError(kCantOpenOutFile, fullProcessedPath)); - return S_OK; - } - } - - if (_ntOptions.PreAllocateOutFile && !_isSplit && _curSizeDefined && _curSize > (1 << 12)) - { - // UInt64 ticks = GetCpuTicks(); - bool res = _outFileStreamSpec->File.SetLength(_curSize); - _fileLengthWasSet = res; - - // ticks = GetCpuTicks() - ticks; - // printf("\nticks = %10d\n", (unsigned)ticks); - if (!res) - { - RINOK(SendMessageError_with_LastError(kCantSetFileLen, fullProcessedPath)); - } - - /* - _outFileStreamSpec->File.Close(); - ticks = GetCpuTicks() - ticks; - printf("\nticks = %10d\n", (unsigned)ticks); - return S_FALSE; - */ - - /* - File.SetLength() on FAT (xp64): is fast, but then File.Close() can be slow, - if we don't write any data. - File.SetLength() for remote share file (exFAT) can be slow in some cases, - and the Windows can return "network error" after 1 minute, - while remote file still can grow. - We need some way to detect such bad cases and disable PreAllocateOutFile mode. - */ - - res = _outFileStreamSpec->File.SeekToBegin(); - if (!res) - { - RINOK(SendMessageError_with_LastError("Can not seek to begin of file", fullProcessedPath)); - } - } - - #ifdef SUPPORT_ALT_STREAMS - if (isRenamed && !_item.IsAltStream) - { - CIndexToPathPair pair(index, fullProcessedPath); - unsigned oldSize = _renamedFiles.Size(); - unsigned insertIndex = _renamedFiles.AddToUniqueSorted(pair); - if (oldSize == _renamedFiles.Size()) - _renamedFiles[insertIndex].Path = fullProcessedPath; - } - #endif - - if (_isSplit) - { - RINOK(_outFileStreamSpec->Seek(_position, STREAM_SEEK_SET, NULL)); - } - - _outFileStream = outStreamLoc2; - } - } - } - - outStreamLoc = _outFileStream; } -} #ifndef _SFX - if (_hashStream) { if (askExtractMode == NArchive::NExtract::NAskMode::kExtract || @@ -1439,28 +1560,22 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) _hashStreamWasUsed = true; } } + #endif // _SFX - #endif - - if (outStreamLoc) { /* #ifdef SUPPORT_LINKS - if (!_CopyFile_Path.IsEmpty()) { RINOK(PrepareOperation(askExtractMode)); RINOK(MyCopyFile(outStreamLoc)); return SetOperationResult(NArchive::NExtract::NOperationResult::kOK); } - - if (isCopyLink && _testMode) + if (_link.isCopyLink && _testMode) return S_OK; - #endif */ - *outStream = outStreamLoc.Detach(); } @@ -1470,6 +1585,15 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) } + + + + + + + + + STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) { COM_TRY_BEGIN @@ -1498,33 +1622,352 @@ STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) } + + + HRESULT CArchiveExtractCallback::CloseFile() { if (!_outFileStream) return S_OK; HRESULT hres = S_OK; - _outFileStreamSpec->SetTime( - (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, - (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, - (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); const UInt64 processedSize = _outFileStreamSpec->ProcessedSize; - if (_fileLengthWasSet && _curSize > processedSize) + if (_fileLengthWasSet && _fileLength_that_WasSet > processedSize) { bool res = _outFileStreamSpec->File.SetLength(processedSize); _fileLengthWasSet = res; if (!res) - hres = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path)); + { + HRESULT hres2 = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path)); + if (hres == S_OK) + hres = hres2; + } } + _curSize = processedSize; _curSizeDefined = true; + + // #ifdef _WIN32 + _outFileStreamSpec->SetTime( + (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, + (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, + (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); + // #endif + RINOK(_outFileStreamSpec->Close()); _outFileStream.Release(); return hres; } +#ifdef SUPPORT_LINKS + + +HRESULT CArchiveExtractCallback::SetFromLinkPath( + const FString &fullProcessedPath, + const CLinkInfo &linkInfo, + bool &linkWasSet) +{ + linkWasSet = false; + if (!_ntOptions.SymLinks.Val && !linkInfo.isHardLink) + return S_OK; + + UString relatPath; + + /* if (linkInfo.isRelative) + linkInfo.linkPath is final link path that must be stored to file link field + else + linkInfo.linkPath is path from root of archive. So we must add _dirPathPrefix_Full before linkPath. + */ + + if (linkInfo.isRelative) + relatPath = GetDirPrefixOf(_item.Path); + relatPath += linkInfo.linkPath; + + if (!IsSafePath(relatPath)) + { + return SendMessageError2( + 0, // errorCode + "Dangerous link path was ignored", + us2fs(_item.Path), + us2fs(linkInfo.linkPath)); // us2fs(relatPath) + } + + FString existPath; + if (linkInfo.isHardLink /* || linkInfo.IsCopyLink */ || !linkInfo.isRelative) + { + if (!NName::GetFullPath(_dirPathPrefix_Full, us2fs(relatPath), existPath)) + { + RINOK(SendMessageError("Incorrect path", us2fs(relatPath))); + } + } + else + { + existPath = us2fs(linkInfo.linkPath); + // printf("\nlinkPath = : %s\n", GetOemString(linkInfo.linkPath).Ptr()); + } + + if (existPath.IsEmpty()) + return SendMessageError("Empty link", fullProcessedPath); + + if (linkInfo.isHardLink /* || linkInfo.IsCopyLink */) + { + // if (linkInfo.isHardLink) + { + if (!MyCreateHardLink(fullProcessedPath, existPath)) + { + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantCreateHardLink, fullProcessedPath, existPath)); + } + linkWasSet = true; + return S_OK; + } + /* + // IsCopyLink + { + NFind::CFileInfo fi; + if (!fi.Find(existPath)) + { + RINOK(SendMessageError2("Cannot find the file for copying", existPath, fullProcessedPath)); + } + else + { + if (_curSizeDefined && _curSize == fi.Size) + _CopyFile_Path = existPath; + else + { + RINOK(SendMessageError2("File size collision for file copying", existPath, fullProcessedPath)); + } + // RINOK(MyCopyFile(existPath, fullProcessedPath)); + } + } + */ + } + + // is Symbolic link + + /* + if (_item.IsDir && !isRelative) + { + // Windows before Vista doesn't support symbolic links. + // we could convert such symbolic links to Junction Points + // isJunction = true; + // convertToAbs = true; + } + */ + + if (!_ntOptions.SymLinks_AllowDangerous.Val) + { + #ifdef _WIN32 + if (_item.IsDir) + #endif + if (linkInfo.isRelative) + { + CLinkLevelsInfo levelsInfo; + levelsInfo.Parse(linkInfo.linkPath); + if (levelsInfo.FinalLevel < 1 || levelsInfo.IsAbsolute) + { + return SendMessageError2( + 0, // errorCode + "Dangerous symbolic link path was ignored", + us2fs(_item.Path), + us2fs(linkInfo.linkPath)); + } + } + } + + + #ifdef _WIN32 + + CByteBuffer data; + // printf("\nFillLinkData(): %s\n", GetOemString(existPath).Ptr()); + if (!FillLinkData(data, fs2us(existPath), !linkInfo.isJunction, linkInfo.isWSL)) + return SendMessageError("Cannot fill link data", us2fs(_item.Path)); + + /* + if (NtReparse_Size != data.Size() || memcmp(NtReparse_Data, data, data.Size()) != 0) + { + SendMessageError("reconstructed Reparse is different", fs2us(existPath)); + } + */ + + CReparseAttr attr; + if (!attr.Parse(data, data.Size())) + { + RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path))); + return S_OK; + } + if (!NFile::NIO::SetReparseData(fullProcessedPath, _item.IsDir, data, (DWORD)data.Size())) + { + RINOK(SendMessageError_with_LastError(kCantCreateSymLink, fullProcessedPath)); + return S_OK; + } + linkWasSet = true; + + return S_OK; + + + #else // ! _WIN32 + + if (!NFile::NIO::SetSymLink(fullProcessedPath, existPath)) + { + RINOK(SendMessageError_with_LastError(kCantCreateSymLink, fullProcessedPath)); + return S_OK; + } + linkWasSet = true; + + return S_OK; + + #endif // ! _WIN32 +} + + +bool CLinkInfo::Parse(const Byte *data, size_t dataSize, bool isLinuxData) +{ + // this->isLinux = isLinuxData; + + if (isLinuxData) + { + isJunction = false; + isHardLink = false; + AString utf; + if (dataSize >= (1 << 12)) + return false; + utf.SetFrom_CalcLen((const char *)data, (unsigned)dataSize); + UString u; + if (!ConvertUTF8ToUnicode(utf, u)) + return false; + linkPath = u; + + // in linux symbolic data: we expect that linux separator '/' is used + // if windows link was created, then we also must use linux separator + if (u.IsEmpty()) + return false; + wchar_t c = u[0]; + isRelative = !IS_PATH_SEPAR(c); + return true; + } + + CReparseAttr reparse; + if (!reparse.Parse(data, dataSize)) + return false; + isHardLink = false; + // isCopyLink = false; + linkPath = reparse.GetPath(); + isJunction = reparse.IsMountPoint(); + + if (reparse.IsSymLink_WSL()) + { + isWSL = true; + isRelative = reparse.IsRelative_WSL(); + } + else + isRelative = reparse.IsRelative_Win(); + + // FIXME !!! + #ifndef _WIN32 + linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); + #endif + + return true; +} + +#endif // SUPPORT_LINKS + + +HRESULT CArchiveExtractCallback::CloseReparseAndFile() +{ + HRESULT res = S_OK; + + #ifdef SUPPORT_LINKS + + size_t reparseSize = 0; + bool repraseMode = false; + bool needSetReparse = false; + CLinkInfo linkInfo; + + if (_bufPtrSeqOutStream) + { + repraseMode = true; + reparseSize = _bufPtrSeqOutStream_Spec->GetPos(); + if (_curSizeDefined && reparseSize == _outMemBuf.Size()) + { + /* + CReparseAttr reparse; + DWORD errorCode = 0; + needSetReparse = reparse.Parse(_outMemBuf, reparseSize, errorCode); + if (needSetReparse) + { + UString linkPath = reparse.GetPath(); + #ifndef _WIN32 + linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); + #endif + } + */ + needSetReparse = linkInfo.Parse(_outMemBuf, reparseSize, _is_SymLink_in_Data_Linux); + if (!needSetReparse) + res = SendMessageError_with_LastError("Incorrect reparse stream", us2fs(_item.Path)); + } + else + { + res = SendMessageError_with_LastError("Unknown reparse stream", us2fs(_item.Path)); + } + if (!needSetReparse && _outFileStream) + { + HRESULT res2 = WriteStream(_outFileStream, _outMemBuf, reparseSize); + if (res == S_OK) + res = res2; + } + _bufPtrSeqOutStream.Release(); + } + + #endif // SUPPORT_LINKS + + + HRESULT res2 = CloseFile(); + + if (res == S_OK) + res = res2; + + RINOK(res); + + #ifdef SUPPORT_LINKS + if (repraseMode) + { + _curSize = reparseSize; + _curSizeDefined = true; + + #ifdef SUPPORT_LINKS + if (needSetReparse) + { + // in Linux : we must delete empty file before symbolic link creation + // in Windows : we can create symbolic link even without file deleting + if (!DeleteFileAlways(_diskFilePath)) + { + RINOK(SendMessageError_with_LastError("can't delete file", _diskFilePath)); + } + { + bool linkWasSet = false; + RINOK(SetFromLinkPath(_diskFilePath, linkInfo, linkWasSet)); + if (!linkWasSet) + _fileWasExtracted = false; + } + /* + if (!NFile::NIO::SetReparseData(_diskFilePath, _item.IsDir, )) + { + res = SendMessageError_with_LastError(kCantCreateSymLink, _diskFilePath); + } + */ + } + #endif + } + #endif + return res; +} + + + STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) { COM_TRY_BEGIN @@ -1551,9 +1994,9 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) _hashStreamWasUsed = false; } - #endif + #endif // _SFX - RINOK(CloseFile()); + RINOK(CloseReparseAndFile()); #ifdef _USE_SECURITY_CODE if (!_stdOutMode && _extractMode && _ntOptions.NtSecurity.Val && _arc->GetRawProps) @@ -1571,11 +2014,11 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) SECURITY_INFORMATION securInfo = DACL_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION | OWNER_SECURITY_INFORMATION; if (_saclEnabled) securInfo |= SACL_SECURITY_INFORMATION; - ::SetFileSecurityW(fs2us(_diskFilePath), securInfo, (PSECURITY_DESCRIPTOR)(void *)data); + ::SetFileSecurityW(fs2us(_diskFilePath), securInfo, (PSECURITY_DESCRIPTOR)(void *)(const Byte *)(data)); } } } - #endif + #endif // _USE_SECURITY_CODE if (!_curSizeDefined) GetUnpackSize(); @@ -1599,8 +2042,16 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) else NumFiles++; + if (_fileWasExtracted) if (!_stdOutMode && _extractMode && _fi.AttribDefined) - SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); + { + bool res = SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); + if (!res) + { + // do we need error message here in Windows and in posix? + SendMessageError_with_LastError("Cannot set file attribute", _diskFilePath); + } + } RINOK(_extractCallback2->SetOperationResult(opRes, BoolToInt(_encrypted))); @@ -1609,6 +2060,8 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) COM_TRY_END } + + STDMETHODIMP CArchiveExtractCallback::ReportExtractResult(UInt32 indexType, UInt32 index, Int32 opRes) { if (_folderArchiveExtractCallback2) @@ -1666,7 +2119,7 @@ void CDirPathSortPair::SetNumSlashes(const FChar *s) } -bool CDirPathTime::SetDirTime() +bool CDirPathTime::SetDirTime() const { return NDir::SetDirTime(Path, CTimeDefined ? &CTime : NULL, @@ -1693,20 +2146,44 @@ HRESULT CArchiveExtractCallback::SetDirsTimes() pairs.Sort2(); + HRESULT res = S_OK; + for (i = 0; i < pairs.Size(); i++) { - _extractedFolders[pairs[i].Index].SetDirTime(); - // if (!) return GetLastError(); + const CDirPathTime &dpt = _extractedFolders[pairs[i].Index]; + if (!dpt.SetDirTime()) + { + // result = E_FAIL; + // do we need error message here in Windows and in posix? + // SendMessageError_with_LastError("Cannot set directory time", dpt.Path); + } } + /* + #ifndef _WIN32 + for (i = 0; i < _delayedSymLinks.Size(); i++) + { + const CDelayedSymLink &link = _delayedSymLinks[i]; + if (!link.Create()) + { + if (res == S_OK) + res = GetLastError_noZero_HRESULT(); + // res = E_FAIL; + // do we need error message here in Windows and in posix? + SendMessageError_with_LastError("Cannot create Symbolic Link", link._source); + } + } + #endif // _WIN32 + */ + ClearExtractedDirsInfo(); - return S_OK; + return res; } HRESULT CArchiveExtractCallback::CloseArc() { - HRESULT res = CloseFile(); + HRESULT res = CloseReparseAndFile(); HRESULT res2 = SetDirsTimes(); if (res == S_OK) res = res2; diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/CPP/7zip/UI/Common/ArchiveExtractCallback.h index dd5c98c0..25eb6444 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -4,12 +4,14 @@ #define __ARCHIVE_EXTRACT_CALLBACK_H #include "../../../Common/MyCom.h" +#include "../../../Common/MyLinux.h" #include "../../../Common/Wildcard.h" #include "../../IPassword.h" #include "../../Common/FileStreams.h" #include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" #include "../../Archive/IArchive.h" @@ -52,6 +54,7 @@ struct CExtractNtOptions { CBoolPair NtSecurity; CBoolPair SymLinks; + CBoolPair SymLinks_AllowDangerous; CBoolPair HardLinks; CBoolPair AltStreams; bool ReplaceColonForAltStream; @@ -64,9 +67,10 @@ struct CExtractNtOptions WriteToAltStreamIfColon(false) { SymLinks.Val = true; + SymLinks_AllowDangerous.Val = false; HardLinks.Val = true; AltStreams.Val = true; - + PreAllocateOutFile = #ifdef _WIN32 true; @@ -165,9 +169,35 @@ struct CDirPathTime FString Path; - bool SetDirTime(); + bool SetDirTime() const; +}; + + +#ifdef SUPPORT_LINKS + +struct CLinkInfo +{ + // bool isCopyLink; + bool isHardLink; + bool isJunction; + bool isRelative; + bool isWSL; + UString linkPath; + + void Clear() + { + // IsCopyLink = false; + isHardLink = false; + isJunction = false; + isRelative = false; + isWSL = false; + linkPath.Empty(); + } + + bool Parse(const Byte *data, size_t dataSize, bool isLinuxData); }; +#endif // SUPPORT_LINKS class CArchiveExtractCallback: @@ -225,15 +255,53 @@ class CArchiveExtractCallback: bool ATimeDefined; bool MTimeDefined; bool AttribDefined; + + bool IsReparse() const + { + return (AttribDefined && (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0); + } + + bool IsLinuxSymLink() const + { + return (AttribDefined && MY_LIN_S_ISLNK(Attrib >> 16)); + } + + void SetFromPosixAttrib(UInt32 a) + { + // here we set only part of combined attribute required by SetFileAttrib() call + #ifdef _WIN32 + // Windows sets FILE_ATTRIBUTE_NORMAL, if we try to set 0 as attribute. + Attrib = MY_LIN_S_ISDIR(a) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((a & 0222) == 0) // (& S_IWUSR) in p7zip + Attrib |= FILE_ATTRIBUTE_READONLY; + #else + Attrib = (a << 16) | FILE_ATTRIBUTE_UNIX_EXTENSION; + #endif + AttribDefined = true; + } } _fi; + bool _is_SymLink_in_Data; + bool _is_SymLink_in_Data_Linux; // false = WIN32, true = LINUX + + bool _fileWasExtracted; + UInt32 _index; UInt64 _curSize; bool _curSizeDefined; bool _fileLengthWasSet; + UInt64 _fileLength_that_WasSet; + COutFileStream *_outFileStreamSpec; CMyComPtr<ISequentialOutStream> _outFileStream; + CByteBuffer _outMemBuf; + CBufPtrSeqOutStream *_bufPtrSeqOutStream_Spec; + CMyComPtr<ISequentialOutStream> _bufPtrSeqOutStream; + + #ifndef _SFX COutStreamWithHash *_hashStreamSpec; @@ -261,6 +329,10 @@ class CArchiveExtractCallback: bool _progressTotal_Defined; CObjectVector<CDirPathTime> _extractedFolders; + + #ifndef _WIN32 + // CObjectVector<NWindows::NFile::NDir::CDelayedSymLink> _delayedSymLinks; + #endif #if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) bool _saclEnabled; @@ -272,7 +344,7 @@ class CArchiveExtractCallback: HRESULT SendMessageError(const char *message, const FString &path); HRESULT SendMessageError_with_LastError(const char *message, const FString &path); - HRESULT SendMessageError2(const char *message, const FString &path1, const FString &path2); + HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2); public: @@ -335,10 +407,12 @@ public: private: CHardLinks _hardLinks; - UString linkPath; + CLinkInfo _link; // FString _CopyFile_Path; // HRESULT MyCopyFile(ISequentialOutStream *outStream); + HRESULT Link(const FString &fullProcessedPath); + HRESULT ReadLink(); public: // call PrepareHardLinks() after Init() @@ -367,10 +441,33 @@ private: void ClearExtractedDirsInfo() { _extractedFolders.Clear(); + #ifndef _WIN32 + // _delayedSymLinks.Clear(); + #endif } + HRESULT Read_fi_Props(); + void CorrectPathParts(); + void CreateFolders(); + + bool _isRenamed; + HRESULT CheckExistFile(FString &fullProcessedPath, bool &needExit); + HRESULT GetExtractStream(CMyComPtr<ISequentialOutStream> &outStreamLoc, bool &needExit); + HRESULT CloseFile(); + HRESULT CloseReparseAndFile(); + HRESULT CloseReparseAndFile2(); HRESULT SetDirsTimes(); + + const void *NtReparse_Data; + UInt32 NtReparse_Size; + + #ifdef SUPPORT_LINKS + HRESULT SetFromLinkPath( + const FString &fullProcessedPath, + const CLinkInfo &linkInfo, + bool &linkWasSet); + #endif }; diff --git a/CPP/7zip/UI/Common/ArchiveName.cpp b/CPP/7zip/UI/Common/ArchiveName.cpp index 9048edce..1baf3e1e 100644 --- a/CPP/7zip/UI/Common/ArchiveName.cpp +++ b/CPP/7zip/UI/Common/ArchiveName.cpp @@ -21,7 +21,7 @@ static UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) int dotPos = resultName.ReverseFind_Dot(); if (dotPos > 0) { - FString archiveName2 = resultName.Left(dotPos); + FString archiveName2 = resultName.Left((unsigned)dotPos); if (archiveName2.ReverseFind_Dot() < 0) resultName = archiveName2; } @@ -64,7 +64,7 @@ static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepN int dotPos = resultName.ReverseFind_Dot(); if (dotPos > 0) { - FString name2 = resultName.Left(dotPos); + FString name2 = resultName.Left((unsigned)dotPos); if (name2.ReverseFind_Dot() < 0) resultName = name2; } diff --git a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp index 4d9d9e14..d3e0d3ce 100644 --- a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp @@ -63,7 +63,7 @@ STDMETHODIMP COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value) struct CInFileStreamVol: public CInFileStream { - int FileNameIndex; + unsigned FileNameIndex; COpenCallbackImp *OpenCallbackImp; CMyComPtr<IArchiveOpenCallback> OpenCallbackRef; @@ -116,7 +116,7 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre FString fullPath; if (!NFile::NName::GetFullPath(_folderPrefix, us2fs(name2), fullPath)) return S_FALSE; - if (!_fileInfo.Find(fullPath)) + if (!_fileInfo.Find_FollowLink(fullPath)) return S_FALSE; if (_fileInfo.IsDir()) return S_FALSE; @@ -124,10 +124,7 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre CMyComPtr<IInStream> inStreamTemp = inFile; if (!inFile->Open(fullPath)) { - DWORD lastError = ::GetLastError(); - if (lastError == 0) - return E_FAIL; - return HRESULT_FROM_WIN32(lastError); + return GetLastError_noZero_HRESULT(); } FileSizes.Add(_fileInfo.Size); diff --git a/CPP/7zip/UI/Common/ArchiveOpenCallback.h b/CPP/7zip/UI/Common/ArchiveOpenCallback.h index e9575f56..46b26768 100644 --- a/CPP/7zip/UI/Common/ArchiveOpenCallback.h +++ b/CPP/7zip/UI/Common/ArchiveOpenCallback.h @@ -6,6 +6,7 @@ #include "../../../Common/MyCom.h" #include "../../../Windows/FileFind.h" +#include "../../../Windows/FileIO.h" #ifndef _NO_CRYPTO #include "../../IPassword.h" @@ -88,24 +89,28 @@ public: CMyComPtr<IArchiveOpenCallback> ReOpenCallback; // UInt64 TotalSize; - COpenCallbackImp(): Callback(NULL), _subArchiveMode(false) {} + COpenCallbackImp(): _subArchiveMode(false), Callback(NULL) {} - void Init(const FString &folderPrefix, const FString &fileName) + HRESULT Init2(const FString &folderPrefix, const FString &fileName) { - _folderPrefix = folderPrefix; - if (!_fileInfo.Find(_folderPrefix + fileName)) - throw 20121118; FileNames.Clear(); FileNames_WasUsed.Clear(); FileSizes.Clear(); _subArchiveMode = false; // TotalSize = 0; PasswordWasAsked = false; + _folderPrefix = folderPrefix; + if (!_fileInfo.Find_FollowLink(_folderPrefix + fileName)) + { + // throw 20121118; + return GetLastError_noZero_HRESULT(); + } + return S_OK; } bool SetSecondFileInfo(CFSTR newName) { - return _fileInfo.Find(newName) && !_fileInfo.IsDir(); + return _fileInfo.Find_FollowLink(newName) && !_fileInfo.IsDir(); } }; diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index f849a877..4e957331 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -2,19 +2,24 @@ #include "StdAfx.h" -#include <stdio.h> +#include "../../../../C/CpuArch.h" + +// #include <stdio.h> #ifndef _WIN32 + #define USE_POSIX_TIME #define USE_POSIX_TIME2 -#endif +#endif // _WIN32 #ifdef USE_POSIX_TIME #include <time.h> +#include <unistd.h> #ifdef USE_POSIX_TIME2 #include <sys/time.h> +#include <sys/times.h> #endif -#endif +#endif // USE_POSIX_TIME #ifdef _WIN32 #define USE_ALLOCA @@ -29,21 +34,16 @@ #endif #include "../../../../C/7zCrc.h" -#include "../../../../C/CpuArch.h" +#include "../../../../C/RotateDefs.h" #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" #include "../../../Windows/Thread.h" #endif -#if defined(_WIN32) || defined(UNIX_USE_WIN_FILE) -#define USE_WIN_FILE -#endif - -#ifdef USE_WIN_FILE #include "../../../Windows/FileIO.h" -#endif - +#include "../../../Windows/FileFind.h" +#include "../../../Windows/SystemInfo.h" #include "../../../Common/IntToString.h" #include "../../../Common/MyBuffer2.h" @@ -57,7 +57,9 @@ using namespace NWindows; +#ifndef _7ZIP_ST static const UInt32 k_LZMA = 0x030101; +#endif static const UInt64 kComplexInCommands = (UInt64)1 << #ifdef UNDER_CE @@ -66,9 +68,9 @@ static const UInt64 kComplexInCommands = (UInt64)1 << 34; #endif -static const UInt32 kComplexInSeconds = 4; +static const UInt32 kComplexInMs = 4000; -static void SetComplexCommands(UInt32 complexInSeconds, +static void SetComplexCommandsMs(UInt32 complexInMs, bool isSpecifiedFreq, UInt64 cpuFreq, UInt64 &complexInCommands) { complexInCommands = kComplexInCommands; @@ -78,15 +80,15 @@ static void SetComplexCommands(UInt32 complexInSeconds, cpuFreq = kMinFreq; if (cpuFreq < kMaxFreq || isSpecifiedFreq) { - if (complexInSeconds != 0) - complexInCommands = complexInSeconds * cpuFreq; + if (complexInMs != 0) + complexInCommands = complexInMs * cpuFreq / 1000; else complexInCommands = cpuFreq >> 2; } } static const unsigned kNumHashDictBits = 17; -static const UInt32 kFilterUnpackSize = (48 << 10); +static const UInt32 kFilterUnpackSize = (47 << 10); // + 5; // for test static const unsigned kOldLzmaDictBits = 30; @@ -97,8 +99,8 @@ static const UInt32 kMaxLzmaPropSize = 5; #define ALLOC_WITH_HRESULT(_buffer_, _size_) \ - (_buffer_)->Alloc(_size_); \ - if (!(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; + { (_buffer_)->Alloc(_size_); \ + if (!(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; } class CBaseRandomGenerator @@ -109,6 +111,7 @@ class CBaseRandomGenerator public: CBaseRandomGenerator(UInt32 salt = 0): Salt(salt) { Init(); } void Init() { A1 = 362436069; A2 = 521288629;} + MY_FORCE_INLINE UInt32 GetRnd() { return Salt ^ @@ -120,6 +123,26 @@ public: }; +MY_NO_INLINE +static void RandGen(Byte *buf, size_t size) +{ + CBaseRandomGenerator RG; + const size_t size4 = size & ~((size_t)3); + size_t i; + for (i = 0; i < size4; i += 4) + { + const UInt32 v = RG.GetRnd(); + SetUi32(buf + i, v); + } + UInt32 v = RG.GetRnd(); + for (; i < size; i++) + { + buf[i] = (Byte)v; + v >>= 8; + } +} + + class CBenchRandomGenerator: public CAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) @@ -218,6 +241,7 @@ class CBenchmarkInStream: const Byte *Data; size_t Pos; size_t Size; + public: MY_UNKNOWN_IMP void Init(const Byte *data, size_t size) @@ -226,19 +250,22 @@ public: Size = size; Pos = 0; } + bool WasFinished() const { return Pos == Size; } STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); }; STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processedSize) { - size_t remain = Size - Pos; - UInt32 kMaxBlockSize = (1 << 20); + const UInt32 kMaxBlockSize = (1 << 20); if (size > kMaxBlockSize) size = kMaxBlockSize; + const size_t remain = Size - Pos; if (size > remain) size = (UInt32)remain; - for (UInt32 i = 0; i < size; i++) - ((Byte *)data)[i] = Data[Pos + i]; + + if (size != 0) + memcpy(data, Data + Pos, size); + Pos += size; if (processedSize) *processedSize = size; @@ -267,6 +294,16 @@ public: Pos = 0; } + void InitCrc() + { + Crc = CRC_INIT_VAL; + } + + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } + // void Print() { printf("\n%8d %8d\n", (unsigned)BufferSize, (unsigned)Pos); } MY_UNKNOWN_IMP @@ -283,7 +320,7 @@ STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *p if (RealCopy) memcpy(((Byte *)*this) + Pos, data, curSize); if (CalcCrc) - Crc = CrcUpdate(Crc, data, curSize); + Calc(data, curSize); Pos += curSize; } if (processedSize) @@ -296,6 +333,7 @@ STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *p return S_OK; } + class CCrcOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -303,39 +341,46 @@ class CCrcOutStream: public: bool CalcCrc; UInt32 Crc; + UInt64 Pos; + MY_UNKNOWN_IMP CCrcOutStream(): CalcCrc(true) {}; - void Init() { Crc = CRC_INIT_VAL; } + void Init() { Crc = CRC_INIT_VAL; Pos = 0; } + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; STDMETHODIMP CCrcOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { if (CalcCrc) - Crc = CrcUpdate(Crc, data, size); + Calc(data, size); + Pos += size; if (processedSize) *processedSize = size; return S_OK; } +// #include "../../../../C/My_sys_time.h" + static UInt64 GetTimeCount() { #ifdef USE_POSIX_TIME #ifdef USE_POSIX_TIME2 timeval v; if (gettimeofday(&v, 0) == 0) - return (UInt64)(v.tv_sec) * 1000000 + v.tv_usec; + return (UInt64)(v.tv_sec) * 1000000 + (UInt64)v.tv_usec; return (UInt64)time(NULL) * 1000000; #else return time(NULL); #endif #else - /* LARGE_INTEGER value; if (::QueryPerformanceCounter(&value)) - return value.QuadPart; - */ + return (UInt64)value.QuadPart; return GetTickCount(); #endif } @@ -349,15 +394,14 @@ static UInt64 GetFreq() return 1; #endif #else - /* LARGE_INTEGER value; if (::QueryPerformanceFrequency(&value)) - return value.QuadPart; - */ + return (UInt64)value.QuadPart; return 1000; #endif } + #ifdef USE_POSIX_TIME struct CUserTime @@ -367,50 +411,98 @@ struct CUserTime void Init() { - Prev = clock(); + // Prev = clock(); + Sum = 0; + Prev = 0; + Update(); Sum = 0; } - UInt64 GetUserTime() + void Update() { + tms t; + /* clock_t res = */ times(&t); + clock_t newVal = t.tms_utime + t.tms_stime; + Sum += (UInt64)(newVal - Prev); + Prev = newVal; + + /* clock_t v = clock(); - Sum += v - Prev; - Prev = v; + if (v != -1) + { + Sum += v - Prev; + Prev = v; + } + */ + } + UInt64 GetUserTime() + { + Update(); return Sum; } }; #else -static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } -UInt64 GetWinUserTime() -{ - FILETIME creationTime, exitTime, kernelTime, userTime; - if ( - #ifdef UNDER_CE - ::GetThreadTimes(::GetCurrentThread() - #else - ::GetProcessTimes(::GetCurrentProcess() - #endif - , &creationTime, &exitTime, &kernelTime, &userTime) != 0) - return GetTime64(userTime) + GetTime64(kernelTime); - return (UInt64)GetTickCount() * 10000; -} struct CUserTime { - UInt64 StartTime; + bool UseTick; + DWORD Prev_Tick; + UInt64 Prev; + UInt64 Sum; - void Init() { StartTime = GetWinUserTime(); } - UInt64 GetUserTime() { return GetWinUserTime() - StartTime; } + void Init() + { + UseTick = false; + Prev_Tick = 0; + Prev = 0; + Sum = 0; + Update(); + Sum = 0; + } + UInt64 GetUserTime() + { + Update(); + return Sum; + } + void Update(); }; +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +void CUserTime::Update() +{ + DWORD new_Tick = GetTickCount(); + FILETIME creationTime, exitTime, kernelTime, userTime; + if (!UseTick && + #ifdef UNDER_CE + ::GetThreadTimes(::GetCurrentThread() + #else + ::GetProcessTimes(::GetCurrentProcess() + #endif + , &creationTime, &exitTime, &kernelTime, &userTime)) + { + UInt64 newVal = GetTime64(userTime) + GetTime64(kernelTime); + Sum += newVal - Prev; + Prev = newVal; + } + else + { + UseTick = true; + Sum += (UInt64)(new_Tick - (DWORD)Prev_Tick) * 10000; + } + Prev_Tick = new_Tick; +} + + #endif static UInt64 GetUserFreq() { #ifdef USE_POSIX_TIME - return CLOCKS_PER_SEC; + // return CLOCKS_PER_SEC; + return (UInt64)sysconf(_SC_CLK_TCK); #else return 10000000; #endif @@ -551,7 +643,10 @@ UInt64 CBenchInfo::GetRatingPerUsage(UInt64 rating) const if (globalFreq == 0) globalFreq = 1; if (userTime == 0) - userTime = 1; + { + return 0; + // userTime = 1; + } return userFreq * globalTime / globalFreq * rating / userTime; } @@ -577,11 +672,20 @@ struct CBenchProps UInt32 DecComplexCompr; UInt32 DecComplexUnc; - CBenchProps(): LzmaRatingMode(false) {} + unsigned KeySize; + + CBenchProps(): + LzmaRatingMode(false), + KeySize(0) + {} + void SetLzmaCompexity(); UInt64 GeComprCommands(UInt64 unpackSize) { + const UInt32 kMinSize = 100; + if (unpackSize < kMinSize) + unpackSize = kMinSize; return unpackSize * EncComplex; } @@ -637,9 +741,133 @@ UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt } +#ifndef _7ZIP_ST + +#define NUM_CPU_LEVELS_MAX 3 + +struct CAffinityMode +{ + unsigned NumBundleThreads; + unsigned NumLevels; + unsigned NumCoreThreads; + unsigned NumCores; + // unsigned DivideNum; + UInt32 Sizes[NUM_CPU_LEVELS_MAX]; + + void SetLevels(unsigned numCores, unsigned numCoreThreads); + DWORD_PTR GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const; + bool NeedAffinity() const { return NumBundleThreads != 0; } + + WRes CreateThread_WithAffinity(NWindows::CThread &thread, THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID parameter, UInt32 bundleIndex) const + { + if (NeedAffinity()) + { + CCpuSet cpuSet; + GetAffinityMask(bundleIndex, &cpuSet); + return thread.Create_With_CpuSet(startAddress, parameter, &cpuSet); + } + return thread.Create(startAddress, parameter); + } + + CAffinityMode(): + NumBundleThreads(0), + NumLevels(0), + NumCoreThreads(1) + // DivideNum(1) + {} +}; + +void CAffinityMode::SetLevels(unsigned numCores, unsigned numCoreThreads) +{ + NumCores = numCores; + NumCoreThreads = numCoreThreads; + NumLevels = 0; + if (numCoreThreads == 0 || numCores == 0 || numCores % numCoreThreads != 0) + return; + UInt32 c = numCores / numCoreThreads; + UInt32 c2 = 1; + while ((c & 1) == 0) + { + c >>= 1; + c2 <<= 1; + } + if (c2 != 1) + Sizes[NumLevels++] = c2; + if (c != 1) + Sizes[NumLevels++] = c; + if (numCoreThreads != 1) + Sizes[NumLevels++] = numCoreThreads; + if (NumLevels == 0) + Sizes[NumLevels++] = 1; + + /* + printf("\n Cores:"); + for (unsigned i = 0; i < NumLevels; i++) + { + printf(" %d", Sizes[i]); + } + printf("\n"); + */ +} + + +DWORD_PTR CAffinityMode::GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const +{ + CpuSet_Zero(cpuSet); + + if (NumLevels == 0) + return 0; + + // printf("\n%2d", bundleIndex); + + /* + UInt32 low = 0; + if (DivideNum != 1) + { + low = bundleIndex % DivideNum; + bundleIndex /= DivideNum; + } + */ + + UInt32 numGroups = NumCores / NumBundleThreads; + UInt32 m = bundleIndex % numGroups; + UInt32 v = 0; + for (unsigned i = 0; i < NumLevels; i++) + { + UInt32 size = Sizes[i]; + while ((size & 1) == 0) + { + v *= 2; + v |= (m & 1); + m >>= 1; + size >>= 1; + } + v *= size; + v += m % size; + m /= size; + } + + // UInt32 nb = NumBundleThreads / DivideNum; + UInt32 nb = NumBundleThreads; + + DWORD_PTR mask = ((DWORD_PTR)1 << nb) - 1; + // v += low; + mask <<= v; + + // printf(" %2d %8x \n ", v, (unsigned)mask); + #ifdef _WIN32 + *cpuSet = mask; + #else + { + for (unsigned k = 0; k < nb; k++) + CpuSet_Set(cpuSet, v + k); + } + #endif + + return mask; +} -#ifndef _7ZIP_ST struct CBenchSyncCommon { bool ExitMode; @@ -647,9 +875,11 @@ struct CBenchSyncCommon CBenchSyncCommon(): ExitMode(false) {} }; + #endif + struct CEncoderInfo; struct CEncoderInfo @@ -659,6 +889,9 @@ struct CEncoderInfo NSynchronization::CManualResetEvent ReadyEvent; UInt32 NumDecoderSubThreads; CBenchSyncCommon *Common; + UInt32 EncoderIndex; + UInt32 NumEncoderInternalThreads; + CAffinityMode AffinityMode; #endif CMyComPtr<ICompressCoder> _encoder; @@ -673,9 +906,18 @@ struct CEncoderInfo size_t AllocaSize; #endif + unsigned KeySize; Byte _key[32]; Byte _iv[16]; + + HRESULT Set_Key_and_IV(ICryptoProperties *cp) + { + RINOK(cp->SetKey(_key, KeySize)); + return cp->SetInitVector(_iv, sizeof(_iv)); + } + Byte _psw[16]; + bool CheckCrc_Enc; bool CheckCrc_Dec; @@ -726,10 +968,15 @@ struct CEncoderInfo Common(NULL), #endif Salt(0), - fileData(NULL), + KeySize(0), CheckCrc_Enc(true), CheckCrc_Dec(true), - outStreamSpec(NULL), callback(NULL), printCallback(NULL), propStreamSpec(NULL) {} + outStreamSpec(NULL), + callback(NULL), + printCallback(NULL), + fileData(NULL), + propStreamSpec(NULL) + {} #ifndef _7ZIP_ST @@ -775,7 +1022,8 @@ struct CEncoderInfo if (!ReadyEvent.IsCreated()) res = ReadyEvent.Create(); if (res == 0) - res = thread[0].Create(EncodeThreadFunction, this); + res = AffinityMode.CreateThread_WithAffinity(thread[0], EncodeThreadFunction, this, + EncoderIndex); return HRESULT_FROM_WIN32(res); } @@ -794,7 +1042,13 @@ struct CEncoderInfo #endif decoder.CallbackMode = callbackMode; - return thread[index].Create(DecodeThreadFunction, &decoder); + + WRes res = AffinityMode.CreateThread_WithAffinity(thread[index], DecodeThreadFunction, &decoder, + // EncoderIndex * NumEncoderInternalThreads + index + EncoderIndex + ); + + return HRESULT_FROM_WIN32(res); } #endif @@ -864,8 +1118,23 @@ HRESULT CEncoderInfo::Generate() coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = kBufferSize; - RINOK(method.SetCoderProps(scp, &reduceSize)); + const UInt64 reduceSize = kBufferSize; + + /* in posix new thread uses same affinity as parent thread, + so we don't need to send affinity to coder in posix */ + UInt64 affMask; + #if !defined(_7ZIP_ST) && defined(_WIN32) + { + CCpuSet cpuSet; + affMask = AffinityMode.GetAffinityMask(EncoderIndex, &cpuSet); + } + #else + affMask = 0; + #endif + // affMask <<= 3; // debug line: to test no affinity in coder; + // affMask = 0; + + RINOK(method.SetCoderProps_DSReduce_Aff(scp, &reduceSize, (affMask != 0 ? &affMask : NULL))); } else { @@ -971,52 +1240,100 @@ HRESULT CEncoderInfo::Encode() coder.QueryInterface(IID_ICryptoProperties, &cp); CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; CMyComPtr<ISequentialInStream> inStream = inStreamSpec; - UInt64 prev = 0; - - UInt32 crcPrev = 0; if (cp) { - RINOK(cp->SetKey(_key, sizeof(_key))); - RINOK(cp->SetInitVector(_iv, sizeof(_iv))); + RINOK(Set_Key_and_IV(cp)); } - for (UInt64 i = 0; i < NumIterations; i++) + compressedSize = 0; + if (_encoderFilter) + compressedSize = kBufferSize; + + // CBenchmarkOutStream *outStreamSpec = this->outStreamSpec; + UInt64 prev = 0; + const UInt32 mask = (CheckCrc_Enc ? 0 : 0xFFF); + bool useCrc = (mask < NumIterations); + bool crcPrev_defined = false; + UInt32 crcPrev = 0; + UInt64 i = NumIterations; + // printCallback->NewLine(); + + while (i != 0) { - if (printCallback && bi.UnpackSize - prev > (1 << 20)) + i--; + if (printCallback && bi.UnpackSize - prev >= (1 << 24)) { RINOK(printCallback->CheckBreak()); prev = bi.UnpackSize; } + + /* + CBenchInfo info; + progressInfoSpec[0]->SetStartTime(); + */ - bool isLast = (i == NumIterations - 1); - bool calcCrc = ((isLast || (i & 0x7F) == 0 || CheckCrc_Enc) && NumIterations != 1); - outStreamSpec->Init(isLast, calcCrc); - + bool calcCrc = false; + if (useCrc) + calcCrc = (((UInt32)i & mask) == 0); + if (_encoderFilter) { - memcpy((Byte *)rgCopy, uncompressedDataPtr, kBufferSize); + // if (needRealData) + memcpy((Byte *)*outStreamSpec, uncompressedDataPtr, kBufferSize); _encoderFilter->Init(); - My_FilterBench(_encoderFilter, (Byte *)rgCopy, kBufferSize); - RINOK(WriteStream(outStream, (const Byte *)rgCopy, kBufferSize)); + My_FilterBench(_encoderFilter, (Byte *)*outStreamSpec, kBufferSize); + if (calcCrc) + { + outStreamSpec->InitCrc(); + outStreamSpec->Calc((Byte *)*outStreamSpec, kBufferSize); + } } else { + outStreamSpec->Init(true, calcCrc); // write real data for speed consistency at any number of iterations inStreamSpec->Init(uncompressedDataPtr, kBufferSize); RINOK(_encoder->Code(inStream, outStream, NULL, NULL, progressInfo[0])); + if (!inStreamSpec->WasFinished()) + return E_FAIL; + if (compressedSize != outStreamSpec->Pos) + { + if (compressedSize != 0) + return E_FAIL; + compressedSize = outStreamSpec->Pos; + } } // outStreamSpec->Print(); - UInt32 crcNew = CRC_GET_DIGEST(outStreamSpec->Crc); - if (i == 0) - crcPrev = crcNew; - else if (calcCrc && crcPrev != crcNew) - return E_FAIL; + if (calcCrc) + { + const UInt32 crc2 = CRC_GET_DIGEST(outStreamSpec->Crc); + if (crcPrev_defined && crcPrev != crc2) + return E_FAIL; + crcPrev = crc2; + crcPrev_defined = true; + } - compressedSize = outStreamSpec->Pos; bi.UnpackSize += kBufferSize; bi.PackSize += compressedSize; + + /* + { + progressInfoSpec[0]->SetFinishTime(info); + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = 1; + + info.UnpackSize = kBufferSize; + info.PackSize = compressedSize; + // printf("\n%7d\n", encoder.compressedSize); + + RINOK(callback->SetEncodeResult(info, true)); + printCallback->NewLine(); + } + */ + } _encoder.Release(); @@ -1067,7 +1384,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = _uncompressedDataSize; + const UInt64 reduceSize = _uncompressedDataSize; RINOK(_method.SetCoderProps(scp, &reduceSize)); } @@ -1092,40 +1409,84 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (cp) { - RINOK(cp->SetKey(_key, sizeof(_key))); - RINOK(cp->SetInitVector(_iv, sizeof(_iv))); + RINOK(Set_Key_and_IV(cp)); } - for (UInt64 i = 0; i < NumIterations; i++) + CMyComPtr<ICompressSetFinishMode> setFinishMode; + + if (_decoderFilter) { - if (printCallback && pi->BenchInfo.UnpackSize - prev > (1 << 20)) + if (compressedSize > rgCopy.Size()) + return E_FAIL; + } + else + { + decoder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); + } + + const UInt64 numIterations = this->NumIterations; + const UInt32 mask = (CheckCrc_Dec ? 0 : 0xFFF); + + for (UInt64 i = 0; i < numIterations; i++) + { + if (printCallback && pi->BenchInfo.UnpackSize - prev >= (1 << 24)) { RINOK(printCallback->CheckBreak()); prev = pi->BenchInfo.UnpackSize; } - inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); + const UInt64 outSize = kBufferSize; + bool calcCrc = false; + if (((UInt32)i & mask) == 0) + calcCrc = true; crcOutStreamSpec->Init(); - UInt64 outSize = kBufferSize; - crcOutStreamSpec->CalcCrc = ((i & 0x7F) == 0 || CheckCrc_Dec); - if (_decoderFilter) { - if (compressedSize > rgCopy.Size()) - return E_FAIL; + if (calcCrc) // for pure filter speed test without multi-iteration consistency + // if (needRealData) memcpy((Byte *)rgCopy, (const Byte *)*outStreamSpec, compressedSize); _decoderFilter->Init(); My_FilterBench(_decoderFilter, (Byte *)rgCopy, compressedSize); - RINOK(WriteStream(crcOutStream, (const Byte *)rgCopy, compressedSize)); + if (calcCrc) + crcOutStreamSpec->Calc((const Byte *)rgCopy, compressedSize); } else { + crcOutStreamSpec->CalcCrc = calcCrc; + inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); + + if (setFinishMode) + { + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))); + } + RINOK(decoder->Code(inStream, crcOutStream, 0, &outSize, progressInfo[decoderIndex])); + + if (setFinishMode) + { + if (!inStreamSpec->WasFinished()) + return S_FALSE; + + CMyComPtr<ICompressGetInStreamProcessedSize> getInStreamProcessedSize; + decoder.QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); + + if (getInStreamProcessedSize) + { + UInt64 processed; + RINOK(getInStreamProcessedSize->GetInStreamProcessedSize(&processed)); + if (processed != compressedSize) + return S_FALSE; + } + } + + if (crcOutStreamSpec->Pos != outSize) + return S_FALSE; } - if (crcOutStreamSpec->CalcCrc && CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc) + if (calcCrc && CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc) return S_FALSE; + pi->BenchInfo.UnpackSize += kBufferSize; pi->BenchInfo.PackSize += compressedSize; } @@ -1191,10 +1552,8 @@ WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) NWindows::CThread &t = EncodersSpec->encoders[i].thread[0]; if (t.IsCreated()) { - WRes res2 = t.Wait(); - if (res2 == 0) - res2 = t.Close(); - if (res == S_OK) + WRes res2 = t.Wait_Close(); + if (res == 0) res = res2; } } @@ -1202,23 +1561,29 @@ WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) return res; } -#endif +#endif // _7ZIP_ST + + + +static void SetPseudoRand(Byte *data, size_t size, UInt32 startValue) +{ + for (size_t i = 0; i < size; i++) + { + data[i] = (Byte)startValue; + startValue++; + } +} static HRESULT MethodBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - bool - #ifndef _7ZIP_ST - oldLzmaBenchMode - #endif - , - UInt32 - #ifndef _7ZIP_ST - numThreads - #endif - , + #ifndef _7ZIP_ST + bool oldLzmaBenchMode, + UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif const COneMethodInfo &method2, size_t uncompressedDataSize, const Byte *fileData, @@ -1231,7 +1596,7 @@ static HRESULT MethodBench( COneMethodInfo method = method2; UInt64 methodId; UInt32 numStreams; - int codecIndex = FindMethod_Index( + const int codecIndex = FindMethod_Index( EXTERNAL_CODECS_LOC_VARS method.MethodName, true, methodId, numStreams); @@ -1246,19 +1611,21 @@ static HRESULT MethodBench( #ifndef _7ZIP_ST numEncoderThreads = numThreads; - if (oldLzmaBenchMode && methodId == k_LZMA) + if (oldLzmaBenchMode) + if (methodId == k_LZMA) { if (numThreads == 1 && method.Get_NumThreads() < 0) method.AddProp_NumThreads(1); const UInt32 numLzmaThreads = method.Get_Lzma_NumThreads(); if (numThreads > 1 && numLzmaThreads > 1) { - numEncoderThreads = numThreads / 2; + numEncoderThreads = (numThreads + 1) / 2; // 20.03 numSubDecoderThreads = 2; } } - bool mtEncMode = (numEncoderThreads > 1); + bool mtEncMode = (numEncoderThreads > 1) || affinityMode->NeedAffinity(); + #endif CBenchEncoders encodersSpec(numEncoderThreads); @@ -1272,20 +1639,38 @@ static HRESULT MethodBench( encoder.callback = (i == 0) ? callback : 0; encoder.printCallback = printCallback; + #ifndef _7ZIP_ST + encoder.EncoderIndex = i; + encoder.NumEncoderInternalThreads = numSubDecoderThreads; + encoder.AffinityMode = *affinityMode; + + /* + if (numSubDecoderThreads > 1) + if (encoder.AffinityMode.NeedAffinity() + && encoder.AffinityMode.NumBundleThreads == 1) + { + // if old LZMA benchmark uses two threads in coder, we increase (NumBundleThreads) for old LZMA benchmark uses two threads instead of one + if (encoder.AffinityMode.NumBundleThreads * 2 <= encoder.AffinityMode.NumCores) + encoder.AffinityMode.NumBundleThreads *= 2; + } + */ + + #endif + { CCreatedCoder cod; - RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS codecIndex, true, encoder._encoderFilter, cod)); + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS (unsigned)codecIndex, true, encoder._encoderFilter, cod)); encoder._encoder = cod.Coder; if (!encoder._encoder && !encoder._encoderFilter) return E_NOTIMPL; } - encoder.CheckCrc_Enc = (benchProps->EncComplex) > 30 ; - encoder.CheckCrc_Dec = (benchProps->DecComplexCompr + benchProps->DecComplexUnc) > 30 ; + encoder.CheckCrc_Enc = (benchProps->EncComplex) > 30; + encoder.CheckCrc_Dec = (benchProps->DecComplexCompr + benchProps->DecComplexUnc) > 30; - memset(encoder._iv, 0, sizeof(encoder._iv)); - memset(encoder._key, 0, sizeof(encoder._key)); - memset(encoder._psw, 0, sizeof(encoder._psw)); + SetPseudoRand(encoder._iv, sizeof(encoder._iv), 17); + SetPseudoRand(encoder._key, sizeof(encoder._key), 51); + SetPseudoRand(encoder._psw, sizeof(encoder._psw), 123); for (UInt32 j = 0; j < numSubDecoderThreads; j++) { @@ -1334,11 +1719,14 @@ static HRESULT MethodBench( { CEncoderInfo &encoder = encoders[i]; encoder.NumIterations = GetNumIterations(benchProps->GeComprCommands(uncompressedDataSize), complexInCommands); + // encoder.NumIterations = 3; encoder.Salt = g_CrcTable[i & 0xFF]; encoder.Salt ^= (g_CrcTable[(i >> 8) & 0xFF] << 3); // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread // printf(" %8x", encoder.Salt); + encoder.KeySize = benchProps->KeySize; + for (int j = 0; j < 2; j++) { CBenchProgressInfo *spec = new CBenchProgressInfo; @@ -1424,12 +1812,24 @@ static HRESULT MethodBench( status.Res = S_OK; status.EncodeMode = false; - UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads; + const UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads; + #ifndef _7ZIP_ST + const bool mtDecoderMode = (numDecoderThreads > 1) || affinityMode->NeedAffinity(); + #endif for (i = 0; i < numEncoderThreads; i++) { CEncoderInfo &encoder = encoders[i]; + /* + #ifndef _7ZIP_ST + // encoder.affinityMode = *affinityMode; + if (encoder.NumEncoderInternalThreads != 1) + encoder.AffinityMode.DivideNum = encoder.NumEncoderInternalThreads; + #endif + */ + + if (i == 0) { encoder.NumIterations = GetNumIterations(benchProps->GeDecomprCommands(encoder.compressedSize, encoder.kBufferSize), complexInCommands); @@ -1444,9 +1844,9 @@ static HRESULT MethodBench( #ifndef _7ZIP_ST { int numSubThreads = method.Get_NumThreads(); - encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : numSubThreads; + encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : (unsigned)numSubThreads; } - if (numDecoderThreads > 1) + if (mtDecoderMode) { for (UInt32 j = 0; j < numSubDecoderThreads; j++) { @@ -1466,25 +1866,36 @@ static HRESULT MethodBench( } #ifndef _7ZIP_ST - HRESULT res = S_OK; - if (numDecoderThreads > 1) + if (mtDecoderMode) + { + WRes wres = 0; + HRESULT res = S_OK; for (i = 0; i < numEncoderThreads; i++) for (UInt32 j = 0; j < numSubDecoderThreads; j++) { CEncoderInfo &encoder = encoders[i]; - encoder.thread[j].Wait(); - if (encoder.Results[j] != S_OK) - res = encoder.Results[j]; + WRes wres2 = encoder.thread[j]. + // Wait(); // later we can get thread times from thread in UNDER_CE + Wait_Close(); + if (wres == 0 && wres2 != 0) + wres = wres2; + HRESULT res2 = encoder.Results[j]; + if (res == 0 && res2 != 0) + res = res2; } - RINOK(res); - #endif - + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(res); + } + #endif // _7ZIP_ST + RINOK(status.Res); encoders[0].progressInfoSpec[0]->SetFinishTime(info); - + + /* #ifndef _7ZIP_ST #ifdef UNDER_CE - if (numDecoderThreads > 1) + if (mtDecoderMode) for (i = 0; i < numEncoderThreads; i++) for (UInt32 j = 0; j < numSubDecoderThreads; j++) { @@ -1494,6 +1905,7 @@ static HRESULT MethodBench( } #endif #endif + */ info.UnpackSize = 0; info.PackSize = 0; @@ -1541,37 +1953,112 @@ UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary, bool totalBench GetLZMAUsage(lzmaMt, dictionary) + (2 << 20)) * numBigThreads; } -static HRESULT CrcBig(const void *data, UInt32 size, UInt64 numIterations, + + +// ---------- CRC and HASH ---------- + +struct CCrcInfo_Base +{ + CAlignedBuffer Buffer; + const Byte *Data; + size_t Size; + bool CreateLocalBuf; + UInt32 CheckSum_Res; + + CCrcInfo_Base(): CreateLocalBuf(true), CheckSum_Res(0) {} + + HRESULT Generate(const Byte *data, size_t size); + HRESULT CrcProcess(UInt64 numIterations, + const UInt32 *checkSum, IHasher *hf, + IBenchPrintCallback *callback); +}; + + +HRESULT CCrcInfo_Base::Generate(const Byte *data, size_t size) +{ + Size = size; + Data = data; + if (!data || CreateLocalBuf) + { + ALLOC_WITH_HRESULT(&Buffer, size) + Data = Buffer; + } + if (!data) + RandGen(Buffer, size); + else if (CreateLocalBuf && size != 0) + memcpy(Buffer, data, size); + return S_OK; +} + + +HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, const UInt32 *checkSum, IHasher *hf, IBenchPrintCallback *callback) { + MY_ALIGN(16) Byte hash[64]; - UInt64 i; - for (i = 0; i < sizeof(hash); i++) - hash[i] = 0; - for (i = 0; i < numIterations; i++) + memset(hash, 0, sizeof(hash)); + + CheckSum_Res = 0; + + const UInt32 hashSize = hf->GetDigestSize(); + if (hashSize > sizeof(hash)) + return S_FALSE; + + const Byte *buf = Data; + const size_t size = Size; + UInt32 checkSum_Prev = 0; + + UInt64 prev = 0; + UInt64 cur = 0; + + for (UInt64 i = 0; i < numIterations; i++) { - if (callback && (i & 0xFF) == 0) + hf->Init(); + size_t pos = 0; + do { - RINOK(callback->CheckBreak()); + const size_t rem = size - pos; + const UInt32 kStep = ((UInt32)1 << 31); + const UInt32 curSize = (rem < kStep) ? (UInt32)rem : kStep; + hf->Update(buf + pos, curSize); + pos += curSize; } - hf->Init(); - hf->Update(data, size); + while (pos != size); + hf->Final(hash); - UInt32 hashSize = hf->GetDigestSize(); - if (hashSize > sizeof(hash)) - return S_FALSE; UInt32 sum = 0; for (UInt32 j = 0; j < hashSize; j += 4) - sum ^= GetUi32(hash + j); - if (checkSum && sum != *checkSum) { - return S_FALSE; + sum = rotlFixed(sum, 11); + sum += GetUi32(hash + j); + } + if (checkSum) + { + if (sum != *checkSum) + return S_FALSE; + } + else + { + checkSum_Prev = sum; + checkSum = &checkSum_Prev; + } + if (callback) + { + cur += size; + if (cur - prev >= ((UInt32)1 << 30)) + { + prev = cur; + RINOK(callback->CheckBreak()); + } } } + CheckSum_Res = checkSum_Prev; return S_OK; } +extern +UInt32 g_BenchCpuFreqTemp; // we need non-static variavble to disable compiler optimization UInt32 g_BenchCpuFreqTemp = 1; #define YY1 sum += val; sum ^= val; @@ -1596,22 +2083,27 @@ EXTERN_C_END #ifndef _7ZIP_ST -struct CFreqInfo +struct CBaseThreadInfo { NWindows::CThread Thread; IBenchPrintCallback *Callback; HRESULT CallbackRes; - UInt32 ValRes; - UInt32 Size; - UInt64 NumIterations; - void Wait() + WRes Wait_If_Created() { - Thread.Wait(); - Thread.Close(); + if (!Thread.IsCreated()) + return 0; + return Thread.Wait_Close(); } }; +struct CFreqInfo: public CBaseThreadInfo +{ + UInt32 ValRes; + UInt32 Size; + UInt64 NumIterations; +}; + static THREAD_FUNC_DECL FreqThreadFunction(void *param) { CFreqInfo *p = (CFreqInfo *)param; @@ -1634,12 +2126,20 @@ struct CFreqThreads UInt32 NumThreads; CFreqThreads(): Items(NULL), NumThreads(0) {} - void WaitAll() + + WRes WaitAll() { + WRes wres = 0; for (UInt32 i = 0; i < NumThreads; i++) - Items[i].Wait(); + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } NumThreads = 0; + return wres; } + ~CFreqThreads() { WaitAll(); @@ -1647,30 +2147,95 @@ struct CFreqThreads } }; -struct CCrcInfo -{ - NWindows::CThread Thread; - IBenchPrintCallback *Callback; - HRESULT CallbackRes; +static THREAD_FUNC_DECL CrcThreadFunction(void *param); + +struct CCrcInfo: public CBaseThreadInfo +{ const Byte *Data; - UInt32 Size; + size_t Size; UInt64 NumIterations; bool CheckSumDefined; UInt32 CheckSum; CMyComPtr<IHasher> Hasher; HRESULT Res; + UInt32 CheckSum_Res; + + #ifndef _7ZIP_ST + NSynchronization::CManualResetEvent ReadyEvent; + UInt32 ThreadIndex; + CBenchSyncCommon *Common; + CAffinityMode AffinityMode; + #endif + + // we want to call CCrcInfo_Base::Buffer.Free() in main thread. + // so we uses non-local CCrcInfo_Base. + CCrcInfo_Base crcib; + + HRESULT CreateThread() + { + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = AffinityMode.CreateThread_WithAffinity(Thread, CrcThreadFunction, this, + ThreadIndex); + return HRESULT_FROM_WIN32(res); + } #ifdef USE_ALLOCA size_t AllocaSize; #endif - void Wait() + void Process(); + + CCrcInfo(): Res(E_FAIL) {} +}; + +static const bool k_Crc_CreateLocalBuf_For_File = true; // for total BW test +// static const bool k_Crc_CreateLocalBuf_For_File = false; // for shared memory read test + +void CCrcInfo::Process() +{ + crcib.CreateLocalBuf = k_Crc_CreateLocalBuf_For_File; + // we can use additional Generate() passes to reduce some time effects for new page allocation + // for (unsigned y = 0; y < 10; y++) + Res = crcib.Generate(Data, Size); + + // if (Common) { - Thread.Wait(); - Thread.Close(); + WRes wres = ReadyEvent.Set(); + if (wres != 0) + { + if (Res == 0) + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Res != 0) + return; + + wres = Common->StartEvent.Lock(); + + if (wres != 0) + { + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Common->ExitMode) + return; } -}; + + Res = crcib.CrcProcess(NumIterations, + CheckSumDefined ? &CheckSum : NULL, Hasher, + Callback); + CheckSum_Res = crcib.CheckSum_Res; + /* + We don't want to include the time of slow CCrcInfo_Base::Buffer.Free() + to time of benchmark. So we don't free Buffer here + */ + // crcib.Buffer.Free(); +} + static THREAD_FUNC_DECL CrcThreadFunction(void *param) { @@ -1679,34 +2244,52 @@ static THREAD_FUNC_DECL CrcThreadFunction(void *param) #ifdef USE_ALLOCA alloca(p->AllocaSize); #endif - - p->Res = CrcBig(p->Data, p->Size, p->NumIterations, - p->CheckSumDefined ? &p->CheckSum : NULL, p->Hasher, - p->Callback); + p->Process(); return 0; } + struct CCrcThreads { CCrcInfo *Items; - UInt32 NumThreads; + unsigned NumThreads; + CBenchSyncCommon Common; + bool NeedClose; + + CCrcThreads(): Items(NULL), NumThreads(0), NeedClose(false) {} + + WRes StartAndWait(bool exitMode = false); - CCrcThreads(): Items(NULL), NumThreads(0) {} - void WaitAll() - { - for (UInt32 i = 0; i < NumThreads; i++) - Items[i].Wait(); - NumThreads = 0; - } ~CCrcThreads() { - WaitAll(); + StartAndWait(true); delete []Items; } }; + +WRes CCrcThreads::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes wres = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } + NumThreads = 0; + NeedClose = false; + return wres; +} + #endif + static UInt32 CrcCalc1(const Byte *buf, size_t size) { UInt32 crc = CRC_INIT_VAL;; @@ -1715,19 +2298,15 @@ static UInt32 CrcCalc1(const Byte *buf, size_t size) return CRC_GET_DIGEST(crc); } -static void RandGen(Byte *buf, size_t size, CBaseRandomGenerator &RG) -{ - for (size_t i = 0; i < size; i++) - buf[i] = (Byte)RG.GetRnd(); -} - +/* static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) { RandGen(buf, size, RG); return CrcCalc1(buf, size); } +*/ -bool CrcInternalTest() +static bool CrcInternalTest() { CAlignedBuffer buffer; const size_t kBufferSize0 = (1 << 8); @@ -1743,8 +2322,7 @@ bool CrcInternalTest() UInt32 crc1 = CrcCalc1(buf, kBufferSize0); if (crc1 != 0x29058C73) return false; - CBaseRandomGenerator RG; - RandGen(buf + kBufferSize0, kBufferSize1, RG); + RandGen(buf + kBufferSize0, kBufferSize1); for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++) for (unsigned j = 0; j < kCheckSize; j++) if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j)) @@ -1760,11 +2338,25 @@ struct CBenchMethod UInt32 DecComplexCompr; UInt32 DecComplexUnc; const char *Name; + // unsigned KeySize; }; +// #define USE_SW_CMPLX + +#ifdef USE_SW_CMPLX +#define CMPLX(x) ((x) * 1000) +#else +#define CMPLX(x) (x) +#endif + static const CBenchMethod g_Bench[] = { - { 40, 17, 357, 145, 20, "LZMA:x1" }, + // { 40, 17, 357, 145, 20, "LZMA:x1" }, + // { 20, 18, 360, 145, 20, "LZMA2:x1:mt2" }, + + { 20, 18, 360, 145, 20, "LZMA:x1" }, + { 20, 22, 600, 145, 20, "LZMA:x3" }, + { 80, 24, 1220, 145, 20, "LZMA:x5:mt1" }, { 80, 24, 1220, 145, 20, "LZMA:x5:mt2" }, @@ -1778,14 +2370,42 @@ static const CBenchMethod g_Bench[] = { 10, 19, 815, 122, 122, "BZip2:x5:mt2" }, { 10, 19, 2530, 122, 122, "BZip2:x7" }, + // { 10, 18, 1010, 0, 1150, "PPMDZip:x1" }, { 10, 18, 1010, 0, 1150, "PPMD:x1" }, + // { 10, 22, 1655, 0, 1830, "PPMDZip:x5" }, { 10, 22, 1655, 0, 1830, "PPMD:x5" }, - { 2, 0, 6, 0, 6, "Delta:4" }, + // { 2, 0, 3, 0, 4, "Delta:1" }, + // { 2, 0, 3, 0, 4, "Delta:2" }, + // { 2, 0, 3, 0, 4, "Delta:3" }, + { 2, 0, 3, 0, 4, "Delta:4" }, + // { 2, 0, 3, 0, 4, "Delta:8" }, + // { 2, 0, 3, 0, 4, "Delta:32" }, + { 2, 0, 4, 0, 4, "BCJ" }, + // { 10, 0, 18, 0, 18, "AES128CBC:1" }, + // { 10, 0, 21, 0, 21, "AES192CBC:1" }, { 10, 0, 24, 0, 24, "AES256CBC:1" }, - { 2, 0, 8, 0, 2, "AES256CBC:2" } + + // { 10, 0, 18, 0, 18, "AES128CTR:1" }, + // { 10, 0, 21, 0, 21, "AES192CTR:1" }, + // { 10, 0, 24, 0, 24, "AES256CTR:1" }, + // { 2, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:2" }, + // { 2, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:2" }, + { 2, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:2" }, + + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:2" }, + + // { 1, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:3" }, + // { 1, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:3" }, + { 1, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:3" } + + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:3" }, + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:3" }, + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:3" }, }; struct CBenchHash @@ -1796,15 +2416,25 @@ struct CBenchHash const char *Name; }; +// #define ARM_CRC_MUL 100 +#define ARM_CRC_MUL 1 + static const CBenchHash g_Hash[] = { - { 1, 1820, 0x8F8FEDAB, "CRC32:1" }, - { 10, 558, 0x8F8FEDAB, "CRC32:4" }, - { 10, 339, 0x8F8FEDAB, "CRC32:8" }, - { 10, 512, 0xDF1C17CC, "CRC64" }, - { 10, 5100, 0x2D79FF2E, "SHA256" }, - { 10, 2340, 0x4C25132B, "SHA1" }, - { 2, 5500, 0xE084E913, "BLAKE2sp" } + { 1, 1820, 0x21e207bb, "CRC32:1" }, + { 10, 558, 0x21e207bb, "CRC32:4" }, + { 10, 339, 0x21e207bb, "CRC32:8" } , + { 2, 128 *ARM_CRC_MUL, 0x21e207bb, "CRC32:32" }, + { 2, 64 *ARM_CRC_MUL, 0x21e207bb, "CRC32:64" }, + { 10, 512, 0x41b901d1, "CRC64" }, + + { 10, 5100, 0x7913ba03, "SHA256:1" }, + { 2, CMPLX((32 * 4 + 1) * 4 + 4), 0x7913ba03, "SHA256:2" }, + + { 10, 2340, 0xff769021, "SHA1:1" }, + { 2, CMPLX((20 * 6 + 1) * 4 + 4), 0xff769021, "SHA1:2" }, + + { 2, 5500, 0x85189d02, "BLAKE2sp" } }; struct CTotalBenchRes @@ -1842,9 +2472,11 @@ static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) unsigned len = (unsigned)strlen(s + startPos); if (size > len) { - startPos -= (size - len); - if (startPos < 0) + size -= len; + if (startPos < size) startPos = 0; + else + startPos -= size; } f.Print(s + startPos); } @@ -1870,7 +2502,10 @@ static void PrintRating(IBenchPrintCallback &f, UInt64 rating, unsigned size) static void PrintPercents(IBenchPrintCallback &f, UInt64 val, UInt64 divider, unsigned size) { - PrintNumber(f, (val * 100 + divider / 2) / divider, size); + UInt64 v = 0; + if (divider != 0) + v = (val * 100 + divider / 2) / divider; + PrintNumber(f, v, size); } static void PrintChars(IBenchPrintCallback &f, char c, unsigned size) @@ -1886,9 +2521,14 @@ static void PrintSpaces(IBenchPrintCallback &f, unsigned size) PrintChars(f, ' ', size); } +static void PrintUsage(IBenchPrintCallback &f, UInt64 usage, unsigned size) +{ + PrintNumber(f, (usage + 5000) / 10000, size); +} + static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt64 rating, bool showFreq, UInt64 cpuFreq) { - PrintNumber(f, (usage + 5000) / 10000, kFieldSize_Usage); + PrintUsage(f, usage, kFieldSize_Usage); PrintRating(f, rpu, kFieldSize_RU); PrintRating(f, rating, kFieldSize_Rating); if (showFreq) @@ -1898,8 +2538,10 @@ static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt6 else { UInt64 ddd = cpuFreq * usage / 100; + /* if (ddd == 0) ddd = 1; + */ PrintPercents(f, (rating * 10000), ddd, kFieldSize_EU); PrintPercents(f, rating, cpuFreq, kFieldSize_Effec); } @@ -1958,58 +2600,57 @@ AString GetProcessThreadsInfo(const NSystem::CProcessAffinity &ti) { AString s; // s.Add_UInt32(ti.numProcessThreads); - if (ti.processAffinityMask != ti.systemAffinityMask) + unsigned numSysThreads = ti.GetNumSystemThreads(); + if (ti.GetNumProcessThreads() != numSysThreads) { // if (ti.numProcessThreads != ti.numSysThreads) { s += " / "; - s.Add_UInt32(ti.GetNumSystemThreads()); + s.Add_UInt32(numSysThreads); } s += " : "; + #ifdef _WIN32 PrintHex(s, ti.processAffinityMask); s += " / "; PrintHex(s, ti.systemAffinityMask); + #else + unsigned i = (numSysThreads + 3) & ~(unsigned)3; + if (i == 0) + i = 4; + for (; i >= 4; ) + { + i -= 4; + unsigned val = 0; + for (unsigned k = 0; k < 4; k++) + { + const unsigned bit = (ti.IsCpuSet(i + k) ? 1 : 0); + val += (bit << k); + } + PrintHex(s, val); + } + #endif } return s; } -static void PrintSize(AString &s, UInt64 v) -{ - char c = 0; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; - }}}} - else - { - PrintHex(s, v); - return; - } - char temp[32]; - ConvertUInt64ToString(v, temp); - s += temp; - if (c) - s += c; -} - - #ifdef _7ZIP_LARGE_PAGES +#ifdef _WIN32 extern bool g_LargePagesMode; - extern "C" { extern SIZE_T g_LargePageSize; } +#endif void Add_LargePages_String(AString &s) { + #ifdef _WIN32 if (g_LargePagesMode || g_LargePageSize != 0) { s += " (LP-"; - PrintSize(s, g_LargePageSize); + PrintSize_KMGT_Or_Hex(s, g_LargePageSize); #ifdef MY_CPU_X86_OR_AMD64 if (CPU_IsSupported_PageGB()) s += "-1G"; @@ -2018,6 +2659,9 @@ void Add_LargePages_String(AString &s) s += "-NA"; s += ")"; } + #else + s += ""; + #endif } #endif @@ -2140,26 +2784,29 @@ void CBenchCallbackToPrint::NewLine() _file->NewLine(); } -void PrintLeft(IBenchPrintCallback &f, const char *s, unsigned size) +static void PrintLeft(IBenchPrintCallback &f, const char *s, unsigned size) { f.Print(s); - int numSpaces = size - MyStringLen(s); + int numSpaces = (int)size - (int)MyStringLen(s); if (numSpaces > 0) - PrintSpaces(f, numSpaces); + PrintSpaces(f, (unsigned)numSpaces); } -void PrintRight(IBenchPrintCallback &f, const char *s, unsigned size) +static void PrintRight(IBenchPrintCallback &f, const char *s, unsigned size) { - int numSpaces = size - MyStringLen(s); + int numSpaces = (int)size - (int)MyStringLen(s); if (numSpaces > 0) - PrintSpaces(f, numSpaces); + PrintSpaces(f, (unsigned)numSpaces); f.Print(s); } static HRESULT TotalBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, + #ifndef _7ZIP_ST UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif bool forceUnpackSize, size_t unpackSize, const Byte *fileData, @@ -2169,6 +2816,12 @@ static HRESULT TotalBench( { const CBenchMethod &bench = g_Bench[i]; PrintLeft(*callback->_file, bench.Name, kFieldSize_Name); + { + unsigned keySize = 32; + if (IsString1PrefixedByString2(bench.Name, "AES128")) keySize = 16; + else if (IsString1PrefixedByString2(bench.Name, "AES192")) keySize = 24; + callback->BenchProps.KeySize = keySize; + } callback->BenchProps.DecComplexUnc = bench.DecComplexUnc; callback->BenchProps.DecComplexCompr = bench.DecComplexCompr; callback->BenchProps.EncComplex = bench.EncComplex; @@ -2188,7 +2841,10 @@ static HRESULT TotalBench( HRESULT res = MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, - false, numThreads, method, + #ifndef _7ZIP_ST + false, numThreads, affinityMode, + #endif + method, unpackSize2, fileData, bench.DictBits, printCallback, callback, &callback->BenchProps); @@ -2211,20 +2867,38 @@ static HRESULT TotalBench( } -static HRESULT FreqBench( - UInt64 complexInCommands, - UInt32 numThreads, - IBenchPrintCallback *_file, - bool showFreq, - UInt64 specifiedFreq, - UInt64 &cpuFreq, - UInt32 &res) +struct CFreqBench +{ + // in: + UInt64 complexInCommands; + UInt32 numThreads; + bool showFreq; + UInt64 specifiedFreq; + + // out: + UInt64 cpuFreq; + UInt32 res; + + CFreqBench() + {} + + HRESULT FreqBench(IBenchPrintCallback *_file + #ifndef _7ZIP_ST + , const CAffinityMode *affinityMode + #endif + ); +}; + + +HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file + #ifndef _7ZIP_ST + , const CAffinityMode *affinityMode + #endif + ) { res = 0; cpuFreq = 0; - UInt32 bufferSize = 1 << 20; - UInt32 complexity = kNumFreqCommands; if (numThreads == 0) numThreads = 1; @@ -2232,17 +2906,26 @@ static HRESULT FreqBench( numThreads = 1; #endif - UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); - UInt64 numIterations = complexInCommands / complexity / bsize; - if (numIterations == 0) + const UInt32 complexity = kNumFreqCommands; + UInt64 numIterations = complexInCommands / complexity; + UInt32 numIterations2 = 1 << 30; + if (numIterations > numIterations2) + numIterations /= numIterations2; + else + { + numIterations2 = (UInt32)numIterations; numIterations = 1; + } CBenchInfoCalc progressInfoSpec; #ifndef _7ZIP_ST - CFreqThreads threads; - if (numThreads > 1) + + bool mtMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtMode) { + CFreqThreads threads; threads.Items = new CFreqInfo[numThreads]; UInt32 i; for (i = 0; i < numThreads; i++) @@ -2251,16 +2934,22 @@ static HRESULT FreqBench( info.Callback = _file; info.CallbackRes = S_OK; info.NumIterations = numIterations; - info.Size = bufferSize; + info.Size = numIterations2; } progressInfoSpec.SetStartTime(); for (i = 0; i < numThreads; i++) { + // Sleep(10); CFreqInfo &info = threads.Items[i]; - RINOK(info.Thread.Create(FreqThreadFunction, &info)); - threads.NumThreads++; + WRes wres = affinityMode->CreateThread_WithAffinity(info.Thread, FreqThreadFunction, &info, i); + if (info.Thread.IsCreated()) + threads.NumThreads++; + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } - threads.WaitAll(); + WRes wres = threads.WaitAll(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); for (i = 0; i < numThreads; i++) { RINOK(threads.Items[i].CallbackRes); @@ -2273,11 +2962,20 @@ static HRESULT FreqBench( UInt32 sum = g_BenchCpuFreqTemp; for (UInt64 k = numIterations; k > 0; k--) { - RINOK(_file->CheckBreak()); - sum = CountCpuFreq(sum, bufferSize, g_BenchCpuFreqTemp); + sum = CountCpuFreq(sum, numIterations2, g_BenchCpuFreqTemp); + if (_file) + { + RINOK(_file->CheckBreak()); + } } res += sum; } + + if (res == 0x12345678) + if (_file) + { + RINOK(_file->CheckBreak()); + } CBenchInfo info; progressInfoSpec.SetFinishTime(info); @@ -2289,7 +2987,7 @@ static HRESULT FreqBench( if (_file) { { - UInt64 numCommands = (UInt64)numIterations * bufferSize * numThreads * complexity; + UInt64 numCommands = (UInt64)numIterations * numIterations2 * numThreads * complexity; UInt64 rating = info.GetSpeed(numCommands); cpuFreq = rating / numThreads; PrintResults(_file, info, @@ -2308,12 +3006,21 @@ static HRESULT FreqBench( static HRESULT CrcBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - UInt32 numThreads, UInt32 bufferSize, + UInt32 numThreads, + const size_t bufferSize, + const Byte *fileData, + UInt64 &speed, + UInt64 &usage, + UInt32 complexity, unsigned benchWeight, const UInt32 *checkSum, const COneMethodInfo &method, IBenchPrintCallback *_file, + #ifndef _7ZIP_ST + const CAffinityMode *affinityMode, + #endif + bool showRating, CTotalBenchRes *encodeRes, bool showFreq, UInt64 cpuFreq) { @@ -2332,80 +3039,112 @@ static HRESULT CrcBench( methodName, hashID)) return E_NOTIMPL; + /* + // if will generate random data in each thread, instead of global data CAlignedBuffer buffer; - size_t totalSize = (size_t)bufferSize * numThreads; - if (totalSize / numThreads != bufferSize) - return E_OUTOFMEMORY; - ALLOC_WITH_HRESULT(&buffer, totalSize) + if (!fileData) + { + ALLOC_WITH_HRESULT(&buffer, bufferSize) + RandGen(buffer, bufferSize); + fileData = buffer; + } + */ - Byte *buf = (Byte *)buffer; - CBaseRandomGenerator RG; - UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); + const size_t bsize = (bufferSize == 0 ? 1 : bufferSize); UInt64 numIterations = complexInCommands * 256 / complexity / bsize; if (numIterations == 0) numIterations = 1; CBenchInfoCalc progressInfoSpec; + CBenchInfo info; #ifndef _7ZIP_ST - CCrcThreads threads; - if (numThreads > 1) + bool mtEncMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtEncMode) { + CCrcThreads threads; threads.Items = new CCrcInfo[numThreads]; - + { + WRes wres = threads.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + threads.NeedClose = true; + } + UInt32 i; for (i = 0; i < numThreads; i++) { - CCrcInfo &info = threads.Items[i]; + CCrcInfo &ci = threads.Items[i]; AString name; - RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, info.Hasher)); - if (!info.Hasher) + RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, ci.Hasher)); + if (!ci.Hasher) return E_NOTIMPL; CMyComPtr<ICompressSetCoderProperties> scp; - info.Hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); + ci.Hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = 1; - RINOK(method.SetCoderProps(scp, &reduceSize)); + RINOK(method.SetCoderProps(scp)); } - Byte *data = buf + (size_t)bufferSize * i; - info.Callback = _file; - info.Data = data; - info.NumIterations = numIterations; - info.Size = bufferSize; - /* info.Crc = */ RandGenCrc(data, bufferSize, RG); - info.CheckSumDefined = false; + ci.Callback = _file; + ci.Data = fileData; + ci.NumIterations = numIterations; + ci.Size = bufferSize; + ci.CheckSumDefined = false; if (checkSum) { - info.CheckSum = *checkSum; - info.CheckSumDefined = (checkSum && (i == 0)); + ci.CheckSum = *checkSum; + ci.CheckSumDefined = true; } #ifdef USE_ALLOCA - info.AllocaSize = (i * 16 * 21) & 0x7FF; + ci.AllocaSize = (i * 16 * 21) & 0x7FF; #endif } - progressInfoSpec.SetStartTime(); + for (i = 0; i < numThreads; i++) + { + CCrcInfo &ci = threads.Items[i]; + ci.ThreadIndex = i; + ci.Common = &threads.Common; + ci.AffinityMode = *affinityMode; + HRESULT hres = ci.CreateThread(); + if (ci.Thread.IsCreated()) + threads.NumThreads++; + if (hres != 0) + return hres; + } for (i = 0; i < numThreads; i++) { - CCrcInfo &info = threads.Items[i]; - RINOK(info.Thread.Create(CrcThreadFunction, &info)); - threads.NumThreads++; + CCrcInfo &ci = threads.Items[i]; + WRes wres = ci.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(ci.Res); } - threads.WaitAll(); + + progressInfoSpec.SetStartTime(); + + WRes wres = threads.StartAndWait(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + progressInfoSpec.SetFinishTime(info); + for (i = 0; i < numThreads; i++) { RINOK(threads.Items[i].Res); + if (i != 0) + if (threads.Items[i].CheckSum_Res != + threads.Items[i - 1].CheckSum_Res) + return S_FALSE; } } else #endif { - /* UInt32 crc = */ RandGenCrc(buf, bufferSize, RG); - progressInfoSpec.SetStartTime(); CMyComPtr<IHasher> hasher; AString name; RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, hasher)); @@ -2415,14 +3154,16 @@ static HRESULT CrcBench( hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = 1; - RINOK(method.SetCoderProps(scp, &reduceSize)); + RINOK(method.SetCoderProps(scp)); } - RINOK(CrcBig(buf, bufferSize, numIterations, checkSum, hasher, _file)); + CCrcInfo_Base crcib; + crcib.CreateLocalBuf = false; + RINOK(crcib.Generate(fileData, bufferSize)); + progressInfoSpec.SetStartTime(); + RINOK(crcib.CrcProcess(numIterations, checkSum, hasher, _file)); + progressInfoSpec.SetFinishTime(info); } - CBenchInfo info; - progressInfoSpec.SetFinishTime(info); UInt64 unpSize = numIterations * bufferSize; UInt64 unpSizeThreads = unpSize * numThreads; @@ -2432,9 +3173,13 @@ static HRESULT CrcBench( if (_file) { + if (showRating) { - UInt64 numCommands = unpSizeThreads * complexity / 256; - UInt64 rating = info.GetSpeed(numCommands); + UInt64 unpSizeThreads2 = unpSizeThreads; + if (unpSizeThreads2 == 0) + unpSizeThreads2 = numIterations * 1 * numThreads; + const UInt64 numCommands = unpSizeThreads2 * complexity / 256; + const UInt64 rating = info.GetSpeed(numCommands); PrintResults(_file, info, benchWeight, rating, showFreq, cpuFreq, encodeRes); @@ -2443,15 +3188,23 @@ static HRESULT CrcBench( } speed = info.GetSpeed(unpSizeThreads); + usage = info.GetUsage(); return S_OK; } + + static HRESULT TotalBench_Hash( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - UInt32 numThreads, UInt32 bufSize, + UInt32 numThreads, + size_t bufSize, + const Byte *fileData, IBenchPrintCallback *printCallback, CBenchCallbackToPrint *callback, + #ifndef _7ZIP_ST + const CAffinityMode *affinityMode, + #endif CTotalBenchRes *encodeRes, bool showFreq, UInt64 cpuFreq) { @@ -2468,15 +3221,22 @@ static HRESULT TotalBench_Hash( propVariant = bench.Name; RINOK(method.ParseMethodFromPROPVARIANT(UString(), propVariant)); - UInt64 speed; + UInt64 speed, usage; + HRESULT res = CrcBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, - numThreads, bufSize, - speed, + numThreads, bufSize, fileData, + speed, usage, bench.Complex, bench.Weight, - &bench.CheckSum, method, - printCallback, encodeRes, showFreq, cpuFreq); + (!fileData && bufSize == (1 << kNumHashDictBits)) ? &bench.CheckSum : NULL, + method, + printCallback, + #ifndef _7ZIP_ST + affinityMode, + #endif + true, // showRating + encodeRes, showFreq, cpuFreq); if (res == E_NOTIMPL) { // callback->Print(" ---"); @@ -2509,14 +3269,6 @@ static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) prop = result; } -static UInt32 GetNumThreadsNext(unsigned i, UInt32 numThreads) -{ - if (i < 2) - return i + 1; - i -= 1; - UInt32 num = (UInt32)(2 + (i & 1)) << (i >> 1); - return (num <= numThreads) ? num : numThreads; -} static bool AreSameMethodNames(const char *fullName, const char *shortName) { @@ -2524,357 +3276,13 @@ static bool AreSameMethodNames(const char *fullName, const char *shortName) } -#ifdef MY_CPU_X86_OR_AMD64 -static void PrintCpuChars(AString &s, UInt32 v) -{ - for (int j = 0; j < 4; j++) - { - Byte b = (Byte)(v & 0xFF); - v >>= 8; - if (b == 0) - break; - s += (char)b; - } -} -static void x86cpuid_to_String(const Cx86cpuid &c, AString &s) +static void Print_Usage_and_Threads(IBenchPrintCallback &f, UInt64 usage, UInt32 threads) { - s.Empty(); - - UInt32 maxFunc2 = 0; - UInt32 t[3]; - - MyCPUID(0x80000000, &maxFunc2, &t[0], &t[1], &t[2]); - - bool fullNameIsAvail = (maxFunc2 >= 0x80000004); - - if (!fullNameIsAvail) - { - for (int i = 0; i < 3; i++) - PrintCpuChars(s, c.vendor[i]); - } - else - { - for (int i = 0; i < 3; i++) - { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000002 + i, &d[0], &d[1], &d[2], &d[3]); - for (int j = 0; j < 4; j++) - PrintCpuChars(s, d[j]); - } - } - - s.Add_Space_if_NotEmpty(); - { - char temp[32]; - ConvertUInt32ToHex(c.ver, temp); - s += '('; - s += temp; - s += ')'; - } + PrintRequirements(f, "usage:", true, usage, "Benchmark threads: ", threads); } -#endif - - - -static const char * const k_PROCESSOR_ARCHITECTURE[] = -{ - "x86" // "INTEL" - , "MIPS" - , "ALPHA" - , "PPC" - , "SHX" - , "ARM" - , "IA64" - , "ALPHA64" - , "MSIL" - , "x64" // "AMD64" - , "IA32_ON_WIN64" - , "NEUTRAL" - , "ARM64" - , "ARM32_ON_WIN64" -}; - -#define MY__PROCESSOR_ARCHITECTURE_INTEL 0 -#define MY__PROCESSOR_ARCHITECTURE_AMD64 9 - - -#define MY__PROCESSOR_INTEL_PENTIUM 586 -#define MY__PROCESSOR_AMD_X8664 8664 - -/* -static const CUInt32PCharPair k_PROCESSOR[] = -{ - { 2200, "IA64" }, - { 8664, "x64" } -}; - -#define PROCESSOR_INTEL_386 386 -#define PROCESSOR_INTEL_486 486 -#define PROCESSOR_INTEL_PENTIUM 586 -#define PROCESSOR_INTEL_860 860 -#define PROCESSOR_INTEL_IA64 2200 -#define PROCESSOR_AMD_X8664 8664 -#define PROCESSOR_MIPS_R2000 2000 -#define PROCESSOR_MIPS_R3000 3000 -#define PROCESSOR_MIPS_R4000 4000 -#define PROCESSOR_ALPHA_21064 21064 -#define PROCESSOR_PPC_601 601 -#define PROCESSOR_PPC_603 603 -#define PROCESSOR_PPC_604 604 -#define PROCESSOR_PPC_620 620 -#define PROCESSOR_HITACHI_SH3 10003 -#define PROCESSOR_HITACHI_SH3E 10004 -#define PROCESSOR_HITACHI_SH4 10005 -#define PROCESSOR_MOTOROLA_821 821 -#define PROCESSOR_SHx_SH3 103 -#define PROCESSOR_SHx_SH4 104 -#define PROCESSOR_STRONGARM 2577 // 0xA11 -#define PROCESSOR_ARM720 1824 // 0x720 -#define PROCESSOR_ARM820 2080 // 0x820 -#define PROCESSOR_ARM920 2336 // 0x920 -#define PROCESSOR_ARM_7TDMI 70001 -#define PROCESSOR_OPTIL 18767 // 0x494f -*/ - -#ifdef _WIN32 - -static const char * const k_PF[] = -{ - "FP_ERRATA" - , "FP_EMU" - , "CMPXCHG" - , "MMX" - , "PPC_MOVEMEM_64BIT" - , "ALPHA_BYTE" - , "SSE" - , "3DNOW" - , "RDTSC" - , "PAE" - , "SSE2" - , "SSE_DAZ" - , "NX" - , "SSE3" - , "CMPXCHG16B" - , "CMP8XCHG16" - , "CHANNELS" - , "XSAVE" - , "ARM_VFP_32" - , "ARM_NEON" - , "L2AT" - , "VIRT_FIRMWARE" - , "RDWRFSGSBASE" - , "FASTFAIL" - , "ARM_DIVIDE" - , "ARM_64BIT_LOADSTORE_ATOMIC" - , "ARM_EXTERNAL_CACHE" - , "ARM_FMAC" - , "RDRAND" - , "ARM_V8" - , "ARM_V8_CRYPTO" - , "ARM_V8_CRC32" - , "RDTSCP" -}; - -#endif - - - - -static void PrintPage(AString &s, UInt32 v) -{ - if ((v & 0x3FF) == 0) - { - s.Add_UInt32(v >> 10); - s += "K"; - } - else - s.Add_UInt32(v >> 10); -} - -static AString TypeToString2(const char * const table[], unsigned num, UInt32 value) -{ - char sz[16]; - const char *p = NULL; - if (value < num) - p = table[value]; - if (!p) - { - ConvertUInt32ToString(value, sz); - p = sz; - } - return (AString)p; -} - -#ifdef _WIN32 - -static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) -{ - s += TypeToString2(k_PROCESSOR_ARCHITECTURE, ARRAY_SIZE(k_PROCESSOR_ARCHITECTURE), si.wProcessorArchitecture); - - if (!( si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_INTEL && si.dwProcessorType == MY__PROCESSOR_INTEL_PENTIUM - || si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_AMD64 && si.dwProcessorType == MY__PROCESSOR_AMD_X8664)) - { - s += " "; - // s += TypePairToString(k_PROCESSOR, ARRAY_SIZE(k_PROCESSOR), si.dwProcessorType); - s.Add_UInt32(si.dwProcessorType); - } - s += " "; - PrintHex(s, si.wProcessorLevel); - s += "."; - PrintHex(s, si.wProcessorRevision); - if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) - if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) - { - s += " act:"; - PrintHex(s, si.dwActiveProcessorMask); - } - s += " cpus:"; - s.Add_UInt32(si.dwNumberOfProcessors); - if (si.dwPageSize != 1 << 12) - { - s += " page:"; - PrintPage(s, si.dwPageSize); - } - if (si.dwAllocationGranularity != 1 << 16) - { - s += " gran:"; - PrintPage(s, si.dwAllocationGranularity); - } - s += " "; - - DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress; - UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1; - const UInt32 kReserveSize = ((UInt32)1 << 16); - if (minAdd != kReserveSize) - { - PrintSize(s, minAdd); - s += "-"; - } - else - { - if ((maxSize & (kReserveSize - 1)) == 0) - maxSize += kReserveSize; - } - PrintSize(s, maxSize); -} - -#ifndef _WIN64 -typedef VOID (WINAPI *Func_GetNativeSystemInfo)(LPSYSTEM_INFO lpSystemInfo); -#endif - -#endif - -void GetSysInfo(AString &s1, AString &s2) -{ - s1.Empty(); - s2.Empty(); - - #ifdef _WIN32 - SYSTEM_INFO si; - GetSystemInfo(&si); - { - SysInfo_To_String(s1, si); - // s += " : "; - } - - #if !defined(_WIN64) && !defined(UNDER_CE) - Func_GetNativeSystemInfo fn_GetNativeSystemInfo = (Func_GetNativeSystemInfo)GetProcAddress( - GetModuleHandleA("kernel32.dll"), "GetNativeSystemInfo"); - if (fn_GetNativeSystemInfo) - { - SYSTEM_INFO si2; - fn_GetNativeSystemInfo(&si2); - // if (memcmp(&si, &si2, sizeof(si)) != 0) - { - // s += " - "; - SysInfo_To_String(s2, si2); - } - } - #endif - #endif -} - - -void GetCpuName(AString &s) -{ - s.Empty(); - - #ifdef MY_CPU_X86_OR_AMD64 - { - Cx86cpuid cpuid; - if (x86cpuid_CheckAndRead(&cpuid)) - { - AString s2; - x86cpuid_to_String(cpuid, s2); - s += s2; - } - else - { - #ifdef MY_CPU_AMD64 - s += "x64"; - #else - s += "x86"; - #endif - } - } - #else - - #ifdef MY_CPU_LE - s += "LE"; - #elif defined(MY_CPU_BE) - s += "BE"; - #endif - - #endif - - #ifdef _7ZIP_LARGE_PAGES - Add_LargePages_String(s); - #endif -} - - -void GetCpuFeatures(AString &s) -{ - s.Empty(); - - #ifdef _WIN32 - const unsigned kNumFeatures_Extra = 32; // we check also for unknown features - const unsigned kNumFeatures = ARRAY_SIZE(k_PF) + kNumFeatures_Extra; - for (unsigned i = 0; i < kNumFeatures; i++) - { - if (IsProcessorFeaturePresent(i)) - { - s.Add_Space_if_NotEmpty(); - s += TypeToString2(k_PF, ARRAY_SIZE(k_PF), i); - } - } - #endif -} - - -#ifdef _WIN32 -#ifndef UNDER_CE - -typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); - -static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) -{ - HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); - if (!ntdll) - return FALSE; - Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); - if (!func) - return FALSE; - func(vi); - return TRUE; -} - -#endif -#endif - HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS @@ -2896,22 +3304,41 @@ HRESULT Bench( #ifndef _7ZIP_ST - if (threadsInfo.Get() && threadsInfo.processAffinityMask != 0) + if (threadsInfo.Get() && threadsInfo.GetNumProcessThreads() != 0) numCPUs = threadsInfo.GetNumProcessThreads(); else numCPUs = NSystem::GetNumberOfProcessors(); #endif + + // numCPUs = 24; + /* + { + DWORD_PTR mask = (1 << 0); + DWORD_PTR old = SetThreadAffinityMask(GetCurrentThread(), mask); + old = old; + DWORD_PTR old2 = SetThreadAffinityMask(GetCurrentThread(), mask); + old2 = old2; + return 0; + } + */ bool ramSize_Defined = NSystem::GetRamSize(ramSize); UInt32 numThreadsSpecified = numCPUs; - - UInt32 testTime = kComplexInSeconds; - + bool needSetComplexity = false; + UInt32 testTimeMs = kComplexInMs; + UInt32 startDicLog = 22; + bool startDicLog_Defined = false; UInt64 specifiedFreq = 0; - bool multiThreadTests = false; + UInt64 complexInCommands = kComplexInCommands; + UInt32 numThreads_Start = 1; + + #ifndef _7ZIP_ST + CAffinityMode affinityMode; + #endif + COneMethodInfo method; @@ -2919,6 +3346,25 @@ HRESULT Bench( { unsigned i; + + if (printCallback) + { + for (i = 0; i < props.Size(); i++) + { + const CProperty &property = props[i]; + printCallback->Print(" "); + printCallback->Print(GetAnsiString(property.Name)); + if (!property.Value.IsEmpty()) + { + printCallback->Print("="); + printCallback->Print(GetAnsiString(property.Value)); + } + } + if (!props.IsEmpty()) + printCallback->NewLine(); + } + + for (i = 0; i < props.Size(); i++) { const CProperty &property = props[i]; @@ -2930,34 +3376,35 @@ HRESULT Bench( if (property.Value.IsEmpty()) return E_INVALIDARG; - #ifdef USE_WIN_FILE - NFile::NIO::CInFile file; if (!file.Open(us2fs(property.Value))) - return E_INVALIDARG; - UInt64 len; - if (!file.GetLength(len)) - return E_FAIL; - if (len >= ((UInt32)1 << 31) || len == 0) - return E_INVALIDARG; - ALLOC_WITH_HRESULT(&fileDataBuffer, (size_t)len); - UInt32 processedSize; - file.Read((Byte *)fileDataBuffer, (UInt32)len, processedSize); - if (processedSize != len) - return E_FAIL; - if (printCallback) + return GetLastError_noZero_HRESULT(); + size_t len; { - printCallback->Print("file size ="); - PrintNumber(*printCallback, len, 0); - printCallback->NewLine(); + UInt64 len64; + if (!file.GetLength(len64)) + return GetLastError_noZero_HRESULT(); + if (printCallback) + { + printCallback->Print("file size ="); + PrintNumber(*printCallback, len64, 0); + printCallback->NewLine(); + } + len = (size_t)len64; + if (len != len64) + return E_INVALIDARG; } - continue; - #else - - return E_NOTIMPL; + ALLOC_WITH_HRESULT(&fileDataBuffer, len); - #endif + { + size_t processed; + if (!file.ReadFull((Byte *)fileDataBuffer, len, processed)) + return GetLastError_noZero_HRESULT(); + if (processed != len) + return E_FAIL; + } + continue; } NCOM::CPropVariant propVariant; @@ -2966,10 +3413,56 @@ HRESULT Bench( if (name.IsEqualTo("time")) { - RINOK(ParsePropToUInt32(UString(), propVariant, testTime)); + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)); + needSetComplexity = true; + testTimeMs *= 1000; continue; } - + + if (name.IsEqualTo("timems")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)); + needSetComplexity = true; + continue; + } + + if (name.IsEqualTo("tic")) + { + UInt32 v; + RINOK(ParsePropToUInt32(UString(), propVariant, v)); + if (v >= 64) + return E_INVALIDARG; + complexInCommands = (UInt64)1 << v; + continue; + } + + if (name.IsEqualTo("ds")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, startDicLog)); + startDicLog_Defined = true; + continue; + } + + if (name.IsEqualTo("mts")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, numThreads_Start)); + continue; + } + + if (name.IsEqualTo("af")) + { + UInt32 bundle; + RINOK(ParsePropToUInt32(UString(), propVariant, bundle)); + if (bundle > 0 && bundle < numCPUs) + { + #ifndef _7ZIP_ST + affinityMode.SetLevels(numCPUs, 2); + affinityMode.NumBundleThreads = bundle; + #endif + } + continue; + } + if (name.IsEqualTo("freq")) { UInt32 freq32 = 0; @@ -2992,7 +3485,9 @@ HRESULT Bench( { UString s = name.Ptr(2); if (s.IsEqualTo("*") - || s.IsEmpty() && propVariant.vt == VT_BSTR && StringsAreEqual_Ascii(propVariant.bstrVal, "*")) + || (s.IsEmpty() + && propVariant.vt == VT_BSTR + && StringsAreEqual_Ascii(propVariant.bstrVal, "*"))) { multiThreadTests = true; continue; @@ -3009,62 +3504,10 @@ HRESULT Bench( if (printCallback) { - #ifdef _WIN32 - #ifndef UNDER_CE - { - AString s; - // OSVERSIONINFO vi; - OSVERSIONINFOEXW vi; - vi.dwOSVersionInfoSize = sizeof(vi); - // if (::GetVersionEx(&vi)) - if (My_RtlGetVersion(&vi)) - { - s += "Windows"; - if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) - s.Add_UInt32(vi.dwPlatformId); - s += " "; s.Add_UInt32(vi.dwMajorVersion); - s += "."; s.Add_UInt32(vi.dwMinorVersion); - s += " "; s.Add_UInt32(vi.dwBuildNumber); - // s += " "; s += GetAnsiString(vi.szCSDVersion); - } - printCallback->Print(s); - printCallback->NewLine(); - } - #endif - #endif - - { - AString s1, s2; - GetSysInfo(s1, s2); - if (!s1.IsEmpty() || !s2.IsEmpty()) - { - printCallback->Print(s1); - if (s1 != s2 && !s2.IsEmpty()) - { - printCallback->Print(" - "); - printCallback->Print(s2); - } - printCallback->NewLine(); - } - } - { - AString s; - GetCpuFeatures(s); - if (!s.IsEmpty()) - { - printCallback->Print(s); - printCallback->NewLine(); - } - } - { - AString s; - GetCpuName(s); - if (!s.IsEmpty()) - { - printCallback->Print(s); - printCallback->NewLine(); - } - } + AString s; + GetSystemInfoText(s); + printCallback->Print(s); + printCallback->NewLine(); } if (printCallback) @@ -3072,8 +3515,6 @@ HRESULT Bench( printCallback->Print("CPU Freq:"); } - UInt64 complexInCommands = kComplexInCommands; - if (printCallback /* || freqCallback */) { UInt64 numMilCommands = 1 << 6; @@ -3098,7 +3539,9 @@ HRESULT Bench( start = realDelta; if (start == 0) start = 1; - UInt64 freq = GetFreq(); + if (start > (UInt64)1 << 61) + start = 1; + const UInt64 freq = GetFreq(); // mips is constant in some compilers const UInt64 mipsVal = numMilCommands * freq / start; if (printCallback) @@ -3118,12 +3561,25 @@ HRESULT Bench( freqCallback->AddCpuFreq(mipsVal); */ - if (jj >= 3) + if (jj >= 1) { - SetComplexCommands(testTime, false, mipsVal * 1000000, complexInCommands); - if (jj >= 8 || start >= freq) + bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (start >= freq * 16) + { + printCallback->Print(" (Cmplx)"); + needSetComplexity = true; + needStop = true; + } + if (needSetComplexity) + SetComplexCommandsMs(testTimeMs, false, mipsVal * 1000000, complexInCommands); + if (needStop) break; - // break; // change it numMilCommands <<= 1; } } @@ -3156,7 +3612,10 @@ HRESULT Bench( return MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, + #ifndef _7ZIP_ST true, numThreadsSpecified, + &affinityMode, + #endif method, uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, benchCallback, &benchProps); @@ -3173,9 +3632,17 @@ HRESULT Bench( if (!printCallback) return S_FALSE; IBenchPrintCallback &f = *printCallback; - if (!dictIsDefined) - dict = (1 << 24); + UInt64 dict64 = dict; + if (!dictIsDefined) + dict64 = (1 << 27); + if (fileDataBuffer.IsAllocated()) + { + if (!dictIsDefined) + dict64 = fileDataBuffer.Size(); + else if (dict64 > fileDataBuffer.Size()) + dict64 = fileDataBuffer.Size(); + } // methhodName.RemoveChar(L'-'); UInt32 complexity = 10000; @@ -3190,87 +3657,172 @@ HRESULT Bench( int propPos = benchMethod.Find(':'); if (propPos >= 0) { - benchProps = benchMethod.Ptr(propPos + 1); - benchMethod.DeleteFrom(propPos); + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); } if (AreSameMethodNames(benchMethod, methodName)) { - if (benchProps.IsEmpty() - || benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps == "8" && method.PropsString.IsEmpty() - || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) + bool isMainMathed = method.PropsString.IsEmpty(); + if (isMainMathed) + isMainMathed = !checkSum + || (benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps.IsEqualTo_Ascii_NoCase("8")); + const bool sameProps = method.PropsString.IsEqualTo_Ascii_NoCase(benchProps); + if (sameProps || isMainMathed) { complexity = h.Complex; checkSum = &h.CheckSum; - if (method.PropsString.IsEqualTo_Ascii_NoCase(benchProps)) + if (sameProps) break; } } } - if (i == ARRAY_SIZE(g_Hash)) + if (!checkSum) return E_NOTIMPL; } + { + UInt64 usage = 1 << 20; + UInt64 bufSize = dict64; + if (fileDataBuffer.IsAllocated()) + { + usage += fileDataBuffer.Size(); + if (bufSize > fileDataBuffer.Size()) + bufSize = fileDataBuffer.Size(); + if (numThreadsSpecified != 1) + usage += bufSize * numThreadsSpecified * (k_Crc_CreateLocalBuf_For_File ? 1 : 0); + } + else + usage += numThreadsSpecified * bufSize; + Print_Usage_and_Threads(f, usage, numThreadsSpecified); + } + f.NewLine(); - f.Print("Size"); - const unsigned kFieldSize_CrcSpeed = 6; - unsigned numThreadsTests = 0; - for (;;) - { - UInt32 t = GetNumThreadsNext(numThreadsTests, numThreadsSpecified); - PrintNumber(f, t, kFieldSize_CrcSpeed); - numThreadsTests++; - if (t >= numThreadsSpecified) - break; + + const unsigned kFieldSize_CrcSpeed = 7; + CUIntVector numThreadsVector; + { + unsigned nt = numThreads_Start; + for (;;) + { + if (nt > numThreadsSpecified) + break; + numThreadsVector.Add(nt); + unsigned next = nt * 2; + UInt32 ntHalf= numThreadsSpecified / 2; + if (ntHalf > nt && ntHalf < next) + numThreadsVector.Add(ntHalf); + if (numThreadsSpecified > nt && numThreadsSpecified < next) + numThreadsVector.Add(numThreadsSpecified); + nt = next; + } + { + f.NewLine(); + f.Print("THRD"); + FOR_VECTOR (ti, numThreadsVector) + { + PrintNumber(f, numThreadsVector[ti], 1 + kFieldSize_Usage + kFieldSize_CrcSpeed); + } + } + { + f.NewLine(); + f.Print(" "); + FOR_VECTOR (ti, numThreadsVector) + { + PrintRight(f, "Usage", kFieldSize_Usage + 1); + PrintRight(f, "BW", kFieldSize_CrcSpeed + 1); + } + } + { + f.NewLine(); + f.Print("Size"); + FOR_VECTOR (ti, numThreadsVector) + { + PrintRight(f, "%", kFieldSize_Usage + 1); + PrintRight(f, "MB/s", kFieldSize_CrcSpeed + 1); + } + } } + f.NewLine(); f.NewLine(); - CTempValues speedTotals(numThreadsTests); + + CTempValues speedTotals(numThreadsVector.Size()); + CTempValues usageTotals(numThreadsVector.Size()); { - for (unsigned ti = 0; ti < numThreadsTests; ti++) + FOR_VECTOR (ti, numThreadsVector) + { speedTotals.Values[ti] = 0; + usageTotals.Values[ti] = 0; + } } UInt64 numSteps = 0; + for (UInt32 i = 0; i < numIterations; i++) { - for (unsigned pow = 10; pow < 32; pow++) + unsigned pow = 10; // kNumHashDictBits + if (startDicLog_Defined) + pow = startDicLog; + for (;; pow++) { - UInt32 bufSize = (UInt32)1 << pow; - if (bufSize > dict) - break; + const UInt64 bufSize = (UInt64)1 << pow; char s[16]; ConvertUInt32ToString(pow, s); unsigned pos = MyStringLen(s); s[pos++] = ':'; s[pos++] = ' '; s[pos] = 0; - f.Print(s); + PrintRight(f, s, 4); + + size_t dataSize = fileDataBuffer.Size(); + if (dataSize > bufSize || !fileDataBuffer.IsAllocated()) + dataSize = (size_t)bufSize; - for (unsigned ti = 0; ti < numThreadsTests; ti++) + FOR_VECTOR (ti, numThreadsVector) { RINOK(f.CheckBreak()); - UInt32 t = GetNumThreadsNext(ti, numThreadsSpecified); + const UInt32 t = numThreadsVector[ti]; UInt64 speed = 0; - RINOK(CrcBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, - t, bufSize, speed, + UInt64 usage = 0; + + HRESULT res = CrcBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, + t, + dataSize, (const Byte *)fileDataBuffer, + speed, usage, complexity, 1, // benchWeight, - (pow == kNumHashDictBits) ? checkSum : NULL, method, NULL, NULL, false, 0)); - PrintNumber(f, (speed >> 20), kFieldSize_CrcSpeed); + (pow == kNumHashDictBits && !fileDataBuffer.IsAllocated()) ? checkSum : NULL, + method, + &f, + #ifndef _7ZIP_ST + &affinityMode, + #endif + false, // showRating + NULL, false, 0); + + RINOK(res); + + PrintUsage(f, usage, kFieldSize_Usage); + PrintNumber(f, speed / 1000000, kFieldSize_CrcSpeed); speedTotals.Values[ti] += speed; + usageTotals.Values[ti] += usage; } + f.NewLine(); numSteps++; + if (dataSize >= dict64) + break; } } if (numSteps != 0) { f.NewLine(); f.Print("Avg:"); - for (unsigned ti = 0; ti < numThreadsTests; ti++) + for (unsigned ti = 0; ti < numThreadsVector.Size(); ti++) { - PrintNumber(f, ((speedTotals.Values[ti] / numSteps) >> 20), kFieldSize_CrcSpeed); + PrintUsage(f, usageTotals.Values[ti] / numSteps, kFieldSize_Usage); + PrintNumber(f, speedTotals.Values[ti] / numSteps / 1000000, kFieldSize_CrcSpeed); } f.NewLine(); } @@ -3352,7 +3904,7 @@ HRESULT Bench( } } - PrintRequirements(f, "usage:", true, GetBenchMemoryUsage(numThreads, dict, totalBenchMode), "Benchmark threads: ", numThreads); + Print_Usage_and_Threads(f, GetBenchMemoryUsage(numThreads, dict, totalBenchMode), numThreads); f.NewLine(); f.NewLine(); @@ -3443,60 +3995,114 @@ HRESULT Bench( { if (i != 0) printCallback->NewLine(); - HRESULT res; const unsigned kNumCpuTests = 3; for (unsigned freqTest = 0; freqTest < kNumCpuTests; freqTest++) { PrintLeft(f, "CPU", kFieldSize_Name); - UInt32 resVal; - RINOK(FreqBench(complexInCommands, numThreads, printCallback, - (freqTest == kNumCpuTests - 1 || specifiedFreq != 0), // showFreq - specifiedFreq, - cpuFreq, resVal)); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + HRESULT res = fb.FreqBench(printCallback + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); + + cpuFreq = fb.cpuFreq; callback.NewLine(); if (specifiedFreq != 0) cpuFreq = specifiedFreq; + if (testTimeMs >= 1000) if (freqTest == kNumCpuTests - 1) - SetComplexCommands(testTime, specifiedFreq != 0, cpuFreq, complexInCommands); + { + // SetComplexCommandsMs(testTimeMs, specifiedFreq != 0, cpuFreq, complexInCommands); + } } callback.NewLine(); + // return S_OK; // change it + callback.SetFreq(true, cpuFreq); if (!onlyHashBench) { - res = TotalBench(EXTERNAL_CODECS_LOC_VARS - complexInCommands, numThreads, + size_t dataSize = dict; + if (fileDataBuffer.IsAllocated()) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = dict; + } + + HRESULT res = TotalBench(EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef _7ZIP_ST + numThreads, + &affinityMode, + #endif dictIsDefined || fileDataBuffer.IsAllocated(), // forceUnpackSize - fileDataBuffer.IsAllocated() ? fileDataBuffer.Size() : dict, + dataSize, (const Byte *)fileDataBuffer, printCallback, &callback); RINOK(res); } - res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, - 1 << kNumHashDictBits, printCallback, &callback, &callback.EncodeRes, true, cpuFreq); - RINOK(res); + { + size_t dataSize = (size_t)1 << kNumHashDictBits; + if (dictIsDefined) + dataSize = dict; + if (fileDataBuffer.IsAllocated()) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = dict; + } + + HRESULT res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, + dataSize, (const Byte *)fileDataBuffer, + printCallback, &callback, + #ifndef _7ZIP_ST + &affinityMode, + #endif + &callback.EncodeRes, true, cpuFreq); + RINOK(res); + } callback.NewLine(); { PrintLeft(f, "CPU", kFieldSize_Name); - UInt32 resVal; - UInt64 cpuFreqLastTemp = cpuFreq; - RINOK(FreqBench(complexInCommands, numThreads, printCallback, - specifiedFreq != 0, // showFreq - specifiedFreq, - cpuFreqLastTemp, resVal)); + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + HRESULT res = fb.FreqBench(printCallback + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); callback.NewLine(); } } } else { - bool needSetComplexity = true; + needSetComplexity = true; if (!methodName.IsEqualTo_Ascii_NoCase("LZMA")) { unsigned i; @@ -3508,14 +4114,14 @@ HRESULT Bench( int propPos = benchMethod.Find(':'); if (propPos >= 0) { - benchProps = benchMethod.Ptr(propPos + 1); - benchMethod.DeleteFrom(propPos); + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); } if (AreSameMethodNames(benchMethod, methodName)) { if (benchProps.IsEmpty() - || benchProps == "x5" && method.PropsString.IsEmpty() + || (benchProps == "x5" && method.PropsString.IsEmpty()) || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) { callback.BenchProps.EncComplex = h.EncComplex; @@ -3532,10 +4138,12 @@ HRESULT Bench( if (needSetComplexity) callback.BenchProps.SetLzmaCompexity(); + if (startDicLog < kBenchMinDicLogSize) + startDicLog = kBenchMinDicLogSize; + for (unsigned i = 0; i < numIterations; i++) { - const unsigned kStartDicLog = 22; - unsigned pow = (dict < ((UInt32)1 << kStartDicLog)) ? kBenchMinDicLogSize : kStartDicLog; + unsigned pow = (dict < ((UInt32)1 << startDicLog)) ? kBenchMinDicLogSize : (unsigned)startDicLog; if (!multiDict) pow = 31; while (((UInt32)1 << pow) > dict && pow > 0) @@ -3576,7 +4184,10 @@ HRESULT Bench( HRESULT res = MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, + #ifndef _7ZIP_ST true, numThreads, + &affinityMode, + #endif method2, uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, &callback, &callback.BenchProps); diff --git a/CPP/7zip/UI/Common/Bench.h b/CPP/7zip/UI/Common/Bench.h index 18a40a84..02f443e3 100644 --- a/CPP/7zip/UI/Common/Bench.h +++ b/CPP/7zip/UI/Common/Bench.h @@ -66,7 +66,7 @@ AString GetProcessThreadsInfo(const NWindows::NSystem::CProcessAffinity &ti); void GetSysInfo(AString &s1, AString &s2); void GetCpuName(AString &s); -void GetCpuFeatures(AString &s); +void AddCpuFeatures(AString &s); #ifdef _7ZIP_LARGE_PAGES void Add_LargePages_String(AString &s); diff --git a/CPP/7zip/UI/Common/CompressCall.cpp b/CPP/7zip/UI/Common/CompressCall.cpp index 42cb0d23..72a61685 100644 --- a/CPP/7zip/UI/Common/CompressCall.cpp +++ b/CPP/7zip/UI/Common/CompressCall.cpp @@ -76,11 +76,12 @@ static HRESULT Call7zGui(const UString ¶ms, imageName += k7zGui; CProcess process; - WRes res = process.Create(imageName, params, NULL); // curDir); - if (res != 0) + const WRes wres = process.Create(imageName, params, NULL); // curDir); + if (wres != 0) { - ErrorMessageHRESULT(res, imageName); - return res; + HRESULT hres = HRESULT_FROM_WIN32(wres); + ErrorMessageHRESULT(hres, imageName); + return hres; } if (waitFinish) process.Wait(); @@ -130,12 +131,11 @@ static HRESULT CreateMap(const UStringVector &names, for (;;) { random.GenerateName(mappingName, "7zMap"); - - WRes res = fileMapping.Create(PAGE_READWRITE, totalSize, GetSystemString(mappingName)); - if (fileMapping.IsCreated() && res == 0) + const WRes wres = fileMapping.Create(PAGE_READWRITE, totalSize, GetSystemString(mappingName)); + if (fileMapping.IsCreated() && wres == 0) break; - if (res != ERROR_ALREADY_EXISTS) - return res; + if (wres != ERROR_ALREADY_EXISTS) + return HRESULT_FROM_WIN32(wres); fileMapping.Close(); } @@ -143,11 +143,11 @@ static HRESULT CreateMap(const UStringVector &names, for (;;) { random.GenerateName(eventName, "7zEvent"); - WRes res = event.CreateWithName(false, GetSystemString(eventName)); - if (event.IsCreated() && res == 0) + const WRes wres = event.CreateWithName(false, GetSystemString(eventName)); + if (event.IsCreated() && wres == 0) break; - if (res != ERROR_ALREADY_EXISTS) - return res; + if (wres != ERROR_ALREADY_EXISTS) + return HRESULT_FROM_WIN32(wres); event.Close(); } diff --git a/CPP/7zip/UI/Common/CompressCall2.cpp b/CPP/7zip/UI/Common/CompressCall2.cpp index 5bad774b..cbb5dcc9 100644 --- a/CPP/7zip/UI/Common/CompressCall2.cpp +++ b/CPP/7zip/UI/Common/CompressCall2.cpp @@ -6,6 +6,8 @@ #include "../../UI/Common/EnumDirItems.h" +#include "../../UI/FileManager/LangUtils.h" + #include "../../UI/GUI/BenchmarkDialog.h" #include "../../UI/GUI/ExtractGUI.h" #include "../../UI/GUI/UpdateGUI.h" diff --git a/CPP/7zip/UI/Common/DefaultName.cpp b/CPP/7zip/UI/Common/DefaultName.cpp index 02f611c3..8c34ffc7 100644 --- a/CPP/7zip/UI/Common/DefaultName.cpp +++ b/CPP/7zip/UI/Common/DefaultName.cpp @@ -20,7 +20,7 @@ static UString GetDefaultName3(const UString &fileName, int dotPos = fileName.ReverseFind_Dot(); if (dotPos > 0) - return fileName.Left(dotPos) + addSubExtension; + return fileName.Left((unsigned)dotPos) + addSubExtension; if (addSubExtension.IsEmpty()) return fileName + L'~'; diff --git a/CPP/7zip/UI/Common/DirItem.h b/CPP/7zip/UI/Common/DirItem.h index 83937103..337cd1a7 100644 --- a/CPP/7zip/UI/Common/DirItem.h +++ b/CPP/7zip/UI/Common/DirItem.h @@ -3,6 +3,10 @@ #ifndef __DIR_ITEM_H #define __DIR_ITEM_H +#ifdef _WIN32 +#include "../../../Common/MyLinux.h" +#endif + #include "../../../Common/MyString.h" #include "../../../Windows/FileFind.h" @@ -84,13 +88,18 @@ struct CDirItem FILETIME MTime; UString Name; - #if defined(_WIN32) && !defined(UNDER_CE) - // UString ShortName; + #ifndef UNDER_CE CByteBuffer ReparseData; - CByteBuffer ReparseData2; // fixed (reduced) absolute links + #ifdef _WIN32 + // UString ShortName; + CByteBuffer ReparseData2; // fixed (reduced) absolute links for WIM format bool AreReparseData() const { return ReparseData.Size() != 0 || ReparseData2.Size() != 0; } - #endif + #else + bool AreReparseData() const { return ReparseData.Size() != 0; } + #endif // _WIN32 + + #endif // !UNDER_CE UInt32 Attrib; int PhyParent; @@ -100,9 +109,23 @@ struct CDirItem bool IsAltStream; CDirItem(): PhyParent(-1), LogParent(-1), SecureIndex(-1), IsAltStream(false) {} - bool IsDir() const { return (Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0 ; } + + bool IsDir() const { return (Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0; } + bool IsReadOnly() const { return (Attrib & FILE_ATTRIBUTE_READONLY) != 0; } + bool Has_Attrib_ReparsePoint() const { return (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0; } + + #ifdef _WIN32 + UInt32 GetPosixAttrib() const + { + UInt32 v = IsDir() ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG; + v |= (IsReadOnly() ? 0555 : 0777); + return v; + } + #endif }; + + class CDirItems { UStringVector Prefixes; @@ -117,17 +140,15 @@ public: CObjectVector<CDirItem> Items; bool SymLinks; - bool ScanAltStreams; CDirItemsStat Stat; - #ifndef UNDER_CE + #if !defined(UNDER_CE) HRESULT SetLinkInfo(CDirItem &dirItem, const NWindows::NFile::NFind::CFileInfo &fi, const FString &phyPrefix); #endif - #if defined(_WIN32) && !defined(UNDER_CE) CUniqBlocks SecureBlocks; @@ -136,6 +157,7 @@ public: bool ReadSecure; HRESULT AddSecurityItem(const FString &path, int &secureIndex); + HRESULT FillFixedReparse(); #endif @@ -157,6 +179,9 @@ public: unsigned AddPrefix(int phyParent, int logParent, const UString &prefix); void DeleteLastPrefix(); + + // HRESULT EnumerateOneDir(const FString &phyPrefix, CObjectVector<NWindows::NFile::NFind::CDirEntry> &files); + HRESULT EnumerateOneDir(const FString &phyPrefix, CObjectVector<NWindows::NFile::NFind::CFileInfo> &files); HRESULT EnumerateItems2( const FString &phyPrefix, @@ -164,13 +189,10 @@ public: const FStringVector &filePaths, FStringVector *requestedPaths); - #if defined(_WIN32) && !defined(UNDER_CE) - void FillFixedReparse(); - #endif - void ReserveDown(); }; + struct CArcItem { UInt64 Size; diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index 088f0777..7a2dd008 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include <wchar.h> +// #include <stdio.h> #include "../../../Common/Wildcard.h" @@ -55,8 +56,13 @@ void CDirItems::AddDirFileInfo(int phyParent, int logParent, int secureIndex, } } +// (DWORD)E_FAIL +#define DI_DEFAULT_ERROR ERROR_INVALID_FUNCTION + HRESULT CDirItems::AddError(const FString &path, DWORD errorCode) { + if (errorCode == 0) + errorCode = DI_DEFAULT_ERROR; Stat.NumErrors++; if (Callback) return Callback->ScanError(path, errorCode); @@ -83,17 +89,17 @@ UString CDirItems::GetPrefixesPath(const CIntVector &parents, int index, const U unsigned len = name.Len(); int i; - for (i = index; i >= 0; i = parents[i]) - len += Prefixes[i].Len(); + for (i = index; i >= 0; i = parents[(unsigned)i]) + len += Prefixes[(unsigned)i].Len(); wchar_t *p = path.GetBuf_SetEnd(len) + len; p -= name.Len(); wmemcpy(p, (const wchar_t *)name, name.Len()); - for (i = index; i >= 0; i = parents[i]) + for (i = index; i >= 0; i = parents[(unsigned)i]) { - const UString &s = Prefixes[i]; + const UString &s = Prefixes[(unsigned)i]; p -= s.Len(); wmemcpy(p, (const wchar_t *)s, s.Len()); } @@ -150,6 +156,7 @@ CDirItems::CDirItems(): #endif } + #ifdef _USE_SECURITY_CODE HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) @@ -166,7 +173,7 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) DWORD errorCode = 0; DWORD secureSize; - BOOL res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + BOOL res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); if (res) { @@ -185,7 +192,7 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) else { TempSecureBuf.Alloc(secureSize); - res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); if (res) { if (secureSize != TempSecureBuf.Size()) @@ -199,33 +206,117 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) if (res) { - secureIndex = SecureBlocks.AddUniq(TempSecureBuf, secureSize); + secureIndex = (int)SecureBlocks.AddUniq(TempSecureBuf, secureSize); return S_OK; } - if (errorCode == 0) - errorCode = ERROR_INVALID_FUNCTION; return AddError(path, errorCode); } -#endif +#endif // _USE_SECURITY_CODE -HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phyPrefix) -{ - RINOK(ScanProgress(phyPrefix)); +HRESULT CDirItems::EnumerateOneDir(const FString &phyPrefix, CObjectVector<NFind::CFileInfo> &files) +{ NFind::CEnumerator enumerator; + // printf("\n enumerator.SetDirPrefix(phyPrefix) \n"); + enumerator.SetDirPrefix(phyPrefix); + + #ifdef _WIN32 + + NFind::CFileInfo fi; + for (unsigned ttt = 0; ; ttt++) { - NFind::CFileInfo fi; bool found; if (!enumerator.Next(fi, found)) + return AddError(phyPrefix); + if (!found) + return S_OK; + files.Add(fi); + if (Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)); + } + } + + #else // _WIN32 + + // enumerator.SolveLinks = !SymLinks; + + CObjectVector<NFind::CDirEntry> entries; + + for (unsigned ttt = 0; ; ttt++) + { + bool found; + NFind::CDirEntry de; + if (!enumerator.Next(de, found)) { return AddError(phyPrefix); } if (!found) - return S_OK; + break; + entries.Add(de); + } + + FOR_VECTOR(i, entries) + { + const NFind::CDirEntry &de = entries[i]; + NFind::CFileInfo fi; + if (!enumerator.Fill_FileInfo(de, fi, !SymLinks)) + // if (!fi.Find_AfterEnumerator(path)) + { + const FString path = phyPrefix + de.Name; + { + RINOK(AddError(path)); + continue; + } + } + + files.Add(fi); + + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)); + } + } + + return S_OK; + + #endif // _WIN32 +} + + + + +HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phyPrefix) +{ + RINOK(ScanProgress(phyPrefix)); + + CObjectVector<NFind::CFileInfo> files; + RINOK(EnumerateOneDir(phyPrefix, files)); + + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; + { + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(AddError(path)); + continue; + } + fi.Name = di.Name; + } + */ + #endif int secureIndex = -1; #ifdef _USE_SECURITY_CODE @@ -237,7 +328,7 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy AddDirFileInfo(phyParent, logParent, secureIndex, fi); - if (Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) { RINOK(ScanProgress(phyPrefix)); } @@ -246,26 +337,45 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy { const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; unsigned parent = AddPrefix(phyParent, logParent, fs2us(name2)); - RINOK(EnumerateDir(parent, parent, phyPrefix + name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + name2)); } } + return S_OK; } + +/* +EnumerateItems2() + const FStringVector &filePaths - are path without tail slashes. + All dir prefixes of filePaths will be not stores in logical paths +fix it: we can scan AltStream also. +*/ + +#ifdef _WIN32 +// #define FOLLOW_LINK_PARAM +// #define FOLLOW_LINK_PARAM2 +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#else +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#endif + HRESULT CDirItems::EnumerateItems2( const FString &phyPrefix, const UString &logPrefix, const FStringVector &filePaths, FStringVector *requestedPaths) { - int phyParent = phyPrefix.IsEmpty() ? -1 : AddPrefix(-1, -1, fs2us(phyPrefix)); - int logParent = logPrefix.IsEmpty() ? -1 : AddPrefix(-1, -1, logPrefix); + const int phyParent = phyPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, fs2us(phyPrefix)); + const int logParent = logPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, logPrefix); FOR_VECTOR (i, filePaths) { const FString &filePath = filePaths[i]; NFind::CFileInfo fi; const FString phyPath = phyPrefix + filePath; - if (!fi.Find(phyPath)) + if (!fi.Find(phyPath FOLLOW_LINK_PARAM)) { RINOK(AddError(phyPath)); continue; @@ -273,13 +383,13 @@ HRESULT CDirItems::EnumerateItems2( if (requestedPaths) requestedPaths->Add(phyPath); - int delimiter = filePath.ReverseFind_PathSepar(); + const int delimiter = filePath.ReverseFind_PathSepar(); FString phyPrefixCur; int phyParentCur = phyParent; if (delimiter >= 0) { - phyPrefixCur.SetFrom(filePath, delimiter + 1); - phyParentCur = AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); + phyPrefixCur.SetFrom(filePath, (unsigned)(delimiter + 1)); + phyParentCur = (int)AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); } int secureIndex = -1; @@ -296,7 +406,7 @@ HRESULT CDirItems::EnumerateItems2( { const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; unsigned parent = AddPrefix(phyParentCur, logParent, fs2us(name2)); - RINOK(EnumerateDir(parent, parent, phyPrefix + phyPrefixCur + name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + phyPrefixCur + name2)); } } @@ -307,34 +417,46 @@ HRESULT CDirItems::EnumerateItems2( - - static HRESULT EnumerateDirItems( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, + const int phyParent, const int logParent, + const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode CDirItems &dirItems, bool enterToSubFolders); + +/* EnumerateDirItems_Spec() + adds new Dir item prefix, and enumerates dir items, + then it can remove that Dir item prefix, if there are no items in that dir. +*/ + + +/* + EnumerateDirItems_Spec() + it's similar to EnumerateDirItems, but phyPrefix doesn't include (curFolderName) +*/ + static HRESULT EnumerateDirItems_Spec( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &curFolderName, - const FString &phyPrefix, - const UStringVector &addArchivePrefix, + const int phyParent, const int logParent, const FString &curFolderName, + const FString &phyPrefix, // without (curFolderName) + const UStringVector &addParts, // (curNode + addParts) includes (curFolderName) CDirItems &dirItems, bool enterToSubFolders) { const FString name2 = curFolderName + FCHAR_PATH_SEPARATOR; - unsigned parent = dirItems.AddPrefix(phyParent, logParent, fs2us(name2)); - unsigned numItems = dirItems.Items.Size(); + const unsigned parent = dirItems.AddPrefix(phyParent, logParent, fs2us(name2)); + const unsigned numItems = dirItems.Items.Size(); HRESULT res = EnumerateDirItems( - curNode, parent, parent, phyPrefix + name2, - addArchivePrefix, dirItems, enterToSubFolders); + curNode, (int)parent, (int)parent, phyPrefix + name2, + addParts, dirItems, enterToSubFolders); if (numItems == dirItems.Items.Size()) dirItems.DeleteLastPrefix(); return res; } + #ifndef UNDER_CE #ifdef _WIN32 @@ -342,98 +464,107 @@ static HRESULT EnumerateDirItems_Spec( static HRESULT EnumerateAltStreams( const NFind::CFileInfo &fi, const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &fullPath, - const UStringVector &addArchivePrefix, // prefix from curNode - bool addAllItems, + const int phyParent, const int logParent, + const FString &phyPath, // with (fi.Name), without tail slash for folders + const UStringVector &addParts, // with (fi.Name), prefix parts from curNode + bool addAllSubStreams, CDirItems &dirItems) { - NFind::CStreamEnumerator enumerator(fullPath); + NFind::CStreamEnumerator enumerator(phyPath); for (;;) { NFind::CStreamInfo si; bool found; if (!enumerator.Next(si, found)) { - return dirItems.AddError(fullPath + FTEXT(":*")); // , (DWORD)E_FAIL + return dirItems.AddError(phyPath + FTEXT(":*")); // , (DWORD)E_FAIL } if (!found) return S_OK; if (si.IsMainStream()) continue; - UStringVector addArchivePrefixNew = addArchivePrefix; - UString reducedName = si.GetReducedName(); - addArchivePrefixNew.Back() += reducedName; - if (curNode.CheckPathToRoot(false, addArchivePrefixNew, true)) + UStringVector parts = addParts; + const UString reducedName = si.GetReducedName(); + parts.Back() += reducedName; + if (curNode.CheckPathToRoot(false, parts, true)) continue; - if (!addAllItems) - if (!curNode.CheckPathToRoot(true, addArchivePrefixNew, true)) + if (!addAllSubStreams) + if (!curNode.CheckPathToRoot(true, parts, true)) continue; NFind::CFileInfo fi2 = fi; fi2.Name += us2fs(reducedName); fi2.Size = si.Size; - fi2.Attrib &= ~FILE_ATTRIBUTE_DIRECTORY; + fi2.Attrib &= ~(DWORD)(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); fi2.IsAltStream = true; dirItems.AddDirFileInfo(phyParent, logParent, -1, fi2); } } -#endif +#endif // _WIN32 + + +/* We get Reparse data and parse it. + If there is Reparse error, we free dirItem.Reparse data. + Do we need to work with empty reparse data? +*/ HRESULT CDirItems::SetLinkInfo(CDirItem &dirItem, const NFind::CFileInfo &fi, const FString &phyPrefix) { - if (!SymLinks || !fi.HasReparsePoint()) + if (!SymLinks) return S_OK; + + #ifdef _WIN32 + if (!fi.HasReparsePoint() || fi.IsAltStream) + #else // _WIN32 + if (!fi.IsPosixLink()) + #endif // _WIN32 + return S_OK; + const FString path = phyPrefix + fi.Name; CByteBuffer &buf = dirItem.ReparseData; - DWORD res = 0; if (NIO::GetReparseData(path, buf)) { - CReparseAttr attr; - if (attr.Parse(buf, buf.Size(), res)) - return S_OK; - // we ignore unknown reparse points - if (res != ERROR_INVALID_REPARSE_DATA) - res = 0; - } - else - { - res = ::GetLastError(); - if (res == 0) - res = ERROR_INVALID_FUNCTION; + // if (dirItem.ReparseData.Size() != 0) + Stat.FilesSize -= fi.Size; + return S_OK; } + DWORD res = ::GetLastError(); buf.Free(); - if (res == 0) - return S_OK; return AddError(path, res); } -#endif +#endif // UNDER_CE + + static HRESULT EnumerateForItem( - NFind::CFileInfo &fi, + const NFind::CFileInfo &fi, const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, // prefix from curNode + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode, without (fi.Name) CDirItems &dirItems, bool enterToSubFolders) { const UString name = fs2us(fi.Name); - bool enterToSubFolders2 = enterToSubFolders; - UStringVector addArchivePrefixNew = addArchivePrefix; - addArchivePrefixNew.Add(name); - { - UStringVector addArchivePrefixNewTemp(addArchivePrefixNew); - if (curNode.CheckPathToRoot(false, addArchivePrefixNewTemp, !fi.IsDir())) - return S_OK; - } - int dirItemIndex = -1; + UStringVector newParts = addParts; + newParts.Add(name); + // check the path in exclude rules + if (curNode.CheckPathToRoot(false, newParts, !fi.IsDir())) + return S_OK; + + #if !defined(UNDER_CE) + int dirItemIndex = -1; + #if defined(_WIN32) bool addAllSubStreams = false; + #endif // _WIN32 + #endif // !defined(UNDER_CE) - if (curNode.CheckPathToRoot(true, addArchivePrefixNew, !fi.IsDir())) + // check the path in inlcude rules + if (curNode.CheckPathToRoot(true, newParts, !fi.IsDir())) { int secureIndex = -1; #ifdef _USE_SECURITY_CODE @@ -443,58 +574,96 @@ static HRESULT EnumerateForItem( } #endif - dirItemIndex = dirItems.Items.Size(); + #if !defined(UNDER_CE) + dirItemIndex = (int)dirItems.Items.Size(); + #if defined(_WIN32) + // we will not check include rules for substreams. + addAllSubStreams = true; + #endif // _WIN32 + #endif // !defined(UNDER_CE) + dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); if (fi.IsDir()) - enterToSubFolders2 = true; - - addAllSubStreams = true; + enterToSubFolders = true; } - #ifndef UNDER_CE - if (dirItems.ScanAltStreams) - { - RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, - phyPrefix + fi.Name, - addArchivePrefixNew, - addAllSubStreams, - dirItems)); - } + #if !defined(UNDER_CE) + + // we don't scan AltStreams for link files if (dirItemIndex >= 0) { - CDirItem &dirItem = dirItems.Items[dirItemIndex]; + CDirItem &dirItem = dirItems.Items[(unsigned)dirItemIndex]; RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)); if (dirItem.ReparseData.Size() != 0) return S_OK; } - #endif + #if defined(_WIN32) + if (dirItems.ScanAltStreams) + { + RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, + phyPrefix + fi.Name, // with (fi.Name) + newParts, // with (fi.Name) + addAllSubStreams, + dirItems)); + } + #endif + + #endif // !defined(UNDER_CE) + + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif if (!fi.IsDir()) return S_OK; - - const NWildcard::CCensorNode *nextNode = 0; - if (addArchivePrefix.IsEmpty()) + + const NWildcard::CCensorNode *nextNode = NULL; + + if (addParts.IsEmpty()) { int index = curNode.FindSubNode(name); if (index >= 0) - nextNode = &curNode.SubNodes[index]; + { + nextNode = &curNode.SubNodes[(unsigned)index]; + newParts.Clear(); + } } - if (!enterToSubFolders2 && nextNode == 0) - return S_OK; - - addArchivePrefixNew = addArchivePrefix; - if (nextNode == 0) + + if (!nextNode) { + if (!enterToSubFolders) + return S_OK; + + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + return S_OK; // we don't follow posix link + } + #endif + + if (dirItems.SymLinks && fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode + NOTE: (curNode and parent nodes) still can have wildcard rules + to include some items of target directory (of reparse point), + but we ignore these rules here. + */ + return S_OK; + } nextNode = &curNode; - addArchivePrefixNew.Add(name); } return EnumerateDirItems_Spec( - *nextNode, phyParent, logParent, fi.Name, phyPrefix, - addArchivePrefixNew, + *nextNode, phyParent, logParent, fi.Name, + phyPrefix, // without (fi.Name) + newParts, // relative to (*nextNode). (*nextNode + newParts) includes (fi.Name) dirItems, - enterToSubFolders2); + enterToSubFolders); } @@ -513,10 +682,13 @@ static bool CanUseFsDirect(const NWildcard::CCensorNode &curNode) /* Windows doesn't support file name with wildcard But if another system supports file name with wildcard, - and wildcard mode is disabled, we can ignore wildcard in name */ + and wildcard mode is disabled, we can ignore wildcard in name + */ /* + #ifndef _WIN32 if (!item.WildcardParsing) continue; + #endif */ if (DoesNameContainWildcard(name)) return false; @@ -532,26 +704,33 @@ static bool IsVirtualFsFolder(const FString &prefix, const UString &name) UString s = fs2us(prefix); s += name; s.Add_PathSepar(); + // it returns (true) for non real FS folder path like - "\\SERVER\" return IsPathSepar(s[0]) && GetRootPrefixSize(s) == 0; } #endif + + static HRESULT EnumerateDirItems( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, // prefix from curNode + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // prefix from curNode including CDirItems &dirItems, bool enterToSubFolders) { if (!enterToSubFolders) + { + /* if there are IncludeItems censor rules that affect items in subdirs, + then we will enter to all subfolders */ if (curNode.NeedCheckSubDirs()) enterToSubFolders = true; + } RINOK(dirItems.ScanProgress(phyPrefix)); // try direct_names case at first - if (addArchivePrefix.IsEmpty() && !enterToSubFolders) + if (addParts.IsEmpty() && !enterToSubFolders) { if (CanUseFsDirect(curNode)) { @@ -616,16 +795,16 @@ static HRESULT EnumerateDirItems( } else #endif - if (!fi.Find(fullPath)) + if (!fi.Find(fullPath FOLLOW_LINK_PARAM2)) { RINOK(dirItems.AddError(fullPath)); continue; } - bool isDir = fi.IsDir(); - if (isDir && !item.ForDir || !isDir && !item.ForFile) + const bool isDir = fi.IsDir(); + if ((isDir && !item.ForDir) || (!isDir && !item.ForFile)) { - RINOK(dirItems.AddError(fullPath, (DWORD)E_FAIL)); + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)); continue; } { @@ -645,55 +824,74 @@ static HRESULT EnumerateDirItems( dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); - #ifndef UNDER_CE + // we don't scan AltStreams for link files + + #if !defined(UNDER_CE) { CDirItem &dirItem = dirItems.Items.Back(); RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)); if (dirItem.ReparseData.Size() != 0) - { - if (fi.IsAltStream) - dirItems.Stat.AltStreamsSize -= fi.Size; - else - dirItems.Stat.FilesSize -= fi.Size; continue; - } } - #endif - - - #ifndef UNDER_CE + + #if defined(_WIN32) if (needAltStreams && dirItems.ScanAltStreams) { UStringVector pathParts; pathParts.Add(fs2us(fi.Name)); RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, - fullPath, pathParts, + fullPath, // including (name) + pathParts, // including (fi.Name) true, /* addAllSubStreams */ dirItems)); } - #endif + #endif // defined(_WIN32) + #endif // !defined(UNDER_CE) + + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif if (!isDir) continue; - - UStringVector addArchivePrefixNew; - const NWildcard::CCensorNode *nextNode = 0; + + UStringVector newParts; + const NWildcard::CCensorNode *nextNode = NULL; int index = curNode.FindSubNode(name); if (index >= 0) { - for (int t = needEnterVector.Size(); t <= index; t++) + for (int t = (int)needEnterVector.Size(); t <= index; t++) needEnterVector.Add(true); - needEnterVector[index] = false; - nextNode = &curNode.SubNodes[index]; + needEnterVector[(unsigned)index] = false; + nextNode = &curNode.SubNodes[(unsigned)index]; } else { + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + continue; // we don't follow posix link + } + #endif + + if (dirItems.SymLinks) + { + if (fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode */ + continue; + } + } nextNode = &curNode; - addArchivePrefixNew.Add(name); // don't change it to fi.Name. It's for shortnames support + newParts.Add(name); // don't change it to fi.Name. It's for shortnames support } RINOK(EnumerateDirItems_Spec(*nextNode, phyParent, logParent, fi.Name, phyPrefix, - addArchivePrefixNew, dirItems, true)); + newParts, dirItems, true)); } for (i = 0; i < curNode.SubNodes.Size(); i++) @@ -718,7 +916,7 @@ static HRESULT EnumerateDirItems( } // we don't want to call fi.Find() for root folder or virtual folder - if (phyPrefix.IsEmpty() && nextNode.Name.IsEmpty() + if ((phyPrefix.IsEmpty() && nextNode.Name.IsEmpty()) #if defined(_WIN32) && !defined(UNDER_CE) || IsVirtualFsFolder(phyPrefix, nextNode.Name) #endif @@ -729,7 +927,7 @@ static HRESULT EnumerateDirItems( } else { - if (!fi.Find(fullPath)) + if (!fi.Find(fullPath FOLLOW_LINK_PARAM2)) { if (!nextNode.AreThereIncludeItems()) continue; @@ -739,7 +937,7 @@ static HRESULT EnumerateDirItems( if (!fi.IsDir()) { - RINOK(dirItems.AddError(fullPath, (DWORD)E_FAIL)); + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)); continue; } } @@ -791,7 +989,7 @@ static HRESULT EnumerateDirItems( fi.Name = driveName; RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, - addArchivePrefix, dirItems, enterToSubFolders)); + addParts, dirItems, enterToSubFolders)); } return S_OK; } @@ -800,43 +998,81 @@ static HRESULT EnumerateDirItems( #endif #endif - NFind::CEnumerator enumerator; - enumerator.SetDirPrefix(phyPrefix); - for (unsigned ttt = 0; ; ttt++) + CObjectVector<NFind::CFileInfo> files; + + // for (int y = 0; y < 1; y++) { - NFind::CFileInfo fi; - bool found; - if (!enumerator.Next(fi, found)) + // files.Clear(); + RINOK(dirItems.EnumerateOneDir(phyPrefix, files)); + /* + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + // const NFind::CFileInfo &fi = files[i]; + #else + NFind::CFileInfo &fi = files[i]; { - RINOK(dirItems.AddError(phyPrefix)); - break; + const NFind::CFileInfo &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; } - if (!found) - break; + #endif - if (dirItems.Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + } + */ + } + + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; { - RINOK(dirItems.ScanProgress(phyPrefix)); + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; } + */ + #endif RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, - addArchivePrefix, dirItems, enterToSubFolders)); + addParts, dirItems, enterToSubFolders)); + if (dirItems.Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(dirItems.ScanProgress(phyPrefix)); + } } return S_OK; } + + + HRESULT EnumerateItems( const NWildcard::CCensor &censor, const NWildcard::ECensorPathMode pathMode, - const UString &addPathPrefix, + const UString &addPathPrefix, // prefix that will be added to Logical Path CDirItems &dirItems) { FOR_VECTOR (i, censor.Pairs) { const NWildcard::CPair &pair = censor.Pairs[i]; - int phyParent = pair.Prefix.IsEmpty() ? -1 : dirItems.AddPrefix(-1, -1, pair.Prefix); + const int phyParent = pair.Prefix.IsEmpty() ? -1 : (int)dirItems.AddPrefix(-1, -1, pair.Prefix); int logParent = -1; if (pathMode == NWildcard::k_AbsPath) @@ -844,7 +1080,7 @@ HRESULT EnumerateItems( else { if (!addPathPrefix.IsEmpty()) - logParent = dirItems.AddPrefix(-1, -1, addPathPrefix); + logParent = (int)dirItems.AddPrefix(-1, -1, addPathPrefix); } RINOK(EnumerateDirItems(pair.Head, phyParent, logParent, us2fs(pair.Prefix), UStringVector(), @@ -855,34 +1091,103 @@ HRESULT EnumerateItems( dirItems.ReserveDown(); #if defined(_WIN32) && !defined(UNDER_CE) - dirItems.FillFixedReparse(); + RINOK(dirItems.FillFixedReparse()); #endif return S_OK; } + + #if defined(_WIN32) && !defined(UNDER_CE) -void CDirItems::FillFixedReparse() +HRESULT CDirItems::FillFixedReparse() { - /* imagex/WIM reduces absolute pathes in links (raparse data), - if we archive non root folder. We do same thing here */ - - if (!SymLinks) - return; - FOR_VECTOR(i, Items) { CDirItem &item = Items[i]; + + if (!SymLinks) + { + // continue; // for debug + if (!item.Has_Attrib_ReparsePoint()) + continue; + + // if (item.IsDir()) continue; + + const FString phyPath = GetPhyPath(i); + + NFind::CFileInfo fi; + if (fi.Fill_From_ByHandleFileInfo(phyPath)) // item.IsDir() + { + item.Size = fi.Size; + item.CTime = fi.CTime; + item.ATime = fi.ATime; + item.MTime = fi.MTime; + item.Attrib = fi.Attrib; + continue; + } + + /* + // we request properties of target file instead of properies of symbolic link + // here we also can manually parse unsupported links (like WSL links) + NIO::CInFile inFile; + if (inFile.Open(phyPath)) + { + BY_HANDLE_FILE_INFORMATION info; + if (inFile.GetFileInformation(&info)) + { + // Stat.FilesSize doesn't contain item.Size already + // Stat.FilesSize -= item.Size; + item.Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + Stat.FilesSize += item.Size; + item.CTime = info.ftCreationTime; + item.ATime = info.ftLastAccessTime; + item.MTime = info.ftLastWriteTime; + item.Attrib = info.dwFileAttributes; + continue; + } + } + */ + + RINOK(AddError(phyPath)); + continue; + } + + // (SymLinks == true) here + if (item.ReparseData.Size() == 0) continue; + + // if (item.Size == 0) + { + // 20.03: we use Reparse Data instead of real data + item.Size = item.ReparseData.Size(); + } CReparseAttr attr; - DWORD errorCode = 0; - if (!attr.Parse(item.ReparseData, item.ReparseData.Size(), errorCode)) + if (!attr.Parse(item.ReparseData, item.ReparseData.Size())) + { + const FString phyPath = GetPhyPath(i); + AddError(phyPath, attr.ErrorCode); continue; - if (attr.IsRelative()) + } + + /* imagex/WIM reduces absolute paths in links (raparse data), + if we archive non root folder. We do same thing here */ + + bool isWSL = false; + if (attr.IsSymLink_WSL()) + { + // isWSL = true; + // we don't change WSL symlinks continue; + } + else + { + if (attr.IsRelative_Win()) + continue; + } const UString &link = attr.GetPath(); if (!IsDrivePath(link)) @@ -892,7 +1197,7 @@ void CDirItems::FillFixedReparse() FString fullPathF; if (!NDir::MyGetFullPathName(GetPhyPath(i), fullPathF)) continue; - UString fullPath = fs2us(fullPathF); + const UString fullPath = fs2us(fullPathF); const UString logPath = GetLogPath(i); if (logPath.Len() >= fullPath.Len()) continue; @@ -903,7 +1208,7 @@ void CDirItems::FillFixedReparse() if (!IsPathSepar(prefix.Back())) continue; - unsigned rootPrefixSize = GetRootPrefixSize(prefix); + const unsigned rootPrefixSize = GetRootPrefixSize(prefix); if (rootPrefixSize == 0) continue; if (rootPrefixSize == prefix.Len()) @@ -919,10 +1224,12 @@ void CDirItems::FillFixedReparse() newLink += link.Ptr(prefix.Len()); CByteBuffer data; - if (!FillLinkData(data, newLink, attr.IsSymLink())) + bool isSymLink = !attr.IsMountPoint(); + if (!FillLinkData(data, newLink, isSymLink, isWSL)) continue; item.ReparseData2 = data; } + return S_OK; } #endif diff --git a/CPP/7zip/UI/Common/EnumDirItems.h b/CPP/7zip/UI/Common/EnumDirItems.h index 6490bd50..9b17c600 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.h +++ b/CPP/7zip/UI/Common/EnumDirItems.h @@ -5,12 +5,8 @@ #include "../../../Common/Wildcard.h" -#include "../../../Windows/FileFind.h" - #include "DirItem.h" -void AddDirFileInfo(int phyParent, int logParent, int secureIndex, - const NWindows::NFile::NFind::CFileInfo &fi, CObjectVector<CDirItem> &dirItems); HRESULT EnumerateItems( const NWildcard::CCensor &censor, diff --git a/CPP/7zip/UI/Common/Extract.cpp b/CPP/7zip/UI/Common/Extract.cpp index c49daa1d..18bd1037 100644 --- a/CPP/7zip/UI/Common/Extract.cpp +++ b/CPP/7zip/UI/Common/Extract.cpp @@ -7,6 +7,7 @@ #include "../../../Common/StringConvert.h" #include "../../../Windows/FileDir.h" +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/PropVariant.h" #include "../../../Windows/PropVariantConv.h" @@ -19,6 +20,19 @@ using namespace NWindows; using namespace NFile; using namespace NDir; + +static void SetErrorMessage(const char *message, + const FString &path, HRESULT errorCode, + UString &s) +{ + s = message; + s += " : "; + s += NError::MyFormatMessage(errorCode); + s += " : "; + s += fs2us(path); +} + + static HRESULT DecompressArchive( CCodecs *codecs, const CArchiveLink &arcLink, @@ -47,7 +61,7 @@ static HRESULT DecompressArchive( // So it extracts different archives to one folder. // We will use top level archive name const CArc &arc0 = arcLink.Arcs[0]; - if (StringsAreEqualNoCase_Ascii(codecs->Formats[arc0.FormatIndex].Name, "pe")) + if (arc0.FormatIndex >= 0 && StringsAreEqualNoCase_Ascii(codecs->Formats[(unsigned)arc0.FormatIndex].Name, "pe")) replaceName = arc0.DefaultName; } @@ -164,11 +178,8 @@ static HRESULT DecompressArchive( */ else if (!CreateComplexDir(outDir)) { - HRESULT res = ::GetLastError(); - if (res == S_OK) - res = E_FAIL; - errorMessage = "Can not create output directory: "; - errorMessage += fs2us(outDir); + const HRESULT res = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot create output directory", outDir, res, errorMessage); return res; } @@ -221,6 +232,7 @@ static HRESULT DecompressArchive( Sorted list for file paths was sorted with case insensitive compare function. But FindInSorted function did binary search via case sensitive compare function */ +int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &name); int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &name) { unsigned left = 0, right = fileName.Size(); @@ -230,7 +242,7 @@ int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &n const UString &midValue = fileName[mid]; int compare = CompareFileNames(name, midValue); if (compare == 0) - return mid; + return (int)mid; if (compare < 0) right = mid; else @@ -239,6 +251,8 @@ int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &n return -1; } + + HRESULT Extract( CCodecs *codecs, const CObjectVector<COpenType> &types, @@ -268,11 +282,19 @@ HRESULT Extract( fi.Size = 0; if (!options.StdInMode) { - const FString &arcPath = us2fs(arcPaths[i]); - if (!fi.Find(arcPath)) - throw "there is no such archive"; + const FString arcPath = us2fs(arcPaths[i]); + if (!fi.Find_FollowLink(arcPath)) + { + const HRESULT errorCode = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot find archive file", arcPath, errorCode, errorMessage); + return errorCode; + } if (fi.IsDir()) - throw "can't decompress folder"; + { + HRESULT errorCode = E_FAIL; + SetErrorMessage("The item is a directory", arcPath, errorCode, errorMessage); + return errorCode; + } } arcSizes.Add(fi.Size); totalPackSize += fi.Size; @@ -314,8 +336,12 @@ HRESULT Extract( } else { - if (!fi.Find(us2fs(arcPath)) || fi.IsDir()) - throw "there is no such archive"; + if (!fi.Find_FollowLink(us2fs(arcPath)) || fi.IsDir()) + { + const HRESULT errorCode = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot find archive file", us2fs(arcPath), errorCode, errorMessage); + return errorCode; + } } /* @@ -379,12 +405,7 @@ HRESULT Extract( { thereAreNotOpenArcs = true; if (!options.StdInMode) - { - NFind::CFileInfo fi2; - if (fi2.Find(us2fs(arcPath))) - if (!fi2.IsDir()) - totalPackProcessed += fi2.Size; - } + totalPackProcessed += fi.Size; continue; } @@ -397,7 +418,7 @@ HRESULT Extract( // numArcs = arcPaths.Size(); if (arcLink.VolumePaths.Size() != 0) { - Int64 correctionSize = arcLink.VolumesSize; + Int64 correctionSize = (Int64)arcLink.VolumesSize; FOR_VECTOR (v, arcLink.VolumePaths) { int index = Find_FileName_InSortedVector(arcPathsFull, arcLink.VolumePaths[v]); @@ -415,7 +436,7 @@ HRESULT Extract( Int64 newPackSize = (Int64)totalPackSize + correctionSize; if (newPackSize < 0) newPackSize = 0; - totalPackSize = newPackSize; + totalPackSize = (UInt64)newPackSize; RINOK(extractCallback->SetTotal(totalPackSize)); } } diff --git a/CPP/7zip/UI/Common/Extract.h b/CPP/7zip/UI/Common/Extract.h index 03ac74b2..bfabe195 100644 --- a/CPP/7zip/UI/Common/Extract.h +++ b/CPP/7zip/UI/Common/Extract.h @@ -53,10 +53,10 @@ struct CExtractOptions: public CExtractOptionsBase #endif CExtractOptions(): - TestMode(false), StdInMode(false), StdOutMode(false), - YesToAll(false) + YesToAll(false), + TestMode(false) {} }; diff --git a/CPP/7zip/UI/Common/ExtractingFilePath.cpp b/CPP/7zip/UI/Common/ExtractingFilePath.cpp index 535530e4..21a306d2 100644 --- a/CPP/7zip/UI/Common/ExtractingFilePath.cpp +++ b/CPP/7zip/UI/Common/ExtractingFilePath.cpp @@ -8,6 +8,8 @@ #include "ExtractingFilePath.h" +extern +bool g_PathTrailReplaceMode; bool g_PathTrailReplaceMode = #ifdef _WIN32 true @@ -17,6 +19,7 @@ bool g_PathTrailReplaceMode = ; +#ifdef _WIN32 static void ReplaceIncorrectChars(UString &s) { { @@ -31,7 +34,10 @@ static void ReplaceIncorrectChars(UString &s) || #endif c == WCHAR_PATH_SEPARATOR) - s.ReplaceOneCharAtPos(i, '_'); + s.ReplaceOneCharAtPos(i, + '_' // default + // (wchar_t)(0xf000 + c) // 21.02 debug: WSL encoding for unsupported characters + ); } } @@ -72,8 +78,7 @@ static void ReplaceIncorrectChars(UString &s) } } } - -#ifdef _WIN32 +#endif /* WinXP-64 doesn't support ':', '\\' and '/' symbols in name of alt stream. But colon in postfix ":$DATA" is allowed. @@ -98,6 +103,8 @@ void Correct_AltStream_Name(UString &s) s = '_'; } +#ifdef _WIN32 + static const unsigned g_ReservedWithNum_Index = 4; static const char * const g_ReservedNames[] = @@ -149,7 +156,7 @@ static void Correct_PathPart(UString &s) if (s.IsEmpty()) return; - if (s[0] == '.' && (s[1] == 0 || s[1] == '.' && s[2] == 0)) + if (s[0] == '.' && (s[1] == 0 || (s[1] == '.' && s[2] == 0))) s.Empty(); #ifdef _WIN32 else diff --git a/CPP/7zip/UI/Common/ExtractingFilePath.h b/CPP/7zip/UI/Common/ExtractingFilePath.h index 81cb8267..8f8f9f1b 100644 --- a/CPP/7zip/UI/Common/ExtractingFilePath.h +++ b/CPP/7zip/UI/Common/ExtractingFilePath.h @@ -5,9 +5,9 @@ #include "../../../Common/MyString.h" -#ifdef _WIN32 +// #ifdef _WIN32 void Correct_AltStream_Name(UString &s); -#endif +// #endif // replaces unsuported characters, and replaces "." , ".." and "" to "[]" UString Get_Correct_FsFile_Name(const UString &name); diff --git a/CPP/7zip/UI/Common/HashCalc.cpp b/CPP/7zip/UI/Common/HashCalc.cpp index c340ac7f..c87be443 100644 --- a/CPP/7zip/UI/Common/HashCalc.cpp +++ b/CPP/7zip/UI/Common/HashCalc.cpp @@ -8,6 +8,7 @@ #include "../../Common/FileStreams.h" #include "../../Common/StreamUtils.h" +#include "../../Common/StreamObjects.h" #include "EnumDirItems.h" #include "HashCalc.h" @@ -211,6 +212,8 @@ HRESULT HashCalc( else { RINOK(callback->StartScanning()); + + dirItems.SymLinks = options.SymLinks.Val; dirItems.ScanAltStreams = options.AltStreamsMode; HRESULT res = EnumerateItems(censor, @@ -258,31 +261,47 @@ HRESULT HashCalc( UString path; bool isDir = false; bool isAltStream = false; + if (options.StdInMode) { inStream = new CStdInFileStream; } else { - CInFileStream *inStreamSpec = new CInFileStream; - inStream = inStreamSpec; - const CDirItem &dirItem = dirItems.Items[i]; - isDir = dirItem.IsDir(); - isAltStream = dirItem.IsAltStream; path = dirItems.GetLogPath(i); - if (!isDir) + const CDirItem &di = dirItems.Items[i]; + isAltStream = di.IsAltStream; + + #ifndef UNDER_CE + // if (di.AreReparseData()) + if (di.ReparseData.Size() != 0) { - FString phyPath = dirItems.GetPhyPath(i); - if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) + CBufInStream *inStreamSpec = new CBufInStream(); + inStream = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + } + else + #endif + { + CInFileStream *inStreamSpec = new CInFileStream; + inStreamSpec->File.PreserveATime = options.PreserveATime; + inStream = inStreamSpec; + isDir = di.IsDir(); + if (!isDir) { - HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); - hb.NumErrors++; - if (res != S_FALSE) - return res; - continue; + const FString phyPath = dirItems.GetPhyPath(i); + if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) + { + HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); + hb.NumErrors++; + if (res != S_FALSE) + return res; + continue; + } } } } + RINOK(callback->GetStream(path, isDir)); UInt64 fileSize = 0; diff --git a/CPP/7zip/UI/Common/HashCalc.h b/CPP/7zip/UI/Common/HashCalc.h index db5b39aa..b6d320b5 100644 --- a/CPP/7zip/UI/Common/HashCalc.h +++ b/CPP/7zip/UI/Common/HashCalc.h @@ -62,6 +62,8 @@ struct CHashBundle: public IHashCalc NumDirs = NumFiles = NumAltStreams = FilesSize = AltStreamsSize = NumErrors = 0; } + virtual ~CHashBundle() {}; + void InitForNewFile(); void Update(const void *data, UInt32 size); void SetSize(UInt64 size); @@ -90,12 +92,20 @@ struct IHashCallbackUI: public IDirItemsCallback struct CHashOptions { UStringVector Methods; + bool PreserveATime; bool OpenShareForWrite; bool StdInMode; bool AltStreamsMode; + CBoolPair SymLinks; + NWildcard::ECensorPathMode PathMode; - CHashOptions(): StdInMode(false), OpenShareForWrite(false), AltStreamsMode(false), PathMode(NWildcard::k_RelatPath) {}; + CHashOptions(): + PreserveATime(false), + OpenShareForWrite(false), + StdInMode(false), + AltStreamsMode(false), + PathMode(NWildcard::k_RelatPath) {}; }; HRESULT HashCalc( diff --git a/CPP/7zip/UI/Common/LoadCodecs.cpp b/CPP/7zip/UI/Common/LoadCodecs.cpp index f1334613..b94720c5 100644 --- a/CPP/7zip/UI/Common/LoadCodecs.cpp +++ b/CPP/7zip/UI/Common/LoadCodecs.cpp @@ -39,6 +39,8 @@ EXPORT_CODECS #include "../../../Common/StringToInt.h" #include "../../../Common/StringConvert.h" +#include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileIO.h" #include "../../../Windows/PropVariant.h" #include "LoadCodecs.h" @@ -82,11 +84,11 @@ using namespace NFile; static CFSTR const kMainDll = - // #ifdef _WIN32 + #ifdef _WIN32 FTEXT("7z.dll"); - // #else - // FTEXT("7z.so"); - // #endif + #else + FTEXT("7z.so"); + #endif #ifdef _WIN32 @@ -110,7 +112,7 @@ static bool ReadPathFromRegistry(HKEY baseKey, LPCWSTR value, FString &path) { path = us2fs(pathU); NName::NormalizeDirPathPrefix(path); - return NFind::DoesFileExist(path + kMainDll); + return NFind::DoesFileExist_Raw(path + kMainDll); } } return false; @@ -163,7 +165,7 @@ int CArcInfoEx::FindExtension(const UString &ext) const { FOR_VECTOR (i, Exts) if (ext.IsEqualTo_NoCase(Exts[i].Ext)) - return i; + return (int)i; return -1; } @@ -206,15 +208,18 @@ static bool ParseSignatures(const Byte *data, unsigned size, CObjectVector<CByte #endif // _SFX +// #include <stdio.h> + #ifdef EXTERNAL_CODECS static FString GetBaseFolderPrefixFromRegistry() { FString moduleFolderPrefix = NDLL::GetModuleDirPrefix(); + #ifdef _WIN32 - if (!NFind::DoesFileExist(moduleFolderPrefix + kMainDll) && - !NFind::DoesDirExist(moduleFolderPrefix + kCodecsFolderName) && - !NFind::DoesDirExist(moduleFolderPrefix + kFormatsFolderName)) + if ( !NFind::DoesFileOrDirExist(moduleFolderPrefix + kMainDll) + && !NFind::DoesFileOrDirExist(moduleFolderPrefix + kCodecsFolderName) + && !NFind::DoesFileOrDirExist(moduleFolderPrefix + kFormatsFolderName)) { FString path; if (ReadPathFromRegistry(HKEY_CURRENT_USER, kProgramPath2Value, path)) return path; @@ -223,6 +228,8 @@ static FString GetBaseFolderPrefixFromRegistry() if (ReadPathFromRegistry(HKEY_LOCAL_MACHINE, kProgramPathValue, path)) return path; } #endif + + // printf("\nmoduleFolderPrefix = %s\n", (const char *)GetAnsiString(moduleFolderPrefix)); return moduleFolderPrefix; } @@ -238,25 +245,29 @@ static HRESULT GetCoderClass(Func_GetMethodProperty getMethodProperty, UInt32 in if (::SysStringByteLen(prop.bstrVal) != sizeof(GUID)) return E_FAIL; isAssigned = true; - clsId = *(const GUID *)prop.bstrVal; + clsId = *(const GUID *)(const void *)prop.bstrVal; } else if (prop.vt != VT_EMPTY) return E_FAIL; return S_OK; } +#define MY_GET_FUNC(dest, type, func) *(void **)(&dest) = (func); +// #define MY_GET_FUNC(dest, type, func) dest = (type)(func); + HRESULT CCodecs::LoadCodecs() { CCodecLib &lib = Libs.Back(); - lib.CreateDecoder = (Func_CreateDecoder)lib.Lib.GetProc("CreateDecoder"); - lib.CreateEncoder = (Func_CreateEncoder)lib.Lib.GetProc("CreateEncoder"); - lib.GetMethodProperty = (Func_GetMethodProperty)lib.Lib.GetProc("GetMethodProperty"); + MY_GET_FUNC (lib.CreateDecoder, Func_CreateDecoder, lib.Lib.GetProc("CreateDecoder")); + MY_GET_FUNC (lib.CreateEncoder, Func_CreateEncoder, lib.Lib.GetProc("CreateEncoder")); + MY_GET_FUNC (lib.GetMethodProperty, Func_GetMethodProperty, lib.Lib.GetProc("GetMethodProperty")); if (lib.GetMethodProperty) { UInt32 numMethods = 1; - Func_GetNumberOfMethods getNumberOfMethods = (Func_GetNumberOfMethods)lib.Lib.GetProc("GetNumberOfMethods"); + Func_GetNumberOfMethods getNumberOfMethods; + MY_GET_FUNC (getNumberOfMethods, Func_GetNumberOfMethods, lib.Lib.GetProc("GetNumberOfMethods")); if (getNumberOfMethods) { RINOK(getNumberOfMethods(&numMethods)); @@ -272,7 +283,8 @@ HRESULT CCodecs::LoadCodecs() } } - Func_GetHashers getHashers = (Func_GetHashers)lib.Lib.GetProc("GetHashers"); + Func_GetHashers getHashers; + MY_GET_FUNC (getHashers, Func_GetHashers, lib.Lib.GetProc("GetHashers")); if (getHashers) { RINOK(getHashers(&lib.ComHashers)); @@ -381,14 +393,17 @@ HRESULT CCodecs::LoadFormats() const NDLL::CLibrary &lib = Libs.Back().Lib; Func_GetHandlerProperty getProp = NULL; - Func_GetHandlerProperty2 getProp2 = (Func_GetHandlerProperty2)lib.GetProc("GetHandlerProperty2"); - Func_GetIsArc getIsArc = (Func_GetIsArc)lib.GetProc("GetIsArc"); + Func_GetHandlerProperty2 getProp2; + MY_GET_FUNC (getProp2, Func_GetHandlerProperty2, lib.GetProc("GetHandlerProperty2")); + Func_GetIsArc getIsArc; + MY_GET_FUNC (getIsArc, Func_GetIsArc, lib.GetProc("GetIsArc")); UInt32 numFormats = 1; if (getProp2) { - Func_GetNumberOfFormats getNumberOfFormats = (Func_GetNumberOfFormats)lib.GetProc("GetNumberOfFormats"); + Func_GetNumberOfFormats getNumberOfFormats; + MY_GET_FUNC (getNumberOfFormats, Func_GetNumberOfFormats, lib.GetProc("GetNumberOfFormats")); if (getNumberOfFormats) { RINOK(getNumberOfFormats(&numFormats)); @@ -396,7 +411,7 @@ HRESULT CCodecs::LoadFormats() } else { - getProp = (Func_GetHandlerProperty)lib.GetProc("GetHandlerProperty"); + MY_GET_FUNC (getProp, Func_GetHandlerProperty, lib.GetProc("GetHandlerProperty")); if (!getProp) return S_OK; } @@ -404,7 +419,7 @@ HRESULT CCodecs::LoadFormats() for (UInt32 i = 0; i < numFormats; i++) { CArcInfoEx item; - item.LibIndex = Libs.Size() - 1; + item.LibIndex = (int)(Libs.Size() - 1); item.FormatIndex = i; RINOK(GetProp_String(getProp, getProp2, i, NArchive::NHandlerPropID::kName, item.Name)); @@ -417,7 +432,7 @@ HRESULT CCodecs::LoadFormats() continue; if (::SysStringByteLen(prop.bstrVal) != sizeof(GUID)) return E_FAIL; - item.ClassID = *(const GUID *)prop.bstrVal; + item.ClassID = *(const GUID *)(const void *)prop.bstrVal; prop.Clear(); } @@ -473,23 +488,53 @@ extern "C" } #endif + +void CCodecs::AddLastError(const FString &path) +{ + HRESULT res = GetLastError_noZero_HRESULT(); + CCodecError &error = Errors.AddNew(); + error.Path = path; + error.ErrorCode = res; +} + HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loadedOK) { if (loadedOK) *loadedOK = false; + // needCheckDll = 1; + + #ifdef _WIN32 if (needCheckDll) { NDLL::CLibrary lib; if (!lib.LoadEx(dllPath, LOAD_LIBRARY_AS_DATAFILE)) + { + /* if is not win32 + // %1 is not a valid Win32 application. + // #define ERROR_BAD_EXE_FORMAT 193L + */ + // return GetLastError_noZero_HRESULT(); + DWORD lastError = GetLastError(); + if (lastError != ERROR_BAD_EXE_FORMAT) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.Message = "cannot load file as datafile library"; + error.ErrorCode = HRESULT_FROM_WIN32(lastError); + } return S_OK; + } } + #else + UNUSED_VAR(needCheckDll) + #endif Libs.AddNew(); CCodecLib &lib = Libs.Back(); lib.Path = dllPath; bool used = false; - HRESULT res = S_OK; + // HRESULT res = S_OK; if (lib.Lib.Load(dllPath)) { @@ -499,10 +544,28 @@ HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loaded lib.LoadIcons(); #endif + /* + { + Func_LibStartup _LibStartup; + MY_GET_FUNC (_LibStartup, Func_LibStartup, lib.Lib.GetProc("LibStartup")); + if (_LibStartup) + { + HRESULT res = _LibStartup(); + if (res != 0) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.ErrorCode = res; + } + } + } + */ + #ifdef _7ZIP_LARGE_PAGES if (g_LargePageSize != 0) { - Func_SetLargePageMode setLargePageMode = (Func_SetLargePageMode)lib.Lib.GetProc("SetLargePageMode"); + Func_SetLargePageMode setLargePageMode; + MY_GET_FUNC (setLargePageMode, Func_SetLargePageMode, lib.Lib.GetProc("SetLargePageMode")); if (setLargePageMode) setLargePageMode(); } @@ -510,16 +573,18 @@ HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loaded if (CaseSensitiveChange) { - Func_SetCaseSensitive setCaseSensitive = (Func_SetCaseSensitive)lib.Lib.GetProc("SetCaseSensitive"); + Func_SetCaseSensitive setCaseSensitive; + MY_GET_FUNC (setCaseSensitive, Func_SetCaseSensitive, lib.Lib.GetProc("SetCaseSensitive")); if (setCaseSensitive) setCaseSensitive(CaseSensitive ? 1 : 0); } - lib.CreateObject = (Func_CreateObject)lib.Lib.GetProc("CreateObject"); + MY_GET_FUNC (lib.CreateObject, Func_CreateObject, lib.Lib.GetProc("CreateObject")); { unsigned startSize = Codecs.Size() + Hashers.Size(); - res = LoadCodecs(); - used = (startSize != Codecs.Size() + Hashers.Size()); + HRESULT res = LoadCodecs(); + if (startSize != Codecs.Size() + Hashers.Size()) + used = true; if (res == S_OK && lib.CreateObject) { startSize = Formats.Size(); @@ -527,22 +592,61 @@ HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loaded if (startSize != Formats.Size()) used = true; } + if (res != S_OK) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.ErrorCode = res; + } } + // plugins can use non-7-zip dlls, so we silently ignore non7zip DLLs + /* + if (!used) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.Message = "no 7-Zip code"; + } + */ + } + else + { + AddLastError(dllPath); } if (!used) Libs.DeleteBack(); - return res; + return S_OK; } -HRESULT CCodecs::LoadDllsFromFolder(const FString &folderPrefix) +HRESULT CCodecs::LoadDllsFromFolder(const FString &folderPath) { + if (!NFile::NFind::DoesDirExist_FollowLink(folderPath)) + // if (!NFile::NFind::DoesDirExist(folderPath)) + { + // AddLastError(folderPath); + return S_OK; + } + + FString folderPrefix = folderPath; + folderPrefix.Add_PathSepar(); + NFile::NFind::CEnumerator enumerator; enumerator.SetDirPrefix(folderPrefix); - NFile::NFind::CFileInfo fi; - while (enumerator.Next(fi)) + NFile::NFind::CDirEntry fi; + for (;;) { + bool found; + if (!enumerator.Next(fi, found)) + { + // it can be wrong Symbolic link to folder here + AddLastError(folderPath); + break; + // return GetLastError_noZero_HRESULT(); + } + if (!found) + break; if (fi.IsDir()) continue; RINOK(LoadDll(folderPrefix + fi.Name, true)); @@ -585,6 +689,7 @@ HRESULT CCodecs::Load() Formats.Clear(); #ifdef EXTERNAL_CODECS + Errors.Clear(); MainDll_ErrorPath.Empty(); Codecs.Clear(); Hashers.Clear(); @@ -627,6 +732,8 @@ HRESULT CCodecs::Load() Formats.Add(item); } + // printf("\nLoad codecs \n"); + #ifdef EXTERNAL_CODECS const FString baseFolder = GetBaseFolderPrefixFromRegistry(); { @@ -635,8 +742,8 @@ HRESULT CCodecs::Load() if (!loadedOK) MainDll_ErrorPath = kMainDll; } - RINOK(LoadDllsFromFolder(baseFolder + kCodecsFolderName FSTRING_PATH_SEPARATOR)); - RINOK(LoadDllsFromFolder(baseFolder + kFormatsFolderName FSTRING_PATH_SEPARATOR)); + RINOK(LoadDllsFromFolder(baseFolder + kCodecsFolderName)); + RINOK(LoadDllsFromFolder(baseFolder + kFormatsFolderName)); NeedSetLibCodecs = true; @@ -659,7 +766,7 @@ HRESULT CCodecs::Load() FOR_VECTOR(i, Libs) { CCodecLib &lib = Libs[i]; - lib.SetCodecs = (Func_SetCodecs)lib.Lib.GetProc("SetCodecs"); + MY_GET_FUNC (lib.SetCodecs, Func_SetCodecs, lib.Lib.GetProc("SetCodecs")); if (lib.SetCodecs) { RINOK(lib.SetCodecs(this)); @@ -679,7 +786,7 @@ int CCodecs::FindFormatForArchiveName(const UString &arcPath) const int dotPos = arcPath.ReverseFind_Dot(); if (dotPos <= arcPath.ReverseFind_PathSepar()) return -1; - const UString ext = arcPath.Ptr(dotPos + 1); + const UString ext = arcPath.Ptr((unsigned)(dotPos + 1)); if (ext.IsEmpty()) return -1; if (ext.IsEqualTo_Ascii_NoCase("exe")) @@ -692,7 +799,7 @@ int CCodecs::FindFormatForArchiveName(const UString &arcPath) const continue; */ if (arc.FindExtension(ext) >= 0) - return i; + return (int)i; } return -1; } @@ -703,7 +810,7 @@ int CCodecs::FindFormatForExtension(const UString &ext) const return -1; FOR_VECTOR (i, Formats) if (Formats[i].FindExtension(ext) >= 0) - return i; + return (int)i; return -1; } @@ -711,7 +818,7 @@ int CCodecs::FindFormatForArchiveType(const UString &arcType) const { FOR_VECTOR (i, Formats) if (Formats[i].Name.IsEqualTo_NoCase(arcType)) - return i; + return (int)i; return -1; } @@ -722,8 +829,8 @@ bool CCodecs::FindFormatForArchiveType(const UString &arcType, CIntVector &forma { int pos2 = arcType.Find(L'.', pos); if (pos2 < 0) - pos2 = arcType.Len(); - const UString name = arcType.Mid(pos, pos2 - pos); + pos2 = (int)arcType.Len(); + const UString name = arcType.Mid(pos, (unsigned)pos2 - pos); if (name.IsEmpty()) return false; int index = FindFormatForArchiveType(name); @@ -733,7 +840,7 @@ bool CCodecs::FindFormatForArchiveType(const UString &arcType, CIntVector &forma return false; } formatIndices.Add(index); - pos = pos2 + 1; + pos = (unsigned)pos2 + 1; } return true; } @@ -756,19 +863,19 @@ void CCodecIcons::LoadIcons(HMODULE m) CIconPair iconPair; iconPair.IconIndex = -1; if (pos < 0) - pos = s.Len(); + pos = (int)s.Len(); else { - UString num = s.Ptr(pos + 1); + const UString num = s.Ptr((unsigned)pos + 1); if (!num.IsEmpty()) { const wchar_t *end; - iconPair.IconIndex = ConvertStringToUInt32(num, &end); + iconPair.IconIndex = (int)ConvertStringToUInt32(num, &end); if (*end != 0) continue; } } - iconPair.Ext = s.Left(pos); + iconPair.Ext = s.Left((unsigned)pos); IconPairs.Add(iconPair); } } @@ -946,7 +1053,7 @@ int CCodecs::GetCodec_LibIndex(UInt32 index) const #ifdef EXTERNAL_CODECS const CDllCodecInfo &ci = Codecs[index - NUM_EXPORT_CODECS]; - return ci.LibIndex; + return (int)ci.LibIndex; #else return -1; #endif @@ -961,7 +1068,7 @@ int CCodecs::GetHasherLibIndex(UInt32 index) #ifdef EXTERNAL_CODECS const CDllHasherInfo &ci = Hashers[index - NUM_EXPORT_HASHERS]; - return ci.LibIndex; + return (int)ci.LibIndex; #else return -1; #endif @@ -1014,7 +1121,8 @@ bool CCodecs::GetCodec_EncoderIsAssigned(UInt32 index) const UInt32 CCodecs::GetCodec_NumStreams(UInt32 index) { NCOM::CPropVariant prop; - RINOK(GetProperty(index, NMethodPropID::kPackStreams, &prop)); + if (GetProperty(index, NMethodPropID::kPackStreams, &prop) != S_OK) + return 0; if (prop.vt == VT_UI4) return (UInt32)prop.ulVal; if (prop.vt == VT_EMPTY) @@ -1065,10 +1173,33 @@ AString CCodecs::GetHasherName(UInt32 index) UInt32 CCodecs::GetHasherDigestSize(UInt32 index) { NCOM::CPropVariant prop; - RINOK(GetHasherProp(index, NMethodPropID::kDigestSize, &prop)); + if (GetHasherProp(index, NMethodPropID::kDigestSize, &prop) != S_OK) + return 0; if (prop.vt != VT_UI4) return 0; return prop.ulVal; } +void CCodecs::GetCodecsErrorMessage(UString &s) +{ + s.Empty(); + FOR_VECTOR (i, Errors) + { + const CCodecError &ce = Errors[i]; + s += "Codec Load Error: "; + s += fs2us(ce.Path); + if (ce.ErrorCode != 0) + { + s += " : "; + s += NWindows::NError::MyFormatMessage(ce.ErrorCode); + } + if (!ce.Message.IsEmpty()) + { + s += " : "; + s += ce.Message; + } + s.Add_LF(); + } +} + #endif // EXTERNAL_CODECS diff --git a/CPP/7zip/UI/Common/LoadCodecs.h b/CPP/7zip/UI/Common/LoadCodecs.h index ac9eeac7..660ddee4 100644 --- a/CPP/7zip/UI/Common/LoadCodecs.h +++ b/CPP/7zip/UI/Common/LoadCodecs.h @@ -132,7 +132,8 @@ struct CArcInfoEx bool Flags_BackwardOpen() const { return (Flags & NArcInfoFlags::kBackwardOpen) != 0; } bool Flags_PreArc() const { return (Flags & NArcInfoFlags::kPreArc) != 0; } bool Flags_PureStartOpen() const { return (Flags & NArcInfoFlags::kPureStartOpen) != 0; } - + bool Flags_ByExtOnlyOpen() const { return (Flags & NArcInfoFlags::kByExtOnlyOpen) != 0; } + UString GetMainExt() const { if (Exts.IsEmpty()) @@ -227,6 +228,13 @@ struct CCodecLib #endif +struct CCodecError +{ + FString Path; + HRESULT ErrorCode; + AString Message; + CCodecError(): ErrorCode(0) {} +}; class CCodecs: #ifdef EXTERNAL_CODECS @@ -243,7 +251,9 @@ public: CObjectVector<CCodecLib> Libs; FString MainDll_ErrorPath; - + CObjectVector<CCodecError> Errors; + + void AddLastError(const FString &path); void CloseLibs(); class CReleaser @@ -272,7 +282,7 @@ public: HRESULT CreateArchiveHandler(const CArcInfoEx &ai, bool outHandler, void **archive) const { - return Libs[ai.LibIndex].CreateObject(&ai.ClassID, outHandler ? &IID_IOutArchive : &IID_IInArchive, (void **)archive); + return Libs[(unsigned)ai.LibIndex].CreateObject(&ai.ClassID, outHandler ? &IID_IOutArchive : &IID_IInArchive, (void **)archive); } #endif @@ -306,11 +316,11 @@ public: const wchar_t *GetFormatNamePtr(int formatIndex) const { - return formatIndex < 0 ? L"#" : (const wchar_t *)Formats[formatIndex].Name; + return formatIndex < 0 ? L"#" : (const wchar_t *)Formats[(unsigned)formatIndex].Name; } HRESULT Load(); - + #ifndef _SFX int FindFormatForArchiveName(const UString &arcPath) const; int FindFormatForExtension(const UString &ext) const; @@ -352,6 +362,8 @@ public: AString GetHasherName(UInt32 index); UInt32 GetHasherDigestSize(UInt32 index); + void GetCodecsErrorMessage(UString &s); + #endif HRESULT CreateInArchive(unsigned formatIndex, CMyComPtr<IInArchive> &archive) const @@ -399,7 +411,7 @@ public: if (!arc.UpdateEnabled) continue; if (arc.Name.IsEqualTo_NoCase(name)) - return i; + return (int)i; } return -1; } diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index 419c29ee..7bec9d53 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -14,6 +14,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../../Windows/FileDir.h" @@ -32,11 +33,13 @@ #include "SetProperties.h" #endif +#ifndef _SFX #ifdef SHOW_DEBUG_INFO #define PRF(x) x #else #define PRF(x) #endif +#endif // increase it, if you need to support larger SFX stubs static const UInt64 kMaxCheckStartPosition = 1 << 23; @@ -64,7 +67,7 @@ Open: - open FAIL: Try to open with all other types from offset 0 only. If some open type is OK and physical archive size is uequal or larger - than file size, then return that archive with warning that can not be open as [extension type]. + than file size, then return that archive with warning that cannot be open as [extension type]. If extension was EXE, it will try to open as unknown_extension case - file has unknown extension (like a.hhh) It tries to open via parser code. @@ -141,14 +144,14 @@ struct CParseItem bool LenIsUnknown; CParseItem(): - LenIsUnknown(false), + // OkSize(0), FileTime_Defined(false), UnpackSize_Defined(false), - NumSubFiles_Defined(false), NumSubDirs_Defined(false), + NumSubFiles_Defined(false), IsSelfExe(false), - IsNotArcType(false) - // OkSize(0) + IsNotArcType(false), + LenIsUnknown(false) {} /* @@ -214,15 +217,17 @@ int CHandler::FindInsertPos(const CParseItem &item) const left = mid + 1; else if (item.Size < midItem.Size) right = mid; + /* else if (item.Size > midItem.Size) left = mid + 1; + */ else { left = mid + 1; // return -1; } } - return left; + return (int)left; } void CHandler::AddUnknownItem(UInt64 next) @@ -260,7 +265,7 @@ void CHandler::AddItem(const CParseItem &item) int pos = FindInsertPos(item); if (pos >= 0) { - _items.Insert(pos, item); + _items.Insert((unsigned)pos, item); UInt64 next = item.Offset + item.Size; if (_maxEndOffset < next) _maxEndOffset = next; @@ -401,7 +406,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; - Int32 index = allFilesMode ? i : indices[i]; + UInt32 index = allFilesMode ? i : indices[i]; const CParseItem &item = _items[index]; RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); @@ -417,7 +422,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, outStreamSpec->Init(skipMode ? 0 : unpackSize, true); Int32 opRes = NExtract::NOperationResult::kOK; - RINOK(_stream->Seek(item.Offset, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)item.Offset, STREAM_SEEK_SET, NULL)); streamSpec->Init(unpackSize); RINOK(copyCoder->Code(inStream, outStream, NULL, NULL, progress)); @@ -496,13 +501,14 @@ static HRESULT Archive_GetArcProp_UInt(IInArchive *arc, PROPID propid, UInt64 &r RINOK(arc->GetArchiveProperty(propid, &prop)); switch (prop.vt) { - case VT_UI4: result = prop.ulVal; defined = true; break; - case VT_I4: result = (Int64)prop.lVal; defined = true; break; - case VT_UI8: result = (UInt64)prop.uhVal.QuadPart; defined = true; break; - case VT_I8: result = (UInt64)prop.hVal.QuadPart; defined = true; break; - case VT_EMPTY: break; + case VT_UI4: result = prop.ulVal; break; + case VT_I4: result = (UInt64)(Int64)prop.lVal; break; + case VT_UI8: result = (UInt64)prop.uhVal.QuadPart; break; + case VT_I8: result = (UInt64)prop.hVal.QuadPart; break; + case VT_EMPTY: return S_OK; default: return E_FAIL; } + defined = true; return S_OK; } @@ -513,13 +519,14 @@ static HRESULT Archive_GetArcProp_Int(IInArchive *arc, PROPID propid, Int64 &res RINOK(arc->GetArchiveProperty(propid, &prop)); switch (prop.vt) { - case VT_UI4: result = prop.ulVal; defined = true; break; - case VT_I4: result = prop.lVal; defined = true; break; - case VT_UI8: result = (Int64)prop.uhVal.QuadPart; defined = true; break; - case VT_I8: result = (Int64)prop.hVal.QuadPart; defined = true; break; - case VT_EMPTY: break; + case VT_UI4: result = prop.ulVal; break; + case VT_I4: result = prop.lVal; break; + case VT_UI8: result = (Int64)prop.uhVal.QuadPart; break; + case VT_I8: result = (Int64)prop.hVal.QuadPart; break; + case VT_EMPTY: return S_OK; default: return E_FAIL; } + defined = true; return S_OK; } @@ -607,6 +614,8 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa #endif + + HRESULT CArc::GetItemPath(UInt32 index, UString &result) const { #ifdef MY_CPU_LE @@ -621,19 +630,42 @@ HRESULT CArc::GetItemPath(UInt32 index, UString &result) const propType == NPropDataType::kUtf16z) { unsigned len = size / 2 - 1; + // (len) doesn't include null terminator + + /* + #if WCHAR_MAX > 0xffff + len = (unsigned)Utf16LE__Get_Num_WCHARs(p, len); + + wchar_t *s = result.GetBuf(len); + wchar_t *sEnd = Utf16LE__To_WCHARs_Sep(p, len, s); + if (s + len != sEnd) return E_FAIL; + *sEnd = 0; + + #else + */ + wchar_t *s = result.GetBuf(len); for (unsigned i = 0; i < len; i++) { wchar_t c = GetUi16(p); p = (const void *)((const Byte *)p + 2); + #if WCHAR_PATH_SEPARATOR != L'/' if (c == L'/') c = WCHAR_PATH_SEPARATOR; + else if (c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme #endif + *s++ = c; } *s = 0; + + // #endif + result.ReleaseBuf_SetLen(len); + + Convert_UnicodeEsc16_To_UnicodeEscHigh(result); if (len != 0) return S_OK; } @@ -721,6 +753,8 @@ HRESULT CArc::GetItemPath(UInt32 index, UString &result) const if (result.IsEmpty()) return GetDefaultItemPath(index, result); + + Convert_UnicodeEsc16_To_UnicodeEscHigh(result); return S_OK; } @@ -772,7 +806,7 @@ int FindAltStreamColon_in_Path(const wchar_t *path) if (c == ':') { if (colonPos < 0) - colonPos = i; + colonPos = (int)i; continue; } if (c == WCHAR_PATH_SEPARATOR) @@ -865,8 +899,8 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const int colon = FindAltStreamColon_in_Path(item.Path); if (colon >= 0) { - item.MainPath.DeleteFrom(colon); - item.AltStreamName = item.Path.Ptr(colon + 1); + item.MainPath.DeleteFrom((unsigned)colon); + item.AltStreamName = item.Path.Ptr((unsigned)(colon + 1)); item.MainIsDir = (colon == 0 || IsPathSepar(item.Path[(unsigned)colon - 1])); item.IsAltStream = true; } @@ -877,7 +911,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const #ifndef _SFX if (item._use_baseParentFolder_mode) { - RINOK(GetItemPathToParent(mainIndex, item._baseParentFolder, item.PathParts)); + RINOK(GetItemPathToParent(mainIndex, (unsigned)item._baseParentFolder, item.PathParts)); #ifdef SUPPORT_ALT_STREAMS if ((item.WriteToAltStreamIfColon || needFindAltStream) && !item.PathParts.IsEmpty()) @@ -888,10 +922,10 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const colon = FindAltStreamColon_in_Path(s); if (colon >= 0) { - item.AltStreamName = s.Ptr(colon + 1); + item.AltStreamName = s.Ptr((unsigned)(colon + 1)); item.MainIsDir = (colon == 0 || IsPathSepar(s[(unsigned)colon - 1])); item.IsAltStream = true; - s.DeleteFrom(colon); + s.DeleteFrom((unsigned)colon); } } if (colon == 0) @@ -1007,9 +1041,9 @@ static void MakeCheckOrder(CCodecs *codecs, FOR_VECTOR (k, sigs) { const CByteBuffer &sig = sigs[k]; - if (sig.Size() == 0 && dataSize == 0 || - sig.Size() != 0 && sig.Size() <= dataSize && - TestSignature(data, sig, sig.Size())) + if ((sig.Size() == 0 && dataSize == 0) + || (sig.Size() != 0 && sig.Size() <= dataSize + && TestSignature(data, sig, sig.Size()))) { orderIndices2.Add(index); orderIndices[i] = -1; @@ -1019,8 +1053,6 @@ static void MakeCheckOrder(CCodecs *codecs, } } -#endif - #ifdef UNDER_CE static const unsigned kNumHashBytes = 1; #define HASH_VAL(buf) ((buf)[0]) @@ -1030,9 +1062,6 @@ static void MakeCheckOrder(CCodecs *codecs, #define HASH_VAL(buf) GetUi16(buf) #endif - -#ifndef _SFX - static bool IsExeExt(const UString &ext) { return ext.IsEqualTo_Ascii_NoCase("exe"); @@ -1243,11 +1272,11 @@ HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openR bool offsetDefined; RINOK(Archive_GetArcProp_Int(archive, kpidOffset, Offset, offsetDefined)); - Int64 globalOffset = startPos + Offset; - AvailPhySize = FileSize - globalOffset; + Int64 globalOffset = (Int64)startPos + Offset; + AvailPhySize = (UInt64)((Int64)FileSize - globalOffset); if (PhySizeDefined) { - UInt64 endPos = globalOffset + PhySize; + UInt64 endPos = (UInt64)(globalOffset + (Int64)PhySize); if (endPos < FileSize) { AvailPhySize = PhySize; @@ -1263,11 +1292,12 @@ HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openR } /* -static PrintNumber(const char *s, int n) +static void PrintNumber(const char *s, int n) { char temp[100]; sprintf(temp, "%s %d", s, n); - OutputDebugStringA(temp); + // OutputDebugStringA(temp); + printf(temp); } */ @@ -1286,7 +1316,7 @@ HRESULT CArc::PrepareToOpen(const COpenOptions &op, unsigned formatIndex, CMyCom { const CArcInfoEx &ai = op.codecs->Formats[formatIndex]; if (ai.LibIndex >= 0 ? - !op.codecs->Libs[ai.LibIndex].SetCodecs : + !op.codecs->Libs[(unsigned)ai.LibIndex].SetCodecs : !op.codecs->Libs.IsEmpty()) { CMyComPtr<ISetCompressCodecsInfo> setCompressCodecsInfo; @@ -1437,7 +1467,7 @@ HRESULT CArc::CheckZerosTail(const COpenOptions &op, UInt64 offset) { if (!op.stream) return S_OK; - RINOK(op.stream->Seek(offset, STREAM_SEEK_SET, NULL)); + RINOK(op.stream->Seek((Int64)offset, STREAM_SEEK_SET, NULL)); const UInt32 kBufSize = 1 << 11; Byte buf[kBufSize]; @@ -1463,6 +1493,8 @@ HRESULT CArc::CheckZerosTail(const COpenOptions &op, UInt64 offset) } } + + #ifndef _SFX class CExtractCallback_To_OpenCallback: @@ -1510,7 +1542,7 @@ STDMETHODIMP CExtractCallback_To_OpenCallback::SetRatioInfo(const UInt64 *inSize STDMETHODIMP CExtractCallback_To_OpenCallback::GetStream(UInt32 /* index */, ISequentialOutStream **outStream, Int32 /* askExtractMode */) { - *outStream = 0; + *outStream = NULL; return S_OK; } @@ -1524,6 +1556,7 @@ STDMETHODIMP CExtractCallback_To_OpenCallback::SetOperationResult(Int32 /* opera return S_OK; } + static HRESULT OpenArchiveSpec(IInArchive *archive, bool needPhySize, IInStream *stream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *openCallback, @@ -1547,22 +1580,32 @@ static HRESULT OpenArchiveSpec(IInArchive *archive, bool needPhySize, if (phySize_Defined) return S_OK; - bool phySizeCantBeDetected = false;; + bool phySizeCantBeDetected = false; RINOK(Archive_GetArcBoolProp(archive, kpidPhySizeCantBeDetected, phySizeCantBeDetected)); if (!phySizeCantBeDetected) { - RINOK(archive->Extract(0, (UInt32)(Int32)-1, BoolToInt(true), extractCallback)); + PRF(printf("\n-- !phySize_Defined after Open, call archive->Extract()")); + // It's for bzip2/gz and some xz archives, where Open operation doesn't know phySize. + // But the Handler will know phySize after full archive testing. + RINOK(archive->Extract(NULL, (UInt32)(Int32)-1, BoolToInt(true), extractCallback)); + PRF(printf("\n-- OK")); } } return S_OK; } + + static int FindFormatForArchiveType(CCodecs *codecs, CIntVector orderIndices, const char *name) { FOR_VECTOR (i, orderIndices) - if (StringsAreEqualNoCase_Ascii(codecs->Formats[orderIndices[i]].Name, name)) - return i; + { + int oi = orderIndices[i]; + if (oi >= 0) + if (StringsAreEqualNoCase_Ascii(codecs->Formats[(unsigned)oi].Name, name)) + return (int)i; + } return -1; } @@ -1590,7 +1633,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { int dotPos = fileName.ReverseFind_Dot(); if (dotPos >= 0) - extension = fileName.Ptr(dotPos + 1); + extension = fileName.Ptr((unsigned)(dotPos + 1)); } CIntVector orderIndices; @@ -1615,13 +1658,18 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool isUnknownExt = false; #endif + #ifndef _SFX bool isForced = false; + #endif + unsigned numMainTypes = 0; int formatIndex = op.openType.FormatIndex; if (formatIndex >= 0) { + #ifndef _SFX isForced = true; + #endif orderIndices.Add(formatIndex); numMainTypes = 1; isMainFormatArr[(unsigned)formatIndex] = true; @@ -1658,10 +1706,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) isNumber = true; } if (isNumber) + { if (c == 'z' || c == 'Z') isZip = true; else isRar = true; + } } #endif @@ -1673,7 +1723,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (IgnoreSplit || !op.openType.CanReturnArc) if (ai.IsSplit()) continue; - if (op.excludedFormats->FindInSorted(i) >= 0) + if (op.excludedFormats->FindInSorted((int)i) >= 0) continue; #ifndef _SFX @@ -1683,17 +1733,17 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (ai.FindExtension(extension) >= 0 #ifndef _SFX - || isZip && StringsAreEqualNoCase_Ascii(ai.Name, "zip") - || isRar && StringsAreEqualNoCase_Ascii(ai.Name, "rar") + || (isZip && StringsAreEqualNoCase_Ascii(ai.Name, "zip")) + || (isRar && StringsAreEqualNoCase_Ascii(ai.Name, "rar")) #endif ) { // PrintNumber("orderIndices.Insert", i); - orderIndices.Insert(numFinded++, i); + orderIndices.Insert(numFinded++, (int)i); isMainFormatArr[i] = true; } else - orderIndices.Add(i); + orderIndices.Add((int)i); } } @@ -1739,8 +1789,8 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) const Byte kRarHeader[] = { 0x52 , 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00 }; if (TestSignature(buf, kRarHeader, 7) && buf[9] == 0x73 && (buf[10] & 1) != 0) { - orderIndices2.Add(orderIndices[i]); - orderIndices[i] = -1; + orderIndices2.Add(orderIndices[(unsigned)i]); + orderIndices[(unsigned)i] = -1; if (i >= (int)numFinded) numFinded++; } @@ -1785,10 +1835,10 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) int iUdf = FindFormatForArchiveType(op.codecs, orderIndices, "udf"); if (iUdf > iIso && iIso >= 0) { - int isoIndex = orderIndices[iIso]; - int udfIndex = orderIndices[iUdf]; - orderIndices[iUdf] = isoIndex; - orderIndices[iIso] = udfIndex; + int isoIndex = orderIndices[(unsigned)iIso]; + int udfIndex = orderIndices[(unsigned)iUdf]; + orderIndices[(unsigned)iUdf] = isoIndex; + orderIndices[(unsigned)iIso] = udfIndex; } } @@ -1842,12 +1892,14 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (unsigned i = 0; i < numCheckTypes; i++) { FormatIndex = orderIndices[i]; + + // orderIndices[] item cannot be negative here bool exactOnly = false; #ifndef _SFX - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; // OutputDebugStringW(ai.Name); if (i >= numMainTypes) { @@ -1871,7 +1923,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) CMyComPtr<IInArchive> archive; - RINOK(PrepareToOpen(op, FormatIndex, archive)); + RINOK(PrepareToOpen(op, (unsigned)FormatIndex, archive)); if (!archive) continue; @@ -1948,7 +2000,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool thereIsTail = ErrorInfo.ThereIsTail; if (thereIsTail && mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)(Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) thereIsTail = false; } @@ -2063,16 +2115,22 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (i = 0; i < orderIndices.Size(); i++) { - unsigned form = orderIndices[i]; + // orderIndices[] item cannot be negative here + unsigned form = (unsigned)orderIndices[i]; if (skipFrontalFormat[form]) continue; + const CArcInfoEx &ai = op.codecs->Formats[form]; + if (ai.IsSplit()) { - splitIndex = form; + splitIndex = (int)form; continue; } + if (ai.Flags_ByExtOnlyOpen()) + continue; + if (ai.IsArcFunc) { UInt32 isArcRes = ai.IsArcFunc(byteBuffer, processedSize); @@ -2118,12 +2176,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } if (splitIndex >= 0) - sortedFormats.Insert(0, splitIndex); + sortedFormats.Insert(0, (unsigned)splitIndex); for (i = 0; i < sortedFormats.Size(); i++) { - FormatIndex = sortedFormats[i]; - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + FormatIndex = (int)sortedFormats[i]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; if (op.callback) RINOK(op.callback->SetTotal(NULL, &fileSize)); @@ -2131,7 +2189,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) RINOK(op.stream->Seek(0, STREAM_SEEK_SET, NULL)); CMyComPtr<IInArchive> archive; - RINOK(PrepareToOpen(op, FormatIndex, archive)); + RINOK(PrepareToOpen(op, (unsigned)FormatIndex, archive)); if (!archive) continue; @@ -2144,7 +2202,10 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) result = archive->Open(op.stream, &searchLimit, op.callback); else */ - result = OpenArchiveSpec(archive, !mode.CanReturnArc, op.stream, &searchLimit, op.callback, extractCallback_To_OpenCallback); + // if (!CanReturnArc), it's ParserMode, and we need phy size + result = OpenArchiveSpec(archive, + !mode.CanReturnArc, // needPhySize + op.stream, &searchLimit, op.callback, extractCallback_To_OpenCallback); } if (result == S_FALSE) @@ -2166,7 +2227,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } NArchive::NParser::CParseItem pi; - pi.Offset = Offset; + pi.Offset = (UInt64)Offset; pi.Size = AvailPhySize; // bool needScan = false; @@ -2203,7 +2264,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { if (mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)(Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) openCur = true; } @@ -2299,6 +2360,8 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (index < 0) continue; const CArcInfoEx &ai = op.codecs->Formats[(unsigned)index]; + if (ai.Flags_ByExtOnlyOpen()) + continue; bool isDifficult = false; // if (ai.Version < 0x91F) // we don't use parser with old DLL (before 9.31) if (!ai.NewInterface) @@ -2329,7 +2392,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } if (isDifficult) { - difficultFormats.Add(index); + difficultFormats.Add((unsigned)index); difficultBools[(unsigned)index] = true; } } @@ -2398,7 +2461,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) size_t processedSize = kBufSize - bytesInBuf; // printf("\nRead ask = %d", (unsigned)processedSize); UInt64 seekPos = bufPhyPos + bytesInBuf; - RINOK(op.stream->Seek(bufPhyPos + bytesInBuf, STREAM_SEEK_SET, NULL)); + RINOK(op.stream->Seek((Int64)(bufPhyPos + bytesInBuf), STREAM_SEEK_SET, NULL)); RINOK(ReadStream(op.stream, byteBuffer + bytesInBuf, &processedSize)); // printf(" processed = %d", (unsigned)processedSize); if (processedSize == 0) @@ -2471,7 +2534,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } } - size_t availSize = bytesInBuf - (size_t)posInBuf; + const size_t availSize = bytesInBuf - (size_t)posInBuf; if (availSize < kNumHashBytes) break; size_t scanSize = availSize - @@ -2502,7 +2565,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (!needCheckStartOpen) { for (; buf < bufLimit && hash[HASH_VAL(buf)] == 0xFF; buf++); - ppp = buf - (byteBuffer + (size_t)posInBuf); + ppp = (size_t)(buf - (byteBuffer + (size_t)posInBuf)); pos += ppp; if (buf == bufLimit) continue; @@ -2599,13 +2662,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) // printf("\nIsArc OK: %S", (const wchar_t *)ai.Name); } - /* - if (pos == 67109888) - pos = pos; - */ PRF(printf("\npos = %9I64d : %S", pos, (const wchar_t *)ai.Name)); - bool isMainFormat = isMainFormatArr[index]; + const bool isMainFormat = isMainFormatArr[index]; const COpenSpecFlags &specFlags = mode.GetSpec(isForced, isMainFormat, isUnknownExt); CMyComPtr<IInArchive> archive; @@ -2615,14 +2674,14 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) // OutputDebugStringW(ai.Name); - UInt64 rem = fileSize - startArcPos; + const UInt64 rem = fileSize - startArcPos; UInt64 arcStreamOffset = 0; if (ai.Flags_UseGlobalOffset()) { limitedStreamSpec->InitAndSeek(0, fileSize); - limitedStream->Seek(startArcPos, STREAM_SEEK_SET, NULL); + limitedStream->Seek((Int64)startArcPos, STREAM_SEEK_SET, NULL); } else { @@ -2642,20 +2701,23 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) extractCallback_To_OpenCallback_Spec->Files = 0; extractCallback_To_OpenCallback_Spec->Offset = startArcPos; - HRESULT result = OpenArchiveSpec(archive, true, limitedStream, &maxCheckStartPosition, + HRESULT result = OpenArchiveSpec(archive, + true, // needPhySize + limitedStream, &maxCheckStartPosition, useOffsetCallback ? (IArchiveOpenCallback *)openCallback_Offset : (IArchiveOpenCallback *)op.callback, extractCallback_To_OpenCallback); RINOK(ReadBasicProps(archive, ai.Flags_UseGlobalOffset() ? 0 : startArcPos, result)); bool isOpen = false; + if (result == S_FALSE) { if (!mode.CanReturnParser) { if (formatIndex < 0 && ErrorInfo.IsArc_After_NonOpen()) { - ErrorInfo.ErrorFormatIndex = index; + ErrorInfo.ErrorFormatIndex = (int)index; NonOpen_ErrorInfo = ErrorInfo; // if archive was detected, we don't need additional open attempts return S_FALSE; @@ -2667,6 +2729,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } else { + if (PhySizeDefined && PhySize == 0) + { + PRF(printf(" phySizeDefined && PhySize == 0 ")); + // we skip that epmty archive case with unusual unexpected (PhySize == 0) from Code function. + continue; + } isOpen = true; RINOK(result); PRF(printf(" OK ")); @@ -2680,9 +2748,10 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) pi.Offset = startArcPos; if (ai.Flags_UseGlobalOffset()) - pi.Offset = Offset; + pi.Offset = (UInt64)Offset; else if (Offset != 0) return E_FAIL; + UInt64 arcRem = FileSize - pi.Offset; UInt64 phySize = arcRem; bool phySizeDefined = PhySizeDefined; @@ -2714,7 +2783,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (isOpen && !phySizeDefined) { - // it's for Z format + // it's for Z format, or bzip2,gz,xz with phySize that was not detected pi.LenIsUnknown = true; needScan = true; phySize = arcRem; @@ -2786,7 +2855,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool thereIsTail = ErrorInfo.ThereIsTail; if (thereIsTail && mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, arcStreamOffset + Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)((Int64)arcStreamOffset + Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) thereIsTail = false; } @@ -2794,10 +2863,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (pi.Offset != 0) { if (!pi.IsNotArcType) + { if (thereIsTail) openCur = specFlags.CanReturnMid; else openCur = specFlags.CanReturnTail; + } } else { @@ -2805,11 +2876,11 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) openCur = true; else openCur = specFlags.CanReturnFrontal; - if (formatIndex >= -2) openCur = true; } + if (formatIndex < 0 && pi.IsSelfExe /* && mode.SkipSfxStub */) openCur = false; @@ -2836,7 +2907,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { InStream = op.stream; Archive = archive; - FormatIndex = index; + FormatIndex = (int)index; ArcStreamOffset = arcStreamOffset; return S_OK; } @@ -2850,7 +2921,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } */ - pi.FormatIndex = index; + pi.FormatIndex = (int)index; // printf("\nAdd offset = %d", (int)pi.Offset); handlerSpec->AddItem(pi); @@ -2905,6 +2976,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) return S_OK; } + + + HRESULT CArc::OpenStream(const COpenOptions &op) { RINOK(OpenStream2(op)); @@ -2929,13 +3003,13 @@ HRESULT CArc::OpenStream(const COpenOptions &op) { int dotPos = fileName.ReverseFind_Dot(); if (dotPos >= 0) - extension = fileName.Ptr(dotPos + 1); + extension = fileName.Ptr((unsigned)(dotPos + 1)); } DefaultName.Empty(); if (FormatIndex >= 0) { - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; if (ai.Exts.Size() == 0) DefaultName = GetDefaultName2(fileName, UString(), UString()); else @@ -2943,7 +3017,7 @@ HRESULT CArc::OpenStream(const COpenOptions &op) int subExtIndex = ai.FindExtension(extension); if (subExtIndex < 0) subExtIndex = 0; - const CArcExtInfo &extInfo = ai.Exts[subExtIndex]; + const CArcExtInfo &extInfo = ai.Exts[(unsigned)subExtIndex]; DefaultName = GetDefaultName2(fileName, extInfo.Ext, extInfo.AddExt); } } @@ -2981,9 +3055,7 @@ HRESULT CArc::OpenStreamOrFile(COpenOptions &op) fileStream = fileStreamSpec; Path = filePath; if (!fileStreamSpec->Open(us2fs(Path))) - { - return GetLastError(); - } + return GetLastError_noZero_HRESULT(); op.stream = fileStream; #ifdef _SFX IgnoreSplit = true; @@ -3288,7 +3360,7 @@ HRESULT CArchiveLink::Open2(COpenOptions &op, IOpenCallbackUI *callbackUI) if (!op.stream && !op.stdInMode) { NFile::NDir::GetFullPathAndSplit(us2fs(op.filePath), prefix, name); - openCallbackSpec->Init(prefix, name); + RINOK(openCallbackSpec->Init2(prefix, name)); } else { @@ -3340,7 +3412,7 @@ HRESULT CArc::ReOpen(const COpenOptions &op) CTailInStream *tailStreamSpec = new CTailInStream; stream2 = tailStreamSpec; tailStreamSpec->Stream = op.stream; - tailStreamSpec->Offset = globalOffset; + tailStreamSpec->Offset = (UInt64)globalOffset; tailStreamSpec->Init(); RINOK(tailStreamSpec->SeekToStart()); } @@ -3352,8 +3424,8 @@ HRESULT CArc::ReOpen(const COpenOptions &op) if (res == S_OK) { - RINOK(ReadBasicProps(Archive, globalOffset, res)); - ArcStreamOffset = globalOffset; + RINOK(ReadBasicProps(Archive, (UInt64)globalOffset, res)); + ArcStreamOffset = (UInt64)globalOffset; if (ArcStreamOffset != 0) InStream = op.stream; } @@ -3393,14 +3465,14 @@ HRESULT CArchiveLink::ReOpen(COpenOptions &op) { FString dirPrefix, fileName; NFile::NDir::GetFullPathAndSplit(us2fs(op.filePath), dirPrefix, fileName); - openCallbackSpec->Init(dirPrefix, fileName); + RINOK(openCallbackSpec->Init2(dirPrefix, fileName)); } CInFileStream *fileStreamSpec = new CInFileStream; CMyComPtr<IInStream> stream(fileStreamSpec); if (!fileStreamSpec->Open(us2fs(op.filePath))) - return GetLastError(); + return GetLastError_noZero_HRESULT(); op.stream = stream; CArc &arc = Arcs[0]; @@ -3415,6 +3487,7 @@ HRESULT CArchiveLink::ReOpen(COpenOptions &op) #ifndef _SFX +bool ParseComplexSize(const wchar_t *s, UInt64 &result); bool ParseComplexSize(const wchar_t *s, UInt64 &result) { result = 0; @@ -3472,7 +3545,7 @@ static bool ParseTypeParams(const UString &s, COpenType &type) return false; } -bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) +static bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) { int pos2 = s.Find(L':'); @@ -3481,11 +3554,11 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) if (pos2 < 0) { name = s; - pos2 = s.Len(); + pos2 = (int)s.Len(); } else { - name = s.Left(pos2); + name = s.Left((unsigned)pos2); pos2++; } @@ -3514,17 +3587,17 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) } - for (unsigned i = pos2; i < s.Len();) + for (unsigned i = (unsigned)pos2; i < s.Len();) { int next = s.Find(L':', i); if (next < 0) - next = s.Len(); - const UString name = s.Mid(i, next - i); + next = (int)s.Len(); + const UString name = s.Mid(i, (unsigned)next - i); if (name.IsEmpty()) return false; if (!ParseTypeParams(name, type)) return false; - i = next + 1; + i = (unsigned)next + 1; } return true; @@ -3537,15 +3610,15 @@ bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector<COpenType> { int pos2 = s.Find(L'.', pos); if (pos2 < 0) - pos2 = s.Len(); - UString name = s.Mid(pos, pos2 - pos); + pos2 = (int)s.Len(); + UString name = s.Mid(pos, (unsigned)pos2 - pos); if (name.IsEmpty()) return false; COpenType type; if (!ParseType(codecs, name, type)) return false; types.Add(type); - pos = pos2 + 1; + pos = (unsigned)pos2 + 1; } return true; } diff --git a/CPP/7zip/UI/Common/OpenArchive.h b/CPP/7zip/UI/Common/OpenArchive.h index 6eb0d395..ebeb91d1 100644 --- a/CPP/7zip/UI/Common/OpenArchive.h +++ b/CPP/7zip/UI/Common/OpenArchive.h @@ -88,9 +88,9 @@ struct COpenType COpenType(): FormatIndex(-1), Recursive(true), - EachPos(false), CanReturnArc(true), CanReturnParser(false), + EachPos(false), // SkipSfxStub(true), // ExeAsUnknown(true), ZerosTailIsAllowed(false), @@ -121,7 +121,7 @@ struct COpenOptions IInStream *stream; ISequentialInStream *seqStream; IArchiveOpenCallback *callback; - COpenCallbackImp *callbackSpec; + COpenCallbackImp *callbackSpec; // it's used for SFX only OPEN_PROPS_DECL // bool openOnlySpecifiedByExtension, @@ -286,7 +286,7 @@ public: UString filePath; UString DefaultName; int FormatIndex; // - 1 means Parser. - int SubfileIndex; + UInt32 SubfileIndex; // (UInt32)(Int32)-1; means no subfile FILETIME MTime; bool MTimeDefined; @@ -302,7 +302,7 @@ public: UInt64 GetEstmatedPhySize() const { return PhySizeDefined ? PhySize : FileSize; } UInt64 ArcStreamOffset; // offset of stream that is open by Archive Handler - Int64 GetGlobalOffset() const { return ArcStreamOffset + Offset; } // it's global offset of archive + Int64 GetGlobalOffset() const { return (Int64)ArcStreamOffset + Offset; } // it's global offset of archive // AString ErrorFlagsText; @@ -397,6 +397,13 @@ struct CArchiveLink IArchiveGetRawProps *GetArchiveGetRawProps() const { return Arcs.Back().GetRawProps; } IArchiveGetRootProps *GetArchiveGetRootProps() const { return Arcs.Back().GetRootProps; } + /* + Open() opens archive and COpenOptions::callback + Open2() uses COpenCallbackImp that implements Volumes and password callback + Open3() calls Open2() and callbackUI->Open_Finished(); + Open_Strict() returns S_FALSE also in case, if there is non-open expected nested archive. + */ + HRESULT Open(COpenOptions &options); HRESULT Open2(COpenOptions &options, IOpenCallbackUI *callbackUI); HRESULT Open3(COpenOptions &options, IOpenCallbackUI *callbackUI); diff --git a/CPP/7zip/UI/Common/PropIDUtils.cpp b/CPP/7zip/UI/Common/PropIDUtils.cpp index 7702e222..30efd53b 100644 --- a/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -14,8 +14,10 @@ #include "PropIDUtils.h" +#ifndef _SFX #define Get16(x) GetUi16(x) #define Get32(x) GetUi32(x) +#endif using namespace NWindows; @@ -63,9 +65,9 @@ static void ConvertPosixAttribToString(char *s, UInt32 a) throw() s[8 - i] = MY_ATTR_CHAR(a, i + 1, 'w'); s[9 - i] = MY_ATTR_CHAR(a, i + 0, 'x'); } - if ((a & 0x800) != 0) s[3] = ((a & (1 << 6)) ? 's' : 'S'); - if ((a & 0x400) != 0) s[6] = ((a & (1 << 3)) ? 's' : 'S'); - if ((a & 0x200) != 0) s[9] = ((a & (1 << 0)) ? 't' : 'T'); + if ((a & 0x800) != 0) s[3] = ((a & (1 << 6)) ? 's' : 'S'); // S_ISUID + if ((a & 0x400) != 0) s[6] = ((a & (1 << 3)) ? 's' : 'S'); // S_ISGID + if ((a & 0x200) != 0) s[9] = ((a & (1 << 0)) ? 't' : 'T'); // S_ISVTX s[10] = 0; a &= ~(UInt32)0xFFFF; @@ -213,13 +215,13 @@ void ConvertPropertyToString2(UString &dest, const PROPVARIANT &prop, PROPID pro dest = temp; } +#ifndef _SFX + static inline unsigned GetHex(unsigned v) { return (v < 10) ? ('0' + v) : ('A' + (v - 10)); } -#ifndef _SFX - static inline void AddHexToString(AString &res, unsigned v) { res += (char)GetHex(v >> 4); @@ -272,7 +274,7 @@ static int FindPairIndex(const CSecID2Name * pairs, unsigned num, UInt32 id) { for (unsigned i = 0; i < num; i++) if (pairs[i].n == id) - return i; + return (int)i; return -1; } @@ -479,11 +481,16 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName */ } +/* #define MY_SE_OWNER_DEFAULTED (0x0001) #define MY_SE_GROUP_DEFAULTED (0x0002) +*/ #define MY_SE_DACL_PRESENT (0x0004) +/* #define MY_SE_DACL_DEFAULTED (0x0008) +*/ #define MY_SE_SACL_PRESENT (0x0010) +/* #define MY_SE_SACL_DEFAULTED (0x0020) #define MY_SE_DACL_AUTO_INHERIT_REQ (0x0100) #define MY_SE_SACL_AUTO_INHERIT_REQ (0x0200) @@ -493,6 +500,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName #define MY_SE_SACL_PROTECTED (0x2000) #define MY_SE_RM_CONTROL_VALID (0x4000) #define MY_SE_SELF_RELATIVE (0x8000) +*/ void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) { @@ -590,25 +598,45 @@ static const CSecID2Name k_ReparseTags[] = { 0x80000014, "NFS" }, { 0x80000015, "FILE_PLACEHOLDER" }, { 0x80000016, "DFM" }, - { 0x80000017, "WOF" } + { 0x80000017, "WOF" }, + { 0x80000018, "WCI" }, + { 0x8000001B, "APPEXECLINK" }, + { 0xA000001D, "LX_SYMLINK" }, + { 0x80000023, "AF_UNIX" }, + { 0x80000024, "LX_FIFO" }, + { 0x80000025, "LX_CHR" }, + { 0x80000026, "LX_BLK" } }; bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) { s.Empty(); NFile::CReparseAttr attr; - DWORD errorCode = 0; - if (attr.Parse(data, size, errorCode)) + + if (attr.Parse(data, size)) { - if (!attr.IsSymLink()) - s += "Junction: "; - s += attr.GetPath(); - if (!attr.IsOkNamePair()) + if (attr.IsSymLink_WSL()) { - s += " : "; - s += attr.PrintName; + s += "WSL: "; + s += attr.GetPath(); + } + else + { + if (!attr.IsSymLink_Win()) + s += "Junction: "; + s += attr.GetPath(); + if (s.IsEmpty()) + s += "Link: "; + if (!attr.IsOkNamePair()) + { + s += " : "; + s += attr.PrintName; + } } + if (attr.MinorError) + s += " : MINOR_ERROR"; return true; + // s += " "; // for debug } if (size < 8) @@ -651,7 +679,7 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) for (UInt32 i = 0; i < len; i++) { - if (i >= 8) + if (i >= 16) { s += "..."; break; diff --git a/CPP/7zip/UI/Common/SetProperties.cpp b/CPP/7zip/UI/Common/SetProperties.cpp index c3de5d5a..4b3037af 100644 --- a/CPP/7zip/UI/Common/SetProperties.cpp +++ b/CPP/7zip/UI/Common/SetProperties.cpp @@ -27,6 +27,21 @@ static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) prop = result; } + +struct CPropPropetiesVector +{ + CPropVariant *values; + CPropPropetiesVector(unsigned num) + { + values = new CPropVariant[num]; + } + ~CPropPropetiesVector() + { + delete []values; + } +}; + + HRESULT SetProperties(IUnknown *unknown, const CObjectVector<CProperty> &properties) { if (properties.IsEmpty()) @@ -37,8 +52,7 @@ HRESULT SetProperties(IUnknown *unknown, const CObjectVector<CProperty> &propert return S_OK; UStringVector realNames; - CPropVariant *values = new CPropVariant[properties.Size()]; - try + CPropPropetiesVector values(properties.Size()); { unsigned i; for (i = 0; i < properties.Size(); i++) @@ -62,19 +76,12 @@ HRESULT SetProperties(IUnknown *unknown, const CObjectVector<CProperty> &propert else ParseNumberString(property.Value, propVariant); realNames.Add(name); - values[i] = propVariant; + values.values[i] = propVariant; } CRecordVector<const wchar_t *> names; for (i = 0; i < realNames.Size(); i++) names.Add((const wchar_t *)realNames[i]); - RINOK(setProperties->SetProperties(&names.Front(), values, names.Size())); - } - catch(...) - { - delete []values; - throw; + return setProperties->SetProperties(&names.Front(), values.values, names.Size()); } - delete []values; - return S_OK; } diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index 8c7ae45f..fc922a70 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -32,7 +32,7 @@ static const char * const kUpdateIsNotSupoorted = "update operations are not supported for this archive"; -static const char * const kUpdateIsNotSupoorted_MultiVol = +static const char * const kUpdateIsNotSupported_MultiVol = "Updating for multivolume archives is not implemented"; using namespace NWindows; @@ -41,8 +41,9 @@ using namespace NFile; using namespace NDir; using namespace NName; +#ifdef _WIN32 static CFSTR const kTempFolderPrefix = FTEXT("7zE"); - +#endif void CUpdateErrorInfo::SetFromLastError(const char *message) { @@ -57,26 +58,12 @@ HRESULT CUpdateErrorInfo::SetFromLastError(const char *message, const FString &f return Get_HRESULT_Error(); } -static bool DeleteEmptyFolderAndEmptySubFolders(const FString &path) +HRESULT CUpdateErrorInfo::SetFromError_DWORD(const char *message, const FString &fileName, DWORD error) { - NFind::CFileInfo fileInfo; - FString pathPrefix = path + FCHAR_PATH_SEPARATOR; - { - NFind::CEnumerator enumerator; - enumerator.SetDirPrefix(pathPrefix); - while (enumerator.Next(fileInfo)) - { - if (fileInfo.IsDir()) - if (!DeleteEmptyFolderAndEmptySubFolders(pathPrefix + fileInfo.Name)) - return false; - } - } - /* - // we don't need clear read-only for folders - if (!MySetFileAttributes(path, 0)) - return false; - */ - return RemoveDir(path); + Message = message; + FileNames.Add(fileName); + SystemError = error; + return Get_HRESULT_Error(); } @@ -175,7 +162,7 @@ STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *pr altStream.StreamSpec = new COutFileStream; altStream.Stream = altStream.StreamSpec; if (!altStream.StreamSpec->Create(name, false)) - return ::GetLastError(); + return GetLastError_noZero_HRESULT(); { // NSynchronization::CCriticalSectionLock lock(g_TempPathsCS); TempFiles->Paths.Add(name); @@ -204,14 +191,14 @@ STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *pr { // CMyComPtr<IOutStream> outStream; // RINOK(altStream.Stream.QueryInterface(IID_IOutStream, &outStream)); - RINOK(altStream.Stream->Seek(_offsetPos, STREAM_SEEK_SET, NULL)); + RINOK(altStream.Stream->Seek((Int64)_offsetPos, STREAM_SEEK_SET, NULL)); altStream.Pos = _offsetPos; } UInt32 curSize = (UInt32)MyMin((UInt64)size, volSize - altStream.Pos); UInt32 realProcessed; RINOK(altStream.Stream->Write(data, curSize, &realProcessed)); - data = (void *)((Byte *)data + realProcessed); + data = (const void *)((const Byte *)data + realProcessed); size -= realProcessed; altStream.Pos += realProcessed; _offsetPos += realProcessed; @@ -240,9 +227,9 @@ STDMETHODIMP COutMultiVolStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *n return STG_E_INVALIDFUNCTION; switch (seekOrigin) { - case STREAM_SEEK_SET: _absPos = offset; break; - case STREAM_SEEK_CUR: _absPos += offset; break; - case STREAM_SEEK_END: _absPos = _length + offset; break; + case STREAM_SEEK_SET: _absPos = (UInt64)offset; break; + case STREAM_SEEK_CUR: _absPos = (UInt64)((Int64)_absPos + offset); break; + case STREAM_SEEK_END: _absPos = (UInt64)((Int64)_length + offset); break; } _offsetPos = _absPos; if (newPosition) @@ -298,11 +285,11 @@ void CArchivePath::ParseFromPath(const UString &path, EArcNameMode mode) Name.DeleteBack(); else { - const UString ext = Name.Ptr(dotPos + 1); + const UString ext = Name.Ptr((unsigned)(dotPos + 1)); if (BaseExtension.IsEqualTo_NoCase(ext)) { BaseExtension = ext; - Name.DeleteFrom(dotPos); + Name.DeleteFrom((unsigned)dotPos); return; } } @@ -392,7 +379,7 @@ bool CUpdateOptions::SetArcPath(const CCodecs *codecs, const UString &arcPath) } else { - const CArcInfoEx &arcInfo = codecs->Formats[formatIndex]; + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; if (!arcInfo.UpdateEnabled) return false; typeExt = arcInfo.GetMainExt(); @@ -417,8 +404,8 @@ bool CUpdateOptions::SetArcPath(const CCodecs *codecs, const UString &arcPath) struct CUpdateProduceCallbackImp: public IUpdateProduceCallback { const CObjectVector<CArcItem> *_arcItems; - IUpdateCallbackUI *_callback; CDirItemsStat *_stat; + IUpdateCallbackUI *_callback; CUpdateProduceCallbackImp( const CObjectVector<CArcItem> *a, @@ -541,7 +528,7 @@ static HRESULT Compress( } else { - RINOK(codecs->CreateOutArchive(formatIndex, outArchive)); + RINOK(codecs->CreateOutArchive((unsigned)formatIndex, outArchive)); #ifdef EXTERNAL_CODECS { @@ -576,7 +563,7 @@ static HRESULT Compress( } { - const CArcInfoEx &arcInfo = codecs->Formats[formatIndex]; + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; if (options.AltStreams.Val && !arcInfo.Flags_AltStreams()) return E_NOTIMPL; if (options.NtSecurity.Val && !arcInfo.Flags_NtSecure()) @@ -614,7 +601,7 @@ static HRESULT Compress( int colonPos = FindAltStreamColon_in_Path(ai.Name); if (colonPos >= 0) { - UString mainName = ai.Name.Left(colonPos); + UString mainName = ai.Name.Left((unsigned)colonPos); /* actually we must improve that code to support cases with folder renaming like: rn arc dir1\ dir2\ @@ -623,7 +610,7 @@ static HRESULT Compress( { needRename = true; dest += ':'; - dest += ai.Name.Ptr(colonPos + 1); + dest += ai.Name.Ptr((unsigned)(colonPos + 1)); break; } } @@ -638,7 +625,7 @@ static HRESULT Compress( { up2.NewProps = true; RINOK(arc->IsItemAnti(i, up2.IsAnti)); - up2.NewNameIndex = newNames.Add(dest); + up2.NewNameIndex = (int)newNames.Add(dest); } updatePairs2.Add(up2); } @@ -664,7 +651,7 @@ static HRESULT Compress( if (up.ExistOnDisk()) { CDirItemsStat2 &stat = stat2.NewData; - const CDirItem &di = dirItems.Items[up.DirIndex]; + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; if (di.IsDir()) { if (up.IsAnti) @@ -697,7 +684,7 @@ static HRESULT Compress( else if (up.ArcIndex >= 0) { CDirItemsStat2 &stat = *(up.NewData ? &stat2.NewData : &stat2.OldData); - const CArcItem &ai = arcItems[up.ArcIndex]; + const CArcItem &ai = arcItems[(unsigned)up.ArcIndex]; if (ai.IsDir) { if (up.IsAnti) @@ -733,6 +720,7 @@ static HRESULT Compress( CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; CMyComPtr<IArchiveUpdateCallback> updateCallback(updateCallbackSpec); + updateCallbackSpec->PreserveATime = options.PreserveATime; updateCallbackSpec->ShareForWrite = options.OpenShareForWrite; updateCallbackSpec->StopAfterOpenError = options.StopAfterOpenError; updateCallbackSpec->StdInMode = options.StdInMode; @@ -922,9 +910,9 @@ static HRESULT Compress( CUpdatePair2 &pair2 = updatePairs2[i]; const FILETIME *ft2 = NULL; if (pair2.NewProps && pair2.DirIndex >= 0) - ft2 = &dirItems.Items[pair2.DirIndex].MTime; + ft2 = &dirItems.Items[(unsigned)pair2.DirIndex].MTime; else if (pair2.UseArcProps && pair2.ArcIndex >= 0) - ft2 = &arcItems[pair2.ArcIndex].MTime; + ft2 = &arcItems[(unsigned)pair2.ArcIndex].MTime; if (ft2) { if (::CompareFileTime(&ft, ft2) < 0) @@ -936,7 +924,7 @@ static HRESULT Compress( if (outStreamSpec) outStreamSpec->SetMTime(&ft); else if (volStreamSpec) - volStreamSpec->SetMTime(&ft);; + volStreamSpec->SetMTime(&ft); } } @@ -1041,7 +1029,7 @@ static HRESULT EnumerateInArchiveItems( #if defined(_WIN32) && !defined(UNDER_CE) -#include <mapi.h> +#include <MAPI.h> #endif @@ -1074,7 +1062,7 @@ HRESULT UpdateArchive( if (options.Commands.Size() != 1) return E_NOTIMPL; const CActionSet &as = options.Commands[0].ActionSet; - for (int i = 2; i < NPairState::kNumValues; i++) + for (unsigned i = 2; i < NPairState::kNumValues; i++) if (as.StateActions[i] != NPairAction::kCompress) return E_NOTIMPL; } @@ -1103,7 +1091,7 @@ HRESULT UpdateArchive( if (options.SfxModule.Find(FCHAR_PATH_SEPARATOR) < 0) { const FString fullName = NDLL::GetModuleDirPrefix() + options.SfxModule; - if (NFind::DoesFileExist(fullName)) + if (NFind::DoesFileExist_FollowLink(fullName)) { options.SfxModule = fullName; found = true; @@ -1111,7 +1099,7 @@ HRESULT UpdateArchive( } if (!found) { - if (!NFind::DoesFileExist(options.SfxModule)) + if (!NFind::DoesFileExist_FollowLink(options.SfxModule)) return errorInfo.SetFromLastError("cannot find specified SFX module", options.SfxModule); } } @@ -1143,7 +1131,7 @@ HRESULT UpdateArchive( else { NFind::CFileInfo fi; - if (!fi.Find(us2fs(arcPath))) + if (!fi.Find_FollowLink(us2fs(arcPath))) { if (renameMode) throw "can't find archive";; @@ -1156,24 +1144,35 @@ HRESULT UpdateArchive( else { if (fi.IsDir()) - throw "there is no such archive"; + return errorInfo.SetFromError_DWORD("There is a folder with the name of archive", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EISDIR + #endif + ); if (fi.IsDevice) return E_NOTIMPL; if (!options.StdOutMode && options.UpdateArchiveItself) if (fi.IsReadOnly()) { - errorInfo.SystemError = ERROR_ACCESS_DENIED; - errorInfo.Message = "The file is read-only"; - errorInfo.FileNames.Add(us2fs(arcPath)); - return errorInfo.Get_HRESULT_Error(); + return errorInfo.SetFromError_DWORD("The file is read-only", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EACCES + #endif + ); } if (options.VolumesSizes.Size() > 0) { errorInfo.FileNames.Add(us2fs(arcPath)); - errorInfo.SystemError = (DWORD)E_NOTIMPL; - errorInfo.Message = kUpdateIsNotSupoorted_MultiVol; + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; return E_NOTIMPL; } CObjectVector<COpenType> types2; @@ -1211,8 +1210,8 @@ HRESULT UpdateArchive( if (arcLink.VolumePaths.Size() > 1) { - errorInfo.SystemError = (DWORD)E_NOTIMPL; - errorInfo.Message = kUpdateIsNotSupoorted_MultiVol; + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; return E_NOTIMPL; } @@ -1222,7 +1221,7 @@ HRESULT UpdateArchive( if (arc.ErrorInfo.ThereIsTail) { - errorInfo.SystemError = (DWORD)E_NOTIMPL; + // errorInfo.SystemError = (DWORD)E_NOTIMPL; errorInfo.Message = "There is some data block after the end of the archive"; return E_NOTIMPL; } @@ -1292,7 +1291,7 @@ HRESULT UpdateArchive( HRESULT res = EnumerateItems(censor, options.PathMode, - options.AddPathPrefix, + UString(), // options.AddPathPrefix, dirItems); if (res != S_OK) @@ -1332,8 +1331,6 @@ HRESULT UpdateArchive( dirItems.AddSecurityItem(prefix, secureIndex); #endif parentDirItem.SecureIndex = secureIndex; - - parentDirItem_Ptr = &parentDirItem; } } } @@ -1565,7 +1562,7 @@ HRESULT UpdateArchive( } */ - LPMAPISENDMAIL sendMail = (LPMAPISENDMAIL)mapiLib.GetProc("MAPISendMail"); + LPMAPISENDMAIL sendMail = (LPMAPISENDMAIL)(void *)mapiLib.GetProc("MAPISendMail"); if (sendMail == 0) { errorInfo.SetFromLastError("7-Zip cannot find MAPISendMail function"); @@ -1610,8 +1607,8 @@ HRESULT UpdateArchive( MapiFileDesc &f = files[i]; memset(&f, 0, sizeof(f)); f.nPosition = 0xFFFFFFFF; - f.lpszPathName = (char *)(const char *)paths[i]; - f.lpszFileName = (char *)(const char *)names[i]; + f.lpszPathName = paths[i].Ptr_non_const(); + f.lpszFileName = names[i].Ptr_non_const(); } { @@ -1626,7 +1623,7 @@ HRESULT UpdateArchive( { memset(&rec, 0, sizeof(rec)); rec.ulRecipClass = MAPI_TO; - rec.lpszAddress = (char *)(const char *)addr; + rec.lpszAddress = addr.Ptr_non_const(); m.nRecipCount = 1; m.lpRecips = &rec; } @@ -1660,8 +1657,12 @@ HRESULT UpdateArchive( if (processedItems[i] != 0 || dirItem.Size == 0) { NFind::CFileInfo fileInfo; + /* here we compare Raw FileInfo that can be link with actual file info that was processed. + we can fix it. */ if (fileInfo.Find(phyPath)) { + // FIXME: here we can check Find_FollowLink() also; + // maybe we must exclude also files with archive name: "a a.7z * -sdel" if (fileInfo.Size == dirItem.Size && CompareFileTime(&fileInfo.MTime, &dirItem.MTime) == 0 diff --git a/CPP/7zip/UI/Common/Update.h b/CPP/7zip/UI/Common/Update.h index dc9ff5d3..06d13877 100644 --- a/CPP/7zip/UI/Common/Update.h +++ b/CPP/7zip/UI/Common/Update.h @@ -18,7 +18,7 @@ enum EArcNameMode { k_ArcNameMode_Smart, k_ArcNameMode_Exact, - k_ArcNameMode_Add, + k_ArcNameMode_Add }; struct CArchivePath @@ -91,6 +91,7 @@ struct CUpdateOptions bool SfxMode; FString SfxModule; + bool PreserveATime; bool OpenShareForWrite; bool StopAfterOpenError; @@ -104,7 +105,7 @@ struct CUpdateOptions FString WorkingDir; NWildcard::ECensorPathMode PathMode; - UString AddPathPrefix; + // UString AddPathPrefix; CBoolPair NtSecurity; CBoolPair AltStreams; @@ -122,20 +123,26 @@ struct CUpdateOptions CUpdateOptions(): UpdateArchiveItself(true), + ArcNameMode(k_ArcNameMode_Smart), + SfxMode(false), + + PreserveATime(false), + OpenShareForWrite(false), + StopAfterOpenError(false), + StdInMode(false), StdOutMode(false), + EMailMode(false), EMailRemoveAfter(false), - OpenShareForWrite(false), - StopAfterOpenError(false), - ArcNameMode(k_ArcNameMode_Smart), + PathMode(NWildcard::k_RelatPath), DeleteAfterCompressing(false), SetArcMTime(false) - {}; + {}; void SetActionCommand_Add() { @@ -150,7 +157,7 @@ struct CUpdateOptions struct CUpdateErrorInfo { - DWORD SystemError; + DWORD SystemError; // it's DWORD (WRes) only; AString Message; FStringVector FileNames; @@ -158,6 +165,7 @@ struct CUpdateErrorInfo HRESULT Get_HRESULT_Error() const { return SystemError == 0 ? E_FAIL : HRESULT_FROM_WIN32(SystemError); } void SetFromLastError(const char *message); HRESULT SetFromLastError(const char *message, const FString &fileName); + HRESULT SetFromError_DWORD(const char *message, const FString &fileName, DWORD error); CUpdateErrorInfo(): SystemError(0) {}; }; diff --git a/CPP/7zip/UI/Common/UpdateCallback.cpp b/CPP/7zip/UI/Common/UpdateCallback.cpp index fd46dda8..69cde093 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.cpp +++ b/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +// #include <stdio.h> + #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" #endif @@ -10,6 +12,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Common/Wildcard.h" +#include "../../../Common/UTFConvert.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" @@ -54,6 +57,7 @@ CArchiveUpdateCallback::CArchiveUpdateCallback(): CommentIndex(-1), Comment(NULL), + PreserveATime(false), ShareForWrite(false), StopAfterOpenError(false), StdInMode(false), @@ -124,7 +128,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetUpdateItemInfo(UInt32 index, { *indexInArchive = (UInt32)(Int32)-1; if (up.ExistInArchive()) - *indexInArchive = (ArcItems == 0) ? up.ArcIndex : (*ArcItems)[up.ArcIndex].IndexInServer; + *indexInArchive = ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex; } return S_OK; COM_TRY_END @@ -188,7 +192,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRootRawProp(PROPID { if (ParentDirItem->SecureIndex < 0) return S_OK; - const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[ParentDirItem->SecureIndex]; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)ParentDirItem->SecureIndex]; *data = buf; *dataSize = (UInt32)buf.Size(); *propType = NPropDataType::kRaw; @@ -220,7 +224,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con const CUpdatePair2 &up = (*UpdatePairs)[index]; if (up.UseArcProps && up.ExistInArchive() && Arc->GetRawProps) return Arc->GetRawProps->GetRawProp( - ArcItems ? (*ArcItems)[up.ArcIndex].IndexInServer : up.ArcIndex, + ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, propID, data, dataSize, propType); { /* @@ -230,8 +234,8 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con if (up.IsAnti) return S_OK; - #ifndef UNDER_CE - const CDirItem &di = DirItems->Items[up.DirIndex]; + #if defined(_WIN32) && !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; #endif #ifdef _USE_SECURITY_CODE @@ -241,18 +245,19 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con return S_OK; if (di.SecureIndex < 0) return S_OK; - const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[di.SecureIndex]; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)di.SecureIndex]; *data = buf; *dataSize = (UInt32)buf.Size(); *propType = NPropDataType::kRaw; } else #endif + if (propID == kpidNtReparse) { - // propID == kpidNtReparse if (!StoreSymLinks) return S_OK; - #ifndef UNDER_CE + #if defined(_WIN32) && !defined(UNDER_CE) + // we use ReparseData2 instead of ReparseData for WIM format const CByteBuffer *buf = &di.ReparseData2; if (buf->Size() == 0) buf = &di.ReparseData; @@ -272,7 +277,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con return S_OK; } -#ifndef UNDER_CE +#if defined(_WIN32) && !defined(UNDER_CE) static UString GetRelativePath(const UString &to, const UString &from) { @@ -340,22 +345,25 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR prop.Detach(value); return S_OK; } + + #if !defined(UNDER_CE) + if (up.DirIndex >= 0) { - #ifndef UNDER_CE - const CDirItem &di = DirItems->Items[up.DirIndex]; + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + + #ifdef _WIN32 // if (di.IsDir()) { CReparseAttr attr; - DWORD errorCode = 0; - if (attr.Parse(di.ReparseData, di.ReparseData.Size(), errorCode)) + if (attr.Parse(di.ReparseData, di.ReparseData.Size())) { UString simpleName = attr.GetPath(); - if (attr.IsRelative()) + if (!attr.IsSymLink_WSL() && attr.IsRelative_Win()) prop = simpleName; else { - const FString phyPath = DirItems->GetPhyPath(up.DirIndex); + const FString phyPath = DirItems->GetPhyPath((unsigned)up.DirIndex); FString fullPath; if (NDir::MyGetFullPathName(phyPath, fullPath)) { @@ -366,8 +374,26 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR return S_OK; } } - #endif + + #else // _WIN32 + + if (di.ReparseData.Size() != 0) + { + AString utf; + utf.SetFrom_CalcLen((const char *)(const Byte *)di.ReparseData, (unsigned)di.ReparseData.Size()); + + UString us; + if (ConvertUTF8ToUnicode(utf, us)) + { + prop = us; + prop.Detach(value); + return S_OK; + } + } + + #endif // _WIN32 } + #endif // !defined(UNDER_CE) } else if (propID == kpidHardLink) { @@ -375,7 +401,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR { const CKeyKeyValPair &pair = _map[_hardIndex_To]; const CUpdatePair2 &up2 = (*UpdatePairs)[pair.Value]; - prop = DirItems->GetLogPath(up2.DirIndex); + prop = DirItems->GetLogPath((unsigned)up2.DirIndex); prop.Detach(value); return S_OK; } @@ -399,7 +425,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR } } else if (propID == kpidPath && up.NewNameIndex >= 0) - prop = (*NewNames)[up.NewNameIndex]; + prop = (*NewNames)[(unsigned)up.NewNameIndex]; else if (propID == kpidComment && CommentIndex >= 0 && (unsigned)CommentIndex == index @@ -411,13 +437,13 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR } else if ((up.UseArcProps || (KeepOriginalItemNames && (propID == kpidPath || propID == kpidIsAltStream))) && up.ExistInArchive() && Archive) - return Archive->GetProperty(ArcItems ? (*ArcItems)[up.ArcIndex].IndexInServer : up.ArcIndex, propID, value); + return Archive->GetProperty(ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, propID, value); else if (up.ExistOnDisk()) { - const CDirItem &di = DirItems->Items[up.DirIndex]; + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; switch (propID) { - case kpidPath: prop = DirItems->GetLogPath(up.DirIndex); break; + case kpidPath: prop = DirItems->GetLogPath((unsigned)up.DirIndex); break; case kpidIsDir: prop = di.IsDir(); break; case kpidSize: prop = di.IsDir() ? (UInt64)0 : di.Size; break; case kpidAttrib: prop = di.Attrib; break; @@ -428,6 +454,16 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR #if defined(_WIN32) && !defined(UNDER_CE) // case kpidShortName: prop = di.ShortName; break; #endif + case kpidPosixAttrib: + { + #ifdef _WIN32 + prop = di.GetPosixAttrib(); + #else + if (di.Attrib & FILE_ATTRIBUTE_UNIX_EXTENSION) + prop = (UInt32)(di.Attrib >> 16); + #endif + break; + } } } prop.Detach(value); @@ -456,9 +492,9 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea { UString name; if (up.ArcIndex >= 0) - name = (*ArcItems)[up.ArcIndex].Name; + name = (*ArcItems)[(unsigned)up.ArcIndex].Name; else if (up.DirIndex >= 0) - name = DirItems->GetLogPath(up.DirIndex); + name = DirItems->GetLogPath((unsigned)up.DirIndex); RINOK(Callback->GetStream(name, isDir, true, mode)); /* 9.33: fixed. Handlers expect real stream object for files, even for anti-file. @@ -474,7 +510,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea return S_OK; } - RINOK(Callback->GetStream(DirItems->GetLogPath(up.DirIndex), isDir, false, mode)); + RINOK(Callback->GetStream(DirItems->GetLogPath((unsigned)up.DirIndex), isDir, false, mode)); if (isDir) return S_OK; @@ -491,27 +527,42 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea } else { + #if !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + if (di.AreReparseData()) + { + /* + // we still need DeviceIoControlOut() instead of Read + if (!inStreamSpec->File.OpenReparse(path)) + { + return Callback->OpenFileError(path, ::GetLastError()); + } + */ + // 20.03: we use Reparse Data instead of real data + + CBufInStream *inStreamSpec = new CBufInStream(); + CMyComPtr<ISequentialInStream> inStreamLoc = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + *inStream = inStreamLoc.Detach(); + return S_OK; + } + #endif // !defined(UNDER_CE) + CInFileStream *inStreamSpec = new CInFileStream; CMyComPtr<ISequentialInStream> inStreamLoc(inStreamSpec); inStreamSpec->SupportHardLinks = StoreHardLinks; - inStreamSpec->Callback = this; - inStreamSpec->CallbackRef = index; + inStreamSpec->File.PreserveATime = PreserveATime; - const FString path = DirItems->GetPhyPath(up.DirIndex); + const FString path = DirItems->GetPhyPath((unsigned)up.DirIndex); _openFiles_Indexes.Add(index); _openFiles_Paths.Add(path); - #if defined(_WIN32) && !defined(UNDER_CE) - if (DirItems->Items[up.DirIndex].AreReparseData()) - { - if (!inStreamSpec->File.OpenReparse(path)) - { - return Callback->OpenFileError(path, ::GetLastError()); - } - } - else - #endif + /* 21.02 : we set Callback/CallbackRef after _openFiles_Indexes adding + for correct working if exception was raised in GetPhyPath */ + inStreamSpec->Callback = this; + inStreamSpec->CallbackRef = index; + if (!inStreamSpec->OpenShared(path, ShareForWrite)) { DWORD error = ::GetLastError(); @@ -522,6 +573,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea return hres; } + // #if defined(USE_WIN_FILE) || !defined(_WIN32) if (StoreHardLinks) { CStreamFileProps props; @@ -546,6 +598,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea } } } + // #endif if (ProcessedItemsStatuses) { @@ -592,8 +645,8 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in const CUpdatePair2 &up = (*UpdatePairs)[index]; if (up.ExistOnDisk()) { - name = DirItems->GetLogPath(up.DirIndex); - isDir = DirItems->Items[up.DirIndex].IsDir(); + name = DirItems->GetLogPath((unsigned)up.DirIndex); + isDir = DirItems->Items[(unsigned)up.DirIndex].IsDir(); } } return Callback->ReportUpdateOpeartion(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); @@ -716,7 +769,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOu COutFileStream *streamSpec = new COutFileStream; CMyComPtr<ISequentialOutStream> streamLoc(streamSpec); if (!streamSpec->Create(fileName, false)) - return ::GetLastError(); + return GetLastError_noZero_HRESULT(); *volumeStream = streamLoc.Detach(); return S_OK; COM_TRY_END @@ -738,7 +791,10 @@ STDMETHODIMP CArchiveUpdateCallback::CryptoGetTextPassword(BSTR *password) HRESULT CArchiveUpdateCallback::InFileStream_On_Error(UINT_PTR val, DWORD error) { - if (error == ERROR_LOCK_VIOLATION) + #ifdef _WIN32 // FIX IT !!! + // why did we check only for ERROR_LOCK_VIOLATION ? + // if (error == ERROR_LOCK_VIOLATION) + #endif { MT_LOCK UInt32 index = (UInt32)val; @@ -756,6 +812,7 @@ HRESULT CArchiveUpdateCallback::InFileStream_On_Error(UINT_PTR val, DWORD error) void CArchiveUpdateCallback::InFileStream_On_Destroy(UINT_PTR val) { + { MT_LOCK UInt32 index = (UInt32)val; FOR_VECTOR(i, _openFiles_Indexes) @@ -767,5 +824,11 @@ void CArchiveUpdateCallback::InFileStream_On_Destroy(UINT_PTR val) return; } } - throw 20141125; + } + /* 21.02 : this function can be called in destructor. + And destructor can be called after some exception. + If we don't want to throw exception in desctructors or after another exceptions, + we must disable the code below that raises new exception. + */ + // throw 20141125; } diff --git a/CPP/7zip/UI/Common/UpdateCallback.h b/CPP/7zip/UI/Common/UpdateCallback.h index 9e6925f6..3fe0a647 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.h +++ b/CPP/7zip/UI/Common/UpdateCallback.h @@ -135,6 +135,7 @@ public: int CommentIndex; const UString *Comment; + bool PreserveATime; bool ShareForWrite; bool StopAfterOpenError; bool StdInMode; @@ -152,9 +153,9 @@ public: bool IsDir(const CUpdatePair2 &up) const { if (up.DirIndex >= 0) - return DirItems->Items[up.DirIndex].IsDir(); + return DirItems->Items[(unsigned)up.DirIndex].IsDir(); else if (up.ArcIndex >= 0) - return (*ArcItems)[up.ArcIndex].IsDir; + return (*ArcItems)[(unsigned)up.ArcIndex].IsDir; return false; } }; diff --git a/CPP/7zip/UI/Common/UpdatePair.cpp b/CPP/7zip/UI/Common/UpdatePair.cpp index 82876c1d..31d73f94 100644 --- a/CPP/7zip/UI/Common/UpdatePair.cpp +++ b/CPP/7zip/UI/Common/UpdatePair.cpp @@ -42,7 +42,9 @@ static const char * const k_Duplicate_inArc_Message = "Duplicate filename in arc static const char * const k_Duplicate_inDir_Message = "Duplicate filename on disk:"; static const char * const k_NotCensoredCollision_Message = "Internal file name collision (file on disk, file in archive):"; -static void ThrowError(const char *message, const UString &s1, const UString &s2) +MY_ATTR_NORETURN +static +void ThrowError(const char *message, const UString &s1, const UString &s2) { UString m (message); m.Add_LF(); m += s1; @@ -144,18 +146,18 @@ void GetUpdatePairInfoList( if (dirIndex < numDirItems) { - dirIndex2 = dirIndices[dirIndex]; - di = &dirItems.Items[dirIndex2]; + dirIndex2 = (int)dirIndices[dirIndex]; + di = &dirItems.Items[(unsigned)dirIndex2]; } if (arcIndex < numArcItems) { - arcIndex2 = arcIndices[arcIndex]; - ai = &arcItems[arcIndex2]; + arcIndex2 = (int)arcIndices[arcIndex]; + ai = &arcItems[(unsigned)arcIndex2]; compareResult = 1; if (dirIndex < numDirItems) { - compareResult = CompareFileNames(dirNames[dirIndex2], ai->Name); + compareResult = CompareFileNames(dirNames[(unsigned)dirIndex2], ai->Name); if (compareResult == 0) { if (di->IsDir() != ai->IsDir) @@ -166,7 +168,7 @@ void GetUpdatePairInfoList( if (compareResult < 0) { - name = &dirNames[dirIndex2]; + name = &dirNames[(unsigned)dirIndex2]; pair.State = NUpdateArchive::NPairState::kOnlyOnDisk; pair.DirIndex = dirIndex2; dirIndex++; @@ -184,9 +186,9 @@ void GetUpdatePairInfoList( { int dupl = duplicatedArcItem[arcIndex]; if (dupl != 0) - ThrowError(k_Duplicate_inArc_Message, ai->Name, arcItems[arcIndices[arcIndex + dupl]].Name); + ThrowError(k_Duplicate_inArc_Message, ai->Name, arcItems[arcIndices[(unsigned)((int)arcIndex + dupl)]].Name); - name = &dirNames[dirIndex2]; + name = &dirNames[(unsigned)dirIndex2]; if (!ai->Censored) ThrowError(k_NotCensoredCollision_Message, *name, ai->Name); @@ -222,7 +224,7 @@ void GetUpdatePairInfoList( } else { - prevHostFile = updatePairs.Size(); + prevHostFile = (int)updatePairs.Size(); prevHostName = name; } diff --git a/CPP/7zip/UI/Common/UpdateProduce.cpp b/CPP/7zip/UI/Common/UpdateProduce.cpp index d5052f13..fa4bd69c 100644 --- a/CPP/7zip/UI/Common/UpdateProduce.cpp +++ b/CPP/7zip/UI/Common/UpdateProduce.cpp @@ -28,7 +28,7 @@ void UpdateProduce( { case NPairAction::kIgnore: if (pair.ArcIndex >= 0 && callback) - callback->ShowDeleteFile(pair.ArcIndex); + callback->ShowDeleteFile((unsigned)pair.ArcIndex); continue; case NPairAction::kCopy: @@ -43,7 +43,7 @@ void UpdateProduce( 1) no such alt stream in Disk 2) there is Host file in disk */ - if (updatePairs[pair.HostIndex].DirIndex >= 0) + if (updatePairs[(unsigned)pair.HostIndex].DirIndex >= 0) continue; } } diff --git a/CPP/7zip/UI/Common/UpdateProduce.h b/CPP/7zip/UI/Common/UpdateProduce.h index 64c58cc5..595370fe 100644 --- a/CPP/7zip/UI/Common/UpdateProduce.h +++ b/CPP/7zip/UI/Common/UpdateProduce.h @@ -18,12 +18,12 @@ struct CUpdatePair2 bool IsMainRenameItem; - void SetAs_NoChangeArcItem(int arcIndex) + void SetAs_NoChangeArcItem(unsigned arcIndex) // int { NewData = NewProps = false; UseArcProps = true; IsAnti = false; - ArcIndex = arcIndex; + ArcIndex = (int)arcIndex; } bool ExistOnDisk() const { return DirIndex != -1; } diff --git a/CPP/7zip/UI/Common/WorkDir.cpp b/CPP/7zip/UI/Common/WorkDir.cpp index 284eaa16..1307ceeb 100644 --- a/CPP/7zip/UI/Common/WorkDir.cpp +++ b/CPP/7zip/UI/Common/WorkDir.cpp @@ -5,6 +5,7 @@ #include "../../../Common/StringConvert.h" #include "../../../Common/Wildcard.h" +#include "../../../Windows/FileFind.h" #include "../../../Windows/FileName.h" #include "WorkDir.h" @@ -39,13 +40,13 @@ FString GetWorkDir(const NWorkDir::CInfo &workDirInfo, const FString &path, FStr #endif int pos = path.ReverseFind_PathSepar() + 1; - fileName = path.Ptr(pos); + fileName = path.Ptr((unsigned)pos); switch (mode) { case NWorkDir::NMode::kCurrent: { - return path.Left(pos); + return path.Left((unsigned)pos); } case NWorkDir::NMode::kSpecified: { @@ -75,8 +76,7 @@ HRESULT CWorkDirTempFile::CreateTempFile(const FString &originalPath) OutStream = _outStreamSpec; if (!_tempFile.Create(workDir + namePart, &_outStreamSpec->File)) { - DWORD error = GetLastError(); - return error ? error : E_FAIL; + return GetLastError_noZero_HRESULT(); } _originalPath = originalPath; return S_OK; @@ -87,8 +87,7 @@ HRESULT CWorkDirTempFile::MoveToOriginal(bool deleteOriginal) OutStream.Release(); if (!_tempFile.MoveTo(_originalPath, deleteOriginal)) { - DWORD error = GetLastError(); - return error ? error : E_FAIL; + return GetLastError_noZero_HRESULT(); } return S_OK; } diff --git a/CPP/7zip/UI/Console/Console.dsp b/CPP/7zip/UI/Console/Console.dsp index 34918e38..53ec5525 100644 --- a/CPP/7zip/UI/Console/Console.dsp +++ b/CPP/7zip/UI/Console/Console.dsp @@ -357,6 +357,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Console/ConsoleClose.cpp b/CPP/7zip/UI/Console/ConsoleClose.cpp index 703f8213..9e4c040d 100644 --- a/CPP/7zip/UI/Console/ConsoleClose.cpp +++ b/CPP/7zip/UI/Console/ConsoleClose.cpp @@ -4,8 +4,13 @@ #include "ConsoleClose.h" -#if !defined(UNDER_CE) && defined(_WIN32) +#ifndef UNDER_CE + +#ifdef _WIN32 #include "../../../Common/MyWindows.h" +#else +#include <stdlib.h> +#include <signal.h> #endif namespace NConsoleClose { @@ -13,7 +18,8 @@ namespace NConsoleClose { unsigned g_BreakCounter = 0; static const unsigned kBreakAbortThreshold = 2; -#if !defined(UNDER_CE) && defined(_WIN32) +#ifdef _WIN32 + static BOOL WINAPI HandlerRoutine(DWORD ctrlType) { if (ctrlType == CTRL_LOGOFF_EVENT) @@ -37,33 +43,58 @@ static BOOL WINAPI HandlerRoutine(DWORD ctrlType) return FALSE; */ } -#endif - -/* -void CheckCtrlBreak() -{ - if (TestBreakSignal()) - throw CCtrlBreakException(); -} -*/ CCtrlHandlerSetter::CCtrlHandlerSetter() { - #if !defined(UNDER_CE) && defined(_WIN32) if (!SetConsoleCtrlHandler(HandlerRoutine, TRUE)) throw "SetConsoleCtrlHandler fails"; - #endif } CCtrlHandlerSetter::~CCtrlHandlerSetter() { - #if !defined(UNDER_CE) && defined(_WIN32) if (!SetConsoleCtrlHandler(HandlerRoutine, FALSE)) { // warning for throw in destructor. // throw "SetConsoleCtrlHandler fails"; } - #endif } +#else // _WIN32 + +static void HandlerRoutine(int) +{ + g_BreakCounter++; + if (g_BreakCounter < kBreakAbortThreshold) + return; + exit(EXIT_FAILURE); +} + +CCtrlHandlerSetter::CCtrlHandlerSetter() +{ + memo_sig_int = signal(SIGINT, HandlerRoutine); // CTRL-C + if (memo_sig_int == SIG_ERR) + throw "SetConsoleCtrlHandler fails (SIGINT)"; + memo_sig_term = signal(SIGTERM, HandlerRoutine); // for kill -15 (before "kill -9") + if (memo_sig_term == SIG_ERR) + throw "SetConsoleCtrlHandler fails (SIGTERM)"; +} + +CCtrlHandlerSetter::~CCtrlHandlerSetter() +{ + signal(SIGINT, memo_sig_int); // CTRL-C + signal(SIGTERM, memo_sig_term); // kill {pid} +} + +#endif // _WIN32 + +/* +void CheckCtrlBreak() +{ + if (TestBreakSignal()) + throw CCtrlBreakException(); +} +*/ + } + +#endif diff --git a/CPP/7zip/UI/Console/ConsoleClose.h b/CPP/7zip/UI/Console/ConsoleClose.h index 11c1631c..9c9e035c 100644 --- a/CPP/7zip/UI/Console/ConsoleClose.h +++ b/CPP/7zip/UI/Console/ConsoleClose.h @@ -5,28 +5,34 @@ namespace NConsoleClose { +class CCtrlBreakException {}; + +#ifdef UNDER_CE + +inline bool TestBreakSignal() { return false; } +struct CCtrlHandlerSetter {}; + +#else + extern unsigned g_BreakCounter; inline bool TestBreakSignal() { - #ifdef UNDER_CE - return false; - #else return (g_BreakCounter != 0); - #endif } class CCtrlHandlerSetter { + #ifndef _WIN32 + void (*memo_sig_int)(int); + void (*memo_sig_term)(int); + #endif public: CCtrlHandlerSetter(); virtual ~CCtrlHandlerSetter(); }; -class CCtrlBreakException -{}; - -// void CheckCtrlBreak(); +#endif } diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index 21c2f071..23eab615 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -69,6 +69,7 @@ HRESULT CExtractScanConsole::ScanError(const FString &path, DWORD systemError) } +void Print_UInt64_and_String(AString &s, UInt64 val, const char *name); void Print_UInt64_and_String(AString &s, UInt64 val, const char *name) { char temp[32]; @@ -78,6 +79,7 @@ void Print_UInt64_and_String(AString &s, UInt64 val, const char *name) s += name; } +void PrintSize_bytes_Smart(AString &s, UInt64 val); void PrintSize_bytes_Smart(AString &s, UInt64 val) { Print_UInt64_and_String(s, val, "bytes"); @@ -96,7 +98,7 @@ void PrintSize_bytes_Smart(AString &s, UInt64 val) s += ')'; } -void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) +static void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) { if (val == (UInt64)(Int64)-1) return; @@ -106,6 +108,7 @@ void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) +void Print_DirItemsStat(AString &s, const CDirItemsStat &st); void Print_DirItemsStat(AString &s, const CDirItemsStat &st) { if (st.NumDirs != 0) @@ -124,6 +127,7 @@ void Print_DirItemsStat(AString &s, const CDirItemsStat &st) } +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st); void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st) { Print_DirItemsStat(s, (CDirItemsStat &)st); @@ -184,9 +188,9 @@ static const char * const kTestString = "T"; static const char * const kExtractString = "-"; static const char * const kSkipString = "."; -// static const char * const kCantAutoRename = "can not create file with auto name\n"; -// static const char * const kCantRenameFile = "can not rename existing file\n"; -// static const char * const kCantDeleteOutputFile = "can not delete output file "; +// static const char * const kCantAutoRename = "cannot create file with auto name\n"; +// static const char * const kCantRenameFile = "cannot rename existing file\n"; +// static const char * const kCantDeleteOutputFile = "cannot delete output file "; static const char * const kMemoryExceptionMessage = "Can't allocate required memory!"; @@ -394,6 +398,7 @@ STDMETHODIMP CExtractCallbackConsole::MessageError(const wchar_t *message) return CheckBreak2(); } +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest); void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest) { dest.Empty(); @@ -436,7 +441,7 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest) else { dest += "Error #"; - dest.Add_UInt32(opRes); + dest.Add_UInt32((UInt32)opRes); } } @@ -566,6 +571,7 @@ static AString GetOpenArcErrorMessage(UInt32 errorFlags) return s; } +void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags); void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags) { if (errorFlags == 0) @@ -573,7 +579,7 @@ void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags) so << s << endl << GetOpenArcErrorMessage(errorFlags) << endl; } -void Add_Messsage_Pre_ArcType(UString &s, const char *pre, const wchar_t *arcType) +static void Add_Messsage_Pre_ArcType(UString &s, const char *pre, const wchar_t *arcType) { s.Add_LF(); s += pre; @@ -582,6 +588,7 @@ void Add_Messsage_Pre_ArcType(UString &s, const char *pre, const wchar_t *arcTyp s += "] archive"; } +void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, const CArc &arc); void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, const CArc &arc) { const CArcErrorInfo &er = arc.ErrorInfo; @@ -596,7 +603,7 @@ void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, c } else { - Add_Messsage_Pre_ArcType(s, "Can not open the file", codecs->GetFormatNamePtr(er.ErrorFormatIndex)); + Add_Messsage_Pre_ArcType(s, "Cannot open the file", codecs->GetFormatNamePtr(er.ErrorFormatIndex)); Add_Messsage_Pre_ArcType(s, "The file is open", codecs->GetFormatNamePtr(arc.FormatIndex)); } @@ -806,7 +813,9 @@ HRESULT CExtractCallbackConsole::ExtractResult(HRESULT result) else { NumArcsWithError++; - if (result == E_ABORT || result == ERROR_DISK_FULL) + if (result == E_ABORT + || result == HRESULT_FROM_WIN32(ERROR_DISK_FULL) + ) return result; if (_se) diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.h b/CPP/7zip/UI/Console/ExtractCallbackConsole.h index dc659521..5ac1d0b0 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.h +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.h @@ -32,6 +32,9 @@ class CExtractScanConsole: public IDirItemsCallback } public: + + virtual ~CExtractScanConsole() {} + void Init(CStdOutStream *outStream, CStdOutStream *errorStream, CStdOutStream *percentStream) { _so = outStream; diff --git a/CPP/7zip/UI/Console/HashCon.cpp b/CPP/7zip/UI/Console/HashCon.cpp index 762b21bb..a70f5f8a 100644 --- a/CPP/7zip/UI/Console/HashCon.cpp +++ b/CPP/7zip/UI/Console/HashCon.cpp @@ -222,7 +222,8 @@ void CHashCallbackConsole::PrintResultLine(UInt64 fileSize, s[0] = 0; if (showHash) AddHashHexToString(s, h.Digests[digestIndex], h.DigestSize); - SetSpacesAndNul(s + strlen(s), (int)GetColumnWidth(h.DigestSize) - (int)strlen(s)); + const unsigned pos = (unsigned)strlen(s); + SetSpacesAndNul(s + pos, GetColumnWidth(h.DigestSize) - pos); if (i != 0) _s.Add_Space(); _s += s; @@ -235,20 +236,15 @@ void CHashCallbackConsole::PrintResultLine(UInt64 fileSize, char s[kSizeField_Len + 32]; char *p = s; + SetSpacesAndNul(s, kSizeField_Len); if (showHash) { p = s + kSizeField_Len; ConvertUInt64ToString(fileSize, p); - int numSpaces = kSizeField_Len - (int)strlen(p); + int numSpaces = (int)kSizeField_Len - (int)strlen(p); if (numSpaces > 0) - { p -= (unsigned)numSpaces; - for (unsigned i = 0; i < (unsigned)numSpaces; i++) - p[i] = ' '; - } } - else - SetSpacesAndNul(s, kSizeField_Len); _s += p; } diff --git a/CPP/7zip/UI/Console/HashCon.h b/CPP/7zip/UI/Console/HashCon.h index 5b30b69a..0731bd18 100644 --- a/CPP/7zip/UI/Console/HashCon.h +++ b/CPP/7zip/UI/Console/HashCon.h @@ -38,7 +38,7 @@ public: PrintName(true) {} - ~CHashCallbackConsole() { } + virtual ~CHashCallbackConsole() {} INTERFACE_IHashCallbackUI(;) }; diff --git a/CPP/7zip/UI/Console/List.cpp b/CPP/7zip/UI/Console/List.cpp index 416ef2c9..d6cb9825 100644 --- a/CPP/7zip/UI/Console/List.cpp +++ b/CPP/7zip/UI/Console/List.cpp @@ -351,7 +351,7 @@ struct CListStat2 AltStreams.Update(st.AltStreams); NumDirs += st.NumDirs; } - const UInt64 GetNumStreams() const { return MainFiles.NumFiles + AltStreams.NumFiles; } + UInt64 GetNumStreams() const { return MainFiles.NumFiles + AltStreams.NumFiles; } CListStat &GetStat(bool altStreamsMode) { return altStreamsMode ? AltStreams : MainFiles; } }; @@ -751,7 +751,7 @@ void CFieldPrinter::PrintSum(const CListStat2 &stat2) PrintSum(stat2.MainFiles, stat2.NumDirs, kString_Files); if (stat2.AltStreams.NumFiles != 0) { - PrintSum(stat2.AltStreams, 0, kString_AltStreams);; + PrintSum(stat2.AltStreams, 0, kString_AltStreams); CListStat st = stat2.MainFiles; st.Update(stat2.AltStreams); PrintSum(st, 0, kString_Streams); @@ -905,7 +905,7 @@ static HRESULT PrintArcProp(CStdOutStream &so, IInArchive *archive, PROPID propI static void PrintArcTypeError(CStdOutStream &so, const UString &type, bool isWarning) { so << "Open " << (isWarning ? "WARNING" : "ERROR") - << ": Can not open the file as [" + << ": Cannot open the file as [" << type << "] archive" << endl; @@ -926,6 +926,7 @@ static void ErrorInfo_Print(CStdOutStream &so, const CArcErrorInfo &er) PrintPropPair(so, "WARNING", er.WarningMessage, true); } +HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) { FOR_VECTOR (r, arcLink.Arcs) @@ -990,11 +991,12 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const return S_OK; } +HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) { #ifndef _NO_CRYPTO if (arcLink.PasswordWasAsked) - so << "Can not open encrypted archive. Wrong password?"; + so << "Cannot open encrypted archive. Wrong password?"; else #endif { @@ -1002,10 +1004,10 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const { so.NormalizePrint_UString(arcLink.NonOpen_ArcPath); so << endl; - PrintArcTypeError(so, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); + PrintArcTypeError(so, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } else - so << "Can not open the file as archive"; + so << "Cannot open the file as archive"; } so << endl; @@ -1065,12 +1067,12 @@ HRESULT ListArchives(CCodecs *codecs, if (!stdInMode) { NFile::NFind::CFileInfo fi; - if (!fi.Find(us2fs(arcPath))) + if (!fi.Find_FollowLink(us2fs(arcPath))) { DWORD errorCode = GetLastError(); if (errorCode == 0) errorCode = ERROR_FILE_NOT_FOUND; - lastError = HRESULT_FROM_WIN32(lastError);; + lastError = HRESULT_FROM_WIN32(errorCode); g_StdOut.Flush(); if (g_ErrStream) { @@ -1279,7 +1281,7 @@ HRESULT ListArchives(CCodecs *codecs, } else { - SplitPathToParts(fp.FilePath, pathParts);; + SplitPathToParts(fp.FilePath, pathParts); bool include; if (!wildcardCensor.CheckPathVect(pathParts, !fp.IsDir, include)) continue; @@ -1331,7 +1333,7 @@ HRESULT ListArchives(CCodecs *codecs, { g_StdOut << "----------\n"; PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath, false); - PrintArcTypeError(g_StdOut, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); + PrintArcTypeError(g_StdOut, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } } diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index 333223bd..8c24aaff 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -6,6 +6,11 @@ #ifdef _WIN32 #include <Psapi.h> +#else +#include <unistd.h> +#include <sys/ioctl.h> +#include <sys/time.h> +#include <sys/times.h> #endif #include "../../../../C/CpuArch.h" @@ -15,12 +20,13 @@ #include "../../../Common/CommandLineParser.h" #include "../../../Common/IntToString.h" #include "../../../Common/MyException.h" +#include "../../../Common/StdInStream.h" +#include "../../../Common/StdOutStream.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/ErrorMsg.h" - #include "../../../Windows/TimeUtils.h" #include "../Common/ArchiveCommandLine.h" @@ -37,12 +43,11 @@ #include "BenchCon.h" #include "ConsoleClose.h" #include "ExtractCallbackConsole.h" +#include "HashCon.h" #include "List.h" #include "OpenCallbackConsole.h" #include "UpdateCallbackConsole.h" -#include "HashCon.h" - #ifdef PROG_VARIANT_R #include "../../../../C/7zVersion.h" #else @@ -66,27 +71,30 @@ extern const CCodecInfo *g_Codecs[]; extern unsigned g_NumHashers; extern const CHasherInfo *g_Hashers[]; -static const char * const kCopyrightString = "\n7-Zip" - #ifndef EXTERNAL_CODECS - #ifdef PROG_VARIANT_R - " (r)" - #else - " (a)" - #endif - #endif +#if defined(PROG_VARIANT_Z) + #define PROG_POSTFIX "z" + #define PROG_POSTFIX_2 " (z)" +#elif defined(PROG_VARIANT_R) + #define PROG_POSTFIX "r" + #define PROG_POSTFIX_2 " (r)" +#elif !defined(EXTERNAL_CODECS) + #define PROG_POSTFIX "a" + #define PROG_POSTFIX_2 " (a)" +#else + #define PROG_POSTFIX "" + #define PROG_POSTFIX_2 "" +#endif + + +static const char * const kCopyrightString = "\n7-Zip" + PROG_POSTFIX_2 " " MY_VERSION_CPU - " : " MY_COPYRIGHT_DATE "\n\n"; + " : " MY_COPYRIGHT_DATE "\n"; static const char * const kHelpString = "Usage: 7z" -#ifndef EXTERNAL_CODECS -#ifdef PROG_VARIANT_R - "r" -#else - "a" -#endif -#endif + PROG_POSTFIX " <command> [<switches>...] <archive_name> [<file_names>...] [@listfile]\n" "\n" "<Commands>\n" @@ -141,6 +149,7 @@ static const char * const kHelpString = " -spf : use fully qualified file paths\n" " -ssc[-] : set sensitive case mode\n" " -sse : stop archive creating, if it can't open some input file\n" + " -ssp : do not change Last Access Time of source files while archiving\n" " -ssw : compress shared files\n" " -stl : set archive timestamp from the most recently modified file\n" " -stm{HexMask} : set CPU thread affinity mask (hexadecimal number)\n" @@ -163,6 +172,7 @@ static const char * const kUnsupportedArcTypeMessage = "Unsupported archive type #define kDefaultSfxModule "7zCon.sfx" +MY_ATTR_NORETURN static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) { if (g_ErrStream) @@ -170,15 +180,109 @@ static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) throw code; } -#ifndef _WIN32 -static void GetArguments(int numArgs, const char *args[], UStringVector &parts) + +#ifdef _WIN32 +#define ShowProgInfo(so) +#else +static void ShowProgInfo(CStdOutStream *so) { - parts.Clear(); - for (int i = 0; i < numArgs; i++) + if (!so) + return; + + *so + + /* + #ifdef __DATE__ + << " " << __DATE__ + #endif + #ifdef __TIME__ + << " " << __TIME__ + #endif + */ + + #ifdef __VERSION__ + << " compiler: " << __VERSION__ + #endif + + #ifdef __GNUC__ + << " GCC " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__ + #endif + + #ifdef __clang__ + << " CLANG " << __clang_major__ << "." << __clang_minor__ + #endif + + #ifdef __xlC__ + << " XLC " << (__xlC__ >> 8) << "." << (__xlC__ & 0xFF) + #ifdef __xlC_ver__ + << "." << (__xlC_ver__ >> 8) << "." << (__xlC_ver__ & 0xFF) + #endif + #endif + + #ifdef _MSC_VER + << " MSC " << _MSC_VER + #endif + + #ifdef __ARM_FEATURE_CRC32 + << " CRC32" + #endif + + << " " << (unsigned)(sizeof(void *)) * 8 << "-bit" + + #ifdef __ILP32__ + << " ILP32" + #endif + + #ifdef __ARM_ARCH + << " arm_v:" << __ARM_ARCH + #ifdef __ARM_ARCH_ISA_THUMB + << " thumb:" << __ARM_ARCH_ISA_THUMB + #endif + #endif + + + #ifdef ENV_HAVE_LOCALE + << " locale=" << GetLocale() + #endif + #ifndef _WIN32 + << " UTF8=" << (IsNativeUTF8() ? "+" : "-") + << " use-UTF8=" << (g_ForceToUTF8 ? "+" : "-") + << " wchar_t=" << (unsigned)(sizeof(wchar_t)) * 8 << "-bit" + << " Files=" << (unsigned)(sizeof(off_t)) * 8 << "-bit" + #endif + ; + { - UString s = MultiByteToUnicodeString(args[i]); - parts.Add(s); + const UInt32 numCpus = NWindows::NSystem::GetNumberOfProcessors(); + *so << " Threads:" << numCpus; } + + { + AString s; + GetCpuName(s); + s.Trim(); + *so << ", " << s; + } + + #ifdef _7ZIP_ASM + *so << ",ASM"; + #endif + + #if (defined MY_CPU_X86_OR_AMD64 || defined(MY_CPU_ARM_OR_ARM64)) + if (CPU_IsSupported_AES()) *so << ",AES"; + #endif + + #ifdef MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_CRC32()) *so << ",CRC32"; + #if defined(_WIN32) + if (CPU_IsSupported_CRYPTO()) *so << ",CRYPTO"; + #else + if (CPU_IsSupported_SHA1()) *so << ",SHA1"; + if (CPU_IsSupported_SHA2()) *so << ",SHA2"; + #endif + #endif + + *so << endl; } #endif @@ -188,6 +292,8 @@ static void ShowCopyrightAndHelp(CStdOutStream *so, bool needHelp) return; *so << kCopyrightString; // *so << "# CPUs: " << (UInt64)NWindows::NSystem::GetNumberOfProcessors() << endl; + ShowProgInfo(so); + *so << endl; if (needHelp) *so << kHelpString; } @@ -211,7 +317,7 @@ static void PrintUInt32(CStdOutStream &so, UInt32 val, unsigned size) static void PrintLibIndex(CStdOutStream &so, int libIndex) { if (libIndex >= 0) - PrintUInt32(so, libIndex, 2); + PrintUInt32(so, (UInt32)libIndex, 2); else so << " "; so << ' '; @@ -329,7 +435,6 @@ static void ThrowException_if_Error(HRESULT res) throw CSystemException(res); } - static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') { char temp[64]; @@ -341,6 +446,8 @@ static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') *g_StdStream << p; } +#ifdef _WIN32 + static void PrintTime(const char *s, UInt64 val, UInt64 total) { *g_StdStream << endl << s << " Time ="; @@ -427,19 +534,19 @@ static void PrintStat() HMODULE kern = ::GetModuleHandleW(L"kernel32.dll"); Func_GetProcessMemoryInfo my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo) - ::GetProcAddress(kern, "K32GetProcessMemoryInfo"); + (void *)::GetProcAddress(kern, "K32GetProcessMemoryInfo"); if (!my_GetProcessMemoryInfo) { HMODULE lib = LoadLibraryW(L"Psapi.dll"); if (lib) - my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo)::GetProcAddress(lib, "GetProcessMemoryInfo"); + my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo)(void *)::GetProcAddress(lib, "GetProcessMemoryInfo"); } if (my_GetProcessMemoryInfo) memDefined = my_GetProcessMemoryInfo(GetCurrentProcess(), &m, sizeof(m)); // FreeLibrary(lib); Func_QueryProcessCycleTime my_QueryProcessCycleTime = (Func_QueryProcessCycleTime) - ::GetProcAddress(kern, "QueryProcessCycleTime"); + (void *)::GetProcAddress(kern, "QueryProcessCycleTime"); if (my_QueryProcessCycleTime) cycleDefined = my_QueryProcessCycleTime(GetCurrentProcess(), &cycleTime); } @@ -455,18 +562,32 @@ static void PrintStat() PrintTime("Kernel ", kernelTime, totalTime); + const UInt64 processTime = kernelTime + userTime; + #ifndef UNDER_CE if (cycleDefined) { - *g_StdStream << " "; - PrintNum(cycleTime / 1000000, 22); + *g_StdStream << " Cnt:"; + PrintNum(cycleTime / 1000000, 15); *g_StdStream << " MCycles"; } #endif PrintTime("User ", userTime, totalTime); + + #ifndef UNDER_CE + if (cycleDefined) + { + *g_StdStream << " Freq (cnt/ptime):"; + UInt64 us = processTime / 10; + if (us == 0) + us = 1; + PrintNum(cycleTime / us, 6); + *g_StdStream << " MHz"; + } + #endif - PrintTime("Process", kernelTime + userTime, totalTime); + PrintTime("Process", processTime, totalTime); #ifndef UNDER_CE if (memDefined) PrintMemUsage("Virtual ", m.PeakPagefileUsage); #endif @@ -475,10 +596,96 @@ static void PrintStat() #ifndef UNDER_CE if (memDefined) PrintMemUsage("Physical", m.PeakWorkingSetSize); #endif +} + + +#else // ! _WIN32 + +static UInt64 Get_timeofday_us() +{ + struct timeval now; + if (gettimeofday(&now, 0 ) == 0) + return (UInt64)now.tv_sec * 1000000 + (UInt64)now.tv_usec; + return 0; +} + +static void PrintTime(const char *s, UInt64 val, UInt64 total_us, UInt64 kFreq) +{ + *g_StdStream << endl << s << " Time ="; + + { + UInt64 sec, ms; + + if (kFreq == 0) + { + sec = val / 1000000; + ms = val % 1000000 / 1000; + } + else + { + sec = val / kFreq; + ms = (UInt32)((val - (sec * kFreq)) * 1000 / kFreq); + } + + PrintNum(sec, 6); + *g_StdStream << '.'; + PrintNum(ms, 3, '0'); + } + if (total_us == 0) + return; + + UInt64 percent = 0; + if (kFreq == 0) + percent = val * 100 / total_us; + else + { + const UInt64 kMaxVal = (UInt64)(Int64)-1; + UInt32 m = 100000000; + for (;;) + { + if (m == 0 || kFreq == 0) + break; + if (kMaxVal / m > val && + kMaxVal / kFreq > total_us) + break; + if (val > m) + val >>= 1; + else + m >>= 1; + if (kFreq > total_us) + kFreq >>= 1; + else + total_us >>= 1; + } + const UInt64 total = kFreq * total_us; + if (total != 0) + percent = val * m / total; + } + *g_StdStream << " ="; + PrintNum(percent, 5); + *g_StdStream << '%'; +} + +static void PrintStat(UInt64 startTime) +{ + tms t; + /* clock_t res = */ times(&t); + const UInt64 totalTime = Get_timeofday_us() - startTime; + const UInt64 kFreq = (UInt64)sysconf(_SC_CLK_TCK); + PrintTime("Kernel ", (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("User ", (UInt64)t.tms_utime, totalTime, kFreq); + PrintTime("Process", (UInt64)t.tms_utime + (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("Global ", totalTime, totalTime, 0); *g_StdStream << endl; } +#endif // ! _WIN32 + + + + + static void PrintHexId(CStdOutStream &so, UInt64 id) { char s[32]; @@ -486,23 +693,62 @@ static void PrintHexId(CStdOutStream &so, UInt64 id) PrintStringRight(so, s, 8); } +#ifndef _WIN32 +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +#endif int Main2( #ifndef _WIN32 int numArgs, char *args[] #endif +); +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] + #endif ) { + #if defined(MY_CPU_SIZEOF_POINTER) + { unsigned k = sizeof(void *); if (k != MY_CPU_SIZEOF_POINTER) throw "incorrect MY_CPU_PTR_SIZE"; } + #endif + #if defined(_WIN32) && !defined(UNDER_CE) SetFileApisToOEM(); #endif + #ifdef ENV_HAVE_LOCALE + // printf("\nBefore SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + MY_SetLocale(); + // printf("\nAfter SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + #endif + + #ifndef _WIN32 + UInt64 startTime = Get_timeofday_us(); + #endif + UStringVector commandStrings; #ifdef _WIN32 NCommandLineParser::SplitCommandLine(GetCommandLineW(), commandStrings); #else - GetArguments(numArgs, args, commandStrings); + { + if (numArgs > 0) + Set_ModuleDirPrefix_From_ProgArg0(args[0]); + + for (int i = 0; i < numArgs; i++) + { + AString a (args[i]); + /* + printf("\n%d %s :", i, a.Ptr()); + for (unsigned k = 0; k < a.Len(); k++) + printf(" %2x", (unsigned)(Byte)a[k]); + */ + const UString s = MultiByteToUnicodeString(a); + commandStrings.Add(s); + } + // printf("\n"); + } + #endif #ifndef UNDER_CE @@ -542,10 +788,53 @@ int Main2( } if (options.EnableHeaders) + { ShowCopyrightAndHelp(g_StdStream, false); + if (!parser.Parse1Log.IsEmpty()) + *g_StdStream << parser.Parse1Log; + } parser.Parse2(options); + { + int cp = options.ConsoleCodePage; + + int stdout_cp = cp; + int stderr_cp = cp; + int stdin_cp = cp; + + /* + // these cases are complicated. + // maybe we must use CRT functions instead of console WIN32. + // different Windows/CRT versions also can work different ways. + // so the following code was not enabled: + if (cp == -1) + { + // we set CodePage only if stream is attached to terminal + // maybe we should set CodePage even if is not terminal? + #ifdef _WIN32 + { + UINT ccp = GetConsoleOutputCP(); + if (ccp != 0) + { + if (options.IsStdOutTerminal) stdout_cp = ccp; + if (options.IsStdErrTerminal) stderr_cp = ccp; + } + } + if (options.IsInTerminal) + { + UINT ccp = GetConsoleCP(); + if (ccp != 0) stdin_cp = ccp; + } + #endif + } + */ + + if (stdout_cp != -1) g_StdOut.CodePage = stdout_cp; + if (stderr_cp != -1) g_StdErr.CodePage = stderr_cp; + if (stdin_cp != -1) g_StdIn.CodePage = stdin_cp; + } + unsigned percentsNameLevel = 1; if (options.LogLevel == 0 || options.Number_for_Percents != options.Number_for_Out) percentsNameLevel = 2; @@ -559,13 +848,13 @@ int Main2( #if !defined(UNDER_CE) CONSOLE_SCREEN_BUFFER_INFO consoleInfo; if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &consoleInfo)) - consoleWidth = consoleInfo.dwSize.X; + consoleWidth = (unsigned)consoleInfo.dwSize.X; #endif #else struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) == ) + if (ioctl(0, TIOCGWINSZ, &w) == 0) consoleWidth = w.ws_col; #endif @@ -577,6 +866,19 @@ int Main2( codecs->CaseSensitive = options.CaseSensitive; ThrowException_if_Error(codecs->Load()); + #ifdef EXTERNAL_CODECS + { + UString s; + codecs->GetCodecsErrorMessage(s); + if (!s.IsEmpty()) + { + CStdOutStream &so = (g_StdStream ? *g_StdStream : g_StdOut); + so << endl << s << endl; + } + } + #endif + + bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); if (codecs->Formats.Size() == 0 && @@ -640,14 +942,14 @@ int Main2( so << endl << "Libs:" << endl; for (i = 0; i < codecs->Libs.Size(); i++) { - PrintLibIndex(so, i); + PrintLibIndex(so, (int)i); so << ' ' << codecs->Libs[i].Path << endl; } #endif so << endl << "Formats:" << endl; - const char * const kArcFlags = "KSNFMGOPBELH"; + const char * const kArcFlags = "KSNFMGOPBELHX"; const unsigned kNumArcFlags = (unsigned)strlen(kArcFlags); for (i = 0; i < codecs->Formats.Size(); i++) @@ -804,6 +1106,7 @@ int Main2( options.Properties, options.NumIterations, (FILE *)so); if (hresultMain == S_FALSE) { + so << endl; if (g_ErrStream) *g_ErrStream << "\nDecoding ERROR\n"; retCode = NExitCode::kFatalError; @@ -863,7 +1166,7 @@ int Main2( } } - if (hresultMain == S_OK) + if (hresultMain == S_OK) { if (isExtractGroupCommand) { CExtractCallbackConsole *ecs = new CExtractCallbackConsole; @@ -992,7 +1295,7 @@ int Main2( if (isError) retCode = NExitCode::kFatalError; - if (so) + if (so) { if (ecs->NumArcsWithError != 0 || ecs->NumFileErrors != 0) { // if (ecs->NumArchives > 1) @@ -1023,8 +1326,9 @@ int Main2( PrintHashStat(*so, hb); } } + } // if (so) } - else + else // if_(!isExtractGroupCommand) { UInt64 numErrors = 0; UInt64 numWarnings = 0; @@ -1060,7 +1364,8 @@ int Main2( g_StdOut << endl << "Errors: " << numErrors << endl; retCode = NExitCode::kFatalError; } - } + } // if_(isExtractGroupCommand) + } // if_(hresultMain == S_OK) } else if (options.Command.IsFromUpdateGroup()) { @@ -1146,7 +1451,11 @@ int Main2( ShowMessageAndThrowException(kUserErrorMessage, NExitCode::kUserError); if (options.ShowTime && g_StdStream) - PrintStat(); + PrintStat( + #ifndef _WIN32 + startTime + #endif + ); ThrowException_if_Error(hresultMain); diff --git a/CPP/7zip/UI/Console/MainAr.cpp b/CPP/7zip/UI/Console/MainAr.cpp index 87bf57e8..0e45312c 100644 --- a/CPP/7zip/UI/Console/MainAr.cpp +++ b/CPP/7zip/UI/Console/MainAr.cpp @@ -19,7 +19,11 @@ using namespace NWindows; +extern +CStdOutStream *g_StdStream; CStdOutStream *g_StdStream = NULL; +extern +CStdOutStream *g_ErrStream; CStdOutStream *g_ErrStream = NULL; extern int Main2( @@ -48,7 +52,9 @@ static void PrintError(const char *message) *g_ErrStream << "\n\n" << message << endl; } +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION *g_StdStream << "Unsupported Windows version"; return NExitCode::kFatalError; +#endif int MY_CDECL main ( diff --git a/CPP/7zip/UI/Console/OpenCallbackConsole.h b/CPP/7zip/UI/Console/OpenCallbackConsole.h index 64c1dad3..075d3741 100644 --- a/CPP/7zip/UI/Console/OpenCallbackConsole.h +++ b/CPP/7zip/UI/Console/OpenCallbackConsole.h @@ -46,6 +46,8 @@ public: #endif {} + + virtual ~COpenCallbackConsole() {} void Init(CStdOutStream *outStream, CStdOutStream *errorStream, CStdOutStream *percentStream) { diff --git a/CPP/7zip/UI/Console/PercentPrinter.cpp b/CPP/7zip/UI/Console/PercentPrinter.cpp index b2426878..4341fd9f 100644 --- a/CPP/7zip/UI/Console/PercentPrinter.cpp +++ b/CPP/7zip/UI/Console/PercentPrinter.cpp @@ -143,7 +143,7 @@ void CPercentPrinter::Print() _tempU = FileName; _so->Normalize_UString(_tempU); - StdOut_Convert_UString_to_AString(_tempU, _temp); + _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() > MaxLen) { unsigned len = FileName.Len(); @@ -157,7 +157,7 @@ void CPercentPrinter::Print() _tempU.Delete(len / 2, _tempU.Len() - len); _tempU.Insert(len / 2, L" . "); _so->Normalize_UString(_tempU); - StdOut_Convert_UString_to_AString(_tempU, _temp); + _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() <= MaxLen) break; } diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index cd232fff..24056072 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -209,6 +209,7 @@ HRESULT CCallbackConsoleBase::OpenFileError_Base(const FString &path, DWORD syst { MT_LOCK FailedFiles.AddError(path, systemError); + NumNonOpenFiles++; /* if (systemError == ERROR_SHARING_VIOLATION) { @@ -282,6 +283,12 @@ HRESULT CUpdateCallbackConsole::StartOpenArchive(const wchar_t *name) HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) { + if (NeedPercents()) + _percent.ClosePrint(true); + + _percent.ClearCurState(); + NumNonOpenFiles = 0; + if (_so) { *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); @@ -302,7 +309,7 @@ HRESULT CUpdateCallbackConsole::FinishArchive(const CFinishArchiveStat &st) { AString s; // Print_UInt64_and_String(s, _percent.Files == 1 ? "file" : "files", _percent.Files); - PrintPropPair(s, "Files read from disk", _percent.Files); + PrintPropPair(s, "Files read from disk", _percent.Files - NumNonOpenFiles); s.Add_LF(); s += "Archive size: "; PrintSize_bytes_Smart(s, st.OutArcFileSize); diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.h b/CPP/7zip/UI/Console/UpdateCallbackConsole.h index ba8614eb..5c205aad 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.h +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.h @@ -56,7 +56,8 @@ public: StdOutMode(false), NeedFlush(false), PercentsNameLevel(1), - LogLevel(0) + LogLevel(0), + NumNonOpenFiles(0) {} void SetWindowWidth(unsigned width) { _percent.MaxLen = width - 1; } @@ -82,9 +83,9 @@ public: _percent.ClosePrint(false); } - CErrorPathCodes FailedFiles; CErrorPathCodes ScanErrors; + UInt64 NumNonOpenFiles; HRESULT PrintProgress(const wchar_t *name, const char *command, bool showInLog); @@ -95,21 +96,23 @@ class CUpdateCallbackConsole: public IUpdateCallbackUI2, public CCallbackConsole // void PrintPropPair(const char *name, const wchar_t *val); public: + bool DeleteMessageWasShown; + #ifndef _NO_CRYPTO bool PasswordIsDefined; UString Password; bool AskPassword; #endif - bool DeleteMessageWasShown; - - CUpdateCallbackConsole() - : DeleteMessageWasShown(false) + CUpdateCallbackConsole(): + DeleteMessageWasShown(false) #ifndef _NO_CRYPTO , PasswordIsDefined(false) , AskPassword(false) #endif {} + + virtual ~CUpdateCallbackConsole() {} /* void Init(CStdOutStream *outStream) diff --git a/CPP/7zip/UI/Console/UserInputUtils.cpp b/CPP/7zip/UI/Console/UserInputUtils.cpp index 0e2d7ac1..93f60eb2 100644 --- a/CPP/7zip/UI/Console/UserInputUtils.cpp +++ b/CPP/7zip/UI/Console/UserInputUtils.cpp @@ -78,7 +78,7 @@ static bool GetPassword(CStdOutStream *outStream, UString &psw) DWORD mode = 0; if (console != INVALID_HANDLE_VALUE && console != 0) if (GetConsoleMode(console, &mode)) - wasChanged = (SetConsoleMode(console, mode & ~ENABLE_ECHO_INPUT) != 0); + wasChanged = (SetConsoleMode(console, mode & ~(DWORD)ENABLE_ECHO_INPUT) != 0); bool res = g_StdIn.ScanUStringUntilNewLine(psw); if (wasChanged) SetConsoleMode(console, mode); diff --git a/CPP/7zip/UI/Console/makefile b/CPP/7zip/UI/Console/makefile index 541b7681..ada782b0 100644 --- a/CPP/7zip/UI/Console/makefile +++ b/CPP/7zip/UI/Console/makefile @@ -31,6 +31,7 @@ WIN_OBJS = \ $O\PropVariantConv.obj \ $O\Registry.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ diff --git a/CPP/7zip/UI/Console/makefile.gcc b/CPP/7zip/UI/Console/makefile.gcc new file mode 100644 index 00000000..e0d996c1 --- /dev/null +++ b/CPP/7zip/UI/Console/makefile.gcc @@ -0,0 +1,171 @@ +PROG = 7z +IS_NOT_STANDALONE = 1 + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + + +LOCAL_FLAGS_ST = +MT_OBJS = + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef SystemDrive +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/Synchronization.o \ + $O/Threads.o \ + +endif + + + +LOCAL_FLAGS_WIN= + +ifdef SystemDrive + +LOCAL_FLAGS_WIN = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + + + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_WIN) \ + $(LOCAL_FLAGS_ST) \ + -DEXTERNAL_CODECS \ + + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/ListFileUtils.o \ + $O/NewHandler.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/MyString.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/MyVector.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + $O/DLL.o \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileLink.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + $O/System.o \ + $O/SystemInfo.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +COMPRESS_OBJS = \ + $O/CopyCoder.o \ + +C_OBJS = \ + $O/Alloc.o \ + $O/CpuArch.o \ + $O/Sort.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + + +OBJS = \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(SYS_OBJS) \ + $(COMPRESS_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + + +include ../../7zip_gcc.mak diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index a99ca81d..0334942a 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -2,6 +2,7 @@ #include "StdAfx.h" +#include "../../../Common/ComTry.h" #include "../../../Common/StringConvert.h" #include "../../../Windows/COM.h" @@ -31,6 +32,12 @@ #include "resource.h" +// #define SHOW_DEBUG_CTX_MENU + +#ifdef SHOW_DEBUG_CTX_MENU +#include <stdio.h> +#endif + using namespace NWindows; using namespace NFile; using namespace NDir; @@ -63,8 +70,8 @@ CZipContextMenu::~CZipContextMenu() HRESULT CZipContextMenu::GetFileNames(LPDATAOBJECT dataObject, UStringVector &fileNames) { fileNames.Clear(); - if (dataObject == NULL) - return E_FAIL; + if (!dataObject) + return E_INVALIDARG; #ifndef UNDER_CE @@ -146,76 +153,24 @@ struct CContextMenuCommand UINT ResourceID; }; +#define CMD_REC(cns, verb, ids) { NContextMenuFlags::cns, CZipContextMenu::cns, verb, ids } + static const CContextMenuCommand g_Commands[] = { - { - NContextMenuFlags::kOpen, - CZipContextMenu::kOpen, - "Open", - IDS_CONTEXT_OPEN - }, - { - NContextMenuFlags::kExtract, - CZipContextMenu::kExtract, - "Extract", - IDS_CONTEXT_EXTRACT - }, - { - NContextMenuFlags::kExtractHere, - CZipContextMenu::kExtractHere, - "ExtractHere", - IDS_CONTEXT_EXTRACT_HERE - }, - { - NContextMenuFlags::kExtractTo, - CZipContextMenu::kExtractTo, - "ExtractTo", - IDS_CONTEXT_EXTRACT_TO - }, - { - NContextMenuFlags::kTest, - CZipContextMenu::kTest, - "Test", - IDS_CONTEXT_TEST - }, - { - NContextMenuFlags::kCompress, - CZipContextMenu::kCompress, - "Compress", - IDS_CONTEXT_COMPRESS - }, - { - NContextMenuFlags::kCompressEmail, - CZipContextMenu::kCompressEmail, - "CompressEmail", - IDS_CONTEXT_COMPRESS_EMAIL - }, - { - NContextMenuFlags::kCompressTo7z, - CZipContextMenu::kCompressTo7z, - "CompressTo7z", - IDS_CONTEXT_COMPRESS_TO - }, - { - NContextMenuFlags::kCompressTo7zEmail, - CZipContextMenu::kCompressTo7zEmail, - "CompressTo7zEmail", - IDS_CONTEXT_COMPRESS_TO_EMAIL - }, - { - NContextMenuFlags::kCompressToZip, - CZipContextMenu::kCompressToZip, - "CompressToZip", - IDS_CONTEXT_COMPRESS_TO - }, - { - NContextMenuFlags::kCompressToZipEmail, - CZipContextMenu::kCompressToZipEmail, - "CompressToZipEmail", - IDS_CONTEXT_COMPRESS_TO_EMAIL - } + CMD_REC( kOpen, "Open", IDS_CONTEXT_OPEN), + CMD_REC( kExtract, "Extract", IDS_CONTEXT_EXTRACT), + CMD_REC( kExtractHere, "ExtractHere", IDS_CONTEXT_EXTRACT_HERE), + CMD_REC( kExtractTo, "ExtractTo", IDS_CONTEXT_EXTRACT_TO), + CMD_REC( kTest, "Test", IDS_CONTEXT_TEST), + CMD_REC( kCompress, "Compress", IDS_CONTEXT_COMPRESS), + CMD_REC( kCompressEmail, "CompressEmail", IDS_CONTEXT_COMPRESS_EMAIL), + CMD_REC( kCompressTo7z, "CompressTo7z", IDS_CONTEXT_COMPRESS_TO), + CMD_REC( kCompressTo7zEmail, "CompressTo7zEmail", IDS_CONTEXT_COMPRESS_TO_EMAIL), + CMD_REC( kCompressToZip, "CompressToZip", IDS_CONTEXT_COMPRESS_TO), + CMD_REC( kCompressToZipEmail, "CompressToZipEmail", IDS_CONTEXT_COMPRESS_TO_EMAIL) }; + struct CHashCommand { CZipContextMenu::ECommandInternalID CommandInternalID; @@ -232,6 +187,7 @@ static const CHashCommand g_HashCommands[] = { CZipContextMenu::kHash_All, "*", "*" } }; + static int FindCommand(CZipContextMenu::ECommandInternalID &id) { for (unsigned i = 0; i < ARRAY_SIZE(g_Commands); i++) @@ -240,22 +196,31 @@ static int FindCommand(CZipContextMenu::ECommandInternalID &id) return -1; } -bool CZipContextMenu::FillCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &commandMapItem) + +void CZipContextMenu::FillCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi) { mainString.Empty(); int i = FindCommand(id); if (i < 0) - return false; - const CContextMenuCommand &command = g_Commands[i]; - commandMapItem.CommandInternalID = command.CommandInternalID; - commandMapItem.Verb = kMainVerb; - commandMapItem.Verb += command.Verb; - // LangString(command.ResourceHelpID, command.LangID + 1, commandMapItem.HelpString); + throw 201908; + const CContextMenuCommand &command = g_Commands[(unsigned)i]; + cmi.CommandInternalID = command.CommandInternalID; + cmi.Verb = kMainVerb; + cmi.Verb += command.Verb; + // cmi.HelpString = cmi.Verb; LangString(command.ResourceID, mainString); - return true; + // return true; } -static bool MyInsertMenu(CMenu &menu, int pos, UINT id, const UString &s, HBITMAP bitmap) + +void CZipContextMenu::AddCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi) +{ + FillCommand(id, mainString, cmi); + _commandMap.Add(cmi); +} + + +static void MyInsertMenu(CMenu &menu, int pos, UINT id, const UString &s, HBITMAP bitmap) { CMenuItem mi; mi.fType = MFT_STRING; @@ -266,12 +231,40 @@ static bool MyInsertMenu(CMenu &menu, int pos, UINT id, const UString &s, HBITMA mi.StringValue = s; mi.hbmpUnchecked = bitmap; // mi.hbmpChecked = bitmap; // do we need hbmpChecked ??? - return menu.InsertItem(pos, true, mi); + if (!menu.InsertItem(pos, true, mi)) + throw 20190816; // SetMenuItemBitmaps also works // ::SetMenuItemBitmaps(menu, pos, MF_BYPOSITION, bitmap, NULL); } + +static void MyAddSubMenu( + CObjectVector<CZipContextMenu::CCommandMapItem> &_commandMap, + const char *verb, + CMenu &menu, int pos, UINT id, const UString &s, HMENU hSubMenu, HBITMAP bitmap) +{ + CZipContextMenu::CCommandMapItem cmi; + cmi.CommandInternalID = CZipContextMenu::kCommandNULL; + cmi.Verb = verb; + // cmi.HelpString = verb; + _commandMap.Add(cmi); + + CMenuItem mi; + mi.fType = MFT_STRING; + mi.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; + if (bitmap) + mi.fMask |= MIIM_CHECKMARKS; + mi.wID = id; + mi.hSubMenu = hSubMenu; + mi.hbmpUnchecked = bitmap; + + mi.StringValue = s; + if (!menu.InsertItem(pos, true, mi)) + throw 20190817; +} + + static const char * const kArcExts[] = { "7z" @@ -289,6 +282,7 @@ static bool IsItArcExt(const UString &ext) return false; } +UString GetSubFolderNameForExtract(const UString &arcName); UString GetSubFolderNameForExtract(const UString &arcName) { int dotPos = arcName.ReverseFind_Dot(); @@ -302,11 +296,11 @@ UString GetSubFolderNameForExtract(const UString &arcName) if (dotPos > 0) { const UString ext2 = res.Ptr(dotPos + 1); - if (ext.IsEqualTo_Ascii_NoCase("001") && IsItArcExt(ext2) - || ext.IsEqualTo_Ascii_NoCase("rar") && + if ((ext.IsEqualTo_Ascii_NoCase("001") && IsItArcExt(ext2)) + || (ext.IsEqualTo_Ascii_NoCase("rar") && ( ext2.IsEqualTo_Ascii_NoCase("part001") || ext2.IsEqualTo_Ascii_NoCase("part01") - || ext2.IsEqualTo_Ascii_NoCase("part1"))) + || ext2.IsEqualTo_Ascii_NoCase("part1")))) res.DeleteFrom(dotPos); res.TrimRight(); } @@ -415,17 +409,44 @@ static bool DoNeedExtract(const FString &name) // we must use diferent Verbs for Popup subMenu. void CZipContextMenu::AddMapItem_ForSubMenu(const char *verb) { - CCommandMapItem commandMapItem; - commandMapItem.CommandInternalID = kCommandNULL; - commandMapItem.Verb = verb; - _commandMap.Add(commandMapItem); + CCommandMapItem cmi; + cmi.CommandInternalID = kCommandNULL; + cmi.Verb = verb; + // cmi.HelpString = verb; + _commandMap.Add(cmi); +} + + +static HRESULT RETURN_WIN32_LastError_AS_HRESULT() +{ + DWORD lastError = ::GetLastError(); + if (lastError == 0) + return E_FAIL; + return HRESULT_FROM_WIN32(lastError); } +/* + we add CCommandMapItem to _commandMap for each new Mene ID. + so then we use _commandMap[offset]. + That way we can execute commands that have menu item. + Another non-implemented way: + We can return the number off all possible commnad in QueryContextMenu(). + so the caller could call InvokeCommand() via string verb aven + without using menu items. +*/ + + + STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, UINT commandIDFirst, UINT commandIDLast, UINT flags) { - // OutputDebugStringA("QueryContextMenu"); + COM_TRY_BEGIN + try { + + #ifdef SHOW_DEBUG_CTX_MENU + { char s[256]; sprintf(s, "QueryContextMenu: index=%d first=%d last=%d flags=%x _files=%d", + indexMenu, commandIDFirst, commandIDLast, flags, _fileNames.Size()); OutputDebugStringA(s); } /* for (UInt32 i = 0; i < _fileNames.Size(); i++) @@ -433,15 +454,29 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, OutputDebugStringW(_fileNames[i]); } */ + #endif LoadLangOneTime(); + if (_fileNames.Size() == 0) - return E_FAIL; + { + return MAKE_HRESULT(SEVERITY_SUCCESS, 0, 0); + // return E_INVALIDARG; + } + + if (commandIDFirst > commandIDLast) + return E_INVALIDARG; + + UINT currentCommandID = commandIDFirst; - if ((flags & 0x000F) != CMF_NORMAL && - (flags & CMF_VERBSONLY) == 0 && - (flags & CMF_EXPLORE) == 0) - return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID); + + if ((flags & 0x000F) != CMF_NORMAL + && (flags & CMF_VERBSONLY) == 0 + && (flags & CMF_EXPLORE) == 0) + return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID - commandIDFirst); + // return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID); + // 19.01 : we changed from (currentCommandID) to (currentCommandID - commandIDFirst) + // why it was so before? _commandMap.Clear(); @@ -462,7 +497,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (ci.Cascaded.Val) { if (!popupMenu.CreatePopup()) - return E_FAIL; + return RETURN_WIN32_LastError_AS_HRESULT(); menuDestroyer.Attach(popupMenu); /* 9.31: we commented the following code. Probably we don't need. @@ -508,7 +543,10 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, #endif { if (!fi0.Find(us2fs(fileName))) - return E_FAIL; + { + throw 20190820; + // return RETURN_WIN32_LastError_AS_HRESULT(); + } GetOnlyDirPrefix(us2fs(fileName), folderPrefix); } } @@ -523,10 +561,9 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, bool thereIsMainOpenItem = ((contextMenuFlags & NContextMenuFlags::kOpen) != 0); if (thereIsMainOpenItem) { - CCommandMapItem commandMapItem; - FillCommand(kOpen, mainString, commandMapItem); + CCommandMapItem cmi; + AddCommand(kOpen, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } if ((contextMenuFlags & NContextMenuFlags::kOpenAs) != 0 // && (!thereIsMainOpenItem || !FindExt(kNoOpenAsExtensions, fi0.Name)) @@ -535,37 +572,26 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, CMenu subMenu; if (subMenu.CreatePopup()) { - CMenuItem mi; - mi.fType = MFT_STRING; - mi.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; - if (bitmap) - mi.fMask |= MIIM_CHECKMARKS; - mi.wID = currentCommandID++; - mi.hSubMenu = subMenu; - mi.hbmpUnchecked = bitmap; - - LangString(IDS_CONTEXT_OPEN, mi.StringValue); - popupMenu.InsertItem(subIndex++, true, mi); - AddMapItem_ForSubMenu(kOpenCascadedVerb); + MyAddSubMenu(_commandMap, kOpenCascadedVerb, popupMenu, subIndex++, currentCommandID++, LangString(IDS_CONTEXT_OPEN), subMenu, bitmap); UINT subIndex2 = 0; for (unsigned i = (thereIsMainOpenItem ? 1 : 0); i < ARRAY_SIZE(kOpenTypes); i++) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; if (i == 0) - FillCommand(kOpen, mainString, commandMapItem); + FillCommand(kOpen, mainString, cmi); else { mainString = kOpenTypes[i]; - commandMapItem.CommandInternalID = kOpen; - commandMapItem.Verb = kMainVerb; - commandMapItem.Verb += ".Open."; - commandMapItem.Verb += mainString; - commandMapItem.HelpString = mainString; - commandMapItem.ArcType = mainString; + cmi.CommandInternalID = kOpen; + cmi.Verb = kMainVerb; + cmi.Verb += ".Open."; + cmi.Verb += mainString; + // cmi.HelpString = cmi.Verb; + cmi.ArcType = mainString; } + _commandMap.Add(cmi); MyInsertMenu(subMenu, subIndex2++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } subMenu.Detach(); @@ -584,7 +610,10 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, { NFind::CFileInfo fi; if (!fi.Find(us2fs(_fileNames[i]))) - return E_FAIL; + { + throw 20190821; + // return RETURN_WIN32_LastError_AS_HRESULT(); + } if (!fi.IsDir() && DoNeedExtract(fi.Name)) { needExtract = true; @@ -610,43 +639,39 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if ((contextMenuFlags & NContextMenuFlags::kExtract) != 0) { // Extract - CCommandMapItem commandMapItem; - FillCommand(kExtract, mainString, commandMapItem); - commandMapItem.Folder = baseFolder + specFolder; + CCommandMapItem cmi; + cmi.Folder = baseFolder + specFolder; + AddCommand(kExtract, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } if ((contextMenuFlags & NContextMenuFlags::kExtractHere) != 0) { // Extract Here - CCommandMapItem commandMapItem; - FillCommand(kExtractHere, mainString, commandMapItem); - commandMapItem.Folder = baseFolder; + CCommandMapItem cmi; + cmi.Folder = baseFolder; + AddCommand(kExtractHere, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } if ((contextMenuFlags & NContextMenuFlags::kExtractTo) != 0) { // Extract To - CCommandMapItem commandMapItem; + CCommandMapItem cmi; UString s; - FillCommand(kExtractTo, s, commandMapItem); - commandMapItem.Folder = baseFolder + specFolder; + cmi.Folder = baseFolder + specFolder; + AddCommand(kExtractTo, s, cmi); MyFormatNew_ReducedName(s, specFolder); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); - _commandMap.Add(commandMapItem); } } if ((contextMenuFlags & NContextMenuFlags::kTest) != 0) { // Test - CCommandMapItem commandMapItem; - FillCommand(kTest, mainString, commandMapItem); + CCommandMapItem cmi; + AddCommand(kTest, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } } @@ -660,26 +685,24 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, // Compress if ((contextMenuFlags & NContextMenuFlags::kCompress) != 0) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; if (_dropMode) - commandMapItem.Folder = _dropPath; + cmi.Folder = _dropPath; else - commandMapItem.Folder = fs2us(folderPrefix); - commandMapItem.ArcName = arcName; - FillCommand(kCompress, mainString, commandMapItem); + cmi.Folder = fs2us(folderPrefix); + cmi.ArcName = arcName; + AddCommand(kCompress, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } #ifdef EMAIL_SUPPORT // CompressEmail if ((contextMenuFlags & NContextMenuFlags::kCompressEmail) != 0 && !_dropMode) { - CCommandMapItem commandMapItem; - commandMapItem.ArcName = arcName; - FillCommand(kCompressEmail, mainString, commandMapItem); + CCommandMapItem cmi; + cmi.ArcName = arcName; + AddCommand(kCompressEmail, mainString, cmi); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, mainString, bitmap); - _commandMap.Add(commandMapItem); } #endif @@ -687,32 +710,30 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (contextMenuFlags & NContextMenuFlags::kCompressTo7z && !arcName7z.IsEqualTo_NoCase(fs2us(fi0.Name))) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; UString s; - FillCommand(kCompressTo7z, s, commandMapItem); if (_dropMode) - commandMapItem.Folder = _dropPath; + cmi.Folder = _dropPath; else - commandMapItem.Folder = fs2us(folderPrefix); - commandMapItem.ArcName = arcName7z; - commandMapItem.ArcType = "7z"; + cmi.Folder = fs2us(folderPrefix); + cmi.ArcName = arcName7z; + cmi.ArcType = "7z"; + AddCommand(kCompressTo7z, s, cmi); MyFormatNew_ReducedName(s, arcName7z); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); - _commandMap.Add(commandMapItem); } #ifdef EMAIL_SUPPORT // CompressTo7zEmail if ((contextMenuFlags & NContextMenuFlags::kCompressTo7zEmail) != 0 && !_dropMode) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; UString s; - FillCommand(kCompressTo7zEmail, s, commandMapItem); - commandMapItem.ArcName = arcName7z; - commandMapItem.ArcType = "7z"; + cmi.ArcName = arcName7z; + cmi.ArcType = "7z"; + AddCommand(kCompressTo7zEmail, s, cmi); MyFormatNew_ReducedName(s, arcName7z); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); - _commandMap.Add(commandMapItem); } #endif @@ -720,32 +741,30 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (contextMenuFlags & NContextMenuFlags::kCompressToZip && !arcNameZip.IsEqualTo_NoCase(fs2us(fi0.Name))) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; UString s; - FillCommand(kCompressToZip, s, commandMapItem); if (_dropMode) - commandMapItem.Folder = _dropPath; + cmi.Folder = _dropPath; else - commandMapItem.Folder = fs2us(folderPrefix); - commandMapItem.ArcName = arcNameZip; - commandMapItem.ArcType = "zip"; + cmi.Folder = fs2us(folderPrefix); + cmi.ArcName = arcNameZip; + cmi.ArcType = "zip"; + AddCommand(kCompressToZip, s, cmi); MyFormatNew_ReducedName(s, arcNameZip); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); - _commandMap.Add(commandMapItem); } #ifdef EMAIL_SUPPORT // CompressToZipEmail if ((contextMenuFlags & NContextMenuFlags::kCompressToZipEmail) != 0 && !_dropMode) { - CCommandMapItem commandMapItem; + CCommandMapItem cmi; UString s; - FillCommand(kCompressToZipEmail, s, commandMapItem); - commandMapItem.ArcName = arcNameZip; - commandMapItem.ArcType = "zip"; + cmi.ArcName = arcNameZip; + cmi.ArcType = "zip"; + AddCommand(kCompressToZipEmail, s, cmi); MyFormatNew_ReducedName(s, arcNameZip); MyInsertMenu(popupMenu, subIndex++, currentCommandID++, s, bitmap); - _commandMap.Add(commandMapItem); } #endif } @@ -757,22 +776,10 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (ci.Cascaded.Val) { - CMenuItem mi; - mi.fType = MFT_STRING; - mi.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; - if (bitmap) - mi.fMask |= MIIM_CHECKMARKS; - mi.wID = currentCommandID++; - mi.hSubMenu = popupMenu.Detach(); - mi.StringValue = "7-Zip"; // LangString(IDS_CONTEXT_POPUP_CAPTION); - mi.hbmpUnchecked = bitmap; - CMenu menu; menu.Attach(hMenu); menuDestroyer.Disable(); - menu.InsertItem(indexMenu++, true, mi); - - AddMapItem_ForSubMenu(kMainVerb); + MyAddSubMenu(_commandMap, kMainVerb, menu, indexMenu++, currentCommandID++, (UString)"7-Zip", popupMenu.Detach(), bitmap); } else { @@ -783,7 +790,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (!_isMenuForFM && ((contextMenuFlags & NContextMenuFlags::kCRC) != 0 - && currentCommandID + 6 <= commandIDLast)) + && currentCommandID + 1 < commandIDLast)) { CMenu subMenu; // CMenuDestroyer menuDestroyer_CRC; @@ -793,40 +800,58 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if (subMenu.CreatePopup()) { // menuDestroyer_CRC.Attach(subMenu); - CMenuItem mi; - mi.fType = MFT_STRING; - mi.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; - if (bitmap) - mi.fMask |= MIIM_CHECKMARKS; - mi.wID = currentCommandID++; - mi.hSubMenu = subMenu; - mi.StringValue = "CRC SHA"; - mi.hbmpUnchecked = bitmap; CMenu menu; menu.Attach(hMenu); // menuDestroyer_CRC.Disable(); - menu.InsertItem(indexMenu++, true, mi); - - AddMapItem_ForSubMenu(kCheckSumCascadedVerb); + MyAddSubMenu(_commandMap, kCheckSumCascadedVerb, menu, indexMenu++, currentCommandID++, (UString)"CRC SHA", subMenu, bitmap); for (unsigned i = 0; i < ARRAY_SIZE(g_HashCommands); i++) { + if (currentCommandID >= commandIDLast) + break; const CHashCommand &hc = g_HashCommands[i]; - CCommandMapItem commandMapItem; - commandMapItem.CommandInternalID = hc.CommandInternalID; - commandMapItem.Verb = kCheckSumCascadedVerb; - commandMapItem.Verb += hc.MethodName; - // commandMapItem.HelpString = hc.Name; + CCommandMapItem cmi; + cmi.CommandInternalID = hc.CommandInternalID; + cmi.Verb = kCheckSumCascadedVerb; + cmi.Verb += '.'; + cmi.Verb += hc.MethodName; + // cmi.HelpString = cmi.Verb; + _commandMap.Add(cmi); MyInsertMenu(subMenu, subIndex_CRC++, currentCommandID++, (UString)hc.UserName, bitmap); - _commandMap.Add(commandMapItem); } subMenu.Detach(); } } + #ifdef SHOW_DEBUG_CTX_MENU + { char s[256]; sprintf(s, "Commands=%d currentCommandID - commandIDFirst = %d", + _commandMap.Size(), currentCommandID - commandIDFirst); OutputDebugStringA(s); } + #endif + + if (_commandMap.Size() != currentCommandID - commandIDFirst) + throw 20190818; return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID - commandIDFirst); + + } + catch(...) + { + /* we added some menu items already : num_added_menu_items, + So we MUST return (number_of_defined_ids), where (number_of_defined_ids >= num_added_menu_items) + This will prevent incorrect menu working, when same IDs can be + assigned in multiple menu items from different subhandlers. + And we must add items to _commandMap before adding to menu. + */ + #ifdef SHOW_DEBUG_CTX_MENU + { char s[256]; sprintf(s, "catch() exception: Commands=%d", + _commandMap.Size()); OutputDebugStringA(s); } + #endif + // if (_commandMap.Size() != 0) + return MAKE_HRESULT(SEVERITY_SUCCESS, 0, _commandMap.Size()); + // throw; + } + COM_TRY_END } @@ -843,35 +868,122 @@ static UString Get7zFmPath() return fs2us(NWindows::NDLL::GetModuleDirPrefix()) + L"7zFM.exe"; } + +#ifdef UNDER_CE + #define MY__IS_INTRESOURCE(_r) ((((ULONG_PTR)(_r)) >> 16) == 0) +#else + #define MY__IS_INTRESOURCE(_r) IS_INTRESOURCE(_r) +#endif + + +#ifdef SHOW_DEBUG_CTX_MENU +static void PrintStringA(const char *name, LPCSTR ptr) +{ + AString m; + m += name; + m += ": "; + char s[32]; + sprintf(s, "%p", ptr); + m += s; + if (!MY__IS_INTRESOURCE(ptr)) + { + m += ": \""; + m += ptr; + m += "\""; + } + OutputDebugStringA(m); +} + +#if !defined(UNDER_CE) +static void PrintStringW(const char *name, LPCWSTR ptr) +{ + UString m; + m += name; + m += ": "; + char s[32]; + sprintf(s, "%p", ptr); + m += s; + if (!MY__IS_INTRESOURCE(ptr)) + { + m += ": \""; + m += ptr; + m += "\""; + } + OutputDebugStringW(m); +} +#endif +#endif + + STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) { - // ::OutputDebugStringA("1"); - int commandOffset; + COM_TRY_BEGIN + + #ifdef SHOW_DEBUG_CTX_MENU + + { char s[1280]; sprintf(s, + #ifdef _WIN64 + "64" + #else + "32" + #endif + ": InvokeCommand: cbSize=%d flags=%x " + , commandInfo->cbSize, commandInfo->fMask); OutputDebugStringA(s); } + + PrintStringA("Verb", commandInfo->lpVerb); + PrintStringA("Parameters", commandInfo->lpParameters); + PrintStringA("Directory", commandInfo->lpDirectory); + #endif + + int commandOffset = -1; - // It's fix for bug: crashing in XP. See example in MSDN: "Creating Context Menu Handlers". + // xp64 / Win10 : explorer.exe sends 0 in lpVerbW + // MSDN: if (IS_INTRESOURCE(lpVerbW)), we must use LOWORD(lpVerb) as sommand offset - #if !defined(UNDER_CE) && defined(_MSC_VER) + // FIXME: MINGW doesn't define CMINVOKECOMMANDINFOEX + #if !defined(UNDER_CE) /* && defined(_MSC_VER) */ + bool unicodeVerb = false; if (commandInfo->cbSize == sizeof(CMINVOKECOMMANDINFOEX) && (commandInfo->fMask & CMIC_MASK_UNICODE) != 0) { LPCMINVOKECOMMANDINFOEX commandInfoEx = (LPCMINVOKECOMMANDINFOEX)commandInfo; - if (HIWORD(commandInfoEx->lpVerbW) == 0) - commandOffset = LOWORD(commandInfo->lpVerb); - else + if (!MY__IS_INTRESOURCE(commandInfoEx->lpVerbW)) + { + unicodeVerb = true; commandOffset = FindVerb(commandInfoEx->lpVerbW); + } + + #ifdef SHOW_DEBUG_CTX_MENU + PrintStringW("VerbW", commandInfoEx->lpVerbW); + PrintStringW("ParametersW", commandInfoEx->lpParametersW); + PrintStringW("DirectoryW", commandInfoEx->lpDirectoryW); + PrintStringW("TitleW", commandInfoEx->lpTitleW); + PrintStringA("Title", commandInfoEx->lpTitle); + #endif } - else + if (!unicodeVerb) #endif - if (HIWORD(commandInfo->lpVerb) == 0) + { + #ifdef SHOW_DEBUG_CTX_MENU + OutputDebugStringA("use non-UNICODE verb"); + #endif + // if (HIWORD(commandInfo->lpVerb) == 0) + if (MY__IS_INTRESOURCE(commandInfo->lpVerb)) commandOffset = LOWORD(commandInfo->lpVerb); else commandOffset = FindVerb(GetUnicodeString(commandInfo->lpVerb)); + } + + #ifdef SHOW_DEBUG_CTX_MENU + { char s[128]; sprintf(s, "commandOffset=%d", + commandOffset); OutputDebugStringA(s); } + #endif if (commandOffset < 0 || (unsigned)commandOffset >= _commandMap.Size()) - return E_FAIL; + return E_INVALIDARG; - const CCommandMapItem commandMapItem = _commandMap[commandOffset]; - ECommandInternalID cmdID = commandMapItem.CommandInternalID; + const CCommandMapItem &cmi = _commandMap[(unsigned)commandOffset]; + ECommandInternalID cmdID = cmi.CommandInternalID; try { @@ -881,10 +993,10 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) { UString params; params = GetQuotedString(_fileNames[0]); - if (!commandMapItem.ArcType.IsEmpty()) + if (!cmi.ArcType.IsEmpty()) { params += " -t"; - params += commandMapItem.ArcType; + params += cmi.ArcType; } MyCreateProcess(Get7zFmPath(), params); break; @@ -893,7 +1005,7 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) case kExtractHere: case kExtractTo: { - ExtractArchives(_fileNames, commandMapItem.Folder, + ExtractArchives(_fileNames, cmi.Folder, (cmdID == kExtract), // showDialog (cmdID == kExtractTo) && _elimDup.Val // elimDup ); @@ -919,8 +1031,8 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) (cmdID == kCompress) || (cmdID == kCompressEmail); bool addExtension = (cmdID == kCompress || cmdID == kCompressEmail); - CompressFiles(commandMapItem.Folder, - commandMapItem.ArcName, commandMapItem.ArcType, + CompressFiles(cmi.Folder, + cmi.ArcName, cmi.ArcType, addExtension, _fileNames, email, showDialog, false); break; @@ -943,6 +1055,8 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) } break; } + case kCommandNULL: + break; } } catch(...) @@ -950,66 +1064,75 @@ STDMETHODIMP CZipContextMenu::InvokeCommand(LPCMINVOKECOMMANDINFO commandInfo) ::MessageBoxW(0, L"Error", L"7-Zip", MB_ICONERROR); } return S_OK; + COM_TRY_END } -static void MyCopyString(void *dest, const wchar_t *src, bool writeInUnicode) + + +static void MyCopyString(void *dest, const UString &src, bool writeInUnicode, UINT size) { + if (size != 0) + size--; if (writeInUnicode) { - MyStringCopy((wchar_t *)dest, src); + UString s = src; + s.DeleteFrom(size); + MyStringCopy((wchar_t *)dest, s); } else - MyStringCopy((char *)dest, (const char *)GetAnsiString(src)); + { + AString s = GetAnsiString(src); + s.DeleteFrom(size); + MyStringCopy((char *)dest, s); + } } + STDMETHODIMP CZipContextMenu::GetCommandString(UINT_PTR commandOffset, UINT uType, - UINT * /* pwReserved */ , LPSTR pszName, UINT /* cchMax */) + UINT * /* pwReserved */ , LPSTR pszName, UINT cchMax) { - int cmdOffset = (int)commandOffset; - switch (uType) + COM_TRY_BEGIN + + const int cmdOffset = (int)commandOffset; + + #ifdef SHOW_DEBUG_CTX_MENU + { char s[256]; sprintf(s, "GetCommandString: cmdOffset=%d uType=%d cchMax = %d", + cmdOffset, uType, cchMax); OutputDebugStringA(s); } + #endif + + if (uType == GCS_VALIDATEA || uType == GCS_VALIDATEW) { - #ifdef UNDER_CE - case GCS_VALIDATE: - #else - case GCS_VALIDATEA: - case GCS_VALIDATEW: - #endif - if (cmdOffset < 0 || (unsigned)cmdOffset >= _commandMap.Size()) - return S_FALSE; - else - return S_OK; + if (cmdOffset < 0 || (unsigned)cmdOffset >= _commandMap.Size()) + return S_FALSE; + else + return S_OK; } + if (cmdOffset < 0 || (unsigned)cmdOffset >= _commandMap.Size()) - return E_FAIL; - #ifdef UNDER_CE - if (uType == GCS_HELPTEXT) - #else + { + #ifdef SHOW_DEBUG_CTX_MENU + OutputDebugStringA("---------------- cmdOffset: E_INVALIDARG"); + #endif + return E_INVALIDARG; + } + + const CCommandMapItem &cmi = _commandMap[(unsigned)cmdOffset]; + if (uType == GCS_HELPTEXTA || uType == GCS_HELPTEXTW) - #endif { - MyCopyString(pszName, _commandMap[cmdOffset].HelpString, - #ifdef UNDER_CE - true - #else - uType == GCS_HELPTEXTW - #endif - ); - return NO_ERROR; + // we can return "Verb" here for debug purposes. + // HelpString; + MyCopyString(pszName, cmi.Verb, uType == GCS_HELPTEXTW, cchMax); + return S_OK; } - #ifdef UNDER_CE - if (uType == GCS_VERB) - #else + if (uType == GCS_VERBA || uType == GCS_VERBW) - #endif { - MyCopyString(pszName, _commandMap[cmdOffset].Verb, - #ifdef UNDER_CE - true - #else - uType == GCS_VERBW - #endif - ); - return NO_ERROR; + MyCopyString(pszName, cmi.Verb, uType == GCS_VERBW, cchMax); + return S_OK; } - return E_FAIL; + + return E_INVALIDARG; + + COM_TRY_END } diff --git a/CPP/7zip/UI/Explorer/ContextMenu.h b/CPP/7zip/UI/Explorer/ContextMenu.h index 3a0eacc8..dddb9fd1 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.h +++ b/CPP/7zip/UI/Explorer/ContextMenu.h @@ -54,18 +54,18 @@ public: CZipContextMenu(); ~CZipContextMenu(); -private: - struct CCommandMapItem { ECommandInternalID CommandInternalID; UString Verb; - UString HelpString; + // UString HelpString; UString Folder; UString ArcName; UString ArcType; }; +private: + bool _isMenuForFM; UStringVector _fileNames; bool _dropMode; @@ -78,7 +78,8 @@ private: HRESULT GetFileNames(LPDATAOBJECT dataObject, UStringVector &fileNames); int FindVerb(const UString &verb); - bool FillCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &commandMapItem); + void FillCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi); + void AddCommand(ECommandInternalID id, UString &mainString, CCommandMapItem &cmi); void AddMapItem_ForSubMenu(const char *ver); }; diff --git a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp index c38d7257..cb4da016 100644 --- a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp +++ b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp @@ -39,9 +39,16 @@ DEFINE_GUID(CLSID_CZipContextMenu, using namespace NWindows; +extern +HINSTANCE g_hInstance; HINSTANCE g_hInstance = 0; + +extern +HWND g_HWND; HWND g_HWND = 0; +extern +LONG g_DllRefCount; LONG g_DllRefCount = 0; // Reference count of this DLL. @@ -91,7 +98,18 @@ STDMETHODIMP CShellExtClassFactory::LockServer(BOOL /* fLock */) } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION return FALSE; +#endif + +extern "C" +BOOL WINAPI DllMain( + #ifdef UNDER_CE + HANDLE hInstance + #else + HINSTANCE hInstance + #endif + , DWORD dwReason, LPVOID); extern "C" BOOL WINAPI DllMain( diff --git a/CPP/7zip/UI/Explorer/MyMessages.cpp b/CPP/7zip/UI/Explorer/MyMessages.cpp index 70c2a460..c912504c 100644 --- a/CPP/7zip/UI/Explorer/MyMessages.cpp +++ b/CPP/7zip/UI/Explorer/MyMessages.cpp @@ -26,7 +26,7 @@ void ShowErrorMessageRes(UINT resID) ShowErrorMessageHwndRes(0, resID); } -void ShowErrorMessageDWORD(HWND window, DWORD errorCode) +static void ShowErrorMessageDWORD(HWND window, DWORD errorCode) { ShowErrorMessage(window, NError::MyFormatMessage(errorCode)); } diff --git a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp index c9d7910f..9f6e44c8 100644 --- a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp @@ -59,14 +59,16 @@ static const bool k_shellex_Statuses[2][4] = #else */ -typedef WINADVAPI LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); +typedef +// WINADVAPI +LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); static Func_RegDeleteKeyExW func_RegDeleteKeyExW; static void Init_RegDeleteKeyExW() { if (!func_RegDeleteKeyExW) func_RegDeleteKeyExW = (Func_RegDeleteKeyExW) - GetProcAddress(GetModuleHandleW(L"advapi32.dll"), "RegDeleteKeyExW"); + (void *)GetProcAddress(GetModuleHandleW(L"advapi32.dll"), "RegDeleteKeyExW"); } #define INIT_REG_WOW if (wow != 0) Init_RegDeleteKeyExW(); @@ -80,12 +82,12 @@ static LONG MyRegistry_DeleteKey(HKEY parentKey, LPCTSTR name, UInt32 wow) /* #ifdef _WIN64 - return RegDeleteKeyExW + return RegDeleteKeyExW #else */ - if (!func_RegDeleteKeyExW) - return E_NOTIMPL; - return func_RegDeleteKeyExW + if (!func_RegDeleteKeyExW) + return E_NOTIMPL; + return func_RegDeleteKeyExW // #endif (parentKey, GetUnicodeString(name), wow, 0); } diff --git a/CPP/7zip/UI/Explorer/StdAfx.cpp b/CPP/7zip/UI/Explorer/StdAfx.cpp index 2550270c..d0feea85 100644 --- a/CPP/7zip/UI/Explorer/StdAfx.cpp +++ b/CPP/7zip/UI/Explorer/StdAfx.cpp @@ -1,3 +1,3 @@ // StdAfx.cpp -#include "stdafx.h" +#include "StdAfx.h" diff --git a/CPP/7zip/UI/Far/ExtractEngine.cpp b/CPP/7zip/UI/Far/ExtractEngine.cpp index 3567c048..e6fe2d01 100644 --- a/CPP/7zip/UI/Far/ExtractEngine.cpp +++ b/CPP/7zip/UI/Far/ExtractEngine.cpp @@ -167,6 +167,7 @@ STDMETHODIMP CExtractCallbackImp::MessageError(const wchar_t *message) return CheckBreak2(); } +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &s); void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &s) { s.Empty(); diff --git a/CPP/7zip/UI/Far/Far.cpp b/CPP/7zip/UI/Far/Far.cpp index 5d92bc40..a9e47916 100644 --- a/CPP/7zip/UI/Far/Far.cpp +++ b/CPP/7zip/UI/Far/Far.cpp @@ -3,6 +3,10 @@ #include "StdAfx.h" +#ifdef __clang__ + #pragma clang diagnostic ignored "-Wmissing-prototypes" +#endif + #include "../../../Common/MyWindows.h" #include "../../../Common/MyInitGuid.h" @@ -39,7 +43,9 @@ const char *g_PluginName_for_Error = "7-Zip"; } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION return FALSE; +#endif BOOL WINAPI DllMain( #ifdef UNDER_CE diff --git a/CPP/7zip/UI/Far/Far.dsp b/CPP/7zip/UI/Far/Far.dsp index 2433b252..b32ae6fa 100644 --- a/CPP/7zip/UI/Far/Far.dsp +++ b/CPP/7zip/UI/Far/Far.dsp @@ -53,7 +53,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"C:\Program Files\Far\Plugins\7-Zip\7-ZipFar.dll" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"C:\Progs\Far\Plugins\7-Zip\7-ZipFar.dll" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "Far - Win32 Debug" @@ -80,7 +80,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\Far\Plugins\7-Zip\7-ZipFar.dll" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"C:\Progs\Far\Plugins\7-Zip\7-ZipFar.dll" /pdbtype:sept !ENDIF diff --git a/CPP/7zip/UI/Far/OverwriteDialogFar.cpp b/CPP/7zip/UI/Far/OverwriteDialogFar.cpp index 1171453e..b8fc565f 100644 --- a/CPP/7zip/UI/Far/OverwriteDialogFar.cpp +++ b/CPP/7zip/UI/Far/OverwriteDialogFar.cpp @@ -26,7 +26,7 @@ struct CFileInfoStrings AString Time; }; -void SetFileInfoStrings(const CFileInfo &fileInfo, +static void SetFileInfoStrings(const CFileInfo &fileInfo, CFileInfoStrings &fileInfoStrings) { char buffer[256]; diff --git a/CPP/7zip/UI/Far/Plugin.cpp b/CPP/7zip/UI/Far/Plugin.cpp index 72f81ac9..92b62369 100644 --- a/CPP/7zip/UI/Far/Plugin.cpp +++ b/CPP/7zip/UI/Far/Plugin.cpp @@ -197,10 +197,12 @@ void CPlugin::EnterToDirectory(const UString &dirName) s = ".."; _folder->BindToFolder(s, &newFolder); if (!newFolder) + { if (dirName.IsEmpty()) return; else throw 40325; + } _folder = newFolder; } @@ -457,7 +459,7 @@ static AString PropToString2(const NCOM::CPropVariant &prop, PROPID propID) return s; } -static void AddPropertyString(InfoPanelLine *lines, int &numItems, PROPID propID, const wchar_t *name, +static void AddPropertyString(InfoPanelLine *lines, unsigned &numItems, PROPID propID, const wchar_t *name, const NCOM::CPropVariant &prop) { if (prop.vt != VT_EMPTY) @@ -472,7 +474,7 @@ static void AddPropertyString(InfoPanelLine *lines, int &numItems, PROPID propID } } -static void InsertSeparator(InfoPanelLine *lines, int &numItems) +static void InsertSeparator(InfoPanelLine *lines, unsigned &numItems) { if (numItems < kNumInfoLinesMax) { @@ -528,7 +530,7 @@ void CPlugin::GetOpenPluginInfo(struct OpenPluginInfo *info) MyStringCopy(m_InfoLines[1].Text, g_StartupInfo.GetMsgString(NMessageID::kArchiveType)); MyStringCopy(m_InfoLines[1].Data, (const char *)UnicodeStringToMultiByte(_archiveTypeName, CP_OEMCP)); - int numItems = 2; + unsigned numItems = 2; { CMyComPtr<IFolderProperties> folderProperties; diff --git a/CPP/7zip/UI/Far/PluginDelete.cpp b/CPP/7zip/UI/Far/PluginDelete.cpp index 95584341..0d19c6e7 100644 --- a/CPP/7zip/UI/Far/PluginDelete.cpp +++ b/CPP/7zip/UI/Far/PluginDelete.cpp @@ -4,6 +4,9 @@ #include <stdio.h> +#include "../../../Common/StringConvert.h" +#include "FarUtils.h" + #include "Messages.h" #include "Plugin.h" #include "UpdateCallbackFar.h" @@ -28,16 +31,37 @@ int CPlugin::DeleteFiles(PluginPanelItem *panelItems, int numItems, int opMode) g_StartupInfo.GetMsgString(NMessageID::kDeleteDelete), g_StartupInfo.GetMsgString(NMessageID::kDeleteCancel) }; - char msg[1024]; + + // char msg[1024]; + AString str1; + if (numItems == 1) { - sprintf(msg, g_StartupInfo.GetMsgString(NMessageID::kDeleteFile), panelItems[0].FindData.cFileName); - msgItems[1] = msg; + str1 = g_StartupInfo.GetMsgString(NMessageID::kDeleteFile); + AString name (panelItems[0].FindData.cFileName); + const unsigned kSizeLimit = 48; + if (name.Len() > kSizeLimit) + { + UString s = MultiByteToUnicodeString(name, CP_OEMCP); + ReduceString(s, kSizeLimit); + name = UnicodeStringToMultiByte(s, CP_OEMCP); + } + str1.Replace(AString ("%.40s"), name); + msgItems[1] = str1; + // sprintf(msg, g_StartupInfo.GetMsgString(NMessageID::kDeleteFile), panelItems[0].FindData.cFileName); + // msgItems[2] = msg; } else if (numItems > 1) { - sprintf(msg, g_StartupInfo.GetMsgString(NMessageID::kDeleteNumberOfFiles), numItems); - msgItems[1] = msg; + str1 = g_StartupInfo.GetMsgString(NMessageID::kDeleteNumberOfFiles); + { + AString n; + n.Add_UInt32(numItems); + str1.Replace(AString ("%d"), n); + } + msgItems[1] = str1; + // sprintf(msg, g_StartupInfo.GetMsgString(NMessageID::kDeleteNumberOfFiles), numItems); + // msgItems[1] = msg; } if (g_StartupInfo.ShowMessage(FMSG_WARNING, NULL, msgItems, ARRAY_SIZE(msgItems), 2) != 0) return (FALSE); diff --git a/CPP/7zip/UI/Far/PluginWrite.cpp b/CPP/7zip/UI/Far/PluginWrite.cpp index ec0f119b..bf1d13df 100644 --- a/CPP/7zip/UI/Far/PluginWrite.cpp +++ b/CPP/7zip/UI/Far/PluginWrite.cpp @@ -89,7 +89,7 @@ NFileOperationReturnCode::EEnum CPlugin::PutFiles( } */ - if (numItems == 0) + if (numItems <= 0) return NFileOperationReturnCode::kError; if (_agent->IsThereReadOnlyArc()) @@ -104,14 +104,18 @@ NFileOperationReturnCode::EEnum CPlugin::PutFiles( NCompression::CInfo compressionInfo; compressionInfo.Load(); - int methodIndex = 0; - int i; - for (i = ARRAY_SIZE(g_MethodMap) - 1; i >= 0; i--) + unsigned methodIndex = 0; + + unsigned i; + for (i = ARRAY_SIZE(g_MethodMap); i != 0;) + { + i--; if (compressionInfo.Level >= g_MethodMap[i]) { methodIndex = i; break; } + } const int kMethodRadioIndex = 2; const int kModeRadioIndex = kMethodRadioIndex + 7; @@ -200,10 +204,10 @@ NFileOperationReturnCode::EEnum CPlugin::PutFiles( UStringVector fileNames; fileNames.ClearAndReserve(numItems); - for (i = 0; i < numItems; i++) + for (i = 0; i < (unsigned)numItems; i++) fileNames.AddInReserved(MultiByteToUnicodeString(panelItems[i].FindData.cFileName, CP_OEMCP)); CObjArray<const wchar_t *> fileNamePointers(numItems); - for (i = 0; i < numItems; i++) + for (i = 0; i < (unsigned)numItems; i++) fileNamePointers[i] = fileNames[i]; CMyComPtr<IOutFolderArchive> outArchive; @@ -459,29 +463,36 @@ HRESULT CompressFiles(const CObjectVector<PluginPanelItem> &pluginPanelItems) const int kMethodRadioIndex = kArchiveNameIndex + 2; const int kModeRadioIndex = kMethodRadioIndex + 7; - - char updateAddToArchiveString[512]; + // char updateAddToArchiveString[512]; + AString str1; { const CArcInfoEx &arcInfo = codecs->Formats[archiverIndex]; const AString s (UnicodeStringToMultiByte(arcInfo.Name, CP_OEMCP)); + str1 = g_StartupInfo.GetMsgString(NMessageID::kUpdateAddToArchive); + str1.Replace(AString ("%s"), s); + /* sprintf(updateAddToArchiveString, g_StartupInfo.GetMsgString(NMessageID::kUpdateAddToArchive), (const char *)s); + */ } - int methodIndex = 0; - int i; - for (i = ARRAY_SIZE(g_MethodMap) - 1; i >= 0; i--) + unsigned methodIndex = 0; + unsigned i; + for (i = ARRAY_SIZE(g_MethodMap); i != 0;) + { + i--; if (compressionInfo.Level >= g_MethodMap[i]) { methodIndex = i; break; } + } const struct CInitDialogItem initItems[]= { { DI_DOUBLEBOX, 3, 1, 72, kYSize - 2, false, false, 0, false, NMessageID::kUpdateTitle, NULL, NULL }, - { DI_TEXT, 5, 2, 0, 0, false, false, 0, false, -1, updateAddToArchiveString, NULL }, + { DI_TEXT, 5, 2, 0, 0, false, false, 0, false, -1, str1, NULL }, { DI_EDIT, 5, 3, 70, 3, true, false, DIF_HISTORY, false, -1, archiveNameA, kArchiveHistoryKeyName}, // { DI_EDIT, 5, 3, 70, 3, true, false, 0, false, -1, arcName, NULL}, diff --git a/CPP/7zip/UI/Far/makefile b/CPP/7zip/UI/Far/makefile index 621c8d9e..14be13d6 100644 --- a/CPP/7zip/UI/Far/makefile +++ b/CPP/7zip/UI/Far/makefile @@ -20,7 +20,6 @@ CURRENT_OBJS = \ $O\PluginWrite.obj \ $O\ProgressBox.obj \ $O\UpdateCallbackFar.obj \ - $O\UTFConvert.obj \ COMMON_OBJS = \ $O\IntToString.obj \ @@ -29,6 +28,7 @@ COMMON_OBJS = \ $O\StringConvert.obj \ $O\StringToInt.obj \ $O\MyVector.obj \ + $O\UTFConvert.obj \ $O\Wildcard.obj \ WIN_OBJS = \ diff --git a/CPP/7zip/UI/FileManager/AboutDialog.cpp b/CPP/7zip/UI/FileManager/AboutDialog.cpp index a644474b..b3455cf5 100644 --- a/CPP/7zip/UI/FileManager/AboutDialog.cpp +++ b/CPP/7zip/UI/FileManager/AboutDialog.cpp @@ -6,6 +6,8 @@ #include "../../MyVersion.h" +#include "../Common/LoadCodecs.h" + #include "AboutDialog.h" #include "PropertyNameRes.h" @@ -23,8 +25,20 @@ static const UInt32 kLangIDs[] = #define LLL_(quote) L##quote #define LLL(quote) LLL_(quote) +extern CCodecs *g_CodecsObj; + bool CAboutDialog::OnInit() { + #ifdef EXTERNAL_CODECS + if (g_CodecsObj) + { + UString s; + g_CodecsObj->GetCodecsErrorMessage(s); + if (!s.IsEmpty()) + MessageBoxW(GetParent(), s, L"7-Zip", MB_ICONERROR); + } + #endif + LangSetDlgItems(*this, kLangIDs, ARRAY_SIZE(kLangIDs)); SetItemText(IDT_ABOUT_VERSION, UString("7-Zip " MY_VERSION_CPU)); SetItemText(IDT_ABOUT_DATE, LLL(MY_DATE)); diff --git a/CPP/7zip/UI/FileManager/AboutDialog.rc b/CPP/7zip/UI/FileManager/AboutDialog.rc index 6953c141..b235df0b 100644 --- a/CPP/7zip/UI/FileManager/AboutDialog.rc +++ b/CPP/7zip/UI/FileManager/AboutDialog.rc @@ -18,7 +18,7 @@ CAPTION "About 7-Zip" { DEFPUSHBUTTON "OK", IDOK, bx1, by, bxs, bys PUSHBUTTON "www.7-zip.org", IDB_ABOUT_HOMEPAGE, bx2, by, bxs, bys - ICON IDI_LOGO, -1, m, m, 32, 32, SS_REALSIZEIMAGE, + ICON IDI_LOGO, -1, m, m, 32, 32, SS_REALSIZEIMAGE LTEXT "", IDT_ABOUT_VERSION, m, 54, xc, 8 LTEXT "", IDT_ABOUT_DATE, m, 67, xc, 8 LTEXT MY_COPYRIGHT, -1, m, 80, xc, 8 diff --git a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp index 7069e1aa..e1c99d3c 100644 --- a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp +++ b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp @@ -86,7 +86,7 @@ HRESULT CAltStreamsFolder::Init(const FString &path /* , IFolderFolder *parentFo { CFileInfo fi; if (!fi.Find(_pathBaseFile)) - return GetLastError(); + return GetLastError_noZero_HRESULT(); } unsigned prefixSize = GetFsParentPrefixSize(_pathBaseFile); @@ -612,7 +612,7 @@ static HRESULT CopyStream( FString destPath = destPathSpec; if (CompareFileNames(destPath, srcPath) == 0) { - RINOK(SendMessageError(callback, "can not copy file onto itself", destPath)); + RINOK(SendMessageError(callback, "Cannot copy file onto itself", destPath)); return E_ABORT; } @@ -716,7 +716,7 @@ STDMETHODIMP CAltStreamsFolder::CopyTo(Int32 moveMode, const UInt32 *indices, UI if (destPath.IsEmpty() /* && !ExtractToStreamCallback */) return E_INVALIDARG; - bool isAltDest = NName::IsAltPathPrefix(destPath);; + bool isAltDest = NName::IsAltPathPrefix(destPath); bool isDirectPath = (!isAltDest && !IsPathSepar(destPath.Back())); if (isDirectPath) @@ -764,7 +764,7 @@ STDMETHODIMP CAltStreamsFolder::CopyFrom(Int32 /* moveMode */, const wchar_t * / if (CompareFileNames(fromFolderPath, fs2us(_pathPrefix)) == 0) { - RINOK(SendMessageError(callback, "can not copy file onto itself", _pathPrefix)); + RINOK(SendMessageError(callback, "Cannot copy file onto itself", _pathPrefix)); return E_ABORT; } diff --git a/CPP/7zip/UI/FileManager/App.cpp b/CPP/7zip/UI/FileManager/App.cpp index 4c1ea548..512acc53 100644 --- a/CPP/7zip/UI/FileManager/App.cpp +++ b/CPP/7zip/UI/FileManager/App.cpp @@ -117,7 +117,7 @@ void CApp::SetListSettings() HRESULT CApp::CreateOnePanel(int panelIndex, const UString &mainPath, const UString &arcFormat, bool needOpenArc, - bool &archiveIsOpened, bool &encrypted) + COpenResult &openRes) { if (Panels[panelIndex].PanelCreated) return S_OK; @@ -138,7 +138,7 @@ HRESULT CApp::CreateOnePanel(int panelIndex, const UString &mainPath, const UStr return Panels[panelIndex].Create(_window, _window, id, path, arcFormat, &m_PanelCallbackImp[panelIndex], &AppState, needOpenArc, - archiveIsOpened, encrypted); + openRes); } @@ -282,7 +282,7 @@ void CApp::SaveToolbarChanges() } -HRESULT CApp::Create(HWND hwnd, const UString &mainPath, const UString &arcFormat, int xSizes[2], bool needOpenArc, bool &archiveIsOpened, bool &encrypted) +HRESULT CApp::Create(HWND hwnd, const UString &mainPath, const UString &arcFormat, int xSizes[2], bool needOpenArc, COpenResult &openRes) { _window.Attach(hwnd); @@ -334,21 +334,19 @@ HRESULT CApp::Create(HWND hwnd, const UString &mainPath, const UString &arcForma { if (NumPanels == 1) Panels[panelIndex]._xSize = xSizes[0] + xSizes[1]; - bool archiveIsOpened2 = false; - bool encrypted2 = false; + + COpenResult openRes2; UString path; if (isMainPanel) path = mainPath; RINOK(CreateOnePanel(panelIndex, path, arcFormat, isMainPanel && needOpenArc, - archiveIsOpened2, encrypted2)); + *(isMainPanel ? &openRes : &openRes2))); if (isMainPanel) { - archiveIsOpened = archiveIsOpened2; - encrypted = encrypted2; - if (needOpenArc && !archiveIsOpened2) + if (needOpenArc && !openRes.ArchiveIsOpened) return S_OK; } } @@ -365,10 +363,10 @@ HRESULT CApp::SwitchOnOffOnePanel() if (NumPanels == 1) { NumPanels++; - bool archiveIsOpened, encrypted; + COpenResult openRes; RINOK(CreateOnePanel(1 - LastFocusedPanel, UString(), UString(), false, // needOpenArc - archiveIsOpened, encrypted)); + openRes)); Panels[1 - LastFocusedPanel].Enable(true); Panels[1 - LastFocusedPanel].Show(SW_SHOWNORMAL); } @@ -414,13 +412,13 @@ void CApp::Release() // reduces path to part that exists on disk (or root prefix of path) // output path is normalized (with WCHAR_PATH_SEPARATOR) -static void ReducePathToRealFileSystemPath(UString &path) +static void Reduce_Path_To_RealFileSystemPath(UString &path) { unsigned prefixSize = GetRootPrefixSize(path); while (!path.IsEmpty()) { - if (NFind::DoesDirExist(us2fs(path))) + if (NFind::DoesDirExist_FollowLink(us2fs(path))) { NName::NormalizeDirPathPrefix(path); break; @@ -431,10 +429,10 @@ static void ReducePathToRealFileSystemPath(UString &path) path.Empty(); break; } - path.DeleteFrom(pos + 1); + path.DeleteFrom((unsigned)(pos + 1)); if ((unsigned)pos + 1 == prefixSize) break; - path.DeleteFrom(pos); + path.DeleteFrom((unsigned)pos); } } @@ -443,7 +441,7 @@ static void ReducePathToRealFileSystemPath(UString &path) static bool CheckFolderPath(const UString &path) { UString pathReduced = path; - ReducePathToRealFileSystemPath(pathReduced); + Reduce_Path_To_RealFileSystemPath(pathReduced); return (pathReduced == path); } */ @@ -463,6 +461,7 @@ static void AddValuePair1(UString &s, UINT resourceID, UInt64 size) s.Add_LF(); } +void AddValuePair2(UString &s, UINT resourceID, UInt64 num, UInt64 size); void AddValuePair2(UString &s, UINT resourceID, UInt64 num, UInt64 size) { if (num == 0) @@ -601,7 +600,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) return; destPath = destPanel.GetFsPath(); if (NumPanels == 1) - ReducePathToRealFileSystemPath(destPath); + Reduce_Path_To_RealFileSystemPath(destPath); } } @@ -660,7 +659,7 @@ void CApp::OnCopy(bool move, bool copyToSame, int srcPanelIndex) { if (NumPanels == 1 || CompareFileNames(destPath, srcPanel.GetFsPath()) == 0) { - srcPanel.MessageBox_Error(L"Can not copy files onto itself"); + srcPanel.MessageBox_Error(L"Cannot copy files onto itself"); return; } @@ -882,9 +881,11 @@ void CApp::OnSetSubFolder(int srcPanelIndex) return; if (!newFolder) { - const UString parentPrefix = srcPanel.GetParentDirPrefix(); - bool archiveIsOpened, encrypted; - destPanel.BindToPath(parentPrefix, UString(), archiveIsOpened, encrypted); + { + const UString parentPrefix = srcPanel.GetParentDirPrefix(); + COpenResult openRes; + destPanel.BindToPath(parentPrefix, UString(), openRes); + } destPanel.RefreshListCtrl(); return; } @@ -936,7 +937,7 @@ void CApp::OnNotify(int /* ctrlID */, LPNMHDR pnmh) g_ToolTipBuffer.Empty(); SetButtonText((int)info->hdr.idFrom, g_ToolTipBuffer); g_ToolTipBufferSys = GetSystemString(g_ToolTipBuffer); - info->lpszText = (LPTSTR)(LPCTSTR)g_ToolTipBufferSys; + info->lpszText = g_ToolTipBufferSys.Ptr_non_const(); return; } #ifndef _UNICODE @@ -946,7 +947,7 @@ void CApp::OnNotify(int /* ctrlID */, LPNMHDR pnmh) info->hinst = 0; g_ToolTipBuffer.Empty(); SetButtonText((int)info->hdr.idFrom, g_ToolTipBuffer); - info->lpszText = (LPWSTR)(LPCWSTR)g_ToolTipBuffer; + info->lpszText = g_ToolTipBuffer.Ptr_non_const(); return; } #endif @@ -971,7 +972,7 @@ void CApp::RefreshTitlePanel(unsigned panelIndex, bool always) RefreshTitle(always); } -void AddUniqueStringToHead(UStringVector &list, const UString &s) +static void AddUniqueStringToHead(UStringVector &list, const UString &s) { for (unsigned i = 0; i < list.Size();) if (s.IsEqualTo_NoCase(list[i])) diff --git a/CPP/7zip/UI/FileManager/App.h b/CPP/7zip/UI/FileManager/App.h index fa8eeaa7..21b160dc 100644 --- a/CPP/7zip/UI/FileManager/App.h +++ b/CPP/7zip/UI/FileManager/App.h @@ -89,22 +89,24 @@ public: STDMETHOD(Drop)(IDataObject * dataObject, DWORD keyState, POINTL pt, DWORD *effect); CDropTarget(): - TargetPanelIndex(-1), - SrcPanelIndex(-1), - m_IsAppTarget(false), - m_Panel(0), - App(0), - m_PanelDropIsAllowed(false), - m_DropIsAllowed(false), m_SelectionIndex(-1), + m_DropIsAllowed(false), + m_PanelDropIsAllowed(false), m_SubFolderIndex(-1), - m_SetPathIsOK(false) {} + m_Panel(NULL), + m_IsAppTarget(false), + m_SetPathIsOK(false), + App(NULL), + SrcPanelIndex(-1), + TargetPanelIndex(-1) + {} CApp *App; int SrcPanelIndex; // index of D&D source_panel int TargetPanelIndex; // what panel to use as target_panel of Application }; + class CApp { public: @@ -173,8 +175,8 @@ public: void OnSetSameFolder(int srcPanelIndex); void OnSetSubFolder(int srcPanelIndex); - HRESULT CreateOnePanel(int panelIndex, const UString &mainPath, const UString &arcFormat, bool needOpenArc, bool &archiveIsOpened, bool &encrypted); - HRESULT Create(HWND hwnd, const UString &mainPath, const UString &arcFormat, int xSizes[2], bool needOpenArc, bool &archiveIsOpened, bool &encrypted); + HRESULT CreateOnePanel(int panelIndex, const UString &mainPath, const UString &arcFormat, bool needOpenArc, COpenResult &openRes); + HRESULT Create(HWND hwnd, const UString &mainPath, const UString &arcFormat, int xSizes[2], bool needOpenArc, COpenResult &openRes); void Read(); void Save(); void Release(); @@ -196,7 +198,12 @@ public: void Delete(bool toRecycleBin) { GetFocusedPanel().DeleteItems(toRecycleBin); } HRESULT CalculateCrc2(const UString &methodName); void CalculateCrc(const char *methodName); + + void DiffFiles(const UString &path1, const UString &path2); void DiffFiles(); + + void VerCtrl(unsigned id); + void Split(); void Combine(); void Properties() { GetFocusedPanel().Properties(); } diff --git a/CPP/7zip/UI/FileManager/BrowseDialog.cpp b/CPP/7zip/UI/FileManager/BrowseDialog.cpp index d8f9ebe8..6d2b6b55 100644 --- a/CPP/7zip/UI/FileManager/BrowseDialog.cpp +++ b/CPP/7zip/UI/FileManager/BrowseDialog.cpp @@ -4,7 +4,7 @@ #include "../../../Common/MyWindows.h" -#include <commctrl.h> +#include <CommCtrl.h> #ifndef UNDER_CE #include "../../../Windows/CommonDialog.h" @@ -139,7 +139,7 @@ public: UStringVector Filters; UString FilterDescription; - CBrowseDialog(): FolderMode(false), _showDots(false), ShowAllFiles(true) {} + CBrowseDialog(): _showDots(false), FolderMode(false), ShowAllFiles(true) {} void SetFilter(const UString &s); INT_PTR Create(HWND parent = 0) { return CModalDialog::Create(IDD_BROWSE, parent); } int CompareItems(LPARAM lParam1, LPARAM lParam2); @@ -235,7 +235,7 @@ bool CBrowseDialog::OnInit() column.fmt = LVCFMT_RIGHT; column.cx = 100; const UString s = LangString(IDS_PROP_SIZE); - column.pszText = (wchar_t *)(const wchar_t *)s; + column.pszText = s.Ptr_non_const(); _list.InsertColumn(2, &column); } @@ -476,7 +476,7 @@ bool CBrowseDialog::GetParentPath(const UString &path, UString &parentPrefix, US return false; int pos = s.ReverseFind_PathSepar(); parentPrefix.SetFrom(s, pos + 1); - name = s.Ptr(pos + 1); + name = s.Ptr((unsigned)(pos + 1)); return true; } @@ -604,7 +604,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected int subItem = 0; item.iSubItem = subItem++; item.lParam = kParentIndex; - item.pszText = (wchar_t *)(const wchar_t *)itemName; + item.pszText = itemName.Ptr_non_const(); item.iImage = _extToIconMap.GetIconIndex(FILE_ATTRIBUTE_DIRECTORY, DirPrefix); if (item.iImage < 0) item.iImage = 0; @@ -626,7 +626,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected int subItem = 0; item.iSubItem = subItem++; item.lParam = i; - item.pszText = (wchar_t *)(const wchar_t *)name; + item.pszText = name.Ptr_non_const(); const UString fullPath = DirPrefix + name; #ifndef UNDER_CE @@ -953,7 +953,7 @@ bool CorrectFsPath(const UString &relBase, const UString &path2, UString &result if (path.Back() == WCHAR_PATH_SEPARATOR) { path.DeleteBack(); - result.Insert(0, WCHAR_PATH_SEPARATOR);; + result.Insert(0, WCHAR_PATH_SEPARATOR); } int pos = path.ReverseFind(WCHAR_PATH_SEPARATOR) + 1; UString cur = path.Ptr(pos); diff --git a/CPP/7zip/UI/FileManager/EnumFormatEtc.cpp b/CPP/7zip/UI/FileManager/EnumFormatEtc.cpp index cba9aa21..389aa3e8 100644 --- a/CPP/7zip/UI/FileManager/EnumFormatEtc.cpp +++ b/CPP/7zip/UI/FileManager/EnumFormatEtc.cpp @@ -58,7 +58,7 @@ CEnumFormatEtc::~CEnumFormatEtc() for (ULONG i = 0; i < m_NumFormats; i++) if (m_Formats[i].ptd) CoTaskMemFree(m_Formats[i].ptd); - delete[]m_Formats; + delete []m_Formats; } } diff --git a/CPP/7zip/UI/FileManager/EnumFormatEtc.h b/CPP/7zip/UI/FileManager/EnumFormatEtc.h index 6c476f1a..93a53cb3 100644 --- a/CPP/7zip/UI/FileManager/EnumFormatEtc.h +++ b/CPP/7zip/UI/FileManager/EnumFormatEtc.h @@ -3,7 +3,7 @@ #ifndef __ENUMFORMATETC_H #define __ENUMFORMATETC_H -#include <windows.h> +#include "../../../Common/MyWindows.h" HRESULT CreateEnumFormatEtc(UINT numFormats, const FORMATETC *formats, IEnumFORMATETC **enumFormat); diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/CPP/7zip/UI/FileManager/ExtractCallback.cpp index 1bc96ae3..0e285f04 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -262,6 +262,7 @@ STDMETHODIMP CExtractCallbackImp::ShowMessage(const wchar_t *s) #endif +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, const wchar_t *fileName, UString &s); void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, const wchar_t *fileName, UString &s) { s.Empty(); @@ -449,6 +450,7 @@ static void AddNewLineString(UString &s, const UString &m) s.Add_LF(); } +UString GetOpenArcErrorMessage(UInt32 errorFlags); UString GetOpenArcErrorMessage(UInt32 errorFlags) { UString s; @@ -523,6 +525,7 @@ static UString GetBracedType(const wchar_t *type) return s; } +void OpenResult_GUI(UString &s, const CCodecs *codecs, const CArchiveLink &arcLink, const wchar_t *name, HRESULT result); void OpenResult_GUI(UString &s, const CCodecs *codecs, const CArchiveLink &arcLink, const wchar_t *name, HRESULT result) { FOR_VECTOR (level, arcLink.Arcs) @@ -632,7 +635,9 @@ HRESULT CExtractCallbackImp::ExtractResult(HRESULT result) if (result == S_OK) return result; NumArchiveErrors++; - if (result == E_ABORT || result == ERROR_DISK_FULL) + if (result == E_ABORT + || result == HRESULT_FROM_WIN32(ERROR_DISK_FULL) + ) return result; Add_ArchiveName_Error(); @@ -702,7 +707,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( { if (!destFileInfo.IsDir()) { - RINOK(MessageError("can not replace file with folder with same name", destPathSys)); + RINOK(MessageError("Cannot replace file with folder with same name", destPathSys)); return E_ABORT; } *writeAnswer = BoolToInt(false); @@ -711,7 +716,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( if (destFileInfo.IsDir()) { - RINOK(MessageError("can not replace folder with file with same name", destPathSys)); + RINOK(MessageError("Cannot replace folder with file with same name", destPathSys)); *writeAnswer = BoolToInt(false); return S_OK; } @@ -725,7 +730,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( Int32 overwriteResult; UString destPathSpec = destPath; int slashPos = destPathSpec.ReverseFind_PathSepar(); - destPathSpec.DeleteFrom(slashPos + 1); + destPathSpec.DeleteFrom((unsigned)(slashPos + 1)); destPathSpec += fs2us(destFileInfo.Name); RINOK(AskOverwrite( @@ -746,25 +751,28 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( default: return E_FAIL; } + break; } + default: + break; } if (OverwriteMode == NExtract::NOverwriteMode::kRename) { if (!AutoRenamePath(destPathSys)) { - RINOK(MessageError("can not create name for file", destPathSys)); + RINOK(MessageError("Cannot create name for file", destPathSys)); return E_ABORT; } destPathResultTemp = fs2us(destPathSys); } else { - if (NFind::DoesFileExist(destPathSys)) + if (NFind::DoesFileExist_Raw(destPathSys)) if (!NDir::DeleteFileAlways(destPathSys)) if (GetLastError() != ERROR_FILE_NOT_FOUND) { - RINOK(MessageError("can not delete output file", destPathSys)); + RINOK(MessageError("Cannot delete output file", destPathSys)); return E_ABORT; } } @@ -950,9 +958,8 @@ STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypt } -static const size_t k_SizeT_MAX = (size_t)((size_t)0 - 1); -static const UInt32 kBlockSize = ((UInt32)1 << 31); +// static const UInt32 kBlockSize = ((UInt32)1 << 31); STDMETHODIMP CVirtFileSystem::Write(const void *data, UInt32 size, UInt32 *processedSize) { @@ -977,8 +984,9 @@ STDMETHODIMP CVirtFileSystem::Write(const void *data, UInt32 size, UInt32 *proce if (b < a) b = a; useMem = false; - if (b <= k_SizeT_MAX && b <= MaxTotalAllocSize) - useMem = file.Data.ReAlloc_KeepData((size_t)b, (size_t)file.Size); + const size_t b_sizet = (size_t)b; + if (b == b_sizet && b <= MaxTotalAllocSize) + useMem = file.Data.ReAlloc_KeepData(b_sizet, (size_t)file.Size); } if (useMem) { diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.h b/CPP/7zip/UI/FileManager/ExtractCallback.h index 6cd8d0aa..297492eb 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.h +++ b/CPP/7zip/UI/FileManager/ExtractCallback.h @@ -289,21 +289,19 @@ public: bool MultiArcMode; CExtractCallbackImp(): + #ifndef _SFX + _hashCalc(NULL), + #endif + ProcessAltStreams(true), + StreamMode(false), + OverwriteMode(NExtract::NOverwriteMode::kAsk), #ifndef _NO_CRYPTO PasswordIsDefined(false), PasswordWasAsked(false), #endif - OverwriteMode(NExtract::NOverwriteMode::kAsk), - StreamMode(false), - ProcessAltStreams(true), - _totalFilesDefined(false), _totalBytesDefined(false), MultiArcMode(false) - - #ifndef _SFX - , _hashCalc(NULL) - #endif {} ~CExtractCallbackImp(); diff --git a/CPP/7zip/UI/FileManager/FM.cpp b/CPP/7zip/UI/FileManager/FM.cpp index 3e7f104d..812eff67 100644 --- a/CPP/7zip/UI/FileManager/FM.cpp +++ b/CPP/7zip/UI/FileManager/FM.cpp @@ -4,7 +4,7 @@ #include "../../../Common/MyWindows.h" -#include <shlwapi.h> +#include <Shlwapi.h> #include "../../../../C/Alloc.h" #ifdef _WIN32 @@ -40,19 +40,24 @@ using namespace NWindows; using namespace NFile; using namespace NFind; -#define MAX_LOADSTRING 100 - -#define MENU_HEIGHT 26 +// #define MAX_LOADSTRING 100 +extern +bool g_RAM_Size_Defined; bool g_RAM_Size_Defined; -bool g_LargePagesMode = false; -bool g_OpenArchive = false; + +static bool g_LargePagesMode = false; +// static bool g_OpenArchive = false; static bool g_Maximized = false; +extern +UInt64 g_RAM_Size; UInt64 g_RAM_Size; #ifdef _WIN32 +extern +HINSTANCE g_hInstance; HINSTANCE g_hInstance; #endif @@ -66,6 +71,8 @@ void FreeGlobalCodecs(); #ifndef UNDER_CE +extern +DWORD g_ComCtl32Version; DWORD g_ComCtl32Version; static DWORD GetDllVersion(LPCTSTR dllName) @@ -74,7 +81,7 @@ static DWORD GetDllVersion(LPCTSTR dllName) HINSTANCE hinstDll = LoadLibrary(dllName); if (hinstDll) { - DLLGETVERSIONPROC pDllGetVersion = (DLLGETVERSIONPROC)GetProcAddress(hinstDll, "DllGetVersion"); + DLLGETVERSIONPROC pDllGetVersion = (DLLGETVERSIONPROC)(void *)GetProcAddress(hinstDll, "DllGetVersion"); if (pDllGetVersion) { DLLVERSIONINFO dvi; @@ -93,19 +100,19 @@ static DWORD GetDllVersion(LPCTSTR dllName) bool g_IsSmallScreen = false; +extern +bool g_LVN_ITEMACTIVATE_Support; bool g_LVN_ITEMACTIVATE_Support = true; // LVN_ITEMACTIVATE replaces both NM_DBLCLK & NM_RETURN // Windows 2000 // NT/98 + IE 3 (g_ComCtl32Version >= 4.70) -const int kNumDefaultPanels = 1; - -const int kSplitterWidth = 4; -int kSplitterRateMax = 1 << 16; -int kPanelSizeMin = 120; +static const int kNumDefaultPanels = 1; +static const int kSplitterWidth = 4; +static const int kSplitterRateMax = 1 << 16; +static const int kPanelSizeMin = 120; -// bool OnMenuCommand(HWND hWnd, int id); class CSplitterPos { @@ -343,7 +350,7 @@ typedef BOOL (WINAPI *Func_IsWow64Process)(HANDLE, PBOOL); static void Set_Wow64() { g_Is_Wow64 = false; - Func_IsWow64Process fnIsWow64Process = (Func_IsWow64Process)GetProcAddress( + Func_IsWow64Process fnIsWow64Process = (Func_IsWow64Process)(void *)GetProcAddress( GetModuleHandleA("kernel32.dll"), "IsWow64Process"); if (fnIsWow64Process) { @@ -356,6 +363,7 @@ static void Set_Wow64() #endif +bool IsLargePageSupported(); bool IsLargePageSupported() { #ifdef _WIN64 @@ -383,7 +391,7 @@ static void SetMemoryLock() if (!IsLargePageSupported()) return; // if (ReadLockMemoryAdd()) - NSecurity::AddLockMemoryPrivilege(); + NSecurity::AddLockMemoryPrivilege(); if (ReadLockMemoryEnable()) if (NSecurity::Get_LargePages_RiskLevel() == 0) @@ -393,6 +401,8 @@ static void SetMemoryLock() } } +extern +bool g_SymLink_Supported; bool g_SymLink_Supported = false; static void Set_SymLink_Supported() @@ -443,7 +453,9 @@ static void ErrorMessage(const char *s) } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ErrorMessage("Unsupported Windows version"); return 1; +#endif static int WINAPI WinMain2(int nCmdShow) { @@ -756,12 +768,12 @@ static void ExecuteCommand(UINT commandID) LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) { - int wmId, wmEvent; switch (message) { case WM_COMMAND: - wmId = LOWORD(wParam); - wmEvent = HIWORD(wParam); + { + unsigned wmId = LOWORD(wParam); + unsigned wmEvent = HIWORD(wParam); if ((HWND) lParam != NULL && wmEvent != 0) break; if (wmId >= kMenuCmdID_Toolbar_Start && wmId < kMenuCmdID_Toolbar_End) @@ -772,6 +784,7 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) if (OnMenuCommand(hWnd, wmId)) return 0; break; + } case WM_INITMENUPOPUP: OnMenuActivating(hWnd, HMENU(wParam), LOWORD(lParam)); break; @@ -837,9 +850,8 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) g_App.CreateDragTarget(); - bool archiveIsOpened; - bool encrypted; - bool needOpenFile = false; + COpenResult openRes; + bool needOpenArc = false; UString fullPath = g_MainPath; if (!fullPath.IsEmpty() /* && g_OpenArchive */) @@ -850,29 +862,34 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) if (NFile::NName::GetFullPath(us2fs(fullPath), fullPathF)) fullPath = fs2us(fullPathF); } - if (NFile::NFind::DoesFileExist(us2fs(fullPath))) - needOpenFile = true; + if (NFile::NFind::DoesFileExist_FollowLink(us2fs(fullPath))) + needOpenArc = true; } HRESULT res = g_App.Create(hWnd, fullPath, g_ArcFormat, xSizes, - needOpenFile, - archiveIsOpened, encrypted); + needOpenArc, + openRes); if (res == E_ABORT) return -1; - if (needOpenFile && !archiveIsOpened || res != S_OK) + if ((needOpenArc && !openRes.ArchiveIsOpened) || res != S_OK) { UString m ("Error"); if (res == S_FALSE || res == S_OK) { - m = MyFormatNew(encrypted ? + m = MyFormatNew(openRes.Encrypted ? IDS_CANT_OPEN_ENCRYPTED_ARCHIVE : IDS_CANT_OPEN_ARCHIVE, fullPath); } else if (res != S_OK) m = HResultToMessage(res); + if (!openRes.ErrorMessage.IsEmpty()) + { + m.Add_LF(); + m += openRes.ErrorMessage; + } ErrorMessage(m); return -1; } @@ -954,7 +971,7 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) g_Panel[1]._listView.MoveWindow(xSize - xWidth, 0, xWidth, ySize); */ return 0; - break; + // break; } case WM_SETFOCUS: diff --git a/CPP/7zip/UI/FileManager/FM.dsp b/CPP/7zip/UI/FileManager/FM.dsp index 69d285cb..d87beff5 100644 --- a/CPP/7zip/UI/FileManager/FM.dsp +++ b/CPP/7zip/UI/FileManager/FM.dsp @@ -55,7 +55,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zFM.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "FM - Win32 Debug" @@ -82,7 +82,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zFM.exe" /pdbtype:sept !ELSEIF "$(CFG)" == "FM - Win32 ReleaseU" @@ -109,7 +109,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib /nologo /subsystem:windows /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zFM.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "FM - Win32 DebugU" @@ -137,7 +137,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib /nologo /subsystem:windows /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-ZIP\7zFM.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib Mpr.lib htmlhelp.lib Urlmon.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zFM.exe" /pdbtype:sept !ENDIF @@ -393,6 +393,10 @@ SOURCE=.\PanelSort.cpp SOURCE=.\PanelSplitFile.cpp # End Source File +# Begin Source File + +SOURCE=.\VerCtrl.cpp +# End Source File # End Group # Begin Group "Dialog" diff --git a/CPP/7zip/UI/FileManager/FM.mak b/CPP/7zip/UI/FileManager/FM.mak index 654da627..8b3d97af 100644 --- a/CPP/7zip/UI/FileManager/FM.mak +++ b/CPP/7zip/UI/FileManager/FM.mak @@ -7,8 +7,8 @@ LIBS = $(LIBS) ceshell.lib Commctrl.lib !ELSE LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE -LFLAGS = $(LFLAGS) /DELAYLOAD:mpr.dll -LIBS = $(LIBS) delayimp.lib +LFLAGS = $(LFLAGS) /DELAYLOAD:mpr.dll +LIBS = $(LIBS) delayimp.lib !ENDIF FM_OBJS = \ @@ -70,6 +70,7 @@ FM_OBJS = \ $O\SettingsPage.obj \ $O\SplitDialog.obj \ $O\SystemPage.obj \ + $O\VerCtrl.obj \ !IFNDEF UNDER_CE diff --git a/CPP/7zip/UI/FileManager/FSFolder.cpp b/CPP/7zip/UI/FileManager/FSFolder.cpp index 9e720214..d82ddc27 100644 --- a/CPP/7zip/UI/FileManager/FSFolder.cpp +++ b/CPP/7zip/UI/FileManager/FSFolder.cpp @@ -115,15 +115,15 @@ HRESULT CFsFolderStat::Enumerate() const unsigned len = Path.Len(); CEnumerator enumerator; enumerator.SetDirPrefix(Path); - CFileInfo fi; + CDirEntry fi; while (enumerator.Next(fi)) { if (fi.IsDir()) { + NumFolders++; Path.DeleteFrom(len); Path += fi.Name; RINOK(Enumerate()); - NumFolders++; } else { @@ -136,6 +136,7 @@ HRESULT CFsFolderStat::Enumerate() #ifndef UNDER_CE +bool MyGetCompressedFileSizeW(CFSTR path, UInt64 &size); bool MyGetCompressedFileSizeW(CFSTR path, UInt64 &size) { DWORD highPart; @@ -171,7 +172,7 @@ bool MyGetCompressedFileSizeW(CFSTR path, UInt64 &size) HRESULT CFSFolder::LoadSubItems(int dirItem, const FString &relPrefix) { - unsigned startIndex = Folders.Size(); + const unsigned startIndex = Folders.Size(); { CEnumerator enumerator; enumerator.SetDirPrefix(_path + relPrefix); @@ -261,7 +262,7 @@ HRESULT CFSFolder::LoadSubItems(int dirItem, const FString &relPrefix) if (!_flatMode) return S_OK; - unsigned endIndex = Folders.Size(); + const unsigned endIndex = Folders.Size(); for (unsigned i = startIndex; i < endIndex; i++) LoadSubItems(i, Folders[i]); return S_OK; @@ -293,8 +294,9 @@ bool CFSFolder::LoadComments() return false; AString s; char *p = s.GetBuf((unsigned)(size_t)len); - UInt32 processedSize; - file.Read(p, (UInt32)len, processedSize); + size_t processedSize; + if (!file.ReadFull(p, (unsigned)(size_t)len, processedSize)) + return false; s.ReleaseBuf_CalcLen((unsigned)(size_t)len); if (processedSize != len) return false; @@ -506,7 +508,7 @@ STDMETHODIMP CFSFolder::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *va { int pos = comment.Find((wchar_t)4); if (pos >= 0) - comment.DeleteFrom(pos); + comment.DeleteFrom((unsigned)pos); prop = comment; } break; @@ -784,7 +786,7 @@ STDMETHODIMP CFSFolder::BindToParentFolder(IFolderFolder **resultFolder) return E_FAIL; FString parentPath = _path.Left(pos); pos = parentPath.ReverseFind_PathSepar(); - parentPath.DeleteFrom(pos + 1); + parentPath.DeleteFrom((unsigned)(pos + 1)); if (NName::IsDrivePath_SuperAllowed(parentPath)) { diff --git a/CPP/7zip/UI/FileManager/FSFolder.h b/CPP/7zip/UI/FileManager/FSFolder.h index a7d78993..fac1ef97 100644 --- a/CPP/7zip/UI/FileManager/FSFolder.h +++ b/CPP/7zip/UI/FileManager/FSFolder.h @@ -203,7 +203,7 @@ struct CCopyStateIO int ErrorFileIndex; UString ErrorMessage; - CCopyStateIO(): DeleteSrcFile(false), TotalSize(0), StartPos(0) {} + CCopyStateIO(): TotalSize(0), StartPos(0), DeleteSrcFile(false) {} HRESULT MyCopyFile(CFSTR inPath, CFSTR outPath, DWORD attrib = INVALID_FILE_ATTRIBUTES); }; diff --git a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp index 7f95b1d2..9b78ba0a 100644 --- a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp +++ b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp @@ -4,7 +4,7 @@ #include "../../../Common/MyWindows.h" -#include <Winbase.h> +#include <WinBase.h> #include "../../../Common/Defs.h" #include "../../../Common/StringConvert.h" @@ -397,8 +397,8 @@ static HRESULT CopyFile_Ask( { RINOK(SendMessageError(state.Callback, state.MoveMode ? - "can not move file onto itself" : - "can not copy file onto itself" + "Cannot move file onto itself" : + "Cannot copy file onto itself" , destPath)); return E_ABORT; } @@ -497,8 +497,8 @@ static HRESULT CopyFolder( { RINOK(SendMessageError(state.Callback, state.MoveMode ? - "can not copy folder onto itself" : - "can not move folder onto itself" + "Cannot copy folder onto itself" : + "Cannot move folder onto itself" , destPath)); return E_ABORT; } @@ -513,7 +513,7 @@ static HRESULT CopyFolder( if (!CreateComplexDir(destPath)) { - RINOK(SendMessageError(state.Callback, "can not create folder", destPath)); + RINOK(SendMessageError(state.Callback, "Cannot create folder", destPath)); return E_ABORT; } @@ -547,7 +547,7 @@ static HRESULT CopyFolder( { if (!RemoveDir(srcPath)) { - RINOK(SendMessageError(state.Callback, "can not remove folder", srcPath)); + RINOK(SendMessageError(state.Callback, "Cannot remove folder", srcPath)); return E_ABORT; } } @@ -566,7 +566,7 @@ STDMETHODIMP CFSFolder::CopyTo(Int32 moveMode, const UInt32 *indices, UInt32 num if (destPath.IsEmpty()) return E_INVALIDARG; - bool isAltDest = NName::IsAltPathPrefix(destPath);; + bool isAltDest = NName::IsAltPathPrefix(destPath); bool isDirectPath = (!isAltDest && !IsPathSepar(destPath.Back())); if (isDirectPath) diff --git a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp index 4e964628..a7873fe5 100644 --- a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp +++ b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp @@ -8,10 +8,14 @@ #include "../../../Windows/Thread.h" #include "../Agent/Agent.h" +#include "../GUI/ExtractRes.h" +#include "FileFolderPluginOpen.h" +#include "FormatUtils.h" #include "LangUtils.h" #include "OpenCallback.h" #include "PluginLoader.h" +#include "PropertyName.h" #include "RegistryPlugins.h" using namespace NWindows; @@ -69,18 +73,165 @@ static void SplitNameToPureNameAndExtension(const FString &fullName, { pureName.SetFrom(fullName, index); extensionDelimiter = '.'; - extension = fullName.Ptr(index + 1); + extension = fullName.Ptr((unsigned)index + 1); } } -HRESULT OpenFileFolderPlugin( - IInStream *inStream, - const FString &path, - const UString &arcFormat, - HMODULE *module, - IFolderFolder **resultFolder, - HWND parentWindow, - bool &encrypted, UString &password) + +struct CArcLevelInfo +{ + UString Error; + UString Path; + UString Type; + UString ErrorType; + UString ErrorFlags; +}; + + +struct CArcLevelsInfo +{ + CObjectVector<CArcLevelInfo> Levels; // LastLevel Is NON-OPEN +}; + + +UString GetOpenArcErrorMessage(UInt32 errorFlags); + + +static void GetFolderLevels(CMyComPtr<IFolderFolder> &folder, CArcLevelsInfo &levels) +{ + levels.Levels.Clear(); + + CMyComPtr<IGetFolderArcProps> getFolderArcProps; + folder.QueryInterface(IID_IGetFolderArcProps, &getFolderArcProps); + + if (!getFolderArcProps) + return; + CMyComPtr<IFolderArcProps> arcProps; + getFolderArcProps->GetFolderArcProps(&arcProps); + if (!arcProps) + return; + + UInt32 numLevels; + if (arcProps->GetArcNumLevels(&numLevels) != S_OK) + numLevels = 0; + + for (UInt32 level = 0; level <= numLevels; level++) + { + const PROPID propIDs[] = { kpidError, kpidPath, kpidType, kpidErrorType }; + + CArcLevelInfo lev; + + for (Int32 i = 0; i < 4; i++) + { + CMyComBSTR name; + NCOM::CPropVariant prop; + if (arcProps->GetArcProp(level, propIDs[i], &prop) != S_OK) + continue; + if (prop.vt != VT_EMPTY) + { + UString *s = NULL; + switch (propIDs[i]) + { + case kpidError: s = &lev.Error; break; + case kpidPath: s = &lev.Path; break; + case kpidType: s = &lev.Type; break; + case kpidErrorType: s = &lev.ErrorType; break; + } + *s = (prop.vt == VT_BSTR) ? prop.bstrVal : L"?"; + } + } + + { + NCOM::CPropVariant prop; + if (arcProps->GetArcProp(level, kpidErrorFlags, &prop) == S_OK) + { + UInt32 flags = GetOpenArcErrorFlags(prop); + if (flags != 0) + lev.ErrorFlags = GetOpenArcErrorMessage(flags); + } + } + + levels.Levels.Add(lev); + } +} + +static UString GetBracedType(const wchar_t *type) +{ + UString s ('['); + s += type; + s += ']'; + return s; +} + +static void GetFolderError(CMyComPtr<IFolderFolder> &folder, UString &open_Errors, UString &nonOpen_Errors) +{ + CArcLevelsInfo levs; + GetFolderLevels(folder, levs); + open_Errors.Empty(); + nonOpen_Errors.Empty(); + + FOR_VECTOR (i, levs.Levels) + { + bool isNonOpenLevel = (i == 0); + const CArcLevelInfo &lev = levs.Levels[levs.Levels.Size() - 1 - i]; + + UString m; + + if (!lev.ErrorType.IsEmpty()) + { + m = MyFormatNew(IDS_CANT_OPEN_AS_TYPE, GetBracedType(lev.ErrorType)); + if (!isNonOpenLevel) + { + m.Add_LF(); + m += MyFormatNew(IDS_IS_OPEN_AS_TYPE, GetBracedType(lev.Type)); + } + } + + if (!lev.Error.IsEmpty()) + { + if (!m.IsEmpty()) + m.Add_LF(); + m += GetBracedType(lev.Type); + m += " : "; + m += GetNameOfProperty(kpidError, L"Error"); + m += " : "; + m += lev.Error; + } + + if (!lev.ErrorFlags.IsEmpty()) + { + if (!m.IsEmpty()) + m.Add_LF(); + m += GetNameOfProperty(kpidErrorFlags, L"Errors"); + m += ": "; + m += lev.ErrorFlags; + } + + if (!m.IsEmpty()) + { + if (isNonOpenLevel) + { + UString &s = nonOpen_Errors; + s += lev.Path; + s.Add_LF(); + s += m; + } + else + { + UString &s = open_Errors; + if (!s.IsEmpty()) + s += "--------------------\n"; + s += lev.Path; + s.Add_LF(); + s += m; + } + } + } +} + + +HRESULT CFfpOpen::OpenFileFolderPlugin(IInStream *inStream, + const FString &path, const UString &arcFormat, HWND parentWindow) { CObjectVector<CPluginInfo> plugins; ReadFileFolderPluginInfoList(plugins); @@ -92,8 +243,8 @@ HRESULT OpenFileFolderPlugin( FString fileName; if (slashPos >= 0) { - dirPrefix.SetFrom(path, slashPos + 1); - fileName = path.Ptr(slashPos + 1); + dirPrefix.SetFrom(path, (unsigned)(slashPos + 1)); + fileName = path.Ptr((unsigned)(slashPos + 1)); } else fileName = path; @@ -120,6 +271,8 @@ HRESULT OpenFileFolderPlugin( } */ + ErrorMessage.Empty(); + FOR_VECTOR (i, plugins) { const CPluginInfo &plugin = plugins[i]; @@ -136,24 +289,29 @@ HRESULT OpenFileFolderPlugin( t.OpenCallbackSpec = new COpenArchiveCallback; t.OpenCallback = t.OpenCallbackSpec; - t.OpenCallbackSpec->PasswordIsDefined = encrypted; - t.OpenCallbackSpec->Password = password; + t.OpenCallbackSpec->PasswordIsDefined = Encrypted; + t.OpenCallbackSpec->Password = Password; t.OpenCallbackSpec->ParentWindow = parentWindow; if (inStream) t.OpenCallbackSpec->SetSubArchiveName(fs2us(fileName)); else - t.OpenCallbackSpec->LoadFileInfo(dirPrefix, fileName); + { + RINOK(t.OpenCallbackSpec->LoadFileInfo2(dirPrefix, fileName)); + } t.InStream = inStream; t.Path = fs2us(path); t.ArcFormat = arcFormat; - UString progressTitle = LangString(IDS_OPENNING); - t.OpenCallbackSpec->ProgressDialog.MainWindow = parentWindow; - t.OpenCallbackSpec->ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - t.OpenCallbackSpec->ProgressDialog.MainAddTitle = progressTitle + L' '; - t.OpenCallbackSpec->ProgressDialog.WaitMode = true; + const UString progressTitle = LangString(IDS_OPENNING); + { + CProgressDialog &pd = t.OpenCallbackSpec->ProgressDialog; + pd.MainWindow = parentWindow; + pd.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + pd.MainAddTitle = progressTitle + L' '; + pd.WaitMode = true; + } { NWindows::CThread thread; @@ -161,23 +319,39 @@ HRESULT OpenFileFolderPlugin( t.OpenCallbackSpec->StartProgressDialog(progressTitle, thread); } - if (t.Result == E_ABORT) + if (t.Result != S_FALSE && t.Result != S_OK) return t.Result; - encrypted = t.OpenCallbackSpec->PasswordIsDefined; - if (t.Result == S_OK) + if (t.Folder) { - // if (openCallbackSpec->PasswordWasAsked) + UString open_Errors, nonOpen_Errors; + GetFolderError(t.Folder, open_Errors, nonOpen_Errors); + if (!nonOpen_Errors.IsEmpty()) { - password = t.OpenCallbackSpec->Password; + ErrorMessage = nonOpen_Errors; + // if (t.Result != S_OK) return t.Result; + /* if there are good open leves, and non0open level, + we could force error as critical error and return error here + but it's better to allow to open such rachives */ + // return S_FALSE; } - *module = library.Detach(); - *resultFolder = t.Folder.Detach(); - return S_OK; + } + + // if (openCallbackSpec->PasswordWasAsked) + { + Encrypted = t.OpenCallbackSpec->PasswordIsDefined; + Password = t.OpenCallbackSpec->Password; + } + + if (t.Result == S_OK) + { + Library.Attach(library.Detach()); + // Folder.Attach(t.Folder.Detach()); + Folder = t.Folder; } - if (t.Result != S_FALSE) - return t.Result; + return t.Result; } + return S_FALSE; } diff --git a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.h b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.h index 593880e9..a1f2f104 100644 --- a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.h +++ b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.h @@ -3,7 +3,25 @@ #ifndef __FILE_FOLDER_PLUGIN_OPEN_H #define __FILE_FOLDER_PLUGIN_OPEN_H -HRESULT OpenFileFolderPlugin(IInStream *inStream, const FString &path, const UString &arcFormat, - HMODULE *module, IFolderFolder **resultFolder, HWND parentWindow, bool &encrypted, UString &password); +#include "../../../Windows/DLL.h" + +struct CFfpOpen +{ + CLASS_NO_COPY(CFfpOpen) +public: + // out: + bool Encrypted; + UString Password; + + NWindows::NDLL::CLibrary Library; + CMyComPtr<IFolderFolder> Folder; + UString ErrorMessage; + + CFfpOpen(): Encrypted (false) {} + + HRESULT OpenFileFolderPlugin(IInStream *inStream, + const FString &path, const UString &arcFormat, HWND parentWindow); +}; + #endif diff --git a/CPP/7zip/UI/FileManager/HelpUtils.cpp b/CPP/7zip/UI/FileManager/HelpUtils.cpp index c0f57990..2db95014 100644 --- a/CPP/7zip/UI/FileManager/HelpUtils.cpp +++ b/CPP/7zip/UI/FileManager/HelpUtils.cpp @@ -4,7 +4,7 @@ #include "HelpUtils.h" -#if defined(UNDER_CE) || !defined(_WIN32) +#if defined(UNDER_CE) || !defined(_WIN32) /* || !defined(_MSC_VER) */ void ShowHelpWindow(LPCSTR) { diff --git a/CPP/7zip/UI/FileManager/LangUtils.cpp b/CPP/7zip/UI/FileManager/LangUtils.cpp index bcaa5f1e..83d5aa70 100644 --- a/CPP/7zip/UI/FileManager/LangUtils.cpp +++ b/CPP/7zip/UI/FileManager/LangUtils.cpp @@ -23,6 +23,7 @@ static CLang g_Lang; static bool g_Loaded = false; static NSynchronization::CCriticalSection g_CriticalSection; +bool LangOpen(CLang &lang, CFSTR fileName); bool LangOpen(CLang &lang, CFSTR fileName) { return lang.Open(fileName, "7-Zip"); @@ -145,7 +146,7 @@ static const char * const kLangs = "it.ja.ko.nl.no.=nb.=nn.pl.pt.-br.rm.ro.ru.sr.=hr.-spl.-spc.sk.sq.sv.th.tr." "ur.id.uk.be.sl.et.lv.lt.tg.fa.vi.hy.az.eu.hsb.mk." "st.ts.tn.ve.xh.zu.af.ka.fo.hi.mt.se.ga.yi.ms.kk." - "ky.sw.tk.uz.tt.bn.pa.-in.gu.or.ta.te.kn.ml.as.mr.sa." + "ky.sw.tk.uz.-latn.-cyrl.tt.bn.pa.-in.gu.or.ta.te.kn.ml.as.mr.sa." "mn.=mn.=mng.bo.cy.kh.lo.my.gl.kok..sd.syr.si..iu.am.tzm." "ks.ne.fy.ps.tl.dv..ff.ha..yo.qu.st.ba.lb.kl." "ig.kr.om.ti.gn..la.so.ii..arn..moh..br.." diff --git a/CPP/7zip/UI/FileManager/LinkDialog.cpp b/CPP/7zip/UI/FileManager/LinkDialog.cpp index ddef365c..07f1f061 100644 --- a/CPP/7zip/UI/FileManager/LinkDialog.cpp +++ b/CPP/7zip/UI/FileManager/LinkDialog.cpp @@ -36,46 +36,50 @@ static const UInt32 kLangIDs[] = IDR_LINK_TYPE_HARD, IDR_LINK_TYPE_SYM_FILE, IDR_LINK_TYPE_SYM_DIR, - IDR_LINK_TYPE_JUNCTION + IDR_LINK_TYPE_JUNCTION, + IDR_LINK_TYPE_WSL }; #endif -static bool GetSymLink(CFSTR path, CReparseAttr &attr) -{ - NIO::CInFile file; - if (!file.Open(path, - FILE_SHARE_READ, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)) - return false; - const unsigned kBufSize = MAXIMUM_REPARSE_DATA_BUFFER_SIZE; - CByteArr buf(kBufSize); - DWORD returnedSize; - if (!file.DeviceIoControlOut(my_FSCTL_GET_REPARSE_POINT, buf, kBufSize, &returnedSize)) +static bool GetSymLink(CFSTR path, CReparseAttr &attr, UString &errorMessage) +{ + CByteBuffer buf; + if (!NIO::GetReparseData(path, buf, NULL)) return false; - DWORD errorCode = 0; - if (!attr.Parse(buf, returnedSize, errorCode)) + if (!attr.Parse(buf, buf.Size())) + { + SetLastError(attr.ErrorCode); return false; + } CByteBuffer data2; - if (!FillLinkData(data2, attr.GetPath(), attr.IsSymLink())) + if (!FillLinkData(data2, attr.GetPath(), + !attr.IsMountPoint(), attr.IsSymLink_WSL())) + { + errorMessage = "Cannot reproduce reparse point"; return false; + } - if (data2.Size() != returnedSize || - memcmp(data2, buf, returnedSize) != 0) + if (data2.Size() != buf.Size() || + memcmp(data2, buf, buf.Size()) != 0) + { + errorMessage = "mismatch for reproduced reparse point"; return false; + } return true; } + static const int k_LinkType_Buttons[] = { IDR_LINK_TYPE_HARD, IDR_LINK_TYPE_SYM_FILE, IDR_LINK_TYPE_SYM_DIR, - IDR_LINK_TYPE_JUNCTION + IDR_LINK_TYPE_JUNCTION, + IDR_LINK_TYPE_WSL }; void CLinkDialog::Set_LinkType_Radio(int idb) @@ -104,16 +108,33 @@ bool CLinkDialog::OnInit() if (fi.HasReparsePoint()) { CReparseAttr attr; - bool res = GetSymLink(us2fs(FilePath), attr); + UString error; + bool res = GetSymLink(us2fs(FilePath), attr, error); + if (!res && error.IsEmpty()) + { + DWORD lastError = GetLastError(); + if (lastError != 0) + error = NError::MyFormatMessage(lastError); + } - UString s = attr.PrintName; + UString s = attr.GetPath(); + if (!attr.IsSymLink_WSL()) if (!attr.IsOkNamePair()) { s += " : "; - s += attr.SubsName; + s += attr.PrintName; } + if (!res) + { s.Insert(0, L"ERROR: "); + if (!error.IsEmpty()) + { + s += " : "; + s += error; + } + } + SetItemText(IDT_LINK_PATH_TO_CUR, s); @@ -121,11 +142,13 @@ bool CLinkDialog::OnInit() _pathFromCombo.SetText(FilePath); _pathToCombo.SetText(destPath); - if (res) + // if (res) { if (attr.IsMountPoint()) linkType = IDR_LINK_TYPE_JUNCTION; - if (attr.IsSymLink()) + else if (attr.IsSymLink_WSL()) + linkType = IDR_LINK_TYPE_WSL; + else if (attr.IsSymLink_Win()) { linkType = fi.IsDir() ? @@ -140,6 +163,7 @@ bool CLinkDialog::OnInit() } else { + // no ReparsePoint _pathFromCombo.SetText(AnotherPath); _pathToCombo.SetText(FilePath); if (fi.IsDir()) @@ -258,15 +282,18 @@ void CLinkDialog::OnButton_Link() } NFind::CFileInfo info1, info2; - bool finded1 = info1.Find(us2fs(from)); - bool finded2 = info2.Find(us2fs(to)); + const bool finded1 = info1.Find(us2fs(from)); + const bool finded2 = info2.Find(us2fs(to)); - bool isDirLink = ( + const bool isDirLink = ( idb == IDR_LINK_TYPE_SYM_DIR || idb == IDR_LINK_TYPE_JUNCTION); - if (finded1 && info1.IsDir() != isDirLink || - finded2 && info2.IsDir() != isDirLink) + const bool isWSL = (idb == IDR_LINK_TYPE_WSL); + + if (!isWSL) + if ((finded1 && info1.IsDir() != isDirLink) || + (finded2 && info2.IsDir() != isDirLink)) { ShowError(L"Incorrect link type"); return; @@ -282,25 +309,41 @@ void CLinkDialog::OnButton_Link() } else { - bool isSymLink = (idb != IDR_LINK_TYPE_JUNCTION); + if (finded1 && !info1.IsDir() && !info1.HasReparsePoint() && info1.Size != 0) + { + UString s ("WARNING: reparse point will hide the data of existing file"); + s.Add_LF(); + s += from; + ShowError(s); + return; + } + + const bool isSymLink = (idb != IDR_LINK_TYPE_JUNCTION); CByteBuffer data; - if (!FillLinkData(data, to, isSymLink)) + if (!FillLinkData(data, to, isSymLink, isWSL)) { ShowError(L"Incorrect link"); return; } CReparseAttr attr; - DWORD errorCode = 0; - if (!attr.Parse(data, data.Size(), errorCode)) + if (!attr.Parse(data, data.Size())) { ShowError(L"Internal conversion error"); return; } - - - if (!NIO::SetReparseData(us2fs(from), isDirLink, data, (DWORD)data.Size())) + + bool res; + if (to.IsEmpty()) + { + // res = NIO::SetReparseData(us2fs(from), isDirLink, NULL, 0); + res = NIO::DeleteReparseData(us2fs(from)); + } + else + res = NIO::SetReparseData(us2fs(from), isDirLink, data, (DWORD)data.Size()); + + if (!res) { ShowLastErrorMessage(); return; @@ -350,5 +393,7 @@ void CApp::Link() if (dlg.Create(srcPanel.GetParent()) != IDOK) return; + // fix it: we should refresh panel with changed link + RefreshTitleAlways(); } diff --git a/CPP/7zip/UI/FileManager/LinkDialog.rc b/CPP/7zip/UI/FileManager/LinkDialog.rc index 3d481d4d..a9e220ba 100644 --- a/CPP/7zip/UI/FileManager/LinkDialog.rc +++ b/CPP/7zip/UI/FileManager/LinkDialog.rc @@ -2,7 +2,7 @@ #include "../../GuiCommon.rc" #define xc 288 -#define yc 200 +#define yc 214 #undef xRadioSize #define xRadioSize xc - m - 2 @@ -20,7 +20,7 @@ BEGIN LTEXT "", IDT_LINK_PATH_TO_CUR, m, 78, xc, 8 - GROUPBOX "Link Type", IDG_LINK_TYPE, m, 104, xc, 76 + GROUPBOX "Link Type", IDG_LINK_TYPE, m, 104, xc, 90 CONTROL "Hard Link", IDR_LINK_TYPE_HARD, "Button", BS_AUTORADIOBUTTON | WS_GROUP, m + m, 120, xRadioSize, 10 @@ -30,6 +30,8 @@ BEGIN m + m, 148, xRadioSize, 10 CONTROL "Directory Junction", IDR_LINK_TYPE_JUNCTION, "Button", BS_AUTORADIOBUTTON, m + m, 162, xRadioSize, 10 + CONTROL "WSL", IDR_LINK_TYPE_WSL, "Button", BS_AUTORADIOBUTTON, + m + m, 176, xRadioSize, 10 DEFPUSHBUTTON "Link", IDB_LINK_LINK, bx2, by, bxs, bys PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys diff --git a/CPP/7zip/UI/FileManager/LinkDialogRes.h b/CPP/7zip/UI/FileManager/LinkDialogRes.h index 47b89192..3f7b3f23 100644 --- a/CPP/7zip/UI/FileManager/LinkDialogRes.h +++ b/CPP/7zip/UI/FileManager/LinkDialogRes.h @@ -10,6 +10,7 @@ #define IDR_LINK_TYPE_SYM_FILE 7712 #define IDR_LINK_TYPE_SYM_DIR 7713 #define IDR_LINK_TYPE_JUNCTION 7714 +#define IDR_LINK_TYPE_WSL 7715 #define IDC_LINK_PATH_FROM 100 diff --git a/CPP/7zip/UI/FileManager/MenuPage.cpp b/CPP/7zip/UI/FileManager/MenuPage.cpp index 4067ad75..ac3a7b14 100644 --- a/CPP/7zip/UI/FileManager/MenuPage.cpp +++ b/CPP/7zip/UI/FileManager/MenuPage.cpp @@ -152,7 +152,7 @@ bool CMenuPage::OnInit() ); - if (!NFile::NFind::DoesFileExist(path)) + if (!NFile::NFind::DoesFileExist_Raw(path)) { path.Empty(); EnableItem(dll.ctrl, false); diff --git a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp index 50104950..a133a291 100644 --- a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp +++ b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp @@ -124,7 +124,7 @@ static void MyChangeMenu(HMENU menuLoc, int level, int menuIndex) CMenu menu; menu.Attach(menuLoc); - for (int i = 0;; i++) + for (unsigned i = 0;; i++) { CMenuItem item; item.fMask = Get_fMask_for_String() | MIIM_SUBMENU | MIIM_ID; @@ -214,7 +214,7 @@ static void MyChangeMenu(HMENU menuLoc, int level, int menuIndex) static CMenu g_FileMenu; -struct CFileMenuDestroyer +static struct CFileMenuDestroyer { ~CFileMenuDestroyer() { if ((HMENU)g_FileMenu != 0) g_FileMenu.Destroy(); } } g_FileMenuDestroyer; @@ -461,6 +461,22 @@ void OnMenuUnActivating(HWND hWnd, HMENU hMenu, int id) } */ +static const unsigned g_Zvc_IDs[] = +{ + IDM_VER_EDIT, + IDM_VER_COMMIT, + IDM_VER_REVERT, + IDM_VER_DIFF +}; + +static const char * const g_Zvc_Strings[] = +{ + "Ver Edit (&1)" + , "Ver Commit" + , "Ver Revert" + , "Ver Diff (&0)" +}; + void CFileMenu::Load(HMENU hMenu, unsigned startPos) { CMenu destMenu; @@ -468,7 +484,7 @@ void CFileMenu::Load(HMENU hMenu, unsigned startPos) UString diffPath; ReadRegDiff(diffPath); - + unsigned numRealItems = startPos; for (unsigned i = 0;; i++) @@ -544,11 +560,47 @@ void CFileMenu::Load(HMENU hMenu, unsigned startPos) numRealItems = startPos; } } + + UString vercPath; + if (!diffPath.IsEmpty() && isFsFolder && allAreFiles && numItems == 1) + ReadReg_VerCtrlPath(vercPath); + + if (!vercPath.IsEmpty()) + { + NFile::NFind::CFileInfo fi; + if (fi.Find(FilePath) && fi.Size < ((UInt32)1 << 31) && !fi.IsDir()) + { + for (unsigned k = 0; k < ARRAY_SIZE(g_Zvc_IDs); k++) + { + const unsigned id = g_Zvc_IDs[k]; + if (fi.IsReadOnly()) + { + if (id == IDM_VER_COMMIT || + id == IDM_VER_REVERT || + id == IDM_VER_DIFF) + continue; + } + else + { + if (id == IDM_VER_EDIT) + continue; + } + + CMenuItem item; + UString s (g_Zvc_Strings[k]); + if (destMenu.AppendItem(MF_STRING, id, s)) + { + startPos++; + numRealItems = startPos; + } + } + } + } destMenu.RemoveAllItemsFrom(numRealItems); } -bool ExecuteFileCommand(int id) +bool ExecuteFileCommand(unsigned id) { if (id >= kMenuCmdID_Plugin_Start) { @@ -582,6 +634,13 @@ bool ExecuteFileCommand(int id) case IDM_SHA256: g_App.CalculateCrc("SHA256"); break; case IDM_DIFF: g_App.DiffFiles(); break; + + case IDM_VER_EDIT: + case IDM_VER_COMMIT: + case IDM_VER_REVERT: + case IDM_VER_DIFF: + g_App.VerCtrl(id); break; + case IDM_SPLIT: g_App.Split(); break; case IDM_COMBINE: g_App.Combine(); break; case IDM_PROPERTIES: g_App.Properties(); break; @@ -604,7 +663,7 @@ static void MyBenchmark(bool totalMode) Benchmark(totalMode); } -bool OnMenuCommand(HWND hWnd, int id) +bool OnMenuCommand(HWND hWnd, unsigned id) { if (ExecuteFileCommand(id)) return true; diff --git a/CPP/7zip/UI/FileManager/MyLoadMenu.h b/CPP/7zip/UI/FileManager/MyLoadMenu.h index 02569e52..0a38a732 100644 --- a/CPP/7zip/UI/FileManager/MyLoadMenu.h +++ b/CPP/7zip/UI/FileManager/MyLoadMenu.h @@ -7,7 +7,7 @@ void OnMenuActivating(HWND hWnd, HMENU hMenu, int position); // void OnMenuUnActivating(HWND hWnd, HMENU hMenu, int id); // void OnMenuUnActivating(HWND hWnd); -bool OnMenuCommand(HWND hWnd, int id); +bool OnMenuCommand(HWND hWnd, unsigned id); void MyLoadMenu(); struct CFileMenu @@ -19,6 +19,8 @@ struct CFileMenu bool isAltStreamsSupported; int numItems; + UString FilePath; + CFileMenu(): programMenu(false), readOnly(false), @@ -31,6 +33,6 @@ struct CFileMenu void Load(HMENU hMenu, unsigned startPos); }; -bool ExecuteFileCommand(int id); +bool ExecuteFileCommand(unsigned id); #endif diff --git a/CPP/7zip/UI/FileManager/OpenCallback.cpp b/CPP/7zip/UI/FileManager/OpenCallback.cpp index bf7abfb7..ddec3877 100644 --- a/CPP/7zip/UI/FileManager/OpenCallback.cpp +++ b/CPP/7zip/UI/FileManager/OpenCallback.cpp @@ -92,7 +92,7 @@ STDMETHODIMP COpenArchiveCallback::GetStream(const wchar_t *name, IInStream **in FString fullPath; if (!NFile::NName::GetFullPath(_folderPrefix, us2fs(name), fullPath)) return S_FALSE; - if (!_fileInfo.Find(fullPath)) + if (!_fileInfo.Find_FollowLink(fullPath)) return S_FALSE; if (_fileInfo.IsDir()) return S_FALSE; diff --git a/CPP/7zip/UI/FileManager/OpenCallback.h b/CPP/7zip/UI/FileManager/OpenCallback.h index c952d7b0..9aa5d43c 100644 --- a/CPP/7zip/UI/FileManager/OpenCallback.h +++ b/CPP/7zip/UI/FileManager/OpenCallback.h @@ -74,12 +74,17 @@ public: _subArchiveMode = false; } */ - void LoadFileInfo(const FString &folderPrefix, const FString &fileName) + + HRESULT LoadFileInfo2(const FString &folderPrefix, const FString &fileName) { _folderPrefix = folderPrefix; - if (!_fileInfo.Find(_folderPrefix + fileName)) - throw 1; + if (!_fileInfo.Find_FollowLink(_folderPrefix + fileName)) + { + return GetLastError_noZero_HRESULT(); + } + return S_OK; } + void ShowMessage(const UInt64 *completed); INT_PTR StartProgressDialog(const UString &title, NWindows::CThread &thread) diff --git a/CPP/7zip/UI/FileManager/OptionsDialog.cpp b/CPP/7zip/UI/FileManager/OptionsDialog.cpp index 66e6f3c4..c42e0f87 100644 --- a/CPP/7zip/UI/FileManager/OptionsDialog.cpp +++ b/CPP/7zip/UI/FileManager/OptionsDialog.cpp @@ -28,6 +28,7 @@ using namespace NWindows; +void OptionsDialog(HWND hwndOwner, HINSTANCE hInstance); void OptionsDialog(HWND hwndOwner, HINSTANCE /* hInstance */) { CSystemPage systemPage; diff --git a/CPP/7zip/UI/FileManager/OverwriteDialog.cpp b/CPP/7zip/UI/FileManager/OverwriteDialog.cpp index 1c029132..b455b14a 100644 --- a/CPP/7zip/UI/FileManager/OverwriteDialog.cpp +++ b/CPP/7zip/UI/FileManager/OverwriteDialog.cpp @@ -57,8 +57,8 @@ void COverwriteDialog::SetFileInfoControl(int textID, int iconID, const UString &fileName = fileInfo.Name; int slashPos = fileName.ReverseFind_PathSepar(); - UString s1 = fileName.Left(slashPos + 1); - UString s2 = fileName.Ptr(slashPos + 1); + UString s1 = fileName.Left((unsigned)(slashPos + 1)); + UString s2 = fileName.Ptr((unsigned)(slashPos + 1)); ReduceString(s1); ReduceString(s2); @@ -103,6 +103,22 @@ bool COverwriteDialog::OnInit() SetFileInfoControl(IDT_OVERWRITE_OLD_FILE_SIZE_TIME, IDI_OVERWRITE_OLD_FILE, OldFileInfo); SetFileInfoControl(IDT_OVERWRITE_NEW_FILE_SIZE_TIME, IDI_OVERWRITE_NEW_FILE, NewFileInfo); NormalizePosition(); + + if (!ShowExtraButtons) + { + HideItem(IDB_YES_TO_ALL); + HideItem(IDB_NO_TO_ALL); + HideItem(IDB_AUTO_RENAME); + } + + if (DefaultButton_is_NO) + { + PostMsg(DM_SETDEFID, IDNO); + HWND h = GetItem(IDNO); + PostMsg(WM_NEXTDLGCTL, (WPARAM)h, TRUE); + // ::SetFocus(h); + } + return CModalDialog::OnInit(); } diff --git a/CPP/7zip/UI/FileManager/OverwriteDialog.h b/CPP/7zip/UI/FileManager/OverwriteDialog.h index da7fa55f..24e56cac 100644 --- a/CPP/7zip/UI/FileManager/OverwriteDialog.h +++ b/CPP/7zip/UI/FileManager/OverwriteDialog.h @@ -20,7 +20,7 @@ namespace NOverwriteDialog void SetTime(const FILETIME *t) { - if (t == 0) + if (!t) TimeIsDefined = false; else { @@ -28,15 +28,19 @@ namespace NOverwriteDialog Time = *t; } } + + void SetSize(UInt64 size) + { + SizeIsDefined = true; + Size = size; + } + void SetSize(const UInt64 *size) { - if (size == 0) + if (!size) SizeIsDefined = false; else - { - SizeIsDefined = true; - Size = *size; - } + SetSize(*size); } }; } @@ -51,6 +55,12 @@ class COverwriteDialog: public NWindows::NControl::CModalDialog void ReduceString(UString &s); public: + bool ShowExtraButtons; + bool DefaultButton_is_NO; + + + COverwriteDialog(): ShowExtraButtons(true), DefaultButton_is_NO(false) {} + INT_PTR Create(HWND parent = 0) { BIG_DIALOG_SIZE(280, 200); diff --git a/CPP/7zip/UI/FileManager/Panel.cpp b/CPP/7zip/UI/FileManager/Panel.cpp index afb5649a..6702fa9c 100644 --- a/CPP/7zip/UI/FileManager/Panel.cpp +++ b/CPP/7zip/UI/FileManager/Panel.cpp @@ -2,7 +2,7 @@ #include "StdAfx.h" -#include <Windowsx.h> +#include <WindowsX.h> // #include <stdio.h> #include "../../../Common/IntToString.h" @@ -62,7 +62,7 @@ CPanel::~CPanel() CloseOpenFolders(); } -HWND CPanel::GetParent() +HWND CPanel::GetParent() const { HWND h = CWindow2::GetParent(); return (h == 0) ? _mainWindow : h; @@ -76,7 +76,7 @@ HRESULT CPanel::Create(HWND mainWindow, HWND parentWindow, UINT id, const UString &arcFormat, CPanelCallback *panelCallback, CAppState *appState, bool needOpenArc, - bool &archiveIsOpened, bool &encrypted) + COpenResult &openRes) { _mainWindow = mainWindow; _processTimer = true; @@ -100,9 +100,9 @@ HRESULT CPanel::Create(HWND mainWindow, HWND parentWindow, UINT id, cfp = fs2us(cfpF); } - RINOK(BindToPath(cfp, arcFormat, archiveIsOpened, encrypted)); + RINOK(BindToPath(cfp, arcFormat, openRes)); - if (needOpenArc && !archiveIsOpened) + if (needOpenArc && !openRes.ArchiveIsOpened) return S_OK; if (!CreateEx(0, kClassName, 0, WS_CHILD | WS_VISIBLE, @@ -114,8 +114,11 @@ HRESULT CPanel::Create(HWND mainWindow, HWND parentWindow, UINT id, return S_OK; } +// extern UInt32 g_NumMessages; + LRESULT CPanel::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { + // g_NumMessages++; switch (message) { case kShiftSelectMessage: @@ -309,7 +312,7 @@ LRESULT CMyComboBoxEdit::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) bool shift = IsKeyDown(VK_SHIFT); if (!alt && !ctrl && !shift) { - g_App.SwitchOnOffOnePanel();; + g_App.SwitchOnOffOnePanel(); return 0; } break; @@ -340,7 +343,7 @@ bool CPanel::OnCreate(CREATESTRUCT * /* createStruct */) _ascending = true; _lastFocusedIsList = true; - DWORD style = WS_CHILD | WS_VISIBLE; // | WS_BORDER ; // | LVS_SHAREIMAGELISTS; // | LVS_SHOWSELALWAYS;; + DWORD style = WS_CHILD | WS_VISIBLE; // | WS_BORDER ; // | LVS_SHAREIMAGELISTS; // | LVS_SHOWSELALWAYS; style |= LVS_SHAREIMAGELISTS; // style |= LVS_AUTOARRANGE; @@ -396,7 +399,7 @@ bool CPanel::OnCreate(CREATESTRUCT * /* createStruct */) TBBUTTON tbb [ ] = { // {0, 0, TBSTATE_ENABLED, BTNS_SEP, 0L, 0}, - {VIEW_PARENTFOLDER, kParentFolderID, TBSTATE_ENABLED, BTNS_BUTTON, 0L, 0}, + {VIEW_PARENTFOLDER, kParentFolderID, TBSTATE_ENABLED, BTNS_BUTTON, { 0, 0 }, 0, 0 }, // {0, 0, TBSTATE_ENABLED, BTNS_SEP, 0L, 0}, // {VIEW_NEWFOLDER, kCreateFolderID, TBSTATE_ENABLED, BTNS_BUTTON, 0L, 0}, }; @@ -667,9 +670,14 @@ bool CPanel::OnNotify(UINT /* controlID */, LPNMHDR header, LRESULT &result) return OnNotifyReBar(header, result); else if (header->hwndFrom == _listView) return OnNotifyList(header, result); - else if (::GetParent(header->hwndFrom) == _listView && - header->code == NM_RCLICK) - return OnRightClick((MY_NMLISTVIEW_NMITEMACTIVATE *)header, result); + else if (::GetParent(header->hwndFrom) == _listView) + { + // NMHDR:code is UINT + // NM_RCLICK is unsigned in windows sdk + // NM_RCLICK is int in MinGW + if (header->code == (UINT)NM_RCLICK) + return OnRightClick((MY_NMLISTVIEW_NMITEMACTIVATE *)header, result); + } return false; } @@ -903,8 +911,8 @@ static UString GetSubFolderNameForExtract2(const UString &arcPath) UString name = arcPath; if (slashPos >= 0) { - s = arcPath.Left(slashPos + 1); - name = arcPath.Ptr(slashPos + 1); + s = arcPath.Left((unsigned)(slashPos + 1)); + name = arcPath.Ptr((unsigned)(slashPos + 1)); } s += GetSubFolderNameForExtract(name); return s; @@ -967,7 +975,8 @@ static void AddValuePair(UINT resourceID, UInt64 value, UString &s) s += sz; s.Add_LF(); } -*/ + +// now we don't need CThreadTest, since now we call CopyTo for "test command class CThreadTest: public CProgressThreadVirt { @@ -979,9 +988,6 @@ public: CMyComPtr<IArchiveFolder> ArchiveFolder; }; -// actually now we don't need CThreadTest, since now we call CopyTo for "test command - -/* HRESULT CThreadTest::ProcessVirt() { RINOK(ArchiveFolder->Extract(&Indices[0], Indices.Size(), @@ -1005,9 +1011,7 @@ HRESULT CThreadTest::ProcessVirt() } return S_OK; } -*/ -/* static void AddSizePair(UInt32 langID, UInt64 value, UString &s) { char sz[32]; diff --git a/CPP/7zip/UI/FileManager/Panel.h b/CPP/7zip/UI/FileManager/Panel.h index 03882344..f39d94e2 100644 --- a/CPP/7zip/UI/FileManager/Panel.h +++ b/CPP/7zip/UI/FileManager/Panel.h @@ -48,6 +48,7 @@ const int kParentFolderID = 100; const int kParentIndex = -1; +const UInt32 kParentIndex_UInt32 = (UInt32)(Int32)kParentIndex; #if !defined(_WIN32) || defined(UNDER_CE) #define ROOT_FS_FOLDER L"\\" @@ -225,7 +226,7 @@ struct CSelectedState UString FocusedName; UStringVector SelectedNames; - CSelectedState(): FocusedItem(-1), FocusedName_Defined(false), SelectFocused(true) {} + CSelectedState(): FocusedItem(-1), SelectFocused(true), FocusedName_Defined(false) {} }; #ifdef UNDER_CE @@ -263,6 +264,23 @@ struct CCopyToOptions }; + +struct COpenResult +{ + // bool needOpenArc; + // out: + bool ArchiveIsOpened; + bool Encrypted; + UString ErrorMessage; + + COpenResult(): + // needOpenArc(false), + ArchiveIsOpened(false), Encrypted(false) {} +}; + + + + class CPanel: public NWindows::NControl::CWindow2 { CExtToIconMap _extToIconMap; @@ -403,7 +421,7 @@ public: } } - HWND GetParent(); + HWND GetParent() const; UInt32 GetRealIndex(const LVITEMW &item) const { @@ -483,7 +501,7 @@ public: // PanelFolderChange.cpp void SetToRootFolder(); - HRESULT BindToPath(const UString &fullPath, const UString &arcFormat, bool &archiveIsOpened, bool &encrypted); // can be prefix + HRESULT BindToPath(const UString &fullPath, const UString &arcFormat, COpenResult &openRes); // can be prefix HRESULT BindToPathAndRefresh(const UString &path); void OpenDrivesFolder(); @@ -507,7 +525,8 @@ public: CPanelCallback *panelCallback, CAppState *appState, bool needOpenArc, - bool &archiveIsOpened, bool &encrypted); + COpenResult &openRes); + void SetFocusToList(); void SetFocusToLastRememberedItem(); @@ -515,32 +534,35 @@ public: void SaveListViewInfo(); CPanel() : - // _virtualMode(flase), + _thereAre_ListView_Items(false), _exStyle(0), _showDots(false), _showRealFileIcons(false), - _needSaveInfo(false), - _startGroupSelect(0), - _selectionIsDefined(false), + // _virtualMode(flase), + _enableItemChangeNotify(true), + _mySelectMode(false), + _timestampLevel(kTimestampPrintLevel_MIN), + + _thereAreDeletedItems(false), + _markDeletedItems(true), + PanelCreated(false), + _ListViewMode(3), + _xSize(300), + _flatMode(false), _flatModeForDisk(false), _flatModeForArc(false), - PanelCreated(false), - _thereAre_ListView_Items(false), // _showNtfsStrems_Mode(false), // _showNtfsStrems_ModeForDisk(false), // _showNtfsStrems_ModeForArc(false), - _xSize(300), - _mySelectMode(false), - _thereAreDeletedItems(false), - _markDeletedItems(true), - _enableItemChangeNotify(true), _dontShowMode(false), - _timestampLevel(kTimestampPrintLevel_MIN) + _needSaveInfo(false), + _startGroupSelect(0), + _selectionIsDefined(false) {} void SetExtendedStyle() @@ -590,8 +612,8 @@ public: CMyComPtr<IContextMenu> &systemContextMenu, bool programMenu); void CreateFileMenu(HMENU menu); - bool InvokePluginCommand(int id); - bool InvokePluginCommand(int id, IContextMenu *sevenZipContextMenu, + bool InvokePluginCommand(unsigned id); + bool InvokePluginCommand(unsigned id, IContextMenu *sevenZipContextMenu, IContextMenu *systemContextMenu); void InvokeSystemCommand(const char *command); @@ -737,9 +759,6 @@ public: } }; - // bool _passwordIsDefined; - // UString _password; - void InvalidateList() { _listView.InvalidateRect(NULL, true); } HRESULT RefreshListCtrl(); @@ -773,17 +792,21 @@ public: const CTempFileInfo &tempFileInfo, const UString &virtualFilePath, const UString &arcFormat, - bool &encrypted); + COpenResult &openRes); HRESULT OpenAsArc_Msg(IInStream *inStream, const CTempFileInfo &tempFileInfo, const UString &virtualFilePath, - const UString &arcFormat, - bool &encrypted, - bool showErrorMessage); + const UString &arcFormat + // , bool showErrorMessage + ); - HRESULT OpenAsArc_Name(const UString &relPath, const UString &arcFormat, bool &encrypted, bool showErrorMessage); - HRESULT OpenAsArc_Index(int index, const wchar_t *type /* = NULL */, bool showErrorMessage); + HRESULT OpenAsArc_Name(const UString &relPath, const UString &arcFormat + // , bool showErrorMessage + ); + HRESULT OpenAsArc_Index(int index, const wchar_t *type /* = NULL */ + // , bool showErrorMessage + ); void OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bool editMode, bool useEditor, const wchar_t *type = NULL); diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index c3416cc7..e1bd0117 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -31,14 +31,14 @@ public: CHashBundle Hash; // UString FirstFilePath; - HRESULT Result; + // HRESULT Result2; void ShowFinalResults(HWND hwnd); CPanelCopyThread(): - Result(E_FAIL), ResultsWereShown(false), NeedShowRes(false) + // , Result2(E_FAIL) {} }; @@ -68,6 +68,8 @@ HRESULT CPanelCopyThread::ProcessVirt() } */ + HRESULT result2; + if (options->testMode) { CMyComPtr<IArchiveFolder> archiveFolder; @@ -79,21 +81,21 @@ HRESULT CPanelCopyThread::ProcessVirt() NExtract::NPathMode::EEnum pathMode = NExtract::NPathMode::kCurPaths; // NExtract::NPathMode::kFullPathnames; - Result = archiveFolder->Extract(&Indices.Front(), Indices.Size(), + result2 = archiveFolder->Extract(&Indices.Front(), Indices.Size(), BoolToInt(options->includeAltStreams), BoolToInt(options->replaceAltStreamChars), pathMode, NExtract::NOverwriteMode::kAsk, options->folder, BoolToInt(true), extractCallback2); } else - Result = FolderOperations->CopyTo( + result2 = FolderOperations->CopyTo( BoolToInt(options->moveMode), &Indices.Front(), Indices.Size(), BoolToInt(options->includeAltStreams), BoolToInt(options->replaceAltStreamChars), options->folder, ExtractCallback); - if (Result == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors) + if (result2 == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors) { if (!options->hashMethods.IsEmpty()) NeedShowRes = true; @@ -104,7 +106,7 @@ HRESULT CPanelCopyThread::ProcessVirt() } } - return Result; + return result2; } @@ -232,6 +234,8 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector<UInt32> &ind if (messages) *messages = extracter.Sync.Messages; + + // res = extracter.Result2; res = extracter.Result; if (res == S_OK && extracter.ExtractCallbackSpec->IsOK()) @@ -327,9 +331,11 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri for (i = 0; i < updater.FileNames.Size(); i++) updater.FileNamePointers.AddInReserved(updater.FileNames[i]); - NWindows::CThread thread; - RINOK(thread.Create(CThreadUpdate::MyThreadFunction, &updater)); - updater.ProgressDialog.Create(title, thread, GetParent()); + { + NWindows::CThread thread; + RINOK(thread.Create(CThreadUpdate::MyThreadFunction, &updater)); + updater.ProgressDialog.Create(title, thread, GetParent()); + } if (messages) *messages = updater.ProgressDialog.Sync.Messages; diff --git a/CPP/7zip/UI/FileManager/PanelCrc.cpp b/CPP/7zip/UI/FileManager/PanelCrc.cpp index 1cfbfe5d..6c49e897 100644 --- a/CPP/7zip/UI/FileManager/PanelCrc.cpp +++ b/CPP/7zip/UI/FileManager/PanelCrc.cpp @@ -383,7 +383,7 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) { int pos = basePrefix2.ReverseFind_PathSepar(); if (pos >= 0) - basePrefix2.DeleteFrom(pos + 1); + basePrefix2.DeleteFrom((unsigned)(pos + 1)); } t.Enumerator.BasePrefix = us2fs(basePrefix); diff --git a/CPP/7zip/UI/FileManager/PanelDrag.cpp b/CPP/7zip/UI/FileManager/PanelDrag.cpp index 6a781e22..af8799ca 100644 --- a/CPP/7zip/UI/FileManager/PanelDrag.cpp +++ b/CPP/7zip/UI/FileManager/PanelDrag.cpp @@ -190,7 +190,7 @@ public: HRESULT Result; UStringVector Messages; - CDropSource(): NeedPostCopy(false), Panel(0), Result(S_OK), m_Effect(DROPEFFECT_NONE) {} + CDropSource(): m_Effect(DROPEFFECT_NONE), Panel(NULL), NeedPostCopy(false), Result(S_OK) {} }; STDMETHODIMP CDropSource::QueryContinueDrag(BOOL escapePressed, DWORD keyState) @@ -300,7 +300,7 @@ static bool CopyNamesToHGlobal(NMemory::CGlobal &hgDrop, const UStringVector &na dropFiles->pt.y = 0; dropFiles->pFiles = sizeof(DROPFILES); dropFiles->fWide = TRUE; - WCHAR *p = (WCHAR *)((BYTE *)dropFiles + sizeof(DROPFILES)); + WCHAR *p = (WCHAR *) (void *) ((BYTE *)dropFiles + sizeof(DROPFILES)); for (i = 0; i < names.Size(); i++) { const UString &s = names[i]; diff --git a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp index 38b83124..b91195f4 100644 --- a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp +++ b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp @@ -73,15 +73,15 @@ static bool DoesNameContainWildcard_SkipRoot(const UString &path) return DoesNameContainWildcard(path.Ptr(NName::GetRootPrefixSize(path))); } -HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bool &archiveIsOpened, bool &encrypted) +HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, COpenResult &openRes) { UString path = fullPath; #ifdef _WIN32 path.Replace(L'/', WCHAR_PATH_SEPARATOR); #endif - archiveIsOpened = false; - encrypted = false; + openRes.ArchiveIsOpened = false; + openRes.Encrypted = false; CDisableTimerProcessing disableTimerProcessing(*this); CDisableNotify disableNotify(*this); @@ -169,7 +169,7 @@ HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bo pos++; #endif - sysPath.DeleteFrom(pos); + sysPath.DeleteFrom((unsigned)pos); } } @@ -218,7 +218,7 @@ HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bo tfi.RelPath = fs2us(fileName); tfi.FolderPath = dirPrefix; tfi.FilePath = us2fs(sysPath); - res = OpenAsArc(NULL, tfi, sysPath, arcFormat, encrypted); + res = OpenAsArc(NULL, tfi, sysPath, arcFormat, openRes); } if (res == S_FALSE) @@ -226,7 +226,7 @@ HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bo else { RINOK(res); - archiveIsOpened = true; + openRes.ArchiveIsOpened = true; _parentFolders.Back().ParentFolderPath = fs2us(dirPrefix); path.DeleteFrontal(sysPath.Len()); if (!path.IsEmpty() && IS_PATH_SEPAR(path[0])) @@ -252,7 +252,7 @@ HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bo unsigned skipLen = s.Len(); if (slashPos >= 0) { - s.DeleteFrom(slashPos); + s.DeleteFrom((unsigned)slashPos); skipLen = slashPos + 1; } @@ -266,7 +266,7 @@ HRESULT CPanel::BindToPath(const UString &fullPath, const UString &arcFormat, bo if (pos >= 0) { UString baseName = s; - baseName.DeleteFrom(pos); + baseName.DeleteFrom((unsigned)pos); if (_folderAltStreams->BindToAltStreams(baseName, &newFolder) == S_OK && newFolder) curPos += pos + 1; } @@ -286,7 +286,7 @@ HRESULT CPanel::BindToPathAndRefresh(const UString &path) { CDisableTimerProcessing disableTimerProcessing(*this); CDisableNotify disableNotify(*this); - bool archiveIsOpened, encrypted; + COpenResult openRes; UString s = path; #ifdef _WIN32 @@ -297,7 +297,7 @@ HRESULT CPanel::BindToPathAndRefresh(const UString &path) } #endif - HRESULT res = BindToPath(s, UString(), archiveIsOpened, encrypted); + HRESULT res = BindToPath(s, UString(), openRes); RefreshListCtrl(); return res; } @@ -484,7 +484,7 @@ void CPanel::AddComboBoxItem(const UString &name, int iconIndex, int indent, boo item.mask |= (CBEIF_IMAGE | CBEIF_SELECTEDIMAGE); item.iItem = -1; item.iIndent = indent; - item.pszText = (LPWSTR)(LPCWSTR)name; + item.pszText = name.Ptr_non_const(); _headerComboBox.InsertItem(&item); #endif @@ -559,7 +559,6 @@ bool CPanel::OnComboBoxCommand(UINT code, LPARAM /* param */, LRESULT &result) case CBN_SELENDOK: { - code = code; int index = _headerComboBox.GetCurSel(); if (index >= 0) { @@ -664,7 +663,7 @@ UString CPanel::GetParentDirPrefix() const { int pos = s.ReverseFind_PathSepar(); if (pos >= 0) - s.DeleteFrom(pos + 1); + s.DeleteFrom((unsigned)(pos + 1)); } } } @@ -700,7 +699,7 @@ void CPanel::OpenParentFolder() if (pos >= 0) { parentFolderPrefix = focusedName; - parentFolderPrefix.DeleteFrom(pos + 1); + parentFolderPrefix.DeleteFrom((unsigned)(pos + 1)); focusedName.DeleteFrontal(pos + 1); } } @@ -734,9 +733,8 @@ void CPanel::OpenParentFolder() if (needSetFolder) { { - bool archiveIsOpened; - bool encrypted; - BindToPath(parentFolderPrefix, UString(), archiveIsOpened, encrypted); + COpenResult openRes; + BindToPath(parentFolderPrefix, UString(), openRes); } } } @@ -822,7 +820,12 @@ void CPanel::OpenFolder(int index) return; } CMyComPtr<IFolderFolder> newFolder; - _folder->BindToFolder(index, &newFolder); + HRESULT res = _folder->BindToFolder(index, &newFolder); + if (res != 0) + { + MessageBox_Error_HRESULT(res); + return; + } if (!newFolder) return; SetNewFolder(newFolder); diff --git a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index 89f49bf6..68fe7b02 100644 --- a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -4,7 +4,7 @@ #include "../../../Common/MyWindows.h" -#include <tlhelp32.h> +#include <TlHelp32.h> #include "../../../Common/IntToString.h" @@ -222,14 +222,14 @@ static void My_GetProcessFileName(HANDLE hProcess, UString &path) const char *func_name = "GetProcessImageFileNameW"; Func_GetProcessImageFileNameW my_func = (Func_GetProcessImageFileNameW) - ::GetProcAddress(::GetModuleHandleA("kernel32.dll"), func_name); + (void *)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), func_name); if (!my_func) { if (!g_Psapi_dll_module) g_Psapi_dll_module = LoadLibraryW(L"Psapi.dll"); if (g_Psapi_dll_module) - my_func = (Func_GetProcessImageFileNameW)::GetProcAddress(g_Psapi_dll_module, func_name); + my_func = (Func_GetProcessImageFileNameW)(void *)::GetProcAddress(g_Psapi_dll_module, func_name); } if (my_func) @@ -319,7 +319,7 @@ public: { #ifndef UNDER_CE - Func_GetProcessId func = (Func_GetProcessId)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "GetProcessId"); + Func_GetProcessId func = (Func_GetProcessId)(void *)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "GetProcessId"); if (func) { DWORD id = func(h); @@ -349,7 +349,7 @@ public: GetSnapshot(sps); const int separ = Path.ReverseFind_PathSepar(); - const UString mainName = Path.Ptr(separ + 1); + const UString mainName = Path.Ptr((unsigned)(separ + 1)); if (mainName.IsEmpty()) needFindProcessByPath = false; @@ -444,14 +444,16 @@ public: } }; +void GetFolderError(CMyComPtr<IFolderFolder> &folder, UString &s); + HRESULT CPanel::OpenAsArc(IInStream *inStream, const CTempFileInfo &tempFileInfo, const UString &virtualFilePath, const UString &arcFormat, - bool &encrypted) + COpenResult &openRes) { - encrypted = false; + openRes.Encrypted = false; CFolderLink folderLink; (CTempFileInfo &)folderLink = tempFileInfo; @@ -468,21 +470,18 @@ HRESULT CPanel::OpenAsArc(IInStream *inStream, folderLink.VirtualPath = virtualFilePath; - CMyComPtr<IFolderFolder> newFolder; - - // _passwordIsDefined = false; - // _password.Empty(); + CFfpOpen ffp; + HRESULT res = ffp.OpenFileFolderPlugin(inStream, + folderLink.FilePath.IsEmpty() ? us2fs(virtualFilePath) : folderLink.FilePath, + arcFormat, GetParent()); - NDLL::CLibrary library; + openRes.Encrypted = ffp.Encrypted; + openRes.ErrorMessage = ffp.ErrorMessage; - UString password; - RINOK(OpenFileFolderPlugin(inStream, - folderLink.FilePath.IsEmpty() ? us2fs(virtualFilePath) : folderLink.FilePath, - arcFormat, - &library, &newFolder, GetParent(), encrypted, password)); + RINOK(res); - folderLink.Password = password; - folderLink.UsePassword = encrypted; + folderLink.Password = ffp.Password; + folderLink.UsePassword = ffp.Encrypted; if (_folder) folderLink.ParentFolderPath = GetFolderPath(_folder); @@ -497,75 +496,24 @@ HRESULT CPanel::OpenAsArc(IInStream *inStream, ReleaseFolder(); _library.Free(); - SetNewFolder(newFolder); - _library.Attach(library.Detach()); + SetNewFolder(ffp.Folder); + _library.Attach(ffp.Library.Detach()); _flatMode = _flatModeForArc; - CMyComPtr<IGetFolderArcProps> getFolderArcProps; - _folder.QueryInterface(IID_IGetFolderArcProps, &getFolderArcProps); _thereAreDeletedItems = false; - if (getFolderArcProps) - { - CMyComPtr<IFolderArcProps> arcProps; - getFolderArcProps->GetFolderArcProps(&arcProps); - if (arcProps) - { - /* - UString s; - UInt32 numLevels; - if (arcProps->GetArcNumLevels(&numLevels) != S_OK) - numLevels = 0; - for (UInt32 level2 = 0; level2 <= numLevels; level2++) - { - UInt32 level = numLevels - level2; - PROPID propIDs[] = { kpidError, kpidPath, kpidType, kpidErrorType } ; - UString values[4]; - for (Int32 i = 0; i < 4; i++) - { - CMyComBSTR name; - NCOM::CPropVariant prop; - if (arcProps->GetArcProp(level, propIDs[i], &prop) != S_OK) - continue; - if (prop.vt != VT_EMPTY) - values[i] = (prop.vt == VT_BSTR) ? prop.bstrVal : L"?"; - } - UString s2; - if (!values[3].IsEmpty()) - { - s2 = "Can not open the file as [" + values[3] + "] archive"; - if (level2 != 0) - s2 += "\nThe file is open as [" + values[2] + "] archive"; - } - if (!values[0].IsEmpty()) - { - if (!s2.IsEmpty()) - s2.Add_LF(); - s2 += "["; - s2 += values[2]; - s2 += "] Error: "; - s2 += values[0]; - } - if (!s2.IsEmpty()) - { - if (!s.IsEmpty()) - s += "--------------------\n"; - s += values[1]; - s.Add_LF(); - s += s2; - } - } - */ - /* - if (!s.IsEmpty()) - MessageBox_Warning(s); - else - */ - // after MessageBox_Warning it throws exception in nested archives in Debug Mode. why ?. - // MessageBox_Warning(L"test error"); - } - } + if (!openRes.ErrorMessage.IsEmpty()) + MessageBox_Error(openRes.ErrorMessage); + /* + UString s; + GetFolderError(_folder, s); + if (!s.IsEmpty()) + MessageBox_Error(s); + */ + // we don't show error here by some reasons: + // after MessageBox_Warning it throws exception in nested archives in Debug Mode. why ?. + // MessageBox_Warning(L"test error"); return S_OK; } @@ -574,25 +522,28 @@ HRESULT CPanel::OpenAsArc(IInStream *inStream, HRESULT CPanel::OpenAsArc_Msg(IInStream *inStream, const CTempFileInfo &tempFileInfo, const UString &virtualFilePath, - const UString &arcFormat, - bool &encrypted, - bool showErrorMessage) + const UString &arcFormat + // , bool &encrypted + // , bool showErrorMessage + ) { - HRESULT res = OpenAsArc(inStream, tempFileInfo, virtualFilePath, arcFormat, encrypted); + COpenResult opRes; + + HRESULT res = OpenAsArc(inStream, tempFileInfo, virtualFilePath, arcFormat, opRes); if (res == S_OK) return res; if (res == E_ABORT) return res; - if (showErrorMessage) - if (encrypted || res != S_FALSE) // 17.01 : we show message also for (res != S_FALSE) + // if (showErrorMessage) + if (opRes.Encrypted || res != S_FALSE) // 17.01 : we show message also for (res != S_FALSE) { UString message; if (res == S_FALSE) { message = MyFormatNew( - encrypted ? + opRes.Encrypted ? IDS_CANT_OPEN_ENCRYPTED_ARCHIVE : IDS_CANT_OPEN_ARCHIVE, virtualFilePath); @@ -606,23 +557,28 @@ HRESULT CPanel::OpenAsArc_Msg(IInStream *inStream, } -HRESULT CPanel::OpenAsArc_Name(const UString &relPath, const UString &arcFormat, bool &encrypted, bool showErrorMessage) +HRESULT CPanel::OpenAsArc_Name(const UString &relPath, const UString &arcFormat + // , bool &encrypted, + // , bool showErrorMessage + ) { CTempFileInfo tfi; tfi.RelPath = relPath; tfi.FolderPath = us2fs(GetFsPath()); const UString fullPath = GetFsPath() + relPath; tfi.FilePath = us2fs(fullPath); - return OpenAsArc_Msg(NULL, tfi, fullPath, arcFormat, encrypted, showErrorMessage); + return OpenAsArc_Msg(NULL, tfi, fullPath, arcFormat /* , encrypted, showErrorMessage */); } -HRESULT CPanel::OpenAsArc_Index(int index, const wchar_t *type, bool showErrorMessage) +HRESULT CPanel::OpenAsArc_Index(int index, const wchar_t *type + // , bool showErrorMessage + ) { CDisableTimerProcessing disableTimerProcessing1(*this); CDisableNotify disableNotify(*this); - bool encrypted; - HRESULT res = OpenAsArc_Name(GetItemRelPath2(index), type ? type : L"", encrypted, showErrorMessage); + + HRESULT res = OpenAsArc_Name(GetItemRelPath2(index), type ? type : L"" /* , encrypted, showErrorMessage */); if (res != S_OK) { RefreshTitle(true); // in case of error we must refresh changed title of 7zFM @@ -677,16 +633,24 @@ static const char * const kStartExtensions = " msi doc dot xls ppt pps wps wpt wks xlr wdb vsd pub" " docx docm dotx dotm xlsx xlsm xltx xltm xlsb xps" - " xlam pptx pptm potx potm ppam ppsx ppsm xsn" + " xlam pptx pptm potx potm ppam ppsx ppsm vsdx xsn" " mpp" " msg" " dwf" " flv swf" + " epub" " odt ods" " wb3" " pdf" + " ps" + " txt" + " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml" + " h hpp hxx c cpp cxx m mm go swift" + " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs" + " asm" + " mak clw csproj vcproj sln dsp dsw" " "; static bool FindExt(const char *p, const UString &name) @@ -724,6 +688,7 @@ static bool DoItemAlwaysStart(const UString &name) UString GetQuotedString(const UString &s); +void SplitCmdLineSmart(const UString &cmd, UString &prg, UString ¶ms); void SplitCmdLineSmart(const UString &cmd, UString &prg, UString ¶ms) { params.Empty(); @@ -734,11 +699,11 @@ void SplitCmdLineSmart(const UString &cmd, UString &prg, UString ¶ms) int pos = prg.Find(L'"', 1); if (pos >= 0) { - if ((unsigned)pos + 1 == prg.Len() || prg[pos + 1] == ' ') + if ((unsigned)(pos + 1) == prg.Len() || prg[pos + 1] == ' ') { - params = prg.Ptr(pos + 1); + params = prg.Ptr((unsigned)(pos + 1)); params.Trim(); - prg.DeleteFrom(pos); + prg.DeleteFrom((unsigned)pos); prg.DeleteFrontal(1); } } @@ -841,6 +806,11 @@ void CApp::DiffFiles() else return; + DiffFiles(path1, path2); +} + +void CApp::DiffFiles(const UString &path1, const UString &path2) +{ UString command; ReadRegDiff(command); if (command.IsEmpty()) @@ -894,7 +864,7 @@ static HRESULT StartApplication(const UString &dir, const UString &path, HWND wi execInfo.nShow = SW_SHOWNORMAL; execInfo.hProcess = 0; ShellExecuteExWP shellExecuteExW = (ShellExecuteExWP) - ::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "ShellExecuteExW"); + (void *)::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "ShellExecuteExW"); if (!shellExecuteExW) return 0; shellExecuteExW(&execInfo); @@ -985,7 +955,7 @@ void CPanel::OpenFolderExternal(int index) int pos = prefix.ReverseFind_PathSepar(); if (pos < 0) return; - prefix.DeleteFrom(pos + 1); + prefix.DeleteFrom((unsigned)(pos + 1)); path = prefix; } else @@ -1102,7 +1072,9 @@ void CPanel::OpenItem(int index, bool tryInternal, bool tryExternal, const wchar if (tryInternal) if (!tryExternal || !DoItemAlwaysStart(name)) { - HRESULT res = OpenAsArc_Index(index, type, true); + HRESULT res = OpenAsArc_Index(index, type + // , true + ); disableNotify.Restore(); // we must restore to allow text notification update InvalidateList(); if (res == S_OK || res == E_ABORT) @@ -1299,7 +1271,7 @@ static THREAD_FUNC_DECL MyThreadFunction(void *param) { handles.Add(g_ExitEventLauncher._exitEvent); - DWORD waitResult = ::WaitForMultipleObjects(handles.Size(), &handles.Front(), FALSE, INFINITE); + DWORD waitResult = WaitForMultiObj_Any_Infinite(handles.Size(), &handles.Front()); waitResult -= WAIT_OBJECT_0; @@ -1451,8 +1423,8 @@ static void ReadZoneFile(CFSTR fileName, CByteBuffer &buf) if (fileSize == 0 || fileSize >= ((UInt32)1 << 20)) return; buf.Alloc((size_t)fileSize); - UInt32 processed; - if (file.Read(buf, (UInt32)fileSize, processed) && processed == fileSize) + size_t processed; + if (file.ReadFull(buf, (size_t)fileSize, processed) && processed == fileSize) return; buf.Free(); } @@ -1644,8 +1616,9 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo subSeqStream.QueryInterface(IID_IInStream, &subStream); if (subStream) { - bool encrypted; - HRESULT res = OpenAsArc_Msg(subStream, tempFileInfo, fullVirtPath, type ? type : L"", encrypted, true); + HRESULT res = OpenAsArc_Msg(subStream, tempFileInfo, fullVirtPath, type ? type : L"" + // , true // showErrorMessage + ); if (res == S_OK) { tempDirectory.DisableDeleting(); @@ -1757,9 +1730,12 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo CBufInStream *bufInStreamSpec = new CBufInStream; CMyComPtr<IInStream> bufInStream = bufInStreamSpec; bufInStreamSpec->Init(file.Data, streamSize, virtFileSystem); - bool encrypted; - HRESULT res = OpenAsArc_Msg(bufInStream, tempFileInfo, fullVirtPath, type ? type : L"", encrypted, true); + HRESULT res = OpenAsArc_Msg(bufInStream, tempFileInfo, fullVirtPath, type ? type : L"" + // , encrypted + // , true // showErrorMessage + ); + if (res == S_OK) { tempDirectory.DisableDeleting(); @@ -1782,7 +1758,7 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo #if defined(_WIN32) && !defined(UNDER_CE) if (zoneBuf.Size() != 0) { - if (NFind::DoesFileExist(tempFilePath)) + if (NFind::DoesFileExist_Raw(tempFilePath)) { WriteZoneFile(tempFilePath + k_ZoneId_StreamName, zoneBuf); } @@ -1792,8 +1768,10 @@ void CPanel::OpenItemInArchive(int index, bool tryInternal, bool tryExternal, bo if (tryAsArchive) { - bool encrypted; - HRESULT res = OpenAsArc_Msg(NULL, tempFileInfo, fullVirtPath, type ? type : L"", encrypted, true); + HRESULT res = OpenAsArc_Msg(NULL, tempFileInfo, fullVirtPath, type ? type : L"" + // , encrypted + // , true // showErrorMessage + ); if (res == S_OK) { tempDirectory.DisableDeleting(); diff --git a/CPP/7zip/UI/FileManager/PanelItems.cpp b/CPP/7zip/UI/FileManager/PanelItems.cpp index 178df7c7..33e44588 100644 --- a/CPP/7zip/UI/FileManager/PanelItems.cpp +++ b/CPP/7zip/UI/FileManager/PanelItems.cpp @@ -87,7 +87,7 @@ static int GetColumnAlign(PROPID propID, VARTYPE varType) static int ItemProperty_Compare_NameFirst(void *const *a1, void *const *a2, void * /* param */) { - return (*(*((const CPropColumn **)a1))).Compare_NameFirst(*(*((const CPropColumn **)a2))); + return (*(*((const CPropColumn *const *)a1))).Compare_NameFirst(*(*((const CPropColumn *const *)a2))); } HRESULT CPanel::InitColumns() @@ -154,6 +154,21 @@ HRESULT CPanel::InitColumns() prop.IsRawProp = false; _columns.Add(prop); } + + /* + { + // debug column + CPropColumn prop; + prop.Type = VT_BSTR; + prop.ID = 2000; + prop.Name = "Debug"; + prop.Order = -1; + prop.IsVisible = true; + prop.Width = 300; + prop.IsRawProp = false; + _columns.Add(prop); + } + */ } if (_folderRawProps) @@ -174,7 +189,7 @@ HRESULT CPanel::InitColumns() prop.Name = GetNameOfProperty(propID, name); prop.Order = -1; prop.IsVisible = GetColumnVisible(propID, isFsFolder); - prop.Width = GetColumnWidth(propID, VT_BSTR);; + prop.Width = GetColumnWidth(propID, VT_BSTR); prop.IsRawProp = true; _columns.Add(prop); } @@ -250,7 +265,7 @@ HRESULT CPanel::InitColumns() /* There are restrictions in ListView control: 1) main column (kpidName) must have (LV_COLUMNW::iSubItem = 0) So we need special sorting for columns. - 2) when we add new column, LV_COLUMNW::iOrder can not be larger than already inserted columns) + 2) when we add new column, LV_COLUMNW::iOrder cannot be larger than already inserted columns) So we set column order after all columns are added. */ newColumns.Sort(ItemProperty_Compare_NameFirst, NULL); @@ -435,11 +450,27 @@ void CPanel::SetFocusedSelectedItem(int index, bool select) #endif + +/* + +extern UInt32 g_NumGroups; +extern DWORD g_start_tick; +extern DWORD g_prev_tick; +extern DWORD g_Num_SetItemText; +extern UInt32 g_NumMessages; +*/ + HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) { if (!_folder) return S_OK; + /* + g_start_tick = GetTickCount(); + g_Num_SetItemText = 0; + g_NumMessages = 0; + */ + _dontShowMode = false; LoadFullPathAndShow(); // OutputDebugStringA("=======\n"); @@ -545,6 +576,8 @@ HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) Print_OnNotify("===== Before Load"); + // #define USE_EMBED_ITEM + if (showDots) { UString itemName (".."); @@ -555,8 +588,11 @@ HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) int subItem = 0; item.iSubItem = subItem++; item.lParam = kParentIndex; - // item.pszText = const_cast<wchar_t *>((const wchar_t *)itemName); + #ifdef USE_EMBED_ITEM + item.pszText = const_cast<wchar_t *>((const wchar_t *)itemName); + #else item.pszText = LPSTR_TEXTCALLBACKW; + #endif UInt32 attrib = FILE_ATTRIBUTE_DIRECTORY; item.iImage = _extToIconMap.GetIconIndex(attrib, itemName); if (item.iImage < 0) @@ -591,10 +627,8 @@ HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) if (state.FocusedName_Defined || !state.SelectedNames.IsEmpty()) { relPath.Empty(); - // relPath += GetItemPrefix(i); - // change it (_flatMode) - if (i != kParentIndex && _flatMode) + if (_flatMode) { const wchar_t *prefix = NULL; if (_folderGetItemName) @@ -672,8 +706,11 @@ HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) else */ { - // item.pszText = const_cast<wchar_t *>((const wchar_t *)name); + #ifdef USE_EMBED_ITEM + item.pszText = const_cast<wchar_t *>((const wchar_t *)name); + #else item.pszText = LPSTR_TEXTCALLBACKW; + #endif /* LPSTR_TEXTCALLBACKW works, but in some cases there are problems, since we block notify handler. LPSTR_TEXTCALLBACKW can be 2-3 times faster for loading in this loop. */ diff --git a/CPP/7zip/UI/FileManager/PanelKey.cpp b/CPP/7zip/UI/FileManager/PanelKey.cpp index 5603251f..3ab478eb 100644 --- a/CPP/7zip/UI/FileManager/PanelKey.cpp +++ b/CPP/7zip/UI/FileManager/PanelKey.cpp @@ -29,7 +29,7 @@ static const CVKeyPropIDPair g_VKeyPropIDPairs[] = static int FindVKeyPropIDPair(WORD vKey) { - for (int i = 0; i < ARRAY_SIZE(g_VKeyPropIDPairs); i++) + for (unsigned i = 0; i < ARRAY_SIZE(g_VKeyPropIDPairs); i++) if (g_VKeyPropIDPairs[i].VKey == vKey) return i; return -1; diff --git a/CPP/7zip/UI/FileManager/PanelListNotify.cpp b/CPP/7zip/UI/FileManager/PanelListNotify.cpp index 9061e585..12f471b3 100644 --- a/CPP/7zip/UI/FileManager/PanelListNotify.cpp +++ b/CPP/7zip/UI/FileManager/PanelListNotify.cpp @@ -84,6 +84,7 @@ static void ConvertSizeToString(UInt64 val, wchar_t *s) throw() *s = 0; } +UString ConvertSizeToString(UInt64 value); UString ConvertSizeToString(UInt64 value) { wchar_t s[32]; @@ -110,6 +111,7 @@ static void HexToString(char *dest, const Byte *data, UInt32 size) } */ +bool IsSizeProp(UINT propID) throw(); bool IsSizeProp(UINT propID) throw() { switch (propID) @@ -141,12 +143,39 @@ bool IsSizeProp(UINT propID) throw() return false; } + + +/* +#include <stdio.h> + +UInt64 GetCpuTicks() +{ + #ifdef _WIN64 + return __rdtsc(); + #else + UInt32 lowVal, highVal; + __asm RDTSC; + __asm mov lowVal, EAX; + __asm mov highVal, EDX; + return ((UInt64)highVal << 32) | lowVal; + #endif +} + +UInt32 g_NumGroups; +UInt64 g_start_tick; +UInt64 g_prev_tick; +DWORD g_Num_SetItemText; +UInt32 g_NumMessages; +*/ + LRESULT CPanel::SetItemText(LVITEMW &item) { if (_dontShowMode) return 0; UInt32 realIndex = GetRealIndex(item); + // g_Num_SetItemText++; + /* if ((item.mask & LVIF_IMAGE) != 0) { @@ -190,7 +219,7 @@ LRESULT CPanel::SetItemText(LVITEMW &item) const CPropColumn &property = _visibleColumns[item.iSubItem]; PROPID propID = property.ID; - if (realIndex == kParentIndex) + if (realIndex == kParentIndex_UInt32) { if (propID == kpidName) { @@ -204,7 +233,52 @@ LRESULT CPanel::SetItemText(LVITEMW &item) return 0; } - + /* + // List-view in report-view in Windows 10 is slow (50+ ms) for page change. + // that code shows the time of page reload for items + // if you know how to improve the speed of list view refresh, notify 7-Zip developer + + // if (propID == 2000) + // if (propID == kpidName) + { + // debug column; + // DWORD dw = GetCpuTicks(); + UInt64 dw = GetCpuTicks(); + UInt64 deltaLast = dw - g_prev_tick; + #define conv_ticks(t) ((unsigned)((t) / 100000)) + if (deltaLast > 1000u * 1000 * 1000) + { + UInt64 deltaFull = g_prev_tick - g_start_tick; + char s[128]; + sprintf(s, "%d", conv_ticks(deltaFull)); + OutputDebugStringA(s); + g_start_tick = dw; + g_NumGroups++; + } + g_prev_tick = dw; + UString u; + char s[128]; + UInt64 deltaFull = dw - g_start_tick; + // for (int i = 0; i < 100000; i++) + sprintf(s, "%d %d %d-%d ", g_NumMessages, g_Num_SetItemText, g_NumGroups, conv_ticks(deltaFull)); + // sprintf(s, "%d-%d ", g_NumGroups, conv_ticks(deltaFull)); + u = s; + lstrcpyW(text, u.Ptr()); + text += u.Len(); + + // dw = GetCpuTicks(); + // deltaFull = dw - g_prev_tick; + // sprintf(s, "-%d ", conv_ticks(deltaFull)); + // u = s; + // lstrcpyW(text, u.Ptr()); + // text += u.Len(); + + if (propID != kpidName) + return 0; + } + */ + + if (property.IsRawProp) { const void *data; @@ -525,9 +599,15 @@ bool CPanel::OnNotifyList(LPNMHDR header, LRESULT &result) //is the sub-item information being requested? if ((dispInfo->item.mask & LVIF_TEXT) != 0 || - (dispInfo->item.mask & LVIF_IMAGE) != 0) + (dispInfo->item.mask & LVIF_IMAGE) != 0) SetItemText(dispInfo->item); - return false; + { + // 20.03: + result = 0; + return true; + // old 7-Zip: + // return false; + } } case LVN_KEYDOWN: { diff --git a/CPP/7zip/UI/FileManager/PanelMenu.cpp b/CPP/7zip/UI/FileManager/PanelMenu.cpp index 11c6d7c1..7a88f8f2 100644 --- a/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -25,6 +25,8 @@ using namespace NWindows; +extern +LONG g_DllRefCount; LONG g_DllRefCount = 0; static const UINT kSevenZipStartMenuID = kMenuCmdID_Plugin_Start; @@ -99,18 +101,25 @@ static void AddPropertyString(PROPID propID, const wchar_t *nameBSTR, if (flags != 0) val = GetOpenArcErrorMessage(flags); } + if (val.IsEmpty()) - if ((prop.vt == VT_UI8 || prop.vt == VT_UI4 || prop.vt == VT_UI2) && IsSizeProp(propID)) { - UInt64 v = 0; - ConvertPropVariantToUInt64(prop, v); - val = ConvertSizeToString(v); + if ((prop.vt == VT_UI8 || prop.vt == VT_UI4 || prop.vt == VT_UI2) && IsSizeProp(propID)) + { + UInt64 v = 0; + ConvertPropVariantToUInt64(prop, v); + val = ConvertSizeToString(v); + } + else + ConvertPropertyToString2(val, prop, propID); } - else - ConvertPropertyToString2(val, prop, propID); if (!val.IsEmpty()) { + if (propID == kpidErrorType) + { + AddPropertyPair(L"Open WARNING:", L"Cannot open the file as expected archive type", dialog); + } AddPropertyPair(GetNameOfProperty(propID, nameBSTR), val, dialog); } } @@ -247,7 +256,7 @@ void CPanel::Properties() { numDirs++; numDirs += GetItem_UInt64Prop(index, kpidNumSubDirs); - numFiles += GetItem_UInt64Prop(index, kpidNumSubFiles);; + numFiles += GetItem_UInt64Prop(index, kpidNumSubFiles); } else numFiles++; @@ -343,7 +352,7 @@ void CPanel::Properties() } } - if (level2 != numLevels - 1) + if (level2 < numLevels - 1) { UInt32 level = numLevels - 1 - level2; UInt32 numProps; @@ -365,6 +374,28 @@ void CPanel::Properties() } } } + + { + // we ERROR message for NonOpen level + bool needSep = true; + const int kNumSpecProps = ARRAY_SIZE(kSpecProps); + for (Int32 i = -(int)kNumSpecProps; i < 0; i++) + { + CMyComBSTR name; + PROPID propID = kSpecProps[i + kNumSpecProps]; + NCOM::CPropVariant prop; + if (getProps->GetArcProp(numLevels, propID, &prop) != S_OK) + continue; + if (needSep) + { + AddSeparator(message); + AddSeparator(message); + needSep = false; + } + AddPropertyString(propID, name, prop, message); + } + } + } } @@ -460,7 +491,7 @@ HRESULT CPanel::CreateShellContextMenu( // if (folderPath.IsEmpty()), then ParseDisplayName returns pidls of "My Computer" RINOK(desktopFolder->ParseDisplayName( - GetParent(), NULL, (wchar_t *)(const wchar_t *)folderPath, + GetParent(), NULL, folderPath.Ptr_non_const(), &eaten, &pidls.parent, NULL)); /* @@ -516,14 +547,14 @@ HRESULT CPanel::CreateShellContextMenu( LPITEMIDLIST pidl; const UString fileName = GetItemRelPath2(operatedIndices[i]); RINOK(parentFolder->ParseDisplayName(GetParent(), 0, - (wchar_t *)(const wchar_t *)fileName, &eaten, &pidl, 0)); + fileName.Ptr_non_const(), &eaten, &pidl, 0)); pidls.items.AddInReserved(pidl); } // Get IContextMenu for items RINOK(parentFolder->GetUIObjectOf(GetParent(), pidls.items.Size(), - (LPCITEMIDLIST *)&pidls.items.Front(), IID_IContextMenu, 0, (void**)&systemContextMenu)); + (LPCITEMIDLIST *)(void *)&pidls.items.Front(), IID_IContextMenu, 0, (void**)&systemContextMenu)); if (!systemContextMenu) { @@ -533,6 +564,99 @@ HRESULT CPanel::CreateShellContextMenu( return S_OK; } +// #define SHOW_DEBUG_FM_CTX_MENU + +#ifdef SHOW_DEBUG_FM_CTX_MENU + +#include <stdio.h> + +// #include Common/IntToString.h" + +static void PrintHex(UString &s, UInt32 v) +{ + char sz[32]; + ConvertUInt32ToHex(v, sz); + s += sz; +} + +static void PrintContextStr(UString &s, IContextMenu *ctxm, unsigned i, unsigned id, const char *name) +{ + s += " | "; + name = name; + // s += name; + // s += ": "; + + UString s1; + { + char buf[256]; + buf[0] = 0; + HRESULT res = ctxm->GetCommandString(i, id, + NULL, buf, ARRAY_SIZE(buf) - 1); + if (res != S_OK) + { + PrintHex(s1, res); + s1.Add_Space(); + } + s1 += GetUnicodeString(buf); + } + + UString s2; + { + wchar_t buf2[256]; + buf2[0] = 0; + HRESULT res = ctxm->GetCommandString(i, id | GCS_UNICODE, + NULL, (char *)buf2, ARRAY_SIZE(buf2) - 1); + if (res != S_OK) + { + PrintHex(s2, res); + s2.Add_Space(); + } + s2 += buf2; + } + + s += s1; + if (s2.Compare(s1) != 0) + { + s += " Unicode: "; + s += s2; + } +} + + +static void PrintAllContextItems(IContextMenu *ctxm, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + { + UString s; + s.Add_UInt32(i); + s += ": "; + + /* + UString valid; + { + char name[256]; + HRESULT res = ctxm->GetCommandString(i, GCS_VALIDATEA, + NULL, name, ARRAY_SIZE(name) - 1); + + if (res == S_OK) + { + // valid = "valid"; + } + else if (res == S_FALSE) + valid = "non-valid"; + else + PrintHex(valid, res); + } + s += valid; + */ + + PrintContextStr(s, ctxm, i, GCS_VALIDATEA, "valid"); + PrintContextStr(s, ctxm, i, GCS_VERBA, "v"); + PrintContextStr(s, ctxm, i, GCS_HELPTEXTA, "h"); + OutputDebugStringW(s); + } +} +#endif void CPanel::CreateSystemMenu(HMENU menuSpec, const CRecordVector<UInt32> &operatedIndices, @@ -542,14 +666,16 @@ void CPanel::CreateSystemMenu(HMENU menuSpec, CreateShellContextMenu(operatedIndices, systemContextMenu); - if (systemContextMenu == 0) + if (!systemContextMenu) return; + /* // Set up a CMINVOKECOMMANDINFO structure. CMINVOKECOMMANDINFO ci; ZeroMemory(&ci, sizeof(ci)); ci.cbSize = sizeof(CMINVOKECOMMANDINFO); ci.hwnd = GetParent(); + */ /* if (Sender == GoBtn) @@ -578,7 +704,7 @@ void CPanel::CreateSystemMenu(HMENU menuSpec, { // HMENU hMenu = CreatePopupMenu(); CMenu popupMenu; - // CMenuDestroyer menuDestroyer(popupMenu); + CMenuDestroyer menuDestroyer(popupMenu); if (!popupMenu.CreatePopup()) throw 210503; @@ -592,17 +718,21 @@ void CPanel::CreateSystemMenu(HMENU menuSpec, // commented out but you can uncommnent this // line to show the extended context menu. // Flags |= 0x00000080; - systemContextMenu->QueryContextMenu(hMenu, 0, kSystemStartMenuID, 0x7FFF, Flags); - + HRESULT res = systemContextMenu->QueryContextMenu(hMenu, 0, kSystemStartMenuID, 0x7FFF, Flags); + if (SUCCEEDED(res)) { + #ifdef SHOW_DEBUG_FM_CTX_MENU + PrintAllContextItems(systemContextMenu, (unsigned)res); + #endif + CMenu menu; menu.Attach(menuSpec); CMenuItem menuItem; menuItem.fMask = MIIM_SUBMENU | MIIM_TYPE | MIIM_ID; menuItem.fType = MFT_STRING; menuItem.hSubMenu = popupMenu.Detach(); - // menuDestroyer.Disable(); + menuDestroyer.Disable(); LangString(IDS_SYSTEM, menuItem.StringValue); menu.InsertItem(0, true, menuItem); } @@ -648,8 +778,6 @@ void CPanel::CreateSevenZipMenu(HMENU menuSpec, // CMenuDestroyer menuDestroyer(menu); // menu.CreatePopup(); - bool sevenZipMenuCreated = false; - CZipContextMenu *contextMenuSpec = new CZipContextMenu; CMyComPtr<IContextMenu> contextMenu = contextMenuSpec; // if (contextMenu.CoCreateInstance(CLSID_CZipContextMenu, IID_IContextMenu) == S_OK) @@ -674,9 +802,24 @@ void CPanel::CreateSevenZipMenu(HMENU menuSpec, { HRESULT res = contextMenu->QueryContextMenu(menu, 0, kSevenZipStartMenuID, kSystemStartMenuID - 1, 0); - sevenZipMenuCreated = (HRESULT_SEVERITY(res) == SEVERITY_SUCCESS); + bool sevenZipMenuCreated = SUCCEEDED(res); if (sevenZipMenuCreated) - sevenZipContextMenu = contextMenu; + { + // if (res != 0) + { + // some "non-good" implementation of QueryContextMenu() could add some items to menu, but it return 0. + // so we still allow these items + sevenZipContextMenu = contextMenu; + #ifdef SHOW_DEBUG_FM_CTX_MENU + PrintAllContextItems(contextMenu, (unsigned)res); + #endif + } + } + else + { + // MessageBox_Error_HRESULT_Caption(res, L"QueryContextMenu"); + } + // int code = HRESULT_CODE(res); // int nextItemID = code; } @@ -795,6 +938,9 @@ void CPanel::CreateFileMenu(HMENU menuSpec, fm.isAltStreamsSupported = false; + if (fm.numItems == 1) + fm.FilePath = GetItemFullPath(operatedIndices[0]); + if (_folderAltStreams) { if (operatedIndices.Size() <= 1) @@ -818,7 +964,7 @@ void CPanel::CreateFileMenu(HMENU menuSpec, fm.Load(menu, menu.GetItemCount()); } -bool CPanel::InvokePluginCommand(int id) +bool CPanel::InvokePluginCommand(unsigned id) { return InvokePluginCommand(id, _sevenZipContextMenu, _systemContextMenu); } @@ -827,15 +973,15 @@ bool CPanel::InvokePluginCommand(int id) #define use_CMINVOKECOMMANDINFOEX #endif -bool CPanel::InvokePluginCommand(int id, +bool CPanel::InvokePluginCommand(unsigned id, IContextMenu *sevenZipContextMenu, IContextMenu *systemContextMenu) { UInt32 offset; bool isSystemMenu = (id >= kSystemStartMenuID); if (isSystemMenu) - offset = id - kSystemStartMenuID; + offset = id - kSystemStartMenuID; else - offset = id - kSevenZipStartMenuID; + offset = id - kSevenZipStartMenuID; #ifdef use_CMINVOKECOMMANDINFOEX CMINVOKECOMMANDINFOEX @@ -856,16 +1002,29 @@ bool CPanel::InvokePluginCommand(int id, commandInfo.hwnd = GetParent(); commandInfo.lpVerb = (LPCSTR)(MAKEINTRESOURCE(offset)); commandInfo.lpParameters = NULL; - const CSysString currentFolderSys (GetSystemString(_currentFolderPrefix)); - commandInfo.lpDirectory = (LPCSTR)(LPCTSTR)(currentFolderSys); + // 19.01: fixed CSysString to AString + // MSDN suggest to send NULL: lpDirectory: This member is always NULL for menu items inserted by a Shell extension. + const AString currentFolderA (GetAnsiString(_currentFolderPrefix)); + commandInfo.lpDirectory = (LPCSTR)(currentFolderA); commandInfo.nShow = SW_SHOW; #ifdef use_CMINVOKECOMMANDINFOEX commandInfo.lpParametersW = NULL; commandInfo.lpTitle = ""; - commandInfo.lpVerbW = (LPCWSTR)(MAKEINTRESOURCEW(offset)); - UString currentFolderUnicode = _currentFolderPrefix; + + /* + system ContextMenu handler supports ContextMenu subhandlers. + so InvokeCommand() converts (command_offset) from global number to subhandler number. + XP-64 / win10: + system ContextMenu converts (command_offset) in lpVerb only, + and it keeps lpVerbW unchanged. + also explorer.exe sends 0 in lpVerbW. + We try to keep compatibility with Windows Explorer here. + */ + commandInfo.lpVerbW = NULL; + + const UString currentFolderUnicode = _currentFolderPrefix; commandInfo.lpDirectoryW = currentFolderUnicode; commandInfo.lpTitleW = L""; // commandInfo.ptInvoke.x = xPos; @@ -885,6 +1044,8 @@ bool CPanel::InvokePluginCommand(int id, KillSelection(); return true; } + else + MessageBox_Error_HRESULT_Caption(result, L"InvokeCommand"); return false; } @@ -947,22 +1108,22 @@ bool CPanel::OnContextMenu(HANDLE windowHandle, int xPos, int yPos) CMyComPtr<IContextMenu> systemContextMenu; CreateFileMenu(menu, sevenZipContextMenu, systemContextMenu, false); - int result = menu.Track(TPM_LEFTALIGN + unsigned id = menu.Track(TPM_LEFTALIGN #ifndef UNDER_CE | TPM_RIGHTBUTTON #endif | TPM_RETURNCMD | TPM_NONOTIFY, xPos, yPos, _listView); - if (result == 0) + if (id == 0) return true; - if (result >= kMenuCmdID_Plugin_Start) + if (id >= kMenuCmdID_Plugin_Start) { - InvokePluginCommand(result, sevenZipContextMenu, systemContextMenu); + InvokePluginCommand(id, sevenZipContextMenu, systemContextMenu); return true; } - if (ExecuteFileCommand(result)) + if (ExecuteFileCommand(id)) return true; return true; } diff --git a/CPP/7zip/UI/FileManager/PanelOperations.cpp b/CPP/7zip/UI/FileManager/PanelOperations.cpp index 00bda1a9..d3e2e978 100644 --- a/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -48,9 +48,7 @@ public: CMyComPtr<IProgress> UpdateCallback; CUpdateCallback100Imp *UpdateCallbackSpec; - HRESULT Result; - - CThreadFolderOperations(EFolderOpType opType): OpType(opType), Result(E_FAIL) {} + CThreadFolderOperations(EFolderOpType opType): OpType(opType) {} HRESULT DoOperation(CPanel &panel, const UString &progressTitle, const UString &titleError); }; @@ -60,18 +58,14 @@ HRESULT CThreadFolderOperations::ProcessVirt() switch (OpType) { case FOLDER_TYPE_CREATE_FOLDER: - Result = FolderOperations->CreateFolder(Name, UpdateCallback); - break; + return FolderOperations->CreateFolder(Name, UpdateCallback); case FOLDER_TYPE_DELETE: - Result = FolderOperations->Delete(&Indices.Front(), Indices.Size(), UpdateCallback); - break; + return FolderOperations->Delete(&Indices.Front(), Indices.Size(), UpdateCallback); case FOLDER_TYPE_RENAME: - Result = FolderOperations->Rename(Index, Name, UpdateCallback); - break; + return FolderOperations->Rename(Index, Name, UpdateCallback); default: - Result = E_FAIL; + return E_FAIL; } - return Result; } @@ -83,7 +77,6 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr WaitMode = true; Sync.FinalMessage.ErrorMessage.Title = titleError; - Result = S_OK; UpdateCallbackSpec->Init(); @@ -94,7 +87,6 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr UpdateCallbackSpec->Password = fl.Password; } - MainWindow = panel._mainWindow; // panel.GetParent() MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); MainAddTitle = progressTitle + L' '; @@ -274,9 +266,9 @@ BOOL CPanel::OnBeginLabelEdit(LV_DISPINFOW * lpnmh) return FALSE; } -bool IsCorrectFsName(const UString &name) +static bool IsCorrectFsName(const UString &name) { - const UString lastPart = name.Ptr(name.ReverseFind_PathSepar() + 1); + const UString lastPart = name.Ptr((unsigned)(name.ReverseFind_PathSepar() + 1)); return lastPart != L"." && lastPart != L".."; @@ -410,7 +402,7 @@ void CPanel::CreateFolder() { int pos = newName.Find(WCHAR_PATH_SEPARATOR); if (pos >= 0) - newName.DeleteFrom(pos); + newName.DeleteFrom((unsigned)(pos)); if (!_mySelectMode) state.SelectedNames.Clear(); state.FocusedName = newName; @@ -461,7 +453,7 @@ void CPanel::CreateFile() } int pos = newName.Find(WCHAR_PATH_SEPARATOR); if (pos >= 0) - newName.DeleteFrom(pos); + newName.DeleteFrom((unsigned)pos); if (!_mySelectMode) state.SelectedNames.Clear(); state.FocusedName = newName; diff --git a/CPP/7zip/UI/FileManager/PanelSelect.cpp b/CPP/7zip/UI/FileManager/PanelSelect.cpp index 8cfb23f4..eab9e1ab 100644 --- a/CPP/7zip/UI/FileManager/PanelSelect.cpp +++ b/CPP/7zip/UI/FileManager/PanelSelect.cpp @@ -193,7 +193,7 @@ void CPanel::SelectByType(bool selectMode) else { UString mask ('*'); - mask += name.Ptr(pos); + mask += name.Ptr((unsigned)pos); FOR_VECTOR (i, _selectedStatusVector) if (IsItem_Folder(i) == isItemFolder && DoesWildcardMatchName(mask, GetItemName(i))) _selectedStatusVector[i] = selectMode; @@ -241,6 +241,9 @@ void CPanel::InvertSelection() void CPanel::KillSelection() { SelectAll(false); + // ver 20.01: now we don't like that focused will be selected item. + // So the following code was disabled: + /* if (!_mySelectMode) { int focused = _listView.GetFocusedItem(); @@ -254,6 +257,7 @@ void CPanel::KillSelection() _listView.SetItemState_Selected(focused); } } + */ } void CPanel::OnLeftClick(MY_NMLISTVIEW_NMITEMACTIVATE *itemActivate) diff --git a/CPP/7zip/UI/FileManager/PanelSort.cpp b/CPP/7zip/UI/FileManager/PanelSort.cpp index 4adfc39f..d26acb70 100644 --- a/CPP/7zip/UI/FileManager/PanelSort.cpp +++ b/CPP/7zip/UI/FileManager/PanelSort.cpp @@ -95,7 +95,7 @@ void CPanel::SetSortRawStatus() } -int CALLBACK CompareItems2(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) +static int CALLBACK CompareItems2(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) { if (lpData == 0) return 0; @@ -182,7 +182,7 @@ int CALLBACK CompareItems2(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) return prop1.Compare(prop2); } - +int CALLBACK CompareItems(LPARAM lParam1, LPARAM lParam2, LPARAM lpData); int CALLBACK CompareItems(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) { if (lpData == 0) return 0; diff --git a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp index 7c633323..00a0d801 100644 --- a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp +++ b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp @@ -274,7 +274,7 @@ void CApp::Split() NFind::CFileInfo fileInfo; if (!fileInfo.Find(us2fs(srcPath + itemName))) { - srcPanel.MessageBox_Error(L"Can not find file"); + srcPanel.MessageBox_Error(L"Cannot find file"); return; } if (fileInfo.Size <= splitDialog.VolumeSizes.Front()) diff --git a/CPP/7zip/UI/FileManager/ProgressDialog.h b/CPP/7zip/UI/FileManager/ProgressDialog.h index 0f41b57a..35c182a8 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog.h +++ b/CPP/7zip/UI/FileManager/ProgressDialog.h @@ -137,7 +137,7 @@ public: { _title = title; INT_PTR res = CModalDialog::Create(IDD_PROGRESS, wndParent); - thread.Wait(); + thread.Wait_Close(); return res; } diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 28e3eae7..8e2d7c75 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -536,10 +536,10 @@ bool CProgressDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) labelSize = gSize - valueSize; yPos = my; - for (int i = 0; i < ARRAY_SIZE(kIDs); i += 2) + for (unsigned i = 0; i < ARRAY_SIZE(kIDs); i += 2) { int x = mx; - const int kNumColumn1Items = 5 * 2; + const unsigned kNumColumn1Items = 5 * 2; if (i >= kNumColumn1Items) { if (i == kNumColumn1Items) @@ -583,6 +583,7 @@ void CProgressDialog::SetProgressPos(UInt64 pos) #define UINT_TO_STR_2(val) { s[0] = (wchar_t)('0' + (val) / 10); s[1] = (wchar_t)('0' + (val) % 10); s += 2; } +void GetTimeString(UInt64 timeValue, wchar_t *s); void GetTimeString(UInt64 timeValue, wchar_t *s) { UInt64 hours = timeValue / 3600; @@ -892,8 +893,8 @@ void CProgressDialog::UpdateStatInfo(bool showAll) int slashPos = _filePath.ReverseFind_PathSepar(); if (slashPos >= 0) { - s1.SetFrom(_filePath, slashPos + 1); - s2 = _filePath.Ptr(slashPos + 1); + s1.SetFrom(_filePath, (unsigned)(slashPos + 1)); + s2 = _filePath.Ptr((unsigned)(slashPos + 1)); } else s2 = _filePath; @@ -942,7 +943,7 @@ INT_PTR CProgressDialog::Create(const UString &title, NWindows::CThread &thread, CWaitCursor waitCursor; HANDLE h[] = { thread, _createDialogEvent }; - WRes res2 = WaitForMultipleObjects(ARRAY_SIZE(h), h, FALSE, kCreateDelay); + DWORD res2 = WaitForMultipleObjects(ARRAY_SIZE(h), h, FALSE, kCreateDelay); if (res2 == WAIT_OBJECT_0 && !Sync.ThereIsMessage()) return 0; } @@ -954,9 +955,8 @@ INT_PTR CProgressDialog::Create(const UString &title, NWindows::CThread &thread, { _wasCreated = true; _dialogCreatedEvent.Set(); - res = res; } - thread.Wait(); + thread.Wait_Close(); if (!MessagesDisplayed) MessageBoxW(wndParent, L"Progress Error", L"7-Zip", MB_ICONERROR); return res; diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.h b/CPP/7zip/UI/FileManager/ProgressDialog2.h index 6c4213ac..fc032cd9 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.h +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.h @@ -321,7 +321,7 @@ CProgressThreadVirt::Create() CProgressThreadVirt::Process() { { - ProcessVirt(); // virtual function that must implement real work + Result = ProcessVirt(); // virtual function that must implement real work } if (exceptions) or FinalMessage.ErrorMessage.Message { diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2a.rc b/CPP/7zip/UI/FileManager/ProgressDialog2a.rc index e9713930..c183af82 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2a.rc +++ b/CPP/7zip/UI/FileManager/ProgressDialog2a.rc @@ -44,7 +44,7 @@ DIALOG_ID DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT CAPTION "Progress" { DEFPUSHBUTTON "&Background", IDB_PROGRESS_BACKGROUND, bx3, by, bxs, bys - PUSHBUTTON "&Pause", IDB_PAUSE bx2, by, bxs, bys + PUSHBUTTON "&Pause", IDB_PAUSE, bx2, by, bxs, bys PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys LTEXT "Elapsed time:", IDT_PROGRESS_ELAPSED, m, y0, x0s, 8 diff --git a/CPP/7zip/UI/FileManager/RegistryPlugins.cpp b/CPP/7zip/UI/FileManager/RegistryPlugins.cpp index a0753ca6..76a5787c 100644 --- a/CPP/7zip/UI/FileManager/RegistryPlugins.cpp +++ b/CPP/7zip/UI/FileManager/RegistryPlugins.cpp @@ -62,7 +62,7 @@ static bool ReadPluginInfo(CPluginInfo &pluginInfo, bool needCheckDll) else { pluginInfo.ClassIDDefined = true; - pluginInfo.ClassID = *(const GUID *)prop.bstrVal; + pluginInfo.ClassID = *(const GUID *)(const void *)prop.bstrVal; } prop.Clear(); @@ -75,7 +75,7 @@ static bool ReadPluginInfo(CPluginInfo &pluginInfo, bool needCheckDll) else { pluginInfo.OptionsClassIDDefined = true; - pluginInfo.OptionsClassID = *(const GUID *)prop.bstrVal; + pluginInfo.OptionsClassID = *(const GUID *)(const void *)prop.bstrVal; } prop.Clear(); diff --git a/CPP/7zip/UI/FileManager/RegistryUtils.cpp b/CPP/7zip/UI/FileManager/RegistryUtils.cpp index 91b69a21..a2fc2011 100644 --- a/CPP/7zip/UI/FileManager/RegistryUtils.cpp +++ b/CPP/7zip/UI/FileManager/RegistryUtils.cpp @@ -22,6 +22,7 @@ static LPCWSTR const kLangValueName = L"Lang"; static LPCWSTR const kViewer = L"Viewer"; static LPCWSTR const kEditor = L"Editor"; static LPCWSTR const kDiff = L"Diff"; +static LPCWSTR const kVerCtrlPath = L"7vc"; static LPCTSTR const kShowDots = TEXT("ShowDots"); static LPCTSTR const kShowRealFileIcons = TEXT("ShowRealFileIcons"); @@ -63,6 +64,8 @@ void ReadRegEditor(bool useEditor, UString &path) { ReadCuString(kCU_FMPath, use void SaveRegDiff(const UString &path) { SaveCuString(kCU_FMPath, kDiff, path); } void ReadRegDiff(UString &path) { ReadCuString(kCU_FMPath, kDiff, path); } +void ReadReg_VerCtrlPath(UString &path) { ReadCuString(kCU_FMPath, kVerCtrlPath, path); } + static void Save7ZipOption(LPCTSTR value, bool enabled) { CKey key; diff --git a/CPP/7zip/UI/FileManager/RegistryUtils.h b/CPP/7zip/UI/FileManager/RegistryUtils.h index b731778b..b85d670f 100644 --- a/CPP/7zip/UI/FileManager/RegistryUtils.h +++ b/CPP/7zip/UI/FileManager/RegistryUtils.h @@ -15,6 +15,8 @@ void ReadRegEditor(bool useEditor, UString &path); void SaveRegDiff(const UString &path); void ReadRegDiff(UString &path); +void ReadReg_VerCtrlPath(UString &path); + struct CFmSettings { bool ShowDots; diff --git a/CPP/7zip/UI/FileManager/RootFolder.cpp b/CPP/7zip/UI/FileManager/RootFolder.cpp index 643f1066..84844c7d 100644 --- a/CPP/7zip/UI/FileManager/RootFolder.cpp +++ b/CPP/7zip/UI/FileManager/RootFolder.cpp @@ -46,6 +46,7 @@ static const Byte kProps[] = kpidName }; +UString RootFolder_GetName_Computer(int &iconIndex); UString RootFolder_GetName_Computer(int &iconIndex) { #ifdef USE_WIN_PATHS @@ -56,12 +57,14 @@ UString RootFolder_GetName_Computer(int &iconIndex) return LangString(IDS_COMPUTER); } +UString RootFolder_GetName_Network(int &iconIndex); UString RootFolder_GetName_Network(int &iconIndex) { iconIndex = GetIconIndexForCSIDL(CSIDL_NETWORK); return LangString(IDS_NETWORK); } +UString RootFolder_GetName_Documents(int &iconIndex); UString RootFolder_GetName_Documents(int &iconIndex) { iconIndex = GetIconIndexForCSIDL(CSIDL_PERSONAL); @@ -120,7 +123,7 @@ STDMETHODIMP CRootFolder::GetProperty(UInt32 itemIndex, PROPID propID, PROPVARIA typedef BOOL (WINAPI *SHGetSpecialFolderPathWp)(HWND hwnd, LPWSTR pszPath, int csidl, BOOL fCreate); typedef BOOL (WINAPI *SHGetSpecialFolderPathAp)(HWND hwnd, LPSTR pszPath, int csidl, BOOL fCreate); -UString GetMyDocsPath() +static UString GetMyDocsPath() { UString us; WCHAR s[MAX_PATH + 1]; @@ -136,7 +139,7 @@ UString GetMyDocsPath() else { SHGetSpecialFolderPathAp getA = (SHGetSpecialFolderPathAp) - ::GetProcAddress(::GetModuleHandleA("shell32.dll"), "SHGetSpecialFolderPathA"); + (void *)::GetProcAddress(::GetModuleHandleA("shell32.dll"), "SHGetSpecialFolderPathA"); CHAR s2[MAX_PATH + 1]; if (getA && getA(0, s2, CSIDL_PERSONAL, FALSE)) us = GetUnicodeString(s2); diff --git a/CPP/7zip/UI/FileManager/StdAfx.h b/CPP/7zip/UI/FileManager/StdAfx.h index e6d96041..0e6d4461 100644 --- a/CPP/7zip/UI/FileManager/StdAfx.h +++ b/CPP/7zip/UI/FileManager/StdAfx.h @@ -14,8 +14,8 @@ // #include "../../../Common/MyWindows.h" -// #include <commctrl.h> +// #include <CommCtrl.h> // #include <ShlObj.h> -// #include <shlwapi.h> +// #include <Shlwapi.h> #endif diff --git a/CPP/7zip/UI/FileManager/SysIconUtils.cpp b/CPP/7zip/UI/FileManager/SysIconUtils.cpp index c8ea8b95..b756dfcf 100644 --- a/CPP/7zip/UI/FileManager/SysIconUtils.cpp +++ b/CPP/7zip/UI/FileManager/SysIconUtils.cpp @@ -23,7 +23,7 @@ int GetIconIndexForCSIDL(int csidl) if (pidl) { SHFILEINFO shellInfo; - SHGetFileInfo(LPCTSTR(pidl), FILE_ATTRIBUTE_NORMAL, + SHGetFileInfo((LPCTSTR)(const void *)(pidl), FILE_ATTRIBUTE_NORMAL, &shellInfo, sizeof(shellInfo), SHGFI_PIDL | SHGFI_SYSICONINDEX); IMalloc *pMalloc; @@ -41,7 +41,7 @@ int GetIconIndexForCSIDL(int csidl) #ifndef _UNICODE typedef int (WINAPI * SHGetFileInfoWP)(LPCWSTR pszPath, DWORD attrib, SHFILEINFOW *psfi, UINT cbFileInfo, UINT uFlags); -struct CSHGetFileInfoInit +static struct CSHGetFileInfoInit { SHGetFileInfoWP shGetFileInfoW; CSHGetFileInfoInit() diff --git a/CPP/7zip/UI/FileManager/SysIconUtils.h b/CPP/7zip/UI/FileManager/SysIconUtils.h index f1b27fa1..fde16e46 100644 --- a/CPP/7zip/UI/FileManager/SysIconUtils.h +++ b/CPP/7zip/UI/FileManager/SysIconUtils.h @@ -5,7 +5,7 @@ #include "../../../Common/MyWindows.h" -#include <commctrl.h> +#include <CommCtrl.h> #include "../../../Common/MyString.h" diff --git a/CPP/7zip/UI/FileManager/SystemPage.cpp b/CPP/7zip/UI/FileManager/SystemPage.cpp index 819443ee..a95999f2 100644 --- a/CPP/7zip/UI/FileManager/SystemPage.cpp +++ b/CPP/7zip/UI/FileManager/SystemPage.cpp @@ -34,13 +34,16 @@ static const UInt32 kLangIDs[] = CSysString CModifiedExtInfo::GetString() const { + const char *s; if (State == kExtState_7Zip) - return TEXT("7-Zip"); - if (State == kExtState_Clear) - return TEXT(""); - if (Other7Zip) - return TEXT("[7-Zip]"); - return ProgramKey; + s = "7-Zip"; + else if (State == kExtState_Clear) + s = ""; + else if (Other7Zip) + s = "[7-Zip]"; + else + return ProgramKey; + return CSysString (s); }; @@ -197,14 +200,14 @@ bool CSystemPage::OnInit() ci.mask = LVCF_TEXT | LVCF_FMT | LVCF_WIDTH | LVCF_SUBITEM; ci.cx = 128; ci.fmt = LVCFMT_CENTER; - ci.pszText = (WCHAR *)(const WCHAR *)s; + ci.pszText = s.Ptr_non_const(); ci.iSubItem = 1; _listView.InsertColumn(1, &ci); #if NUM_EXT_GROUPS > 1 { LangString(IDS_SYSTEM_ALL_USERS, s); - ci.pszText = (WCHAR *)(const WCHAR *)s; + ci.pszText = s.Ptr_non_const(); ci.iSubItem = 2; _listView.InsertColumn(2, &ci); } @@ -225,7 +228,7 @@ bool CSystemPage::OnInit() // ListView always uses internal iImage that is 0 by default? // so we always use LVIF_IMAGE. item.iImage = -1; - item.pszText = (wchar_t *)(const wchar_t *)(LPCWSTR)extInfo.Ext; + item.pszText = extInfo.Ext.Ptr_non_const(); CAssoc assoc; const CPluginToIcon &plug = extInfo.Plugins[0]; diff --git a/CPP/7zip/UI/FileManager/SystemPage.h b/CPP/7zip/UI/FileManager/SystemPage.h index 761a49ac..765214cf 100644 --- a/CPP/7zip/UI/FileManager/SystemPage.h +++ b/CPP/7zip/UI/FileManager/SystemPage.h @@ -90,7 +90,7 @@ class CSystemPage: public NWindows::NControl::CPropertyPage bool _needSave; - const HKEY GetHKey(unsigned + HKEY GetHKey(unsigned #if NUM_EXT_GROUPS != 1 group #endif diff --git a/CPP/7zip/UI/FileManager/TextPairs.cpp b/CPP/7zip/UI/FileManager/TextPairs.cpp index d4002b45..6a989fcc 100644 --- a/CPP/7zip/UI/FileManager/TextPairs.cpp +++ b/CPP/7zip/UI/FileManager/TextPairs.cpp @@ -98,7 +98,7 @@ static int ComparePairItems(const CTextPair &p1, const CTextPair &p2) { return ComparePairIDs(p1.ID, p2.ID); } static int ComparePairItems(void *const *a1, void *const *a2, void * /* param */) - { return ComparePairItems(**(const CTextPair **)a1, **(const CTextPair **)a2); } + { return ComparePairItems(**(const CTextPair *const *)a1, **(const CTextPair *const *)a2); } void CPairsStorage::Sort() { Pairs.Sort(ComparePairItems, 0); } diff --git a/CPP/7zip/UI/FileManager/VerCtrl.cpp b/CPP/7zip/UI/FileManager/VerCtrl.cpp new file mode 100644 index 00000000..efed8468 --- /dev/null +++ b/CPP/7zip/UI/FileManager/VerCtrl.cpp @@ -0,0 +1,367 @@ +// VerCtrl.cpp + +#include "StdAfx.h" + +#include "../../../Common/StringToInt.h" + +#include "../../../Windows/FileName.h" +#include "../../../Windows/FileFind.h" + +#include "App.h" +#include "RegistryUtils.h" +#include "OverwriteDialog.h" + +#include "resource.h" + +using namespace NWindows; +using namespace NFile; +using namespace NFind; +using namespace NDir; + +static UString ConvertPath_to_Ctrl(const UString &path) +{ + UString s = path; + s.Replace(L':', L'_'); + return s; +} + +struct CFileDataInfo +{ + CByteBuffer Data; + BY_HANDLE_FILE_INFORMATION Info; + bool IsOpen; + + CFileDataInfo(): IsOpen (false) {} + UInt64 GetSize() const { return (((UInt64)Info.nFileSizeHigh) << 32) + Info.nFileSizeLow; } + bool Read(const UString &path); +}; + + +bool CFileDataInfo::Read(const UString &path) +{ + IsOpen = false; + NIO::CInFile file; + if (!file.Open(path)) + return false; + if (!file.GetFileInformation(&Info)) + return false; + + const UInt64 size = GetSize(); + const size_t size2 = (size_t)size; + if (size2 != size || size2 > (1 << 28)) + { + SetLastError(1); + return false; + } + + Data.Alloc(size2); + + size_t processedSize; + if (!file.ReadFull(Data, size2, processedSize)) + return false; + if (processedSize != size2) + { + SetLastError(1); + return false; + } + IsOpen = true; + return true; +} + + +static bool CreateComplexDir_for_File(const UString &path) +{ + FString resDirPrefix; + FString resFileName; + if (!GetFullPathAndSplit(path, resDirPrefix, resFileName)) + return false; + return CreateComplexDir(resDirPrefix); +} + + +static bool ParseNumberString(const FString &s, UInt32 &number) +{ + const wchar_t *end; + UInt64 result = ConvertStringToUInt64(s, &end); + if (*end != 0 || s.IsEmpty() || result > (UInt32)0x7FFFFFFF) + return false; + number = (UInt32)result; + return true; +} + + +static void WriteFile(const FString &path, bool createAlways, const CFileDataInfo &fdi, const CPanel &panel) +{ + NIO::COutFile outFile; + if (!outFile.Create(path, createAlways)) // (createAlways = false) means CREATE_NEW + { + panel.MessageBox_LastError(); + return; + } + UInt32 processedSize; + if (!outFile.Write(fdi.Data, (UInt32)fdi.Data.Size(), processedSize)) + { + panel.MessageBox_LastError(); + return; + } + if (processedSize != fdi.Data.Size()) + { + panel.MessageBox_Error(L"Write error"); + return; + } + if (!outFile.SetTime( + &fdi.Info.ftCreationTime, + &fdi.Info.ftLastAccessTime, + &fdi.Info.ftLastWriteTime)) + { + panel.MessageBox_LastError(); + return; + } + + if (!SetFileAttrib(path, fdi.Info.dwFileAttributes)) + { + panel.MessageBox_LastError(); + return; + } +} + + +void CApp::VerCtrl(unsigned id) +{ + const CPanel &panel = GetFocusedPanel(); + + if (!panel.Is_IO_FS_Folder()) + { + panel.MessageBox_Error_UnsupportOperation(); + return; + } + + CRecordVector<UInt32> indices; + panel.GetSelectedItemsIndices(indices); + + if (indices.Size() != 1) + { + // panel.MessageBox_Error_UnsupportOperation(); + return; + } + + const UString path = panel.GetItemFullPath(indices[0]); + + UString vercPath; + ReadReg_VerCtrlPath(vercPath); + if (vercPath.IsEmpty()) + return; + NName::NormalizeDirPathPrefix(vercPath); + + FString dirPrefix; + FString fileName; + if (!GetFullPathAndSplit(path, dirPrefix, fileName)) + { + panel.MessageBox_LastError(); + return; + } + + const UString dirPrefix2 = vercPath + ConvertPath_to_Ctrl(dirPrefix); + const UString path2 = dirPrefix2 + fileName; + + bool sameTime = false; + bool sameData = false; + bool areIdentical = false; + + CFileDataInfo fdi, fdi2; + if (!fdi.Read(path)) + { + panel.MessageBox_LastError(); + return; + } + + if (fdi2.Read(path2)) + { + sameData = (fdi.Data == fdi2.Data); + sameTime = (CompareFileTime(&fdi.Info.ftLastWriteTime, &fdi2.Info.ftLastWriteTime) == 0); + areIdentical = (sameData && sameTime); + } + + const bool isReadOnly = NAttributes::IsReadOnly(fdi.Info.dwFileAttributes); + + if (id == IDM_VER_EDIT) + { + if (!isReadOnly) + { + panel.MessageBox_Error(L"File is not read-only"); + return; + } + + if (!areIdentical) + { + if (fdi2.IsOpen) + { + NFind::CEnumerator enumerator; + FString d2 = dirPrefix2; + d2 += "_7vc"; + d2.Add_PathSepar(); + d2 += fileName; + d2.Add_PathSepar(); + enumerator.SetDirPrefix(d2); + NFind::CDirEntry fi; + Int32 maxVal = -1; + while (enumerator.Next(fi)) + { + UInt32 val; + if (!ParseNumberString(fi.Name, val)) + continue; + if ((Int32)val > maxVal) + maxVal = val; + } + + UInt32 next = (UInt32)maxVal + 1; + if (maxVal < 0) + { + next = 1; + if (!::CreateComplexDir_for_File(path2)) + { + panel.MessageBox_LastError(); + return; + } + } + + // we rename old file2 to some name; + FString path_num = d2; + { + AString t; + t.Add_UInt32((UInt32)next); + while (t.Len() < 3) + t.InsertAtFront('0'); + path_num += t; + } + + if (maxVal < 0) + { + if (!::CreateComplexDir_for_File(path_num)) + { + panel.MessageBox_LastError(); + return; + } + } + + if (!NDir::MyMoveFile(path2, path_num)) + { + panel.MessageBox_LastError(); + return; + } + } + else + { + if (!::CreateComplexDir_for_File(path2)) + { + panel.MessageBox_LastError(); + return; + } + } + /* + if (!::CopyFile(fs2fas(path), fs2fas(path2), TRUE)) + { + panel.MessageBox_LastError(); + return; + } + */ + WriteFile(path2, + false, // (createAlways = false) means CREATE_NEW + fdi, panel); + } + + if (!SetFileAttrib(path, fdi.Info.dwFileAttributes & ~(DWORD)FILE_ATTRIBUTE_READONLY)) + { + panel.MessageBox_LastError(); + return; + } + + return; + } + + if (isReadOnly) + { + panel.MessageBox_Error(L"File is read-only"); + return; + } + + if (id == IDM_VER_COMMIT) + { + if (sameData) + { + if (!sameTime) + { + panel.MessageBox_Error( + L"Same data, but different timestamps.\n" + L"Use `Revert` to recover timestamp."); + return; + } + } + if (!SetFileAttrib(path, fdi.Info.dwFileAttributes | FILE_ATTRIBUTE_READONLY)) + { + panel.MessageBox_LastError(); + return; + } + return; + } + + if (id == IDM_VER_REVERT) + { + if (!fdi2.IsOpen) + { + panel.MessageBox_Error(L"No file to revert"); + return; + } + if (!sameData || !sameTime) + { + if (!sameData) + { + /* + UString m; + m = "Are you sure you want to revert file ?"; + m.Add_LF(); + m += path; + if (::MessageBoxW(panel.GetParent(), m, L"Version Control: File Revert", MB_OKCANCEL | MB_ICONQUESTION) != IDOK) + return; + */ + COverwriteDialog dialog; + + dialog.OldFileInfo.SetTime(&fdi.Info.ftLastWriteTime); + dialog.OldFileInfo.SetSize(fdi.GetSize()); + dialog.OldFileInfo.Name = path; + + dialog.NewFileInfo.SetTime(&fdi2.Info.ftLastWriteTime); + dialog.NewFileInfo.SetSize(fdi2.GetSize()); + dialog.NewFileInfo.Name = path2; + + dialog.ShowExtraButtons = false; + dialog.DefaultButton_is_NO = true; + + INT_PTR writeAnswer = dialog.Create(panel.GetParent()); + + if (writeAnswer != IDYES) + return; + } + + WriteFile(path, + true, // (createAlways = true) means CREATE_ALWAYS + fdi2, panel); + } + else + { + if (!SetFileAttrib(path, fdi2.Info.dwFileAttributes | FILE_ATTRIBUTE_READONLY)) + { + panel.MessageBox_LastError(); + return; + } + } + return; + } + + // if (id == IDM_VER_DIFF) + { + if (!fdi2.IsOpen) + return; + DiffFiles(path2, path); + } +} diff --git a/CPP/7zip/UI/FileManager/ViewSettings.cpp b/CPP/7zip/UI/FileManager/ViewSettings.cpp index af61efcd..0f87711f 100644 --- a/CPP/7zip/UI/FileManager/ViewSettings.cpp +++ b/CPP/7zip/UI/FileManager/ViewSettings.cpp @@ -193,14 +193,14 @@ void CWindowInfo::Read(bool &windowPosDefined, bool &panelInfoDefined) } -void SaveUi32Val(const TCHAR *name, UInt32 value) +static void SaveUi32Val(const TCHAR *name, UInt32 value) { CKey key; key.Create(HKEY_CURRENT_USER, kCUBasePath); key.SetValue(name, value); } -bool ReadUi32Val(const TCHAR *name, UInt32 &value) +static bool ReadUi32Val(const TCHAR *name, UInt32 &value) { CKey key; if (key.Open(HKEY_CURRENT_USER, kCUBasePath, KEY_READ) != ERROR_SUCCESS) diff --git a/CPP/7zip/UI/FileManager/resource.h b/CPP/7zip/UI/FileManager/resource.h index bb0d7752..98cb4fd4 100644 --- a/CPP/7zip/UI/FileManager/resource.h +++ b/CPP/7zip/UI/FileManager/resource.h @@ -47,6 +47,11 @@ #define IDM_LINK 558 #define IDM_ALT_STREAMS 559 +#define IDM_VER_EDIT 580 +#define IDM_VER_COMMIT 581 +#define IDM_VER_REVERT 582 +#define IDM_VER_DIFF 583 + #define IDM_OPEN_INSIDE_ONE 590 #define IDM_OPEN_INSIDE_PARSER 591 diff --git a/CPP/7zip/UI/FileManager/resource.rc b/CPP/7zip/UI/FileManager/resource.rc index 6d259eb0..fc2c8408 100644 --- a/CPP/7zip/UI/FileManager/resource.rc +++ b/CPP/7zip/UI/FileManager/resource.rc @@ -167,7 +167,7 @@ BEGIN IDS_FILE_EXIST "File {0} is already exist" IDS_WANT_UPDATE_MODIFIED_FILE "File '{0}' was modified.\nDo you want to update it in the archive?" - IDS_CANNOT_UPDATE_FILE "Can not update file\n'{0}'" + IDS_CANNOT_UPDATE_FILE "Cannot update file\n'{0}'" IDS_CANNOT_START_EDITOR "Cannot start editor." IDS_VIRUS "The file looks like a virus (the file name contains long spaces in name)." IDS_MESSAGE_UNSUPPORTED_OPERATION_FOR_LONG_PATH_FOLDER "The operation cannot be called from a folder that has a long path." @@ -238,8 +238,8 @@ BEGIN IDS_COMBINE_TO "&Combine to:" IDS_COMBINING "Combining..." IDS_COMBINE_SELECT_ONE_FILE "Select only first part of split file" - IDS_COMBINE_CANT_DETECT_SPLIT_FILE "Can not detect file as split file" - IDS_COMBINE_CANT_FIND_MORE_THAN_ONE_PART "Can not find more than one part of split file" + IDS_COMBINE_CANT_DETECT_SPLIT_FILE "Cannot detect file as split file" + IDS_COMBINE_CANT_FIND_MORE_THAN_ONE_PART "Cannot find more than one part of split file" END diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp index e6dc57ba..94dfab4c 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp @@ -164,14 +164,12 @@ bool CBenchmarkDialog::OnInit() s = s2; SetItemText(IDT_BENCH_CPU, s); } - /* { AString s2; - GetCpuFeatures(s2); + AddCpuFeatures(s2); s = s2; SetItemText(IDT_BENCH_CPU_FEATURE, s); } - */ s = "7-Zip " MY_VERSION_CPU; SetItemText(IDT_BENCH_VER, s); @@ -384,7 +382,7 @@ void CBenchmarkDialog::OnChangeSettings() EnableItem(IDB_STOP, true); UInt32 dict = OnChangeDictionary(); - for (int i = 0; i < ARRAY_SIZE(g_IDs); i++) + for (unsigned i = 0; i < ARRAY_SIZE(g_IDs); i++) SetItemText(g_IDs[i], kProcessingString); _startTime = GetTickCount(); PrintTime(); @@ -517,7 +515,7 @@ bool CBenchmarkDialog::OnTimer(WPARAM /* timerID */, LPARAM /* callback */) { UInt32 dicSizeTemp = (UInt32)MyMax(Sync.ProcessedSize, UInt64(1) << 20); - dicSizeTemp = MyMin(dicSizeTemp, Sync.DictionarySize), + dicSizeTemp = MyMin(dicSizeTemp, Sync.DictionarySize); PrintResults(dicSizeTemp, Sync.CompressingInfoTemp, IDT_BENCH_COMPRESS_USAGE1, @@ -932,8 +930,10 @@ HRESULT Benchmark( benchmarker.BenchmarkDialog = &bd; - NWindows::CThread thread; - RINOK(thread.Create(CThreadBenchmark::MyThreadFunction, &benchmarker)); - bd.Create(hwndParent); - return thread.Wait(); + { + NWindows::CThread thread; + RINOK(thread.Create(CThreadBenchmark::MyThreadFunction, &benchmarker)); + bd.Create(hwndParent); + return thread.Wait_Close(); + } } diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.rc b/CPP/7zip/UI/GUI/BenchmarkDialog.rc index f1d37cab..a8455a0f 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.rc +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.rc @@ -117,9 +117,9 @@ BEGIN RTEXT "", IDT_BENCH_CPU, m, 202, xc, 8 - RTEXT "", IDT_BENCH_VER, m + xc - 80, 216, 80, 8 + RTEXT "", IDT_BENCH_VER, m + xc - 100, 216, 100, 8 - LTEXT "", IDT_BENCH_CPU_FEATURE, m, 212, xc - 80, 26 + LTEXT "", IDT_BENCH_CPU_FEATURE, m, 228, xc - 100, 8 LTEXT "", IDT_BENCH_SYS1, m, 238, xc - 140, 8 LTEXT "", IDT_BENCH_SYS2, m, 248, xc - 140, 8 diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index cbb2dda0..9f2cb146 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -248,7 +248,7 @@ static const CFormatInfo g_Formats[] = static bool IsMethodSupportedBySfx(int methodID) { - for (int i = 0; i < ARRAY_SIZE(g_7zSfxMethods); i++) + for (unsigned i = 0; i < ARRAY_SIZE(g_7zSfxMethods); i++) if (methodID == g_7zSfxMethods[i]) return true; return false; @@ -336,6 +336,7 @@ bool CCompressDialog::OnInit() _password1Control.SetText(Info.Password); _password2Control.SetText(Info.Password); _encryptionMethod.Attach(GetItem(IDC_COMPRESS_ENCRYPTION_METHOD)); + _default_encryptionMethod_Index = -1; m_ArchivePath.Attach(GetItem(IDC_COMPRESS_ARCHIVE)); m_Format.Attach(GetItem(IDC_COMPRESS_FORMAT)); @@ -1150,11 +1151,13 @@ bool CCompressDialog::IsXzFormat() void CCompressDialog::SetEncryptionMethod() { _encryptionMethod.ResetContent(); + _default_encryptionMethod_Index = -1; const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; if (ai.Name.IsEqualTo_Ascii_NoCase("7z")) { ComboBox_AddStringAscii(_encryptionMethod, "AES-256"); _encryptionMethod.SetCurSel(0); + _default_encryptionMethod_Index = 0; } else if (ai.Name.IsEqualTo_Ascii_NoCase("zip")) { @@ -1165,9 +1168,15 @@ void CCompressDialog::SetEncryptionMethod() const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; encryptionMethod = fo.EncryptionMethod; } - ComboBox_AddStringAscii(_encryptionMethod, "ZipCrypto"); + int sel = 0; + // if (ZipCryptoIsAllowed) + { + ComboBox_AddStringAscii(_encryptionMethod, "ZipCrypto"); + sel = (encryptionMethod.IsPrefixedBy_Ascii_NoCase("aes") ? 1 : 0); + _default_encryptionMethod_Index = 0; + } ComboBox_AddStringAscii(_encryptionMethod, "AES-256"); - _encryptionMethod.SetCurSel(encryptionMethod.IsPrefixedBy_Ascii_NoCase("aes") ? 1 : 0); + _encryptionMethod.SetCurSel(sel); } } @@ -1189,8 +1198,8 @@ UString CCompressDialog::GetMethodSpec() UString CCompressDialog::GetEncryptionMethodSpec() { UString s; - if (_encryptionMethod.GetCount() > 1 - && _encryptionMethod.GetCurSel() > 0) + if (_encryptionMethod.GetCount() > 0 + && _encryptionMethod.GetCurSel() != _default_encryptionMethod_Index) { _encryptionMethod.GetText(s); s.RemoveChar(L'-'); @@ -1242,18 +1251,17 @@ void CCompressDialog::SetDictionary() case kLZMA: case kLZMA2: { - static const UInt32 kMinDicSize = (1 << 16); if (defaultDict == (UInt32)(Int32)-1) { - if (level >= 9) defaultDict = (1 << 26); - else if (level >= 7) defaultDict = (1 << 25); - else if (level >= 5) defaultDict = (1 << 24); - else if (level >= 3) defaultDict = (1 << 20); - else defaultDict = (kMinDicSize); + defaultDict = + ( level <= 3 ? (1 << (level * 2 + 16)) : + ( level <= 6 ? (1 << (level + 19)) : + ( level <= 7 ? (1 << 25) : (1 << 26) + ))); } - - AddDictionarySize(kMinDicSize); - m_Dictionary.SetCurSel(0); + AddDictionarySize(1 << 16); + AddDictionarySize(1 << 18); + m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); for (unsigned i = 20; i <= 31; i++) for (unsigned j = 0; j < 2; j++) @@ -1286,10 +1294,7 @@ void CCompressDialog::SetDictionary() { if (defaultDict == (UInt32)(Int32)-1) { - if (level >= 9) defaultDict = (192 << 20); - else if (level >= 7) defaultDict = ( 64 << 20); - else if (level >= 5) defaultDict = ( 16 << 20); - else defaultDict = ( 4 << 20); + defaultDict = (UInt32)1 << (level + 19); } for (unsigned i = 20; i < 31; i++) @@ -1355,7 +1360,7 @@ void CCompressDialog::SetDictionary() case kPPMdZip: { if (defaultDict == (UInt32)(Int32)-1) - defaultDict = (1 << (19 + (level > 8 ? 8 : level))); + defaultDict = (UInt32)1 << (level + 19); for (unsigned i = 20; i <= 28; i++) { @@ -1512,6 +1517,21 @@ bool CCompressDialog::GetOrderMode() } +static UInt64 Get_Lzma2_ChunkSize(UInt32 dict) +{ + // we use same default chunk sizes as defined in 7z encoder and lzma2 encoder + UInt64 cs = (UInt64)dict << 2; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + if (cs < kMinSize) cs = kMinSize; + if (cs > kMaxSize) cs = kMaxSize; + if (cs < dict) cs = dict; + cs += (kMinSize - 1); + cs &= ~(UInt64)(kMinSize - 1); + return cs; +} + + void CCompressDialog::SetSolidBlockSize(bool useDictionary) { m_Solid.ResetContent(); @@ -1556,24 +1576,26 @@ void CCompressDialog::SetSolidBlockSize(bool useDictionary) m_Solid.SetCurSel(0); } - UInt64 blockSize; + const UInt64 cs = Get_Lzma2_ChunkSize(dict); + + // Solid Block Size + UInt64 blockSize = cs; // for xz if (is7z) { - blockSize = (UInt64)dict << 7; + // we use same default block sizes as defined in 7z encoder + UInt64 kMaxSize = (UInt64)1 << 32; + if (GetMethodID() == kLZMA2) + { + blockSize = cs << 6; + kMaxSize = (UInt64)1 << 34; + } + else + blockSize = (UInt64)dict << 7; + const UInt32 kMinSize = (UInt32)1 << 24; - const UInt64 kMaxSize = (UInt64)1 << 32; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - } - else - { - blockSize = (UInt64)dict << 2; - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; if (blockSize < kMinSize) blockSize = kMinSize; if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dict) blockSize = dict; } for (unsigned i = 20; i <= 36; i++) @@ -1616,6 +1638,8 @@ void CCompressDialog::SetNumThreads() return; UInt32 numHardwareThreads = NSystem::GetNumberOfProcessors(); + // numHardwareThreads = 64; + UInt32 defaultValue = numHardwareThreads; { @@ -1634,7 +1658,7 @@ void CCompressDialog::SetNumThreads() switch (methodID) { case kLZMA: numAlgoThreadsMax = 2; break; - case kLZMA2: numAlgoThreadsMax = 32; break; + case kLZMA2: numAlgoThreadsMax = 256; break; case kBZip2: numAlgoThreadsMax = 32; break; } if (IsZipFormat()) @@ -1688,9 +1712,12 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) hs |= (hs >> 4); hs |= (hs >> 8); hs >>= 1; - hs |= 0xFFFF; - if (hs > (1 << 24)) + if (hs >= (1 << 24)) hs >>= 1; + hs |= (1 << 16) - 1; + // if (numHashBytes >= 5) + if (level < 5) + hs |= (256 << 10) - 1; hs++; UInt64 size1 = (UInt64)hs * 4; size1 += (UInt64)dict * 4; @@ -1711,10 +1738,7 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) if (methidId != kLZMA && numBlockThreads != 1) { - chunkSize = (UInt64)dict << 2; - chunkSize = MyMax(chunkSize, (UInt64)(1 << 20)); - chunkSize = MyMin(chunkSize, (UInt64)(1 << 28)); - chunkSize = MyMax(chunkSize, (UInt64)dict); + chunkSize = Get_Lzma2_ChunkSize(dict); if (IsXzFormat()) { @@ -1795,10 +1819,19 @@ void CCompressDialog::PrintMemUsage(UINT res, UInt64 value) SetItemText(res, TEXT("?")); return; } - value = (value + (1 << 20) - 1) >> 20; TCHAR s[40]; - ConvertUInt64ToString(value, s); - lstrcat(s, TEXT(" MB")); + if (value <= ((UInt64)16 << 30)) + { + value = (value + (1 << 20) - 1) >> 20; + ConvertUInt64ToString(value, s); + lstrcat(s, TEXT(" MB")); + } + else + { + value = (value + (1 << 30) - 1) >> 30; + ConvertUInt64ToString(value, s); + lstrcat(s, TEXT(" GB")); + } SetItemText(res, s); } diff --git a/CPP/7zip/UI/GUI/CompressDialog.h b/CPP/7zip/UI/GUI/CompressDialog.h index 41466147..6658de5c 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.h +++ b/CPP/7zip/UI/GUI/CompressDialog.h @@ -108,6 +108,7 @@ class CCompressDialog: public NWindows::NControl::CModalDialog NWindows::NControl::CEdit _password1Control; NWindows::NControl::CEdit _password2Control; NWindows::NControl::CComboBox _encryptionMethod; + int _default_encryptionMethod_Index; NCompression::CInfo m_RegistryInfo; diff --git a/CPP/7zip/UI/GUI/Extract.rc b/CPP/7zip/UI/GUI/Extract.rc index f75b2e6f..36bfb009 100644 --- a/CPP/7zip/UI/GUI/Extract.rc +++ b/CPP/7zip/UI/GUI/Extract.rc @@ -7,11 +7,11 @@ BEGIN IDS_MEM_ERROR "The system cannot allocate the required amount of memory" IDS_CANNOT_CREATE_FOLDER "Cannot create folder '{0}'" IDS_UPDATE_NOT_SUPPORTED "Update operations are not supported for this archive." - IDS_CANT_OPEN_ARCHIVE "Can not open file '{0}' as archive" - IDS_CANT_OPEN_ENCRYPTED_ARCHIVE "Can not open encrypted archive '{0}'. Wrong password?" + IDS_CANT_OPEN_ARCHIVE "Cannot open file '{0}' as archive" + IDS_CANT_OPEN_ENCRYPTED_ARCHIVE "Cannot open encrypted archive '{0}'. Wrong password?" IDS_UNSUPPORTED_ARCHIVE_TYPE "Unsupported archive type" - IDS_CANT_OPEN_AS_TYPE "Can not open the file as {0} archive" + IDS_CANT_OPEN_AS_TYPE "Cannot open the file as {0} archive" IDS_IS_OPEN_AS_TYPE "The file is open as {0} archive" IDS_IS_OPEN_WITH_OFFSET "The archive is open with offset" @@ -45,7 +45,7 @@ BEGIN IDS_EXTRACT_MSG_DATA_ERROR "Data error" IDS_EXTRACT_MSG_CRC_ERROR "CRC failed" IDS_EXTRACT_MSG_UNAVAILABLE_DATA "Unavailable data" - IDS_EXTRACT_MSG_UEXPECTED_END "Unexpected end of data"; + IDS_EXTRACT_MSG_UEXPECTED_END "Unexpected end of data" IDS_EXTRACT_MSG_DATA_AFTER_END "There are some data after the end of the payload data" IDS_EXTRACT_MSG_IS_NOT_ARC "Is not archive" IDS_EXTRACT_MSG_HEADERS_ERROR "Headers Error" diff --git a/CPP/7zip/UI/GUI/ExtractDialog.cpp b/CPP/7zip/UI/GUI/ExtractDialog.cpp index b36a4943..5132084d 100644 --- a/CPP/7zip/UI/GUI/ExtractDialog.cpp +++ b/CPP/7zip/UI/GUI/ExtractDialog.cpp @@ -28,6 +28,8 @@ using namespace NName; extern HINSTANCE g_hInstance; +#ifndef _SFX + static const UInt32 kPathMode_IDs[] = { IDS_EXTRACT_PATHS_FULL, @@ -44,8 +46,6 @@ static const UInt32 kOverwriteMode_IDs[] = IDS_EXTRACT_OVERWRITE_RENAME_EXISTING }; -#ifndef _SFX - static const // NExtract::NPathMode::EEnum int @@ -94,6 +94,7 @@ static const unsigned kHistorySize = 16; #ifndef _SFX // it's used in CompressDialog also +void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned numItems, const int *values, int curVal); void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned numItems, const int *values, int curVal) { int curSel = 0; @@ -110,6 +111,7 @@ void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned n } // it's used in CompressDialog also +bool GetBoolsVal(const CBoolPair &b1, const CBoolPair &b2); bool GetBoolsVal(const CBoolPair &b1, const CBoolPair &b2) { if (b1.Def) return b1.Val; @@ -286,6 +288,7 @@ void CExtractDialog::OnButtonSetPath() _path.SetText(resultPath); } +void AddUniqueString(UStringVector &list, const UString &s); void AddUniqueString(UStringVector &list, const UString &s) { FOR_VECTOR (i, list) @@ -357,7 +360,7 @@ void CExtractDialog::OnOK() if (currentItem == CB_ERR) { _path.GetText(s); - if (_path.GetCount() >= kHistorySize) + if (_path.GetCount() >= (int)kHistorySize) currentItem = _path.GetCount() - 1; } else diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index 2674ec98..f5676d85 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -8,7 +8,7 @@ #include "../../../Common/MyWindows.h" -#include <shlwapi.h> +#include <Shlwapi.h> #include "../../../Common/MyInitGuid.h" @@ -34,10 +34,14 @@ using namespace NWindows; +extern +HINSTANCE g_hInstance; HINSTANCE g_hInstance; #ifndef UNDER_CE +extern +DWORD g_ComCtl32Version; DWORD g_ComCtl32Version; static DWORD GetDllVersion(LPCTSTR dllName) @@ -46,7 +50,7 @@ static DWORD GetDllVersion(LPCTSTR dllName) HINSTANCE hinstDll = LoadLibrary(dllName); if (hinstDll) { - DLLGETVERSIONPROC pDllGetVersion = (DLLGETVERSIONPROC)GetProcAddress(hinstDll, "DllGetVersion"); + DLLGETVERSIONPROC pDllGetVersion = (DLLGETVERSIONPROC)(void *)GetProcAddress(hinstDll, "DllGetVersion"); if (pDllGetVersion) { DLLVERSIONINFO dvi; @@ -63,6 +67,8 @@ static DWORD GetDllVersion(LPCTSTR dllName) #endif +extern +bool g_LVN_ITEMACTIVATE_Support; bool g_LVN_ITEMACTIVATE_Support = true; static void ErrorMessage(LPCWSTR message) @@ -90,7 +96,7 @@ static int ShowMemErrorMessage() static int ShowSysErrorMessage(DWORD errorCode) { - if (errorCode == E_OUTOFMEMORY) + if ((HRESULT)errorCode == E_OUTOFMEMORY) return ShowMemErrorMessage(); ErrorMessage(HResultToMessage(errorCode)); return NExitCode::kFatalError; @@ -128,7 +134,17 @@ static int Main2() codecs->CaseSensitiveChange = options.CaseSensitiveChange; codecs->CaseSensitive = options.CaseSensitive; ThrowException_if_Error(codecs->Load()); - + + #ifdef EXTERNAL_CODECS + { + UString s; + codecs->GetCodecsErrorMessage(s); + if (!s.IsEmpty()) + MessageBoxW(0, s, L"7-Zip", MB_ICONERROR); + } + #endif + + bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); if (codecs->Formats.Size() == 0 && @@ -343,7 +359,9 @@ static int Main2() return 0; } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ErrorMessage("Unsupported Windows version"); return NExitCode::kFatalError; +#endif int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, #ifdef UNDER_CE diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index b70e2848..41c1e04d 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -55,7 +55,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-Zip\7zg.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zg.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "GUI - Win32 Debug" @@ -82,7 +82,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-Zip\7zg.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zg.exe" /pdbtype:sept !ELSEIF "$(CFG)" == "GUI - Win32 ReleaseU" @@ -109,7 +109,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386 /out:"C:\UTIL\7zg.exe" -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Program Files\7-Zip\7zgn.exe" /opt:NOWIN98 +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /machine:I386 /out:"C:\Util\7zg.exe" /opt:NOWIN98 # SUBTRACT LINK32 /pdb:none !ELSEIF "$(CFG)" == "GUI - Win32 DebugU" @@ -137,7 +137,7 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\UTIL\7zg.exe" /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Program Files\7-Zip\7zgn.exe" /pdbtype:sept +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib comctl32.lib htmlhelp.lib /nologo /subsystem:windows /debug /machine:I386 /out:"C:\Util\7zg.exe" /pdbtype:sept !ENDIF @@ -1156,6 +1156,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\TimeUtils.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/GUI/StdAfx.h b/CPP/7zip/UI/GUI/StdAfx.h index 64290769..1918c8c4 100644 --- a/CPP/7zip/UI/GUI/StdAfx.h +++ b/CPP/7zip/UI/GUI/StdAfx.h @@ -11,9 +11,9 @@ // #include "../../../Common/MyWindows.h" -// #include <commctrl.h> +// #include <CommCtrl.h> // #include <ShlObj.h> -// #include <shlwapi.h> +// #include <Shlwapi.h> // #define printf(x) NO_PRINTF_(x) // #define sprintf(x) NO_SPRINTF_(x) diff --git a/CPP/7zip/UI/GUI/UpdateGUI.cpp b/CPP/7zip/UI/GUI/UpdateGUI.cpp index 2e738d3a..af07e550 100644 --- a/CPP/7zip/UI/GUI/UpdateGUI.cpp +++ b/CPP/7zip/UI/GUI/UpdateGUI.cpp @@ -56,9 +56,9 @@ HRESULT CThreadUpdating::ProcessVirt() ei, UpdateCallbackGUI, UpdateCallbackGUI, needSetPath); FinalMessage.ErrorMessage.Message = ei.Message.Ptr(); ErrorPaths = ei.FileNames; - if (ei.SystemError != S_OK && ei.SystemError != E_FAIL && ei.SystemError != E_ABORT) - return ei.SystemError; - return res; + if (res != S_OK) + return res; + return HRESULT_FROM_WIN32(ei.SystemError); } static void AddProp(CObjectVector<CProperty> &properties, const char *name, const UString &value) diff --git a/CPP/7zip/UI/GUI/makefile b/CPP/7zip/UI/GUI/makefile index f8416022..255c4f24 100644 --- a/CPP/7zip/UI/GUI/makefile +++ b/CPP/7zip/UI/GUI/makefile @@ -55,6 +55,7 @@ WIN_OBJS = \ $O\Shell.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ $O\Window.obj \ diff --git a/CPP/7zip/cmpl_clang.mak b/CPP/7zip/cmpl_clang.mak new file mode 100644 index 00000000..e62e1e62 --- /dev/null +++ b/CPP/7zip/cmpl_clang.mak @@ -0,0 +1,3 @@ +include ../../var_clang.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_clang_arm64.mak b/CPP/7zip/cmpl_clang_arm64.mak new file mode 100644 index 00000000..3f6b02bf --- /dev/null +++ b/CPP/7zip/cmpl_clang_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_clang_arm64.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_clang_x64.mak b/CPP/7zip/cmpl_clang_x64.mak new file mode 100644 index 00000000..b61e2af6 --- /dev/null +++ b/CPP/7zip/cmpl_clang_x64.mak @@ -0,0 +1,3 @@ +include ../../var_clang_x64.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_clang_x86.mak b/CPP/7zip/cmpl_clang_x86.mak new file mode 100644 index 00000000..0e5cb76c --- /dev/null +++ b/CPP/7zip/cmpl_clang_x86.mak @@ -0,0 +1,3 @@ +include ../../var_clang_x86.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_gcc.mak b/CPP/7zip/cmpl_gcc.mak new file mode 100644 index 00000000..7a1aef2e --- /dev/null +++ b/CPP/7zip/cmpl_gcc.mak @@ -0,0 +1,3 @@ +include ../../var_gcc.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_gcc_arm64.mak b/CPP/7zip/cmpl_gcc_arm64.mak new file mode 100644 index 00000000..53a85844 --- /dev/null +++ b/CPP/7zip/cmpl_gcc_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_arm64.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_gcc_x64.mak b/CPP/7zip/cmpl_gcc_x64.mak new file mode 100644 index 00000000..500c30e4 --- /dev/null +++ b/CPP/7zip/cmpl_gcc_x64.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_x64.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_gcc_x86.mak b/CPP/7zip/cmpl_gcc_x86.mak new file mode 100644 index 00000000..e7687070 --- /dev/null +++ b/CPP/7zip/cmpl_gcc_x86.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_x86.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_mac_arm64.mak b/CPP/7zip/cmpl_mac_arm64.mak new file mode 100644 index 00000000..941028e9 --- /dev/null +++ b/CPP/7zip/cmpl_mac_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_mac_arm64.mak +include ../../warn_clang_mac.mak +include makefile.gcc diff --git a/CPP/7zip/cmpl_mac_x64.mak b/CPP/7zip/cmpl_mac_x64.mak new file mode 100644 index 00000000..d3aa0396 --- /dev/null +++ b/CPP/7zip/cmpl_mac_x64.mak @@ -0,0 +1,3 @@ +include ../../var_mac_x64.mak +include ../../warn_clang_mac.mak +include makefile.gcc diff --git a/CPP/7zip/var_clang.mak b/CPP/7zip/var_clang.mak new file mode 100644 index 00000000..a6df26e7 --- /dev/null +++ b/CPP/7zip/var_clang.mak @@ -0,0 +1,11 @@ +PLATFORM= +O=b/c +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/CPP/7zip/var_clang_arm64.mak b/CPP/7zip/var_clang_arm64.mak new file mode 100644 index 00000000..4b354096 --- /dev/null +++ b/CPP/7zip/var_clang_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/CPP/7zip/var_clang_x64.mak b/CPP/7zip/var_clang_x64.mak new file mode 100644 index 00000000..fefed51c --- /dev/null +++ b/CPP/7zip/var_clang_x64.mak @@ -0,0 +1,12 @@ +PLATFORM=x64 +O=b/c_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/CPP/7zip/var_clang_x86.mak b/CPP/7zip/var_clang_x86.mak new file mode 100644 index 00000000..5f3c2d9c --- /dev/null +++ b/CPP/7zip/var_clang_x86.mak @@ -0,0 +1,12 @@ +PLATFORM=x86 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/CPP/7zip/var_gcc.mak b/CPP/7zip/var_gcc.mak new file mode 100644 index 00000000..664491cf --- /dev/null +++ b/CPP/7zip/var_gcc.mak @@ -0,0 +1,12 @@ +PLATFORM= +O=b/g +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/CPP/7zip/var_gcc_arm64.mak b/CPP/7zip/var_gcc_arm64.mak new file mode 100644 index 00000000..4bbb687d --- /dev/null +++ b/CPP/7zip/var_gcc_arm64.mak @@ -0,0 +1,12 @@ +PLATFORM=arm64 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-mtune=cortex-a53 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/CPP/7zip/var_gcc_x64.mak b/CPP/7zip/var_gcc_x64.mak new file mode 100644 index 00000000..1acf604f --- /dev/null +++ b/CPP/7zip/var_gcc_x64.mak @@ -0,0 +1,10 @@ +PLATFORM=x64 +O=b/g_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ diff --git a/CPP/7zip/var_gcc_x86.mak b/CPP/7zip/var_gcc_x86.mak new file mode 100644 index 00000000..288bf94b --- /dev/null +++ b/CPP/7zip/var_gcc_x86.mak @@ -0,0 +1,11 @@ +PLATFORM=x86 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + diff --git a/CPP/7zip/var_mac_arm64.mak b/CPP/7zip/var_mac_arm64.mak new file mode 100644 index 00000000..adf5fa1d --- /dev/null +++ b/CPP/7zip/var_mac_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/m_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-arch arm64 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/CPP/7zip/var_mac_x64.mak b/CPP/7zip/var_mac_x64.mak new file mode 100644 index 00000000..13d7aa7f --- /dev/null +++ b/CPP/7zip/var_mac_x64.mak @@ -0,0 +1,11 @@ +PLATFORM=x64 +O=b/m_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-arch x86_64 +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/CPP/7zip/warn_clang.mak b/CPP/7zip/warn_clang.mak new file mode 100644 index 00000000..ed4f908f --- /dev/null +++ b/CPP/7zip/warn_clang.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_1 = \ + -Wno-deprecated-copy-dtor \ + + + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_1) diff --git a/CPP/7zip/warn_clang_mac.mak b/CPP/7zip/warn_clang_mac.mak new file mode 100644 index 00000000..41044a2c --- /dev/null +++ b/CPP/7zip/warn_clang_mac.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_MAC = \ + -Wno-poison-system-directories \ + -Wno-c++11-long-long \ + -Wno-atomic-implicit-seq-cst \ + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_MAC) diff --git a/CPP/7zip/warn_gcc.mak b/CPP/7zip/warn_gcc.mak new file mode 100644 index 00000000..5fb747dc --- /dev/null +++ b/CPP/7zip/warn_gcc.mak @@ -0,0 +1,53 @@ +CFLAGS_WARN_GCC_4_5 = \ + +CFLAGS_WARN_GCC_6 = \ + -Waddress \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wcast-align \ + -Wcomment \ + -Wdiv-by-zero \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Winit-self \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + +# -Wno-strict-aliasing + +CFLAGS_WARN_GCC_9 = \ + -Waddress \ + -Waddress-of-packed-member \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wbool-operation \ + -Wcast-align \ + -Wcast-align=strict \ + -Wcomment \ + -Wdangling-else \ + -Wdiv-by-zero \ + -Wduplicated-branches \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Wimplicit-fallthrough=5 \ + -Winit-self \ + -Wint-in-bool-context \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + -Wconversion \ + +# -Wno-sign-conversion \ + +CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ + -Wno-strict-aliasing \ + + +CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) \ + +# $(CFLAGS_WARN_GCC_PPMD_UNALIGNED) + +
\ No newline at end of file diff --git a/CPP/Build.mak b/CPP/Build.mak index f53286b1..1ef676e0 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -12,17 +12,23 @@ O=o !ENDIF !ENDIF +!IF "$(CC)" != "clang-cl" # CFLAGS = $(CFLAGS) -FAsc -Fa$O/asm/ +!ENDIF !IF "$(PLATFORM)" == "x64" -MY_ML = ml64 -Dx64 -WX +MY_ML = ml64 -WX +#-Dx64 !ELSEIF "$(PLATFORM)" == "arm" MY_ML = armasm -WX !ELSE MY_ML = ml -WX +# -DABI_CDECL !ENDIF +# MY_ML = "$(MY_ML) -Fl$O\asm\ + !IFDEF UNDER_CE RFLAGS = $(RFLAGS) -dUNDER_CE @@ -47,6 +53,34 @@ COMPL_ASM = $(MY_ML) -c -Fo$O/ $** CFLAGS = $(CFLAGS) -nologo -c -Fo$O/ -W4 -WX -EHsc -Gy -GR- -GF +!IF "$(CC)" == "clang-cl" + +CFLAGS = $(CFLAGS) \ + -Werror \ + -Wextra \ + -Wall \ + -Weverything \ + -Wno-extra-semi-stmt \ + -Wno-extra-semi \ + -Wno-zero-as-null-pointer-constant \ + -Wno-sign-conversion \ + -Wno-old-style-cast \ + -Wno-reserved-id-macro \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-deprecated-copy-dtor \ + -Wno-exit-time-destructors \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-nonportable-system-include-path \ + -Wno-c++98-compat-pedantic \ + -Wno-cast-qual \ + -Wc++11-extensions \ + +!ENDIF + !IFDEF MY_DYNAMIC_LINK CFLAGS = $(CFLAGS) -MD !ELSE @@ -55,10 +89,15 @@ CFLAGS = $(CFLAGS) -MT !ENDIF !ENDIF + +CFLAGS = $(CFLAGS_COMMON) $(CFLAGS) + !IFNDEF OLD_COMPILER CFLAGS = $(CFLAGS) -GS- -Zc:forScope -Zc:wchar_t !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -MP2 +!IF "$(CC)" != "clang-cl" +CFLAGS = $(CFLAGS) -MP4 +!ENDIF !IFNDEF PLATFORM # CFLAGS = $(CFLAGS) -arch:IA32 !ENDIF @@ -134,6 +173,11 @@ CCOMPL_USE = $(CC) $(CFLAGS_C_ALL) -Yu"Precomp.h" -Fp$O/a.pch $** CCOMPL = $(CC) $(CFLAGS_C_ALL) $** CCOMPLB = $(CC) $(CFLAGS_C_ALL) $< +!IF "$(CC)" == "clang-cl" +COMPL = $(COMPL) -FI StdAfx.h +COMPLB = $(COMPLB) -FI StdAfx.h +CCOMPL_USE = $(CCOMPL_USE) -FI Precomp.h +!ENDIF all: $(PROGPATH) diff --git a/CPP/Common/CRC.cpp b/CPP/Common/CRC.cpp index 9a9f81fb..c6b7d5e4 100644 --- a/CPP/Common/CRC.cpp +++ b/CPP/Common/CRC.cpp @@ -4,4 +4,4 @@ #include "../../C/7zCrc.h" -struct CCRCTableInit { CCRCTableInit() { CrcGenerateTable(); } } g_CRCTableInit; +static struct CCRCTableInit { CCRCTableInit() { CrcGenerateTable(); } } g_CRCTableInit; diff --git a/CPP/Common/C_FileIO.cpp b/CPP/Common/C_FileIO.cpp index 7c629390..4bd3fadc 100644 --- a/CPP/Common/C_FileIO.cpp +++ b/CPP/Common/C_FileIO.cpp @@ -1,92 +1,3 @@ // Common/C_FileIO.cpp -#include "C_FileIO.h" - -#include <fcntl.h> -#ifdef _WIN32 -#include <io.h> -#else -#include <unistd.h> -#endif - -namespace NC { -namespace NFile { -namespace NIO { - -bool CFileBase::OpenBinary(const char *name, int flags) -{ - #ifdef O_BINARY - flags |= O_BINARY; - #endif - Close(); - _handle = ::open(name, flags, 0666); - return _handle != -1; -} - -bool CFileBase::Close() -{ - if (_handle == -1) - return true; - if (close(_handle) != 0) - return false; - _handle = -1; - return true; -} - -bool CFileBase::GetLength(UInt64 &length) const -{ - off_t curPos = Seek(0, SEEK_CUR); - off_t lengthTemp = Seek(0, SEEK_END); - Seek(curPos, SEEK_SET); - length = (UInt64)lengthTemp; - return true; -} - -off_t CFileBase::Seek(off_t distanceToMove, int moveMethod) const -{ - return ::lseek(_handle, distanceToMove, moveMethod); -} - -///////////////////////// -// CInFile - -bool CInFile::Open(const char *name) -{ - return CFileBase::OpenBinary(name, O_RDONLY); -} - -bool CInFile::OpenShared(const char *name, bool) -{ - return Open(name); -} - -ssize_t CInFile::Read(void *data, size_t size) -{ - return read(_handle, data, size); -} - -///////////////////////// -// COutFile - -bool COutFile::Create(const char *name, bool createAlways) -{ - if (createAlways) - { - Close(); - _handle = ::creat(name, 0666); - return _handle != -1; - } - return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY); -} - -bool COutFile::Open(const char *name, DWORD creationDisposition) -{ - return Create(name, false); -} - -ssize_t COutFile::Write(const void *data, size_t size) -{ - return write(_handle, data, size); -} - -}}} +#include "StdAfx.h" diff --git a/CPP/Common/C_FileIO.h b/CPP/Common/C_FileIO.h index ff4ec162..6818558b 100644 --- a/CPP/Common/C_FileIO.h +++ b/CPP/Common/C_FileIO.h @@ -3,51 +3,4 @@ #ifndef __COMMON_C_FILEIO_H #define __COMMON_C_FILEIO_H -#include <stdio.h> -#include <sys/types.h> - -#include "MyTypes.h" -#include "MyWindows.h" - -#ifdef _WIN32 -#ifdef _MSC_VER -typedef size_t ssize_t; -#endif -#endif - -namespace NC { -namespace NFile { -namespace NIO { - -class CFileBase -{ -protected: - int _handle; - bool OpenBinary(const char *name, int flags); -public: - CFileBase(): _handle(-1) {}; - ~CFileBase() { Close(); } - bool Close(); - bool GetLength(UInt64 &length) const; - off_t Seek(off_t distanceToMove, int moveMethod) const; -}; - -class CInFile: public CFileBase -{ -public: - bool Open(const char *name); - bool OpenShared(const char *name, bool shareForWrite); - ssize_t Read(void *data, size_t size); -}; - -class COutFile: public CFileBase -{ -public: - bool Create(const char *name, bool createAlways); - bool Open(const char *name, DWORD creationDisposition); - ssize_t Write(const void *data, size_t size); -}; - -}}} - #endif diff --git a/CPP/Common/CommandLineParser.cpp b/CPP/Common/CommandLineParser.cpp index 145f3435..465e0fde 100644 --- a/CPP/Common/CommandLineParser.cpp +++ b/CPP/Common/CommandLineParser.cpp @@ -84,7 +84,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi if (IsString1PrefixedByString2_NoCase_Ascii((const wchar_t *)s + pos, key)) { switchIndex = i; - maxLen = switchLen; + maxLen = (int)switchLen; } } @@ -94,7 +94,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi return false; } - pos += maxLen; + pos += (unsigned)maxLen; CSwitchResult &sw = _switches[switchIndex]; const CSwitchForm &form = switchForms[switchIndex]; @@ -107,7 +107,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi sw.ThereIs = true; - int rem = s.Len() - pos; + const unsigned rem = s.Len() - pos; if (rem < form.MinLen) { ErrorMessage = "Too short switch:"; @@ -178,7 +178,7 @@ bool CParser::ParseStrings(const CSwitchForm *switchForms, unsigned numSwitches, { if (s.IsEqualTo(kStopSwitchParsing)) { - StopSwitchIndex = NonSwitchStrings.Size(); + StopSwitchIndex = (int)NonSwitchStrings.Size(); continue; } if (!s.IsEmpty() && IsItSwitchChar(s[0])) diff --git a/CPP/Common/CommandLineParser.h b/CPP/Common/CommandLineParser.h index 1dbdd4ea..fbd4fa58 100644 --- a/CPP/Common/CommandLineParser.h +++ b/CPP/Common/CommandLineParser.h @@ -38,7 +38,7 @@ struct CSwitchResult int PostCharIndex; UStringVector PostStrings; - CSwitchResult(): ThereIs(false) {}; + CSwitchResult(): ThereIs(false) {} }; class CParser diff --git a/CPP/Common/Common.h b/CPP/Common/Common.h index 5430a92d..8dac613c 100644 --- a/CPP/Common/Common.h +++ b/CPP/Common/Common.h @@ -40,4 +40,18 @@ you can change this h file or h files included in this file. #define MY_ARRAY_NEW(p, T, size) p = new T[size]; #endif +#if (defined(__GNUC__) && (__GNUC__ >= 8)) + #define MY_ATTR_NORETURN __attribute__((noreturn)) +#elif (defined(__clang__) && (__clang_major__ >= 3)) + #if __has_feature(cxx_attributes) + #define MY_ATTR_NORETURN [[noreturn]] + #else + #define MY_ATTR_NORETURN __attribute__ ((noreturn)) + #endif +#elif (defined(_MSC_VER) && (_MSC_VER >= 1900)) + #define MY_ATTR_NORETURN [[noreturn]] +#else + #define MY_ATTR_NORETURN +#endif + #endif diff --git a/CPP/Common/CrcReg.cpp b/CPP/Common/CrcReg.cpp index 4b662f52..fdbba77b 100644 --- a/CPP/Common/CrcReg.cpp +++ b/CPP/Common/CrcReg.cpp @@ -16,8 +16,10 @@ typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); extern CRC_FUNC g_CrcUpdate; -extern CRC_FUNC g_CrcUpdateT8; extern CRC_FUNC g_CrcUpdateT4; +extern CRC_FUNC g_CrcUpdateT8; +extern CRC_FUNC g_CrcUpdateT0_32; +extern CRC_FUNC g_CrcUpdateT0_64; EXTERN_C_END @@ -41,25 +43,20 @@ public: bool CCrcHasher::SetFunctions(UInt32 tSize) { - _updateFunc = g_CrcUpdate; + CRC_FUNC f = NULL; + if (tSize == 0) f = g_CrcUpdate; + else if (tSize == 1) f = CrcUpdateT1; + else if (tSize == 4) f = g_CrcUpdateT4; + else if (tSize == 8) f = g_CrcUpdateT8; + else if (tSize == 32) f = g_CrcUpdateT0_32; + else if (tSize == 64) f = g_CrcUpdateT0_64; - if (tSize == 1) - _updateFunc = CrcUpdateT1; - else if (tSize == 4) + if (!f) { - if (g_CrcUpdateT4) - _updateFunc = g_CrcUpdateT4; - else - return false; + _updateFunc = g_CrcUpdate; + return false; } - else if (tSize == 8) - { - if (g_CrcUpdateT8) - _updateFunc = g_CrcUpdateT8; - else - return false; - } - + _updateFunc = f; return true; } diff --git a/CPP/Common/Defs.h b/CPP/Common/Defs.h index 1fbd78bd..9adb88cf 100644 --- a/CPP/Common/Defs.h +++ b/CPP/Common/Defs.h @@ -10,6 +10,7 @@ template <class T> inline int MyCompare(T a, T b) { return a == b ? 0 : (a < b ? -1 : 1); } inline int BoolToInt(bool v) { return (v ? 1 : 0); } +inline unsigned BoolToUInt(bool v) { return (v ? (unsigned)1 : (unsigned)0); } inline bool IntToBool(int v) { return (v != 0); } #endif diff --git a/CPP/Common/DynLimBuf.cpp b/CPP/Common/DynLimBuf.cpp index 1d1d99dc..79141040 100644 --- a/CPP/Common/DynLimBuf.cpp +++ b/CPP/Common/DynLimBuf.cpp @@ -51,7 +51,7 @@ CDynLimBuf & CDynLimBuf::operator+=(char c) throw() _chars = newBuf; _size = n; } - _chars[_pos++] = c; + _chars[_pos++] = (Byte)c; return *this; } diff --git a/CPP/Common/DynamicBuffer.h b/CPP/Common/DynamicBuffer.h index 44e3df7f..f6f6b157 100644 --- a/CPP/Common/DynamicBuffer.h +++ b/CPP/Common/DynamicBuffer.h @@ -54,7 +54,7 @@ public: memcpy(GetCurPtrAndGrow(size), data, size * sizeof(T)); } - const size_t GetPos() const { return _pos; } + size_t GetPos() const { return _pos; } // void Empty() { _pos = 0; } }; diff --git a/CPP/Common/IntToString.cpp b/CPP/Common/IntToString.cpp index 05b1c148..c87643c8 100644 --- a/CPP/Common/IntToString.cpp +++ b/CPP/Common/IntToString.cpp @@ -10,20 +10,20 @@ unsigned char temp[tempSize]; unsigned i = 0; \ while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ *s++ = (charType)('0' + (unsigned)val); \ - while (i != 0) { i--; *s++ = temp[i]; } \ - *s = 0; + while (i != 0) { i--; *s++ = (charType)temp[i]; } \ + *s = 0; \ + return s; -void ConvertUInt32ToString(UInt32 val, char *s) throw() +char * ConvertUInt32ToString(UInt32 val, char *s) throw() { CONVERT_INT_TO_STR(char, 16); } -void ConvertUInt64ToString(UInt64 val, char *s) throw() +char * ConvertUInt64ToString(UInt64 val, char *s) throw() { if (val <= (UInt32)0xFFFFFFFF) { - ConvertUInt32ToString((UInt32)val, s); - return; + return ConvertUInt32ToString((UInt32)val, s); } CONVERT_INT_TO_STR(char, 24); } @@ -119,17 +119,16 @@ void ConvertUInt32ToHex8Digits(UInt32 val, wchar_t *s) } */ -void ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() +wchar_t * ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() { CONVERT_INT_TO_STR(wchar_t, 16); } -void ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() +wchar_t * ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() { if (val <= (UInt32)0xFFFFFFFF) { - ConvertUInt32ToString((UInt32)val, s); - return; + return ConvertUInt32ToString((UInt32)val, s); } CONVERT_INT_TO_STR(wchar_t, 24); } @@ -141,7 +140,7 @@ void ConvertInt64ToString(Int64 val, char *s) throw() *s++ = '-'; val = -val; } - ConvertUInt64ToString(val, s); + ConvertUInt64ToString((UInt64)val, s); } void ConvertInt64ToString(Int64 val, wchar_t *s) throw() @@ -151,7 +150,7 @@ void ConvertInt64ToString(Int64 val, wchar_t *s) throw() *s++ = L'-'; val = -val; } - ConvertUInt64ToString(val, s); + ConvertUInt64ToString((UInt64)val, s); } diff --git a/CPP/Common/IntToString.h b/CPP/Common/IntToString.h index d4110d1d..d0a96ef1 100644 --- a/CPP/Common/IntToString.h +++ b/CPP/Common/IntToString.h @@ -5,11 +5,13 @@ #include "MyTypes.h" -void ConvertUInt32ToString(UInt32 value, char *s) throw(); -void ConvertUInt64ToString(UInt64 value, char *s) throw(); +// return: the pointer to the "terminating" null character after written characters -void ConvertUInt32ToString(UInt32 value, wchar_t *s) throw(); -void ConvertUInt64ToString(UInt64 value, wchar_t *s) throw(); +char * ConvertUInt32ToString(UInt32 value, char *s) throw(); +char * ConvertUInt64ToString(UInt64 value, char *s) throw(); + +wchar_t * ConvertUInt32ToString(UInt32 value, wchar_t *s) throw(); +wchar_t * ConvertUInt64ToString(UInt64 value, wchar_t *s) throw(); void ConvertUInt64ToOct(UInt64 value, char *s) throw(); diff --git a/CPP/Common/Lang.cpp b/CPP/Common/Lang.cpp index e959ba48..35d37525 100644 --- a/CPP/Common/Lang.cpp +++ b/CPP/Common/Lang.cpp @@ -31,7 +31,7 @@ bool CLang::OpenFromString(const AString &s2) for (const char *p = kLangSignature;; i++) { - Byte c = *p++; + Byte c = (Byte)(*p++); if (c == 0) break; if (s[i] != c) @@ -122,10 +122,10 @@ bool CLang::Open(CFSTR fileName, const char *id) return false; AString s; - unsigned len = (unsigned)length; + const unsigned len = (unsigned)length; char *p = s.GetBuf(len); - UInt32 processed; - if (!file.Read(p, len, processed)) + size_t processed; + if (!file.ReadFull(p, len, processed)) return false; file.Close(); if (len != processed) @@ -159,5 +159,5 @@ const wchar_t *CLang::Get(UInt32 id) const throw() int index = _ids.FindInSorted(id); if (index < 0) return NULL; - return _text + (size_t)_offsets[index]; + return _text + (size_t)_offsets[(unsigned)index]; } diff --git a/CPP/Common/ListFileUtils.cpp b/CPP/Common/ListFileUtils.cpp index edd37eb1..b361b378 100644 --- a/CPP/Common/ListFileUtils.cpp +++ b/CPP/Common/ListFileUtils.cpp @@ -4,14 +4,19 @@ #include "../../C/CpuArch.h" -#include "../Windows/FileIO.h" - #include "ListFileUtils.h" #include "MyBuffer.h" #include "StringConvert.h" #include "UTFConvert.h" -static const char kQuoteChar = '\"'; +#include "../Windows/FileIO.h" + +#define CSysInFile NWindows::NFile::NIO::CInFile +#define MY_GET_LAST_ERROR ::GetLastError() + + +#define kQuoteChar '\"' + static void AddName(UStringVector &strings, UString &s) { @@ -25,19 +30,37 @@ static void AddName(UStringVector &strings, UString &s) strings.Add(s); } + +static bool My_File_Read(CSysInFile &file, void *data, size_t size, DWORD &lastError) +{ + size_t processed; + if (!file.ReadFull(data, size, processed)) + { + lastError = MY_GET_LAST_ERROR; + return false; + } + if (processed != size) + { + lastError = 1; // error: size of listfile was changed + return false; + } + return true; +} + + bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, DWORD &lastError) { lastError = 0; - NWindows::NFile::NIO::CInFile file; + CSysInFile file; if (!file.Open(fileName)) { - lastError = ::GetLastError(); + lastError = MY_GET_LAST_ERROR; return false; } UInt64 fileSize; if (!file.GetLength(fileSize)) { - lastError = ::GetLastError(); + lastError = MY_GET_LAST_ERROR; return false; } if (fileSize >= ((UInt32)1 << 31) - 32) @@ -48,16 +71,12 @@ bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePag if ((fileSize & 1) != 0) return false; CByteArr buf((size_t)fileSize); - UInt32 processed; - if (!file.Read(buf, (UInt32)fileSize, processed)) - { - lastError = ::GetLastError(); - return false; - } - if (processed != fileSize) + + if (!My_File_Read(file, buf, (size_t)fileSize, lastError)) return false; + file.Close(); - unsigned num = (unsigned)fileSize / 2; + const unsigned num = (unsigned)fileSize / 2; wchar_t *p = u.GetBuf(num); if (codePage == MY__CP_UTF16) for (unsigned i = 0; i < num; i++) @@ -82,22 +101,21 @@ bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePag { AString s; char *p = s.GetBuf((unsigned)fileSize); - UInt32 processed; - if (!file.Read(p, (UInt32)fileSize, processed)) - { - lastError = ::GetLastError(); - return false; - } - if (processed != fileSize) + + if (!My_File_Read(file, p, (size_t)fileSize, lastError)) return false; + file.Close(); - s.ReleaseBuf_CalcLen((unsigned)processed); - if (s.Len() != processed) + s.ReleaseBuf_CalcLen((unsigned)fileSize); + if (s.Len() != fileSize) return false; // #ifdef CP_UTF8 if (codePage == CP_UTF8) { + // we must check UTF8 here, if convert function doesn't check + if (!CheckUTF8_AString(s)) + return false; if (!ConvertUTF8ToUnicode(s, u)) return false; } diff --git a/CPP/Common/MyBuffer.h b/CPP/Common/MyBuffer.h index 5d4e3475..18ab6fa4 100644 --- a/CPP/Common/MyBuffer.h +++ b/CPP/Common/MyBuffer.h @@ -4,6 +4,7 @@ #define __COMMON_MY_BUFFER_H #include "Defs.h" +#include "MyTypes.h" /* 7-Zip now uses CBuffer only as CByteBuffer. So there is no need to use MY_ARRAY_NEW macro in CBuffer code. */ @@ -91,6 +92,12 @@ public: _size = newSize; } + void Wipe() + { + if (_size != 0) + memset(_items, 0, _size * sizeof(T)); + } + CBuffer& operator=(const CBuffer &buffer) { if (&buffer != this) @@ -127,6 +134,17 @@ bool operator!=(const CBuffer<T>& b1, const CBuffer<T>& b2) typedef CBuffer<unsigned char> CByteBuffer; +class CByteBuffer_Wipe: public CByteBuffer +{ + CLASS_NO_COPY(CByteBuffer_Wipe) +public: + // CByteBuffer_Wipe(): CBuffer<unsigned char>() {} + CByteBuffer_Wipe(size_t size): CBuffer<unsigned char>(size) {} + ~CByteBuffer_Wipe() { Wipe(); } +}; + + + template <class T> class CObjArray { protected: diff --git a/CPP/Common/MyBuffer2.h b/CPP/Common/MyBuffer2.h index d61a72ef..de5ebbdd 100644 --- a/CPP/Common/MyBuffer2.h +++ b/CPP/Common/MyBuffer2.h @@ -57,6 +57,15 @@ public: ISzAlloc_Free(&g_AlignedAlloc, _data); } + CAlignedBuffer(size_t size): _size(0) + { + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (!_data) + throw 1; + _size = size; + } + void Free() { ISzAlloc_Free(&g_AlignedAlloc, _data); diff --git a/CPP/Common/MyCom.h b/CPP/Common/MyCom.h index 031921d3..524f6efb 100644 --- a/CPP/Common/MyCom.h +++ b/CPP/Common/MyCom.h @@ -4,6 +4,7 @@ #define __MY_COM_H #include "MyWindows.h" +#include "MyTypes.h" #ifndef RINOK #define RINOK(x) { HRESULT __result_ = (x); if (__result_ != S_OK) return __result_; } @@ -81,7 +82,7 @@ inline HRESULT StringToBstr(LPCOLESTR src, BSTR *bstr) class CMyComBSTR { BSTR m_str; - + CLASS_NO_COPY(CMyComBSTR) public: CMyComBSTR(): m_str(NULL) {} ~CMyComBSTR() { ::SysFreeString(m_str); } @@ -89,13 +90,23 @@ public: operator LPCOLESTR() const { return m_str; } // operator bool() const { return m_str != NULL; } // bool operator!() const { return m_str == NULL; } + + void Wipe_and_Free() + { + if (m_str) + { + memset(m_str, 0, ::SysStringLen(m_str) * sizeof(*m_str)); + Empty(); + } + } + private: // operator BSTR() const { return m_str; } CMyComBSTR(LPCOLESTR src) { m_str = ::SysAllocString(src); } // CMyComBSTR(int nSize) { m_str = ::SysAllocStringLen(NULL, nSize); } // CMyComBSTR(int nSize, LPCOLESTR sz) { m_str = ::SysAllocStringLen(sz, nSize); } - CMyComBSTR(const CMyComBSTR& src) { m_str = src.MyCopy(); } + // CMyComBSTR(const CMyComBSTR& src) { m_str = src.MyCopy(); } /* CMyComBSTR(REFGUID src) @@ -107,6 +118,7 @@ private: } */ + /* CMyComBSTR& operator=(const CMyComBSTR& src) { if (m_str != src.m_str) @@ -117,6 +129,7 @@ private: } return *this; } + */ CMyComBSTR& operator=(LPCOLESTR src) { @@ -158,6 +171,15 @@ private: }; +class CMyComBSTR_Wipe: public CMyComBSTR +{ + CLASS_NO_COPY(CMyComBSTR_Wipe) +public: + CMyComBSTR_Wipe(): CMyComBSTR() {} + ~CMyComBSTR_Wipe() { Wipe_and_Free(); } +}; + + /* If CMyUnknownImp doesn't use virtual destructor, the code size is smaller. @@ -168,17 +190,24 @@ private: virtual ~class_1(); In that case, class_1::Release() calls correct destructor of class_2. - Also you can use virtual ~CMyUnknownImp(), if you want to disable warning + We use virtual ~CMyUnknownImp() to disable warning "class has virtual functions, but destructor is not virtual". + + also we can use virtual ~IUnknown() {} in MyWindows.h */ class CMyUnknownImp { + CLASS_NO_COPY(CMyUnknownImp) public: ULONG __m_RefCount; CMyUnknownImp(): __m_RefCount(0) {} - // virtual + #ifdef _WIN32 + #if defined(__GNUC__) || defined(__clang__) + virtual // to disable GCC/CLANG varnings + #endif + #endif ~CMyUnknownImp() {} }; diff --git a/CPP/Common/MyGuidDef.h b/CPP/Common/MyGuidDef.h index 68745870..38aad6e6 100644 --- a/CPP/Common/MyGuidDef.h +++ b/CPP/Common/MyGuidDef.h @@ -18,6 +18,9 @@ typedef struct { #define REFGUID const GUID * #endif +// typedef GUID IID; +typedef GUID CLSID; + #define REFCLSID REFGUID #define REFIID REFGUID diff --git a/CPP/Common/MyInitGuid.h b/CPP/Common/MyInitGuid.h index 279fba5d..04d77e21 100644 --- a/CPP/Common/MyInitGuid.h +++ b/CPP/Common/MyInitGuid.h @@ -19,13 +19,17 @@ Also we need IID_IUnknown that is initialized in some file for linking: Other: we define IID_IUnknown in this file */ +#ifdef __clang__ + #pragma clang diagnostic ignored "-Wmissing-variable-declarations" +#endif + #ifdef _WIN32 #ifdef UNDER_CE #include <basetyps.h> #endif -#include <initguid.h> +#include <InitGuid.h> #ifdef UNDER_CE DEFINE_GUID(IID_IUnknown, diff --git a/CPP/Common/MyString.cpp b/CPP/Common/MyString.cpp index e2ec8a68..821c9b37 100644 --- a/CPP/Common/MyString.cpp +++ b/CPP/Common/MyString.cpp @@ -237,11 +237,25 @@ bool UString::IsPrefixedBy_Ascii_NoCase(const char *s) const throw() } } +bool StringsAreEqual_Ascii(const char *u, const char *a) throw() +{ + for (;;) + { + char c = *a; + if (c != *u) + return false; + if (c == 0) + return true; + a++; + u++; + } +} + bool StringsAreEqual_Ascii(const wchar_t *u, const char *a) throw() { for (;;) { - unsigned char c = *a; + unsigned char c = (unsigned char)*a; if (c != *u) return false; if (c == 0) @@ -632,9 +646,8 @@ AString &AString::operator+=(const AString &s) void AString::Add_UInt32(UInt32 v) { - char sz[16]; - ConvertUInt32ToString(v, sz); - (*this) += sz; + Grow(10); + _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); } void AString::SetFrom(const char *s, unsigned len) // no check @@ -835,7 +848,7 @@ void AString::Replace(char oldChar, char newChar) throw() char *chars = _chars; while ((unsigned)pos < _len) { - pos = Find(oldChar, pos); + pos = Find(oldChar, (unsigned)pos); if (pos < 0) break; chars[(unsigned)pos] = newChar; @@ -857,11 +870,11 @@ void AString::Replace(const AString &oldString, const AString &newString) int pos = 0; while ((unsigned)pos < _len) { - pos = Find(oldString, pos); + pos = Find(oldString, (unsigned)pos); if (pos < 0) break; - Delete(pos, oldLen); - Insert(pos, newString); + Delete((unsigned)pos, oldLen); + Insert((unsigned)pos, newString); pos += newLen; // number++; } @@ -1150,9 +1163,31 @@ void UString::SetFrom(const wchar_t *s, unsigned len) // no check _len = len; } -void UString::SetFromBstr(BSTR s) +void UString::SetFromBstr(LPCOLESTR s) { - unsigned len = ::SysStringLen(s); + unsigned len = ::SysStringLen((BSTR)(void *)(s)); + + /* + #if WCHAR_MAX > 0xffff + size_t num_wchars = 0; + for (size_t i = 0; i < len;) + { + wchar_t c = s[i++]; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = s[i]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + i++; + } + } + num_wchars++; + } + len = num_wchars; + #endif + */ + if (len > _limit) { wchar_t *newBuf = MY_STRING_NEW_wchar_t(len + 1); @@ -1161,8 +1196,33 @@ void UString::SetFromBstr(BSTR s) _limit = len; } _len = len; + + /* + #if WCHAR_MAX > 0xffff + + wchar_t *chars = _chars; + for (size_t i = 0; i <= len; i++) + { + wchar_t c = *s++; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = *s; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + s++; + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + } + } + chars[i] = c; + } + + #else + */ + // if (s) wmemcpy(_chars, s, len + 1); + + // #endif } UString &UString::operator=(const char *s) @@ -1229,9 +1289,8 @@ UString &UString::operator+=(const char *s) void UString::Add_UInt32(UInt32 v) { - char sz[16]; - ConvertUInt32ToString(v, sz); - (*this) += sz; + Grow(10); + _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); } @@ -1341,7 +1400,7 @@ void UString::InsertAtFront(wchar_t c) } /* -void UString::Insert(unsigned index, wchar_t c) +void UString::Insert_wchar_t(unsigned index, wchar_t c) { InsertSpace(index, 1); _chars[index] = c; @@ -1409,7 +1468,7 @@ void UString::Replace(wchar_t oldChar, wchar_t newChar) throw() wchar_t *chars = _chars; while ((unsigned)pos < _len) { - pos = Find(oldChar, pos); + pos = Find(oldChar, (unsigned)pos); if (pos < 0) break; chars[(unsigned)pos] = newChar; @@ -1431,11 +1490,11 @@ void UString::Replace(const UString &oldString, const UString &newString) int pos = 0; while ((unsigned)pos < _len) { - pos = Find(oldString, pos); + pos = Find(oldString, (unsigned)pos); if (pos < 0) break; - Delete(pos, oldLen); - Insert(pos, newString); + Delete((unsigned)pos, oldLen); + Insert((unsigned)pos, newString); pos += newLen; // number++; } @@ -1609,6 +1668,8 @@ int MyStringCompareNoCase(const char *s1, const char *s2) } */ +#if !defined(USE_UNICODE_FSTRING) || !defined(_UNICODE) + static inline UINT GetCurrentCodePage() { #if defined(UNDER_CE) || !defined(_WIN32) @@ -1618,6 +1679,8 @@ static inline UINT GetCurrentCodePage() #endif } +#endif + #ifdef USE_UNICODE_FSTRING #ifndef _UNICODE @@ -1637,9 +1700,9 @@ FString fas2fs(const AString &s) return MultiByteToUnicodeString(s, GetCurrentCodePage()); } -#endif +#endif // _UNICODE -#else +#else // USE_UNICODE_FSTRING UString fs2us(const FChar *s) { @@ -1656,4 +1719,4 @@ FString us2fs(const wchar_t *s) return UnicodeStringToMultiByte(s, GetCurrentCodePage()); } -#endif +#endif // USE_UNICODE_FSTRING diff --git a/CPP/Common/MyString.h b/CPP/Common/MyString.h index 40de52cd..c242d923 100644 --- a/CPP/Common/MyString.h +++ b/CPP/Common/MyString.h @@ -159,7 +159,7 @@ inline wchar_t MyCharUpper(wchar_t c) throw() return (wchar_t)MyCharUpper_WIN(c); #endif #else - return (wchar_t)towupper(c); + return (wchar_t)towupper((wint_t)c); #endif } @@ -207,6 +207,7 @@ int MyStringCompareNoCase(const wchar_t *s1, const wchar_t *s2) throw(); // ---------- ASCII ---------- // char values in ASCII strings must be less then 128 +bool StringsAreEqual_Ascii(const char *u, const char *a) throw(); bool StringsAreEqual_Ascii(const wchar_t *u, const char *a) throw(); bool StringsAreEqualNoCase_Ascii(const char *s1, const char *s2) throw(); bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const char *s2) throw(); @@ -231,7 +232,7 @@ bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const wchar_t *s2) throw(); cls &operator=(const t *); \ cls &operator+=(t); \ cls &operator+=(const t *); \ - FORBID_STRING_OPS_2(cls, t); \ + FORBID_STRING_OPS_2(cls, t) \ /* cls &operator+(t); \ @@ -266,7 +267,7 @@ class AString AString(const AString &s, char c); // it's for String + char AString(const char *s1, unsigned num1, const char *s2, unsigned num2); - friend AString operator+(const AString &s, char c) { return AString(s, c); } ; + friend AString operator+(const AString &s, char c) { return AString(s, c); } // friend AString operator+(char c, const AString &s); // is not supported friend AString operator+(const AString &s1, const AString &s2); @@ -300,6 +301,7 @@ public: void Empty() { _len = 0; _chars[0] = 0; } operator const char *() const { return _chars; } + char *Ptr_non_const() const { return _chars; } const char *Ptr() const { return _chars; } const char *Ptr(unsigned pos) const { return _chars + pos; } const char *RightPtr(unsigned num) const { return _chars + _len - num; } @@ -438,8 +440,30 @@ public: _chars[index] = 0; } } + + void Wipe_and_Empty() + { + if (_chars) + { + memset(_chars, 0, (_limit + 1) * sizeof(*_chars)); + _len = 0; + } + } +}; + + +class AString_Wipe: public AString +{ + CLASS_NO_COPY(AString_Wipe) +public: + AString_Wipe(): AString() {} + // AString_Wipe(const AString &s): AString(s) {} + // AString_Wipe &operator=(const AString &s) { AString::operator=(s); return *this; } + // AString_Wipe &operator=(const char *s) { AString::operator=(s); return *this; } + ~AString_Wipe() { Wipe_and_Empty(); } }; + bool operator<(const AString &s1, const AString &s2); bool operator>(const AString &s1, const AString &s2); @@ -500,7 +524,7 @@ class UString UString(const UString &s, wchar_t c); // it's for String + char UString(const wchar_t *s1, unsigned num1, const wchar_t *s2, unsigned num2); - friend UString operator+(const UString &s, wchar_t c) { return UString(s, c); } ; + friend UString operator+(const UString &s, wchar_t c) { return UString(s, c); } // friend UString operator+(wchar_t c, const UString &s); // is not supported friend UString operator+(const UString &s1, const UString &s2); @@ -539,6 +563,7 @@ public: void Empty() { _len = 0; _chars[0] = 0; } operator const wchar_t *() const { return _chars; } + wchar_t *Ptr_non_const() const { return _chars; } const wchar_t *Ptr() const { return _chars; } const wchar_t *Ptr(unsigned pos) const { return _chars + pos; } const wchar_t *RightPtr(unsigned num) const { return _chars + _len - num; } @@ -578,7 +603,7 @@ public: UString &operator=(const wchar_t *s); UString &operator=(const UString &s); void SetFrom(const wchar_t *s, unsigned len); // no check - void SetFromBstr(BSTR s); + void SetFromBstr(LPCOLESTR s); UString &operator=(const char *s); UString &operator=(const AString &s) { return operator=(s.Ptr()); } @@ -659,7 +684,7 @@ public: } void InsertAtFront(wchar_t c); - // void Insert(unsigned index, wchar_t c); + // void Insert_wchar_t(unsigned index, wchar_t c); void Insert(unsigned index, const wchar_t *s); void Insert(unsigned index, const UString &s); @@ -680,8 +705,30 @@ public: _chars[index] = 0; } } + + void Wipe_and_Empty() + { + if (_chars) + { + memset(_chars, 0, (_limit + 1) * sizeof(*_chars)); + _len = 0; + } + } +}; + + +class UString_Wipe: public UString +{ + CLASS_NO_COPY(UString_Wipe) +public: + UString_Wipe(): UString() {} + // UString_Wipe(const UString &s): UString(s) {} + // UString_Wipe &operator=(const UString &s) { UString::operator=(s); return *this; } + // UString_Wipe &operator=(const wchar_t *s) { UString::operator=(s); return *this; } + ~UString_Wipe() { Wipe_and_Empty(); } }; + bool operator<(const UString &s1, const UString &s2); bool operator>(const UString &s1, const UString &s2); @@ -866,3 +913,20 @@ typedef const FChar *CFSTR; typedef CObjectVector<FString> FStringVector; #endif + + + +#if defined(_WIN32) + // #include <wchar.h> + // WCHAR_MAX is defined as ((wchar_t)-1) + #define _WCHART_IS_16BIT 1 +#elif (defined(WCHAR_MAX) && (WCHAR_MAX <= 0xffff)) \ + || (defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ == 2)) + #define _WCHART_IS_16BIT 1 +#endif + +#if WCHAR_PATH_SEPARATOR == L'\\' +// WSL scheme +#define WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT ((wchar_t)((unsigned)(0xF000) + (unsigned)'\\')) +// #define WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT '_' +#endif diff --git a/CPP/Common/MyTypes.h b/CPP/Common/MyTypes.h index 75806f37..2b1acce2 100644 --- a/CPP/Common/MyTypes.h +++ b/CPP/Common/MyTypes.h @@ -32,4 +32,15 @@ struct CBoolPair cls(const cls &); \ cls &operator=(const cls &); +class CUncopyable +{ +protected: + CUncopyable() {} // allow constructor + // ~CUncopyable() {} +CLASS_NO_COPY(CUncopyable) +}; + +#define MY_UNCOPYABLE :private CUncopyable +// #define MY_UNCOPYABLE + #endif diff --git a/CPP/Common/MyVector.h b/CPP/Common/MyVector.h index 61dabbd1..c8512342 100644 --- a/CPP/Common/MyVector.h +++ b/CPP/Common/MyVector.h @@ -35,7 +35,7 @@ class CRecordVector public: - CRecordVector(): _items(0), _size(0), _capacity(0) {} + CRecordVector(): _items(NULL), _size(0), _capacity(0) {} CRecordVector(const CRecordVector &v): _items(0), _size(0), _capacity(0) { @@ -257,7 +257,7 @@ public: unsigned mid = (left + right) / 2; const T midVal = (*this)[mid]; if (item == midVal) - return mid; + return (int)mid; if (item < midVal) right = mid; else @@ -274,7 +274,7 @@ public: const T& midVal = (*this)[mid]; int comp = item.Compare(midVal); if (comp == 0) - return mid; + return (int)mid; if (comp < 0) right = mid; else @@ -428,7 +428,7 @@ public: // void Reserve(unsigned newCapacity) { _v.Reserve(newCapacity); } void ClearAndReserve(unsigned newCapacity) { Clear(); _v.ClearAndReserve(newCapacity); } - CObjectVector() {}; + CObjectVector() {} CObjectVector(const CObjectVector &v) { unsigned size = v.Size(); @@ -568,7 +568,7 @@ public: const T& midVal = (*this)[mid]; int comp = item.Compare(midVal); if (comp == 0) - return mid; + return (int)mid; if (comp < 0) right = mid; else @@ -624,9 +624,9 @@ public: { _v.Sort(compare, param); } static int CompareObjectItems(void *const *a1, void *const *a2, void * /* param */) - { return (*(*((const T **)a1))).Compare(*(*((const T **)a2))); } + { return (*(*((const T *const *)a1))).Compare(*(*((const T *const *)a2))); } - void Sort() { _v.Sort(CompareObjectItems, 0); } + void Sort() { _v.Sort(CompareObjectItems, NULL); } }; #define FOR_VECTOR(_i_, _v_) for (unsigned _i_ = 0; _i_ < (_v_).Size(); _i_++) diff --git a/CPP/Common/MyWindows.cpp b/CPP/Common/MyWindows.cpp index 463c77c4..88f312fb 100644 --- a/CPP/Common/MyWindows.cpp +++ b/CPP/Common/MyWindows.cpp @@ -5,6 +5,10 @@ #ifndef _WIN32 #include <stdlib.h> +#include <time.h> +#ifdef __GNUC__ +#include <sys/time.h> +#endif #include "MyWindows.h" @@ -38,11 +42,11 @@ BSTR SysAllocStringByteLen(LPCSTR s, UINT len) /* Original SysAllocStringByteLen in Win32 maybe fills only unaligned null OLECHAR at the end. We provide also aligned null OLECHAR at the end. */ - if (len >= (k_BstrSize_Max - sizeof(OLECHAR) - sizeof(OLECHAR) - sizeof(CBstrSizeType))) + if (len >= (k_BstrSize_Max - (UINT)sizeof(OLECHAR) - (UINT)sizeof(OLECHAR) - (UINT)sizeof(CBstrSizeType))) return NULL; - UINT size = (len + sizeof(OLECHAR) + sizeof(OLECHAR) - 1) & ~(sizeof(OLECHAR) - 1); - void *p = AllocateForBSTR(size + sizeof(CBstrSizeType)); + UINT size = (len + (UINT)sizeof(OLECHAR) + (UINT)sizeof(OLECHAR) - 1) & ~((UINT)sizeof(OLECHAR) - 1); + void *p = AllocateForBSTR(size + (UINT)sizeof(CBstrSizeType)); if (!p) return NULL; *(CBstrSizeType *)p = (CBstrSizeType)len; @@ -56,11 +60,11 @@ BSTR SysAllocStringByteLen(LPCSTR s, UINT len) BSTR SysAllocStringLen(const OLECHAR *s, UINT len) { - if (len >= (k_BstrSize_Max - sizeof(OLECHAR) - sizeof(CBstrSizeType)) / sizeof(OLECHAR)) + if (len >= (k_BstrSize_Max - (UINT)sizeof(OLECHAR) - (UINT)sizeof(CBstrSizeType)) / (UINT)sizeof(OLECHAR)) return NULL; - UINT size = len * sizeof(OLECHAR); - void *p = AllocateForBSTR(size + sizeof(CBstrSizeType) + sizeof(OLECHAR)); + UINT size = len * (UINT)sizeof(OLECHAR); + void *p = AllocateForBSTR(size + (UINT)sizeof(CBstrSizeType) + (UINT)sizeof(OLECHAR)); if (!p) return NULL; *(CBstrSizeType *)p = (CBstrSizeType)size; @@ -98,7 +102,7 @@ UINT SysStringLen(BSTR bstr) { if (!bstr) return 0; - return *((CBstrSizeType *)bstr - 1) / sizeof(OLECHAR); + return *((CBstrSizeType *)bstr - 1) / (UINT)sizeof(OLECHAR); } @@ -139,7 +143,150 @@ LONG CompareFileTime(const FILETIME* ft1, const FILETIME* ft2) DWORD GetLastError() { - return 0; + return (DWORD)errno; +} + +void SetLastError(DWORD dw) +{ + errno = (int)dw; +} + + +static LONG TIME_GetBias() +{ + time_t utc = time(NULL); + struct tm *ptm = localtime(&utc); + int localdaylight = ptm->tm_isdst; /* daylight for local timezone */ + ptm = gmtime(&utc); + ptm->tm_isdst = localdaylight; /* use local daylight, not that of Greenwich */ + LONG bias = (int)(mktime(ptm)-utc); + return bias; +} + +#define TICKS_PER_SEC 10000000 +/* +#define SECS_PER_DAY (24 * 60 * 60) +#define SECS_1601_TO_1970 ((369 * 365 + 89) * (UInt64)SECS_PER_DAY) +#define TICKS_1601_TO_1970 (SECS_1601_TO_1970 * TICKS_PER_SEC) +*/ + +#define GET_TIME_64(pft) ((pft)->dwLowDateTime | ((UInt64)(pft)->dwHighDateTime << 32)) + +#define SET_FILETIME(ft, v64) \ + (ft)->dwLowDateTime = (DWORD)v64; \ + (ft)->dwHighDateTime = (DWORD)(v64 >> 32); + + +BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime) +{ + UInt64 v = GET_TIME_64(fileTime); + v = (UInt64)((Int64)v - (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(localFileTime, v); + return TRUE; +} + +BOOL WINAPI LocalFileTimeToFileTime(const FILETIME *localFileTime, FILETIME *fileTime) +{ + UInt64 v = GET_TIME_64(localFileTime); + v = (UInt64)((Int64)v + (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(fileTime, v); + return TRUE; +} + +/* +VOID WINAPI GetSystemTimeAsFileTime(FILETIME *ft) +{ + UInt64 t = 0; + timeval tv; + if (gettimeofday(&tv, NULL) == 0) + { + t = tv.tv_sec * (UInt64)TICKS_PER_SEC + TICKS_1601_TO_1970; + t += tv.tv_usec * 10; + } + SET_FILETIME(ft, t); +} +*/ + +DWORD WINAPI GetTickCount(VOID) +{ + #ifndef _WIN32 + // gettimeofday() doesn't work in some MINGWs by unknown reason + timeval tv; + if (gettimeofday(&tv, NULL) == 0) + { + // tv_sec and tv_usec are (long) + return (DWORD)((UInt64)(Int64)tv.tv_sec * (UInt64)1000 + (UInt64)(Int64)tv.tv_usec / 1000); + } + #endif + return (DWORD)time(NULL) * 1000; +} + + +#define PERIOD_4 (4 * 365 + 1) +#define PERIOD_100 (PERIOD_4 * 25 - 1) +#define PERIOD_400 (PERIOD_100 * 4 + 1) + +BOOL WINAPI FileTimeToSystemTime(const FILETIME *ft, SYSTEMTIME *st) +{ + UInt32 v; + UInt64 v64 = GET_TIME_64(ft); + v64 /= 10000; + st->wMilliseconds = (WORD)(v64 % 1000); v64 /= 1000; + st->wSecond = (WORD)(v64 % 60); v64 /= 60; + st->wMinute = (WORD)(v64 % 60); v64 /= 60; + v = (UInt32)v64; + st->wHour = (WORD)(v % 24); v /= 24; + + // 1601-01-01 was Monday + st->wDayOfWeek = (WORD)((v + 1) % 7); + + UInt32 leaps, year, day, mon; + leaps = (3 * ((4 * v + (365 - 31 - 28) * 4 + 3) / PERIOD_400) + 3) / 4; + v += 28188 + leaps; + // leaps - the number of exceptions from PERIOD_4 rules starting from 1600-03-01 + // (1959 / 64) - converts day from 03-01 to month + year = (20 * v - 2442) / (5 * PERIOD_4); + day = v - (year * PERIOD_4) / 4; + mon = (64 * day) / 1959; + st->wDay = (WORD)(day - (1959 * mon) / 64); + mon -= 1; + year += 1524; + if (mon > 12) + { + mon -= 12; + year++; + } + st->wMonth = (WORD)mon; + st->wYear = (WORD)year; + + /* + unsigned year, mon; + unsigned char ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + unsigned t; + + year = (WORD)(1601 + v / PERIOD_400 * 400); + v %= PERIOD_400; + + t = v / PERIOD_100; if (t == 4) t = 3; year += t * 100; v -= t * PERIOD_100; + t = v / PERIOD_4; if (t == 25) t = 24; year += t * 4; v -= t * PERIOD_4; + t = v / 365; if (t == 4) t = 3; year += t; v -= t * 365; + + st->wYear = (WORD)year; + + if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) + ms[1] = 29; + for (mon = 0;; mon++) + { + unsigned d = ms[mon]; + if (v < d) + break; + v -= d; + } + st->wDay = (WORD)(v + 1); + st->wMonth = (WORD)(mon + 1); + */ + + return TRUE; } #endif diff --git a/CPP/Common/MyWindows.h b/CPP/Common/MyWindows.h index db3f35d8..0664a5e0 100644 --- a/CPP/Common/MyWindows.h +++ b/CPP/Common/MyWindows.h @@ -5,14 +5,14 @@ #ifdef _WIN32 -#include <windows.h> +#include <Windows.h> #ifdef UNDER_CE #undef VARIANT_TRUE #define VARIANT_TRUE ((VARIANT_BOOL)-1) #endif -#else +#else // _WIN32 #include <stddef.h> // for wchar_t #include <string.h> @@ -20,7 +20,9 @@ #include "MyGuidDef.h" +// WINAPI is __stdcall in Windows-MSVC in windef.h #define WINAPI +#define EXTERN_C MY_EXTERN_C typedef char CHAR; typedef unsigned char UCHAR; @@ -35,17 +37,12 @@ typedef unsigned short USHORT; typedef unsigned short WORD; typedef short VARIANT_BOOL; -typedef int INT; -typedef Int32 INT32; -typedef unsigned int UINT; -typedef UInt32 UINT32; -typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit -typedef UINT32 ULONG; - -#undef DWORD -typedef UINT32 DWORD; +#define LOWORD(l) ((WORD)((DWORD_PTR)(l) & 0xffff)) +#define HIWORD(l) ((WORD)((DWORD_PTR)(l) >> 16)) -typedef long BOOL; +// MS uses long for BOOL, but long is 32-bit in MS. So we use int. +// typedef long BOOL; +typedef int BOOL; #ifndef FALSE #define FALSE 0 @@ -53,7 +50,7 @@ typedef long BOOL; #endif // typedef size_t ULONG_PTR; -typedef size_t DWORD_PTR; +// typedef size_t DWORD_PTR; // typedef uintptr_t UINT_PTR; // typedef ptrdiff_t UINT_PTR; @@ -80,28 +77,33 @@ typedef struct _FILETIME } FILETIME; #define HRESULT LONG -#define FAILED(Status) ((HRESULT)(Status)<0) +#define SUCCEEDED(hr) ((HRESULT)(hr) >= 0) +#define FAILED(hr) ((HRESULT)(hr) < 0) typedef ULONG PROPID; typedef LONG SCODE; -#define ERROR_NEGATIVE_SEEK 131L #define S_OK ((HRESULT)0x00000000L) #define S_FALSE ((HRESULT)0x00000001L) -#define E_NOTIMPL ((HRESULT)0x80004001L) +#define E_NOTIMPL ((HRESULT)0x80004001L) #define E_NOINTERFACE ((HRESULT)0x80004002L) -#define E_ABORT ((HRESULT)0x80004004L) -#define E_FAIL ((HRESULT)0x80004005L) -#define STG_E_INVALIDFUNCTION ((HRESULT)0x80030001L) -#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) -#define E_INVALIDARG ((HRESULT)0x80070057L) +#define E_ABORT ((HRESULT)0x80004004L) +#define E_FAIL ((HRESULT)0x80004005L) +#define STG_E_INVALIDFUNCTION ((HRESULT)0x80030001L) +#define CLASS_E_CLASSNOTAVAILABLE ((HRESULT)0x80040111L) + #ifdef _MSC_VER #define STDMETHODCALLTYPE __stdcall +#define STDAPICALLTYPE __stdcall #else +// do we need __export here? #define STDMETHODCALLTYPE +#define STDAPICALLTYPE #endif +#define STDAPI EXTERN_C HRESULT STDAPICALLTYPE + #define STDMETHOD_(t, f) virtual t STDMETHODCALLTYPE f #define STDMETHOD(f) STDMETHOD_(HRESULT, f) #define STDMETHODIMP_(type) type STDMETHODCALLTYPE @@ -120,9 +122,8 @@ struct IUnknown STDMETHOD(QueryInterface) (REFIID iid, void **outObject) PURE; STDMETHOD_(ULONG, AddRef)() PURE; STDMETHOD_(ULONG, Release)() PURE; - #ifndef _WIN32 virtual ~IUnknown() {} - #endif + // We use virtual ~IUnknown() here for binary compatibility with 7z.so from p7zip }; typedef IUnknown *LPUNKNOWN; @@ -214,8 +215,14 @@ MY_EXTERN_C UINT SysStringByteLen(BSTR bstr); MY_EXTERN_C UINT SysStringLen(BSTR bstr); MY_EXTERN_C DWORD GetLastError(); +MY_EXTERN_C void SetLastError(DWORD dwCode); MY_EXTERN_C LONG CompareFileTime(const FILETIME* ft1, const FILETIME* ft2); +MY_EXTERN_C DWORD GetCurrentThreadId(); +MY_EXTERN_C DWORD GetCurrentProcessId(); + +#define MAX_PATH 1024 + #define CP_ACP 0 #define CP_OEMCP 1 #define CP_UTF8 65001 @@ -227,5 +234,35 @@ typedef enum tagSTREAM_SEEK STREAM_SEEK_END = 2 } STREAM_SEEK; -#endif + + +typedef struct _SYSTEMTIME +{ + WORD wYear; + WORD wMonth; + WORD wDayOfWeek; + WORD wDay; + WORD wHour; + WORD wMinute; + WORD wSecond; + WORD wMilliseconds; +} SYSTEMTIME; + +BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime); +BOOL WINAPI LocalFileTimeToFileTime(const FILETIME *localFileTime, FILETIME *fileTime); +BOOL WINAPI FileTimeToSystemTime(const FILETIME *fileTime, SYSTEMTIME *systemTime); +// VOID WINAPI GetSystemTimeAsFileTime(FILETIME *systemTimeAsFileTime); + +DWORD GetTickCount(); + + +#define CREATE_NEW 1 +#define CREATE_ALWAYS 2 +#define OPEN_EXISTING 3 +#define OPEN_ALWAYS 4 +#define TRUNCATE_EXISTING 5 + + +#endif // _WIN32 + #endif diff --git a/CPP/Common/MyXml.cpp b/CPP/Common/MyXml.cpp index f34a745e..e0145188 100644 --- a/CPP/Common/MyXml.cpp +++ b/CPP/Common/MyXml.cpp @@ -7,9 +7,9 @@ static bool IsValidChar(char c) { return - c >= 'a' && c <= 'z' || - c >= 'A' && c <= 'Z' || - c >= '0' && c <= '9' || + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-'; } @@ -24,7 +24,7 @@ int CXmlItem::FindProp(const char *propName) const throw() { FOR_VECTOR (i, Props) if (Props[i].Name == propName) - return i; + return (int)i; return -1; } @@ -32,7 +32,7 @@ AString CXmlItem::GetPropVal(const char *propName) const { int index = FindProp(propName); if (index >= 0) - return Props[index].Value; + return Props[(unsigned)index].Value; return AString(); } @@ -45,7 +45,7 @@ int CXmlItem::FindSubTag(const char *tag) const throw() { FOR_VECTOR (i, SubItems) if (SubItems[i].IsTagged(tag)) - return i; + return (int)i; return -1; } @@ -75,7 +75,7 @@ AString CXmlItem::GetSubStringForTag(const char *tag) const { int index = FindSubTag(tag); if (index >= 0) - return SubItems[index].GetSubString(); + return SubItems[(unsigned)index].GetSubString(); return AString(); } diff --git a/CPP/Common/Sha1Prepare.cpp b/CPP/Common/Sha1Prepare.cpp new file mode 100644 index 00000000..2652f009 --- /dev/null +++ b/CPP/Common/Sha1Prepare.cpp @@ -0,0 +1,7 @@ +// Sha1Prepare.cpp + +#include "StdAfx.h" + +#include "../../C/Sha1.h" + +static struct CSha1Prepare { CSha1Prepare() { Sha1Prepare(); } } g_Sha1Prepare; diff --git a/CPP/Common/Sha1Reg.cpp b/CPP/Common/Sha1Reg.cpp index 1400c989..0cb2baf7 100644 --- a/CPP/Common/Sha1Reg.cpp +++ b/CPP/Common/Sha1Reg.cpp @@ -4,37 +4,67 @@ #include "../../C/Sha1.h" +#include "../Common/MyBuffer2.h" #include "../Common/MyCom.h" #include "../7zip/Common/RegisterCodec.h" class CSha1Hasher: public IHasher, + public ICompressSetCoderProperties, public CMyUnknownImp { - CSha1 _sha; + CAlignedBuffer _buf; Byte mtDummy[1 << 7]; + CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_buf; } public: - CSha1Hasher() { Sha1_Init(&_sha); } + CSha1Hasher(): + _buf(sizeof(CSha1)) + { + Sha1_SetFunction(Sha(), 0); + Sha1_InitState(Sha()); + } - MY_UNKNOWN_IMP1(IHasher) + MY_UNKNOWN_IMP2(IHasher, ICompressSetCoderProperties) INTERFACE_IHasher(;) + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); }; STDMETHODIMP_(void) CSha1Hasher::Init() throw() { - Sha1_Init(&_sha); + Sha1_InitState(Sha()); } STDMETHODIMP_(void) CSha1Hasher::Update(const void *data, UInt32 size) throw() { - Sha1_Update(&_sha, (const Byte *)data, size); + Sha1_Update(Sha(), (const Byte *)data, size); } STDMETHODIMP_(void) CSha1Hasher::Final(Byte *digest) throw() { - Sha1_Final(&_sha, digest); + Sha1_Final(Sha(), digest); +} + + +STDMETHODIMP CSha1Hasher::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) +{ + unsigned algo = 0; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + if (propIDs[i] == NCoderPropID::kDefaultProp) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + if (prop.ulVal > 2) + return E_NOTIMPL; + algo = (unsigned)prop.ulVal; + } + } + if (!Sha1_SetFunction(Sha(), algo)) + return E_NOTIMPL; + return S_OK; } REGISTER_HASHER(CSha1Hasher, 0x201, "SHA1", SHA1_DIGEST_SIZE) diff --git a/CPP/Common/Sha256Prepare.cpp b/CPP/Common/Sha256Prepare.cpp new file mode 100644 index 00000000..1ec242b5 --- /dev/null +++ b/CPP/Common/Sha256Prepare.cpp @@ -0,0 +1,7 @@ +// Sha256Prepare.cpp + +#include "StdAfx.h" + +#include "../../C/Sha256.h" + +static struct CSha256Prepare { CSha256Prepare() { Sha256Prepare(); } } g_Sha256Prepare; diff --git a/CPP/Common/Sha256Reg.cpp b/CPP/Common/Sha256Reg.cpp index 66941699..5f3a35b0 100644 --- a/CPP/Common/Sha256Reg.cpp +++ b/CPP/Common/Sha256Reg.cpp @@ -4,37 +4,67 @@ #include "../../C/Sha256.h" +#include "../Common/MyBuffer2.h" #include "../Common/MyCom.h" #include "../7zip/Common/RegisterCodec.h" class CSha256Hasher: public IHasher, + public ICompressSetCoderProperties, public CMyUnknownImp { - CSha256 _sha; + CAlignedBuffer _buf; Byte mtDummy[1 << 7]; + CSha256 *Sha() { return (CSha256 *)(void *)(Byte *)_buf; } public: - CSha256Hasher() { Sha256_Init(&_sha); } + CSha256Hasher(): + _buf(sizeof(CSha256)) + { + Sha256_SetFunction(Sha(), 0); + Sha256_InitState(Sha()); + } - MY_UNKNOWN_IMP1(IHasher) + MY_UNKNOWN_IMP2(IHasher, ICompressSetCoderProperties) INTERFACE_IHasher(;) + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); }; STDMETHODIMP_(void) CSha256Hasher::Init() throw() { - Sha256_Init(&_sha); + Sha256_InitState(Sha()); } STDMETHODIMP_(void) CSha256Hasher::Update(const void *data, UInt32 size) throw() { - Sha256_Update(&_sha, (const Byte *)data, size); + Sha256_Update(Sha(), (const Byte *)data, size); } STDMETHODIMP_(void) CSha256Hasher::Final(Byte *digest) throw() { - Sha256_Final(&_sha, digest); + Sha256_Final(Sha(), digest); +} + + +STDMETHODIMP CSha256Hasher::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) +{ + unsigned algo = 0; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + if (propIDs[i] == NCoderPropID::kDefaultProp) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + if (prop.ulVal > 2) + return E_NOTIMPL; + algo = (unsigned)prop.ulVal; + } + } + if (!Sha256_SetFunction(Sha(), algo)) + return E_NOTIMPL; + return S_OK; } REGISTER_HASHER(CSha256Hasher, 0xA, "SHA256", SHA256_DIGEST_SIZE) diff --git a/CPP/Common/StdInStream.cpp b/CPP/Common/StdInStream.cpp index 422a96e5..abad34b6 100644 --- a/CPP/Common/StdInStream.cpp +++ b/CPP/Common/StdInStream.cpp @@ -2,7 +2,9 @@ #include "StdAfx.h" +#ifdef _WIN32 #include <tchar.h> +#endif #include "StdInStream.h" #include "StringConvert.h" @@ -14,14 +16,18 @@ #define kFileOpenMode TEXT("r") -extern int g_CodePage; - CStdInStream g_StdIn(stdin); bool CStdInStream::Open(LPCTSTR fileName) throw() { Close(); - _stream = _tfopen(fileName, kFileOpenMode); + _stream = + #ifdef _WIN32 + _tfopen + #else + fopen + #endif + (fileName, kFileOpenMode); _streamIsOpen = (_stream != 0); return _streamIsOpen; } @@ -56,7 +62,7 @@ bool CStdInStream::ScanUStringUntilNewLine(UString &dest) dest.Empty(); AString s; bool res = ScanAStringUntilNewLine(s); - int codePage = g_CodePage; + int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; if (codePage == CP_UTF8) diff --git a/CPP/Common/StdInStream.h b/CPP/Common/StdInStream.h index 698ebec1..71578eb4 100644 --- a/CPP/Common/StdInStream.h +++ b/CPP/Common/StdInStream.h @@ -13,8 +13,14 @@ class CStdInStream FILE *_stream; bool _streamIsOpen; public: - CStdInStream(): _stream(0), _streamIsOpen(false) {}; - CStdInStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; + int CodePage; + + CStdInStream(FILE *stream = NULL): + _stream(stream), + _streamIsOpen(false), + CodePage(-1) + {}; + ~CStdInStream() { Close(); } bool Open(LPCTSTR fileName) throw(); diff --git a/CPP/Common/StdOutStream.cpp b/CPP/Common/StdOutStream.cpp index 8236072d..40799e22 100644 --- a/CPP/Common/StdOutStream.cpp +++ b/CPP/Common/StdOutStream.cpp @@ -2,7 +2,9 @@ #include "StdAfx.h" +#ifdef _WIN32 #include <tchar.h> +#endif #include "IntToString.h" #include "StdOutStream.h" @@ -11,8 +13,6 @@ #define kFileOpenMode "wt" -extern int g_CodePage; - CStdOutStream g_StdOut(stdout); CStdOutStream g_StdErr(stderr); @@ -47,32 +47,27 @@ CStdOutStream & endl(CStdOutStream & outStream) throw() CStdOutStream & CStdOutStream::operator<<(const wchar_t *s) { - int codePage = g_CodePage; - if (codePage == -1) - codePage = CP_OEMCP; - AString dest; - if (codePage == CP_UTF8) - ConvertUnicodeToUTF8(s, dest); - else - UnicodeStringToMultiByte2(dest, s, (UINT)codePage); - return operator<<((const char *)dest); + AString temp; + UString s2(s); + PrintUString(s2, temp); + return *this; } -void StdOut_Convert_UString_to_AString(const UString &s, AString &temp) +void CStdOutStream::PrintUString(const UString &s, AString &temp) { - int codePage = g_CodePage; + Convert_UString_to_AString(s, temp); + *this << (const char *)temp; +} + +void CStdOutStream::Convert_UString_to_AString(const UString &src, AString &dest) +{ + int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; if (codePage == CP_UTF8) - ConvertUnicodeToUTF8(s, temp); + ConvertUnicodeToUTF8(src, dest); else - UnicodeStringToMultiByte2(temp, s, (UINT)codePage); -} - -void CStdOutStream::PrintUString(const UString &s, AString &temp) -{ - StdOut_Convert_UString_to_AString(s, temp); - *this << (const char *)temp; + UnicodeStringToMultiByte2(dest, src, (UINT)codePage); } diff --git a/CPP/Common/StdOutStream.h b/CPP/Common/StdOutStream.h index 2e637e8f..93f1dfa4 100644 --- a/CPP/Common/StdOutStream.h +++ b/CPP/Common/StdOutStream.h @@ -14,9 +14,15 @@ class CStdOutStream bool _streamIsOpen; public: bool IsTerminalMode; + int CodePage; + + CStdOutStream(FILE *stream = 0): + _stream(stream), + _streamIsOpen(false), + IsTerminalMode(false), + CodePage(-1) + {}; - CStdOutStream(): _stream(0), _streamIsOpen(false), IsTerminalMode(false) {}; - CStdOutStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; ~CStdOutStream() { Close(); } // void AttachStdStream(FILE *stream) { _stream = stream; _streamIsOpen = false; } @@ -52,6 +58,7 @@ public: CStdOutStream & operator<<(const wchar_t *s); void PrintUString(const UString &s, AString &temp); + void Convert_UString_to_AString(const UString &src, AString &dest); void Normalize_UString__LF_Allowed(UString &s); void Normalize_UString(UString &s); @@ -66,6 +73,4 @@ CStdOutStream & endl(CStdOutStream & outStream) throw(); extern CStdOutStream g_StdOut; extern CStdOutStream g_StdErr; -void StdOut_Convert_UString_to_AString(const UString &s, AString &temp); - #endif diff --git a/CPP/Common/StringConvert.cpp b/CPP/Common/StringConvert.cpp index 2a73d688..c0bde0fa 100644 --- a/CPP/Common/StringConvert.cpp +++ b/CPP/Common/StringConvert.cpp @@ -5,9 +5,18 @@ #include "StringConvert.h" #ifndef _WIN32 +// #include <stdio.h> #include <stdlib.h> #endif +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) +#include "UTFConvert.h" +#endif + +#ifdef ENV_HAVE_LOCALE +#include <locale.h> +#endif + static const char k_DefultChar = '_'; #ifdef _WIN32 @@ -71,7 +80,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) d[i] = 0; dest.ReleaseBuf_SetLen(i); */ - unsigned len = MultiByteToWideChar(codePage, 0, src, src.Len(), NULL, 0); + unsigned len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), NULL, 0); if (len == 0) { if (GetLastError() != 0) @@ -79,7 +88,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) } else { - len = MultiByteToWideChar(codePage, 0, src, src.Len(), dest.GetBuf(len), len); + len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), dest.GetBuf(len), (int)len); if (len == 0) throw 282228; dest.ReleaseBuf_SetEnd(len); @@ -175,7 +184,7 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co } */ - unsigned len = WideCharToMultiByte(codePage, 0, src, src.Len(), NULL, 0, NULL, NULL); + unsigned len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), NULL, 0, NULL, NULL); if (len == 0) { if (GetLastError() != 0) @@ -186,8 +195,8 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co BOOL defUsed = FALSE; bool isUtf = (codePage == CP_UTF8 || codePage == CP_UTF7); // defaultChar = defaultChar; - len = WideCharToMultiByte(codePage, 0, src, src.Len(), - dest.GetBuf(len), len, + len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), + dest.GetBuf(len), (int)len, (isUtf ? NULL : &defaultChar), (isUtf ? NULL : &defUsed) ); @@ -213,23 +222,137 @@ AString SystemStringToOemString(const CSysString &src) #endif */ -#else +#else // _WIN32 + +// #include <stdio.h> +/* + if (wchar_t is 32-bit (#if WCHAR_MAX > 0xffff), + and utf-8 string contains big unicode character > 0xffff), + then we still use 16-bit surrogate pair in UString. + It simplifies another code where utf-16 encoding is used. + So we use surrogate-conversion code only in is file. +*/ + +/* + mbstowcs() returns error if there is error in utf-8 stream, + mbstowcs() returns error if there is single surrogates point (d800-dfff) in utf-8 stream +*/ -void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePage */) +/* +static void MultiByteToUnicodeString2_Native(UString &dest, const AString &src) { dest.Empty(); if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 2; + const size_t limit = ((size_t)src.Len() + 1) * 2; wchar_t *d = dest.GetBuf((unsigned)limit); - size_t len = mbstowcs(d, src, limit); + const size_t len = mbstowcs(d, src, limit); if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); return; } + dest.ReleaseBuf_SetEnd(0); +} +*/ + +bool g_ForceToUTF8 = true; // false; + +void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) +{ + dest.Empty(); + if (src.IsEmpty()) + return; + + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + ConvertUTF8ToUnicode(src, dest); + return; + } + + const size_t limit = ((size_t)src.Len() + 1) * 2; + wchar_t *d = dest.GetBuf((unsigned)limit); + const size_t len = mbstowcs(d, src, limit); + if (len != (size_t)-1) + { + dest.ReleaseBuf_SetEnd((unsigned)len); + + #if WCHAR_MAX > 0xffff + d = dest.GetBuf(); + for (size_t i = 0;; i++) + { + // wchar_t c = dest[i]; + wchar_t c = d[i]; + if (c == 0) + break; + if (c >= 0x10000 && c < 0x110000) + { + /* + c -= 0x10000; + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + dest.ReplaceOneCharAtPos(i, c0); + i++; + c = 0xdc00 + (c & 0x3FF); + dest.Insert_wchar_t(i, c); + */ + UString temp = d + i; + + for (size_t t = 0;; t++) + { + wchar_t w = temp[t]; + if (w == 0) + break; + if (i == limit) + break; // unexpected error + if (w >= 0x10000 && w < 0x110000) + { + if (i + 1 == limit) + break; // unexpected error + w -= 0x10000; + d[i++] = (unsigned)0xd800 + (((unsigned)w >> 10) & 0x3FF); + w = 0xdc00 + (w & 0x3FF); + } + d[i++] = w; + } + dest.ReleaseBuf_SetEnd((unsigned)i); + } + } + + #endif + + /* + printf("\nMultiByteToUnicodeString2 (%d) %s\n", (int)src.Len(), src.Ptr()); + printf("char: "); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)(Byte)src[i]); + printf("\n"); + printf("\n-> (%d) %ls\n", (int)dest.Len(), dest.Ptr()); + printf("wchar_t: "); + for (unsigned i = 0; i < dest.Len(); i++) + { + printf (" %02x", (int)dest[i]); + } + printf("\n"); + */ + + return; + } + + /* if there is mbstowcs() error, we have two ways: + + 1) change 0x80+ characters to some character: '_' + in that case we lose data, but we have correct UString() + and that scheme can show errors to user in early stages, + when file converted back to mbs() cannot be found + + 2) transfer bad characters in some UTF-16 range. + it can be non-original Unicode character. + but later we still can restore original character. + */ + + // printf("\nmbstowcs ERROR !!!!!! s=%s\n", src.Ptr()); { unsigned i; const char *s = (const char *)src; @@ -238,6 +361,8 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa Byte c = (Byte)s[i]; if (c == 0) break; + // we can use ascii compatibilty character '_' + // if (c > 0x7F) c = '_'; // we replace "bad: character d[i++] = (wchar_t)c; } d[i] = 0; @@ -245,43 +370,131 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa } } -static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT /* codePage */, char defaultChar, bool &defaultCharWasUsed) +static void UnicodeStringToMultiByte2_Native(AString &dest, const UString &src) { dest.Empty(); - defaultCharWasUsed = false; if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 6; + const size_t limit = ((size_t)src.Len() + 1) * 6; char *d = dest.GetBuf((unsigned)limit); - size_t len = wcstombs(d, src, limit); + + const size_t len = wcstombs(d, src, limit); + if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); return; } + dest.ReleaseBuf_SetEnd(0); +} + + +static void UnicodeStringToMultiByte2(AString &dest, const UString &src2, UINT codePage, char defaultChar, bool &defaultCharWasUsed) +{ + // if (codePage == 1234567) // for debug purposes + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + defaultCharWasUsed = false; + ConvertUnicodeToUTF8(src2, dest); + return; + } + + UString src = src2; + #if WCHAR_MAX > 0xffff + { + src.Empty(); + for (unsigned i = 0; i < src2.Len();) + { + wchar_t c = src2[i]; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != src2.Len()) + { + const wchar_t c2 = src2[i + 1]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + src += c; + i++; + } + } + #endif + + dest.Empty(); + defaultCharWasUsed = false; + if (src.IsEmpty()) + return; + + const size_t len = wcstombs(NULL, src, 0); + + if (len != (size_t)-1) + { + const unsigned limit = ((unsigned)len); + if (limit == len) + { + char *d = dest.GetBuf(limit); + + /* + { + printf("\nwcstombs; len = %d %ls \n", (int)src.Len(), src.Ptr()); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)src[i]); + printf("\n"); + printf("\ndest Limit = %d \n", limit); + } + */ + + const size_t len2 = wcstombs(d, src, len + 1); + + if (len2 != (size_t)-1 && len2 <= limit) + { + /* + printf("\nOK : destLen = %d : %s\n", (int)len, dest.Ptr()); + for (unsigned i = 0; i < len2; i++) + printf(" %02x", (int)(Byte)dest[i]); + printf("\n"); + */ + dest.ReleaseBuf_SetEnd((unsigned)len2); + return; + } + } + } { const wchar_t *s = (const wchar_t *)src; + char *d = dest.GetBuf(src.Len()); + unsigned i; for (i = 0;;) { wchar_t c = s[i]; if (c == 0) break; - if (c >= 0x100) + if (c >= + 0x100 + // 0x80 + ) { c = defaultChar; defaultCharWasUsed = true; } + d[i++] = (char)c; } d[i] = 0; dest.ReleaseBuf_SetLen(i); + /* + printf("\nUnicodeStringToMultiByte2; len = %d \n", (int)src.Len()); + printf("ERROR: %s\n", dest.Ptr()); + */ } } -#endif +#endif // _WIN32 UString MultiByteToUnicodeString(const AString &src, UINT codePage) @@ -317,3 +530,228 @@ AString UnicodeStringToMultiByte(const UString &src, UINT codePage) UnicodeStringToMultiByte2(dest, src, codePage, k_DefultChar, defaultCharWasUsed); return dest; } + + + + + +#ifdef _WIN32 +#define U_to_A(a, b, c) UnicodeStringToMultiByte2 +// #define A_to_U(a, b, c) MultiByteToUnicodeString2 +#else +// void MultiByteToUnicodeString2_Native(UString &dest, const AString &src); +#define U_to_A(a, b, c) UnicodeStringToMultiByte2_Native(a, b) +// #define A_to_U(a, b, c) MultiByteToUnicodeString2_Native(a, b) +#endif + +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) + +bool IsNativeUTF8() +{ + UString u; + AString a, a2; + // for (unsigned c = 0x80; c < (UInt32)0x10000; c += (c >> 9) + 1) + for (unsigned c = 0x80; c < (UInt32)0xD000; c += (c >> 2) + 1) + { + u.Empty(); + u += (wchar_t)c; + /* + if (Unicode_Is_There_Utf16SurrogateError(u)) + continue; + #ifndef _WIN32 + if (Unicode_Is_There_BmpEscape(u)) + continue; + #endif + */ + ConvertUnicodeToUTF8(u, a); + U_to_A(a2, u, CP_OEMCP); + if (a != a2) + return false; + } + return true; +} + +#endif + + +#ifdef ENV_HAVE_LOCALE + +const char *GetLocale(void) +{ + #ifdef ENV_HAVE_LOCALE + // printf("\n\nsetlocale(LC_CTYPE, NULL) : return : "); + const char *s = setlocale(LC_CTYPE, NULL); + if (!s) + { + // printf("[NULL]\n"); + s = "C"; + } + else + { + // ubuntu returns "C" after program start + // printf("\"%s\"\n", s); + } + return s; + #elif defined(LOCALE_IS_UTF8) + return "utf8"; + #else + return "C"; + #endif +} + +#ifdef _WIN32 + static void Set_ForceToUTF8(bool) {} +#else + static void Set_ForceToUTF8(bool val) { g_ForceToUTF8 = val; } +#endif + +static bool Is_Default_Basic_Locale(const char *locale) +{ + const AString a (locale); + if (a.IsEqualTo_Ascii_NoCase("") + || a.IsEqualTo_Ascii_NoCase("C") + || a.IsEqualTo_Ascii_NoCase("POSIX")) + return true; + return false; +} + +static bool Is_Default_Basic_Locale() +{ + return Is_Default_Basic_Locale(GetLocale()); +} + + +void MY_SetLocale() +{ + #ifdef ENV_HAVE_LOCALE + /* + { + const char *s = GetLocale(); + printf("\nGetLocale() : returned : \"%s\"\n", s); + } + */ + + unsigned start = 0; + // unsigned lim = 0; + unsigned lim = 3; + + /* + #define MY_SET_LOCALE_FLAGS__FROM_ENV 1 + #define MY_SET_LOCALE_FLAGS__TRY_UTF8 2 + + unsigned flags = + MY_SET_LOCALE_FLAGS__FROM_ENV | + MY_SET_LOCALE_FLAGS__TRY_UTF8 + + if (flags != 0) + { + if (flags & MY_SET_LOCALE_FLAGS__FROM_ENV) + lim = (flags & MY_SET_LOCALE_FLAGS__TRY_UTF8) ? 3 : 1; + else + { + start = 1; + lim = 2; + } + } + */ + + for (unsigned i = start; i < lim; i++) + { + /* + man7: "If locale is an empty string, "", each part of the locale that + should be modified is set according to the environment variables. + for glibc: glibc, first from the user's environment variables: + 1) the environment variable LC_ALL, + 2) environment variable with the same name as the category (see the + 3) the environment variable LANG + The locale "C" or "POSIX" is a portable locale; it exists on all conforming systems. + + for WIN32 : MSDN : + Sets the locale to the default, which is the user-default + ANSI code page obtained from the operating system. + The locale name is set to the value returned by GetUserDefaultLocaleName. + The code page is set to the value returned by GetACP + */ + const char *newLocale = ""; + + #ifdef __APPLE__ + + /* look also CFLocale + there is no C.UTF-8 in macos + macos has UTF-8 locale only with some language like en_US.UTF-8 + what is best way to set UTF-8 locale in macos? */ + if (i == 1) + newLocale = "en_US.UTF-8"; + + /* file open with non-utf8 sequencies return + #define EILSEQ 92 // "Illegal byte sequence" + */ +#else + // newLocale = "C"; + if (i == 1) + { + newLocale = "C.UTF-8"; // main UTF-8 locale in ubuntu + // newLocale = ".utf8"; // supported in new Windows 10 build 17134 (April 2018 Update), the Universal C Runtime + // newLocale = "en_US.utf8"; // supported by ubuntu ? + // newLocale = "en_US.UTF-8"; + /* setlocale() in ubuntu allows locales with minor chracter changes in strings + "en_US.UTF-8" / "en_US.utf8" */ + } + +#endif + + // printf("\nsetlocale(LC_ALL, \"%s\") : returned: ", newLocale); + + // const char *s = + setlocale(LC_ALL, newLocale); + + /* + if (!s) + printf("NULL: can't set locale"); + else + printf("\"%s\"\n", s); + */ + + // request curent locale of program + const char *locale = GetLocale(); + if (locale) + { + AString a (locale); + a.MakeLower_Ascii(); + // if (a.Find("utf") >= 0) + { + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + } + if (!Is_Default_Basic_Locale(locale)) + { + // if there is some non-default and non-utf locale, we want to use it + break; // comment it for debug + } + } + } + + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + + if (Is_Default_Basic_Locale()) + { + Set_ForceToUTF8(true); + return; + } + + Set_ForceToUTF8(false); + + #elif defined(LOCALE_IS_UTF8) + // assume LC_CTYPE="utf8" + #else + // assume LC_CTYPE="C" + #endif +} +#endif diff --git a/CPP/Common/StringConvert.h b/CPP/Common/StringConvert.h index 25fe503f..865c0254 100644 --- a/CPP/Common/StringConvert.h +++ b/CPP/Common/StringConvert.h @@ -85,4 +85,26 @@ inline AString GetOemString(const UString &u) AString SystemStringToOemString(const CSysString &src); #endif + +#ifdef _WIN32 +/* we don't need locale functions in Windows + but we can define ENV_HAVE_LOCALE here for debug purposes */ +// #define ENV_HAVE_LOCALE +#else +#define ENV_HAVE_LOCALE +#endif + +#ifdef ENV_HAVE_LOCALE +void MY_SetLocale(); +const char *GetLocale(void); +#endif + +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) +bool IsNativeUTF8(); +#endif + +#ifndef _WIN32 +extern bool g_ForceToUTF8; +#endif + #endif diff --git a/CPP/Common/StringToInt.cpp b/CPP/Common/StringToInt.cpp index dfa5cc3b..839867a4 100644 --- a/CPP/Common/StringToInt.cpp +++ b/CPP/Common/StringToInt.cpp @@ -17,7 +17,7 @@ static const UInt64 k_UInt64_max = UINT64_CONST(0xFFFFFFFFFFFFFFFF); if (c < '0' || c > '9') { if (end) *end = s; return res; } \ if (res > (k_ ## uintType ## _max) / 10) return 0; \ res *= 10; \ - unsigned v = (c - '0'); \ + unsigned v = (unsigned)(c - '0'); \ if (res > (k_ ## uintType ## _max) - v) return 0; \ res += v; }} diff --git a/CPP/Common/UTFConvert.cpp b/CPP/Common/UTFConvert.cpp index b772164a..ac069dba 100644 --- a/CPP/Common/UTFConvert.cpp +++ b/CPP/Common/UTFConvert.cpp @@ -2,11 +2,17 @@ #include "StdAfx.h" +// #include <stdio.h> + #include "MyTypes.h" #include "UTFConvert.h" -#ifdef _WIN32 -#define _WCHART_IS_16BIT 1 + +#ifndef _WCHART_IS_16BIT +#ifndef __APPLE__ + // we define it if the system supports files with non-utf8 symbols: + #define _UTF8_RAW_NON_UTF8_SUPPORTED +#endif #endif /* @@ -18,78 +24,332 @@ 1 : 0xC0 : 11 : 2 : 0xE0 : 16 : Basic Multilingual Plane 3 : 0xF0 : 21 : Unicode space - 3 : 0xF8 : 26 : - 5 : 0xFC : 31 : UCS-4 + 4 : 0xF8 : 26 : + 5 : 0xFC : 31 : UCS-4 : wcstombs() in ubuntu is limited to that value 6 : 0xFE : 36 : We can use it, if we want to encode any 32-bit value 7 : 0xFF : */ #define _UTF8_START(n) (0x100 - (1 << (7 - (n)))) -#define _UTF8_HEAD_PARSE2(n) if (c < _UTF8_START((n) + 1)) { numBytes = (n); c -= _UTF8_START(n); } +#define _UTF8_HEAD_PARSE2(n) \ + if (c < _UTF8_START((n) + 1)) \ + { numBytes = (n); val -= _UTF8_START(n); } + +#ifndef _WCHART_IS_16BIT + +/* + if (wchar_t is 32-bit), we can support large points in long UTF-8 sequence, + when we convert wchar_t strings to UTF-8: + (_UTF8_NUM_TAIL_BYTES_MAX == 3) : (21-bits points) - Unicode + (_UTF8_NUM_TAIL_BYTES_MAX == 5) : (31-bits points) - UCS-4 + (_UTF8_NUM_TAIL_BYTES_MAX == 6) : (36-bit hack) +*/ + +#define _UTF8_NUM_TAIL_BYTES_MAX 5 +#endif +/* #define _UTF8_HEAD_PARSE \ + UInt32 val = c; \ _UTF8_HEAD_PARSE2(1) \ else _UTF8_HEAD_PARSE2(2) \ else _UTF8_HEAD_PARSE2(3) \ else _UTF8_HEAD_PARSE2(4) \ else _UTF8_HEAD_PARSE2(5) \ + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else _UTF8_HEAD_PARSE2(6) + #endif +*/ + +#define _UTF8_HEAD_PARSE_MAX_3_BYTES \ + UInt32 val = c; \ + _UTF8_HEAD_PARSE2(1) \ + else _UTF8_HEAD_PARSE2(2) \ + else { numBytes = 3; val -= _UTF8_START(3); } + + +#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + + +#define START_POINT_FOR_SURROGATE 0x10000 + + +/* we use 128 bytes block in 16-bit BMP-PLANE to encode non-UTF-8 Escapes + Also we can use additional HIGH-PLANE (we use 21-bit points above 0x1f0000) + to simplify internal intermediate conversion in Linux: + RAW-UTF-8 <-> internal wchar_t utf-16 strings <-> RAW-UTF-UTF-8 +*/ + + +#if defined(_WCHART_IS_16BIT) + +#define UTF_ESCAPE_PLANE 0 + +#else + +/* +we can place 128 ESCAPE chars to + ef 80 - ee be 80 (3-bytes utf-8) : similar to WSL + ef ff - ee bf bf + +1f ef 80 - f7 be be 80 (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +1f ef ff - f7 be bf bf (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +*/ + +// #define UTF_ESCAPE_PLANE_HIGH (0x1f << 16) +// #define UTF_ESCAPE_PLANE UTF_ESCAPE_PLANE_HIGH +#define UTF_ESCAPE_PLANE 0 + +/* + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is set) + { + if (UTF_ESCAPE_PLANE is UTF_ESCAPE_PLANE_HIGH) + { + we can restore any 8-bit Escape from ESCAPE-PLANE-21 plane. + But ESCAPE-PLANE-21 point cannot be stored to utf-16 (7z archive) + So we still need a way to extract 8-bit Escapes and BMP-Escapes-8 + from same BMP-Escapes-16 stored in 7z. + And if we want to restore any 8-bit from 7z archive, + we still must use UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT for (utf-8 -> utf-16) + Also we need additional Conversions to tranform from utf-16 to utf-16-With-Escapes-21 + } + else (UTF_ESCAPE_PLANE == 0) + { + we must convert original 3-bytes utf-8 BMP-Escape point to sequence + of 3 BMP-Escape-16 points with UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + so we can extract original RAW-UTF-8 from UTFD-16 later. + } + } +*/ + +#endif + + + +#define UTF_ESCAPE_BASE 0xef00 + + +#ifdef UTF_ESCAPE_BASE +#define IS_ESCAPE_POINT(v, plane) (((v) & (UInt32)0xffffff80) == (plane) + UTF_ESCAPE_BASE + 0x80) +#endif + +#define IS_SURROGATE_POINT(v) (((v) & (UInt32)0xfffff800) == 0xd800) +#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffC00) == 0xdc00) + + +#define _ERROR_UTF8_CHECK \ + { NonUtf = true; continue; } + +void CUtf8Check::Check_Buf(const char *src, size_t size) throw() +{ + Clear(); + // Byte maxByte = 0; + + for (;;) + { + if (size == 0) + break; + + const Byte c = (Byte)(*src++); + size--; + + if (c == 0) + { + ZeroChar = true; + continue; + } + + /* + if (c > maxByte) + maxByte = c; + */ + + if (c < 0x80) + continue; + + if (c < 0xc0 + 2)// it's limit for 0x140000 unicode codes : win32 compatibility + _ERROR_UTF8_CHECK + + unsigned numBytes; - // else _UTF8_HEAD_PARSE2(6) + UInt32 val = c; + _UTF8_HEAD_PARSE2(1) + else _UTF8_HEAD_PARSE2(2) + else _UTF8_HEAD_PARSE2(4) + else _UTF8_HEAD_PARSE2(5) + else + { + _ERROR_UTF8_CHECK + } + unsigned pos = 0; + do + { + if (pos == size) + break; + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) + break; + val <<= 6; + val |= c2; + if (pos == 0) + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + pos++; + } + while (--numBytes); + + if (numBytes != 0) + { + if (pos == size) + Truncated = true; + else + _ERROR_UTF8_CHECK + } + + #ifdef UTF_ESCAPE_BASE + if (IS_ESCAPE_POINT(val, 0)) + Escape = true; + #endif + + if (MaxHighPoint < val) + MaxHighPoint = val; + + if (IS_SURROGATE_POINT(val)) + SingleSurrogate = true; + + src += pos; + size -= pos; + } + + // MaxByte = maxByte; +} + +bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw() +{ + CUtf8Check check; + check.Check_Buf(src, size); + return check.IsOK(allowReduced); +} + +/* +bool CheckUTF8_chars(const char *src, bool allowReduced) throw() +{ + CUtf8Check check; + check.CheckBuf(src, strlen(src)); + return check.IsOK(allowReduced); +} +*/ + +bool CheckUTF8_AString(const AString &s) throw() +{ + CUtf8Check check; + check.Check_AString(s); + return check.IsOK(); +} + + +/* bool CheckUTF8(const char *src, bool allowReduced) throw() { + // return Check_UTF8_Buf(src, strlen(src), allowReduced); + for (;;) { - Byte c = *src++; + const Byte c = (Byte)(*src++); if (c == 0) return true; if (c < 0x80) continue; - if (c < 0xC0) // (c < 0xC0 + 2) // if we support only optimal encoding chars + if (c < 0xC0 + 2 || c >= 0xf5) return false; unsigned numBytes; _UTF8_HEAD_PARSE else return false; - - UInt32 val = c; + unsigned pos = 0; + do { - Byte c2 = *src++; + Byte c2 = (Byte)(*src++); if (c2 < 0x80 || c2 >= 0xC0) return allowReduced && c2 == 0; val <<= 6; val |= (c2 - 0x80); + pos++; } while (--numBytes); - + + if (val < _UTF8_RANGE(pos - 1)) + return false; + if (val >= 0x110000) return false; } } +*/ + +// in case of UTF-8 error we have two ways: +// 21.01- : old : 0xfffd: REPLACEMENT CHARACTER : old version +// 21.02+ : new : 0xef00 + (c) : similar to WSL scheme for low symbols + +#define UTF_REPLACEMENT_CHAR 0xfffd + +#define UTF_ESCAPE(c) \ + ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) ? \ + UTF_ESCAPE_PLANE + UTF_ESCAPE_BASE + (c) : UTF_REPLACEMENT_CHAR) + +/* +#define _HARD_ERROR_UTF8 + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; ok = false; continue; } +*/ + +// we ignore utf errors, and don't change (ok) variable! + #define _ERROR_UTF8 \ - { if (dest) dest[destPos] = (wchar_t)0xFFFD; destPos++; ok = false; continue; } + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; continue; } -static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim) throw() +// we store UTF-16 in wchar_t strings. So we use surrogates for big unicode points: + +// for debug puposes only we can store UTF-32 in wchar_t: +// #define START_POINT_FOR_SURROGATE ((UInt32)0 - 1) + + +/* + WIN32 MultiByteToWideChar(CP_UTF8) emits 0xfffd point, if utf-8 error was found. + Ant it can emit single 0xfffd from 2 src bytes. + It doesn't emit single 0xfffd from 3-4 src bytes. + We can + 1) emit Escape point for each incorrect byte. So we can data recover later + 2) emit 0xfffd for each incorrect byte. + That scheme is similar to Escape scheme, but we emit 0xfffd + instead of each Escape point. + 3) emit single 0xfffd from 1-2 incorrect bytes, as WIN32 MultiByteToWideChar scheme +*/ + +static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim, unsigned flags) throw() { size_t destPos = 0; bool ok = true; for (;;) { - Byte c; if (src == srcLim) { *destLen = destPos; return ok; } - c = *src++; + + const Byte c = (Byte)(*src++); if (c < 0x80) { @@ -98,68 +358,127 @@ static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const destPos++; continue; } - if (c < 0xC0) + + if (c < 0xc0 + 2 + || c >= 0xf5) // it's limit for 0x140000 unicode codes : win32 compatibility + { _ERROR_UTF8 + } unsigned numBytes; - _UTF8_HEAD_PARSE - else - _ERROR_UTF8 - - UInt32 val = c; + _UTF8_HEAD_PARSE_MAX_3_BYTES + + unsigned pos = 0; do { - Byte c2; - if (src == srcLim) + if (src + pos == srcLim) break; - c2 = *src; - if (c2 < 0x80 || c2 >= 0xC0) + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) break; - src++; val <<= 6; - val |= (c2 - 0x80); + val |= c2; + pos++; + if (pos == 1) + { + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + if (numBytes == 2) + { + if (flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + if ((val & (0xF800 >> 6)) == (0xd800 >> 6)) + break; + } + else if (numBytes == 3 && val >= (0x110000 >> 12)) + break; + } } while (--numBytes); if (numBytes != 0) + { + if ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) == 0) + { + // the following code to emit the 0xfffd chars as win32 Utf8 function. + // disable the folling line, if you need 0xfffd for each incorrect byte as in Escape mode + src += pos; + } + _ERROR_UTF8 + } + + /* + if (val < _UTF8_RANGE(pos - 1)) _ERROR_UTF8 + */ + + #ifdef UTF_ESCAPE_BASE + + if ((flags & UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT) + && IS_ESCAPE_POINT(val, 0)) + { + // We will emit 3 utf16-Escape-16-21 points from one Escape-16 point (3 bytes) + _ERROR_UTF8 + } + + #endif - if (val < 0x10000) + /* + We don't expect virtual Escape-21 points in UTF-8 stream. + And we don't check for Escape-21. + So utf8-Escape-21 will be converted to another 3 utf16-Escape-21 points. + Maybe we could convert virtual utf8-Escape-21 to one utf16-Escape-21 point in some cases? + */ + + if (val < START_POINT_FOR_SURROGATE) { + /* + if ((flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + && IS_SURROGATE_POINT(val)) + { + // We will emit 3 utf16-Escape-16-21 points from one Surrogate-16 point (3 bytes) + _ERROR_UTF8 + } + */ if (dest) dest[destPos] = (wchar_t)val; destPos++; } else { - val -= 0x10000; - if (val >= 0x100000) + /* + if (val >= 0x110000) + { + // We will emit utf16-Escape-16-21 point from each source byte _ERROR_UTF8 + } + */ if (dest) { - dest[destPos + 0] = (wchar_t)(0xD800 + (val >> 10)); - dest[destPos + 1] = (wchar_t)(0xDC00 + (val & 0x3FF)); + dest[destPos + 0] = (wchar_t)(0xd800 - (0x10000 >> 10) + (val >> 10)); + dest[destPos + 1] = (wchar_t)(0xdc00 + (val & 0x3ff)); } destPos += 2; } + src += pos; } } -#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + #define _UTF8_HEAD(n, val) ((char)(_UTF8_START(n) + (val >> (6 * (n))))) #define _UTF8_CHAR(n, val) ((char)(0x80 + (((val) >> (6 * (n))) & 0x3F))) -static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) +static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim, unsigned flags) { - size_t size = srcLim - src; + size_t size = (size_t)(srcLim - src); for (;;) { if (src == srcLim) return size; - UInt32 val = *src++; + UInt32 val = (UInt32)(*src++); if (val < 0x80) continue; @@ -170,15 +489,32 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) + continue; + #endif + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + continue; + + #endif + + if (IS_SURROGATE_POINT(val)) { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + // it's hack to UTF-8 encoding + + if (val < 0xdc00 && src != srcLim) { - src++; - size += 2; - continue; + const UInt32 c2 = (UInt32)*src; + if (c2 >= 0xdc00 && c2 < 0xe000) + src++; } + size += 2; + continue; } #ifdef _WCHART_IS_16BIT @@ -191,20 +527,26 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) else if (val < _UTF8_RANGE(3)) size += 3; else if (val < _UTF8_RANGE(4)) size += 4; else if (val < _UTF8_RANGE(5)) size += 5; - else size += 6; + else + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + size += 6; + #else + size += 3; + #endif #endif } } -static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim) + +static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim, unsigned flags) { for (;;) { if (src == srcLim) return dest; - UInt32 val = *src++; + UInt32 val = (UInt32)*src++; if (val < 0x80) { @@ -220,22 +562,57 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + /* + if (wchar_t is 32-bit) + && (UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE is set) + && (point is virtual escape plane) + we extract 8-bit byte from virtual HIGH-ESCAPE PLANE. + */ + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) { - src++; - val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000; - dest[0] = _UTF8_HEAD(3, val); - dest[1] = _UTF8_CHAR(2, val); - dest[2] = _UTF8_CHAR(1, val); - dest[3] = _UTF8_CHAR(0, val); - dest += 4; + *dest++ = (char)(val); + continue; + } + #endif // UTF_ESCAPE_PLANE != 0 + + /* if (UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE is defined) + we extract 8-bit byte from BMP-ESCAPE PLANE. */ + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + { + *dest++ = (char)(val); continue; } - } + #endif // UTF_ESCAPE_BASE + + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val < 0xdc00 && src != srcLim) + { + const UInt32 c2 = (UInt32)*src; + if (IS_LOW_SURROGATE_POINT(c2)) + { + src++; + val = (((val - 0xd800) << 10) | (c2 - 0xdc00)) + 0x10000; + dest[0] = _UTF8_HEAD(3, val); + dest[1] = _UTF8_CHAR(2, val); + dest[2] = _UTF8_CHAR(1, val); + dest[3] = _UTF8_CHAR(0, val); + dest += 4; + continue; + } + } + if (flags & UTF_FLAG__TO_UTF8__SURROGATE_ERROR) + val = UTF_REPLACEMENT_CHAR; // WIN32 function does it + } + #ifndef _WCHART_IS_16BIT if (val < _UTF8_RANGE(2)) #endif @@ -249,14 +626,25 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim #ifndef _WCHART_IS_16BIT - UInt32 b; + // we don't expect this case. so we can throw exception + // throw 20210407; + + char b; unsigned numBits; if (val < _UTF8_RANGE(3)) { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } else if (val < _UTF8_RANGE(4)) { numBits = 6 * 4; b = _UTF8_HEAD(4, val); } else if (val < _UTF8_RANGE(5)) { numBits = 6 * 5; b = _UTF8_HEAD(5, val); } - else { numBits = 6 * 6; b = _UTF8_START(6); } - - *dest++ = (Byte)b; + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else { numBits = 6 * 6; b = (char)_UTF8_START(6); } + #else + else + { + val = UTF_REPLACEMENT_CHAR; + { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } + } + #endif + + *dest++ = b; do { @@ -269,20 +657,207 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim } } -bool ConvertUTF8ToUnicode(const AString &src, UString &dest) +bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags) { dest.Empty(); size_t destLen = 0; - Utf8_To_Utf16(NULL, &destLen, src, src.Ptr(src.Len())); - bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src.Ptr(src.Len())); + Utf8_To_Utf16(NULL, &destLen, src, src + srcSize, flags); + bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src + srcSize, flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); return res; } -void ConvertUnicodeToUTF8(const UString &src, AString &dest) +bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags) +{ + return Convert_UTF8_Buf_To_Unicode(src, src.Len(), dest, flags); +} + + +static +unsigned g_UTF8_To_Unicode_Flags = + UTF_FLAG__FROM_UTF8__USE_ESCAPE + #ifndef _WCHART_IS_16BIT + | UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + #endif + #endif + ; + + +/* +bool ConvertUTF8ToUnicode_boolRes(const AString &src, UString &dest) +{ + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} +*/ + +bool ConvertUTF8ToUnicode(const AString &src, UString &dest) { + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} + +void Print_UString(const UString &a); + +void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags) +{ + /* + if (src.Len()== 24) + throw "202104"; + */ dest.Empty(); - size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len())); - Utf16_To_Utf8(dest.GetBuf((unsigned)destLen), src, src.Ptr(src.Len())); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + char *destStart = dest.GetBuf((unsigned)destLen); + const char *destEnd = Utf16_To_Utf8(destStart, src, src.Ptr(src.Len()), flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); + // printf("\nlen = %d\n", src.Len()); + if (destLen != (size_t)(destEnd - destStart)) + { + /* + // dest.ReleaseBuf_SetEnd((unsigned)(destEnd - destStart)); + printf("\nlen = %d\n", (unsigned)destLen); + printf("\n(destEnd - destStart) = %d\n", (unsigned)(destEnd - destStart)); + printf("\n"); + // Print_UString(src); + printf("\n"); + // printf("\nlen = %d\n", destLen); + */ + throw 20210406; + } +} + + + +unsigned g_Unicode_To_UTF8_Flags = + // UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE + 0 + #ifndef _WIN32 + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE + #else + | UTF_FLAG__TO_UTF8__SURROGATE_ERROR; + #endif + #endif + ; + +void ConvertUnicodeToUTF8(const UString &src, AString &dest) +{ + ConvertUnicodeToUTF8_Flags(src, dest, g_Unicode_To_UTF8_Flags); +} + +void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest) +{ + const unsigned flags = g_Unicode_To_UTF8_Flags; + dest.Free(); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + dest.Alloc(destLen); + const char *destEnd = Utf16_To_Utf8((char *)(void *)(Byte *)dest, src, src.Ptr(src.Len()), flags); + if (destLen != (size_t)(destEnd - (char *)(void *)(Byte *)dest)) + throw 202104; } + +/* + +#ifndef _WIN32 +void Convert_UTF16_To_UTF32(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t c = src[i++]; + if (c >= 0xd800 && c < 0xdc00 && i < src.Len()) + { + const wchar_t c2 = src[i]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + dest += c; + } +} + +void Convert_UTF32_To_UTF16(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t w = src[i++]; + if (w >= 0x10000 && w < 0x110000) + { + w -= 0x10000; + dest += (wchar_t)((unsigned)0xd800 + (((unsigned)w >> 10) & 0x3ff)); + w = 0xdc00 + (w & 0x3ff); + } + dest += w; + } +} + +bool UTF32_IsThere_BigPoint(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (c >= 0x110000) + return true; + } + return false; +} + +bool Unicode_IsThere_BmpEscape(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (IS_ESCAPE_POINT(c, 0)) + return true; + } + return false; +} + + +#endif + +bool Unicode_IsThere_Utf16SurrogateError(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 val = (UInt32)src[i++]; + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val >= 0xdc00 || i == src.Len()) + return true; + const UInt32 c2 = (UInt32)*src; + if (!IS_LOW_SURROGATE_POINT(c2)) + return true; + } + } + return false; +} +*/ + +#ifndef _WCHART_IS_16BIT + +void Convert_UnicodeEsc16_To_UnicodeEscHigh +#if UTF_ESCAPE_PLANE == 0 + (UString &) {} +#else + (UString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = s[i]; + if (IS_ESCAPE_POINT(c, 0)) + { + c += UTF_ESCAPE_PLANE; + s.ReplaceOneCharAtPos(i, c); + } + } +} +#endif +#endif diff --git a/CPP/Common/UTFConvert.h b/CPP/Common/UTFConvert.h index 827f3dcf..37c4975a 100644 --- a/CPP/Common/UTFConvert.h +++ b/CPP/Common/UTFConvert.h @@ -3,10 +3,382 @@ #ifndef __COMMON_UTF_CONVERT_H #define __COMMON_UTF_CONVERT_H +#include "MyBuffer.h" #include "MyString.h" -bool CheckUTF8(const char *src, bool allowReduced = false) throw(); -bool ConvertUTF8ToUnicode(const AString &utfString, UString &resultString); -void ConvertUnicodeToUTF8(const UString &unicodeString, AString &resultString); +struct CUtf8Check +{ + // Byte MaxByte; // in original src stream + bool NonUtf; + bool ZeroChar; + bool SingleSurrogate; + bool Escape; + bool Truncated; + UInt32 MaxHighPoint; // only for points >= 0x80 + + CUtf8Check() { Clear(); } + + void Clear() + { + // MaxByte = 0; + NonUtf = false; + ZeroChar = false; + SingleSurrogate = false; + Escape = false; + Truncated = false; + MaxHighPoint = 0; + } + + void Update(const CUtf8Check &c) + { + if (c.NonUtf) NonUtf = true; + if (c.ZeroChar) ZeroChar = true; + if (c.SingleSurrogate) SingleSurrogate = true; + if (c.Escape) Escape = true; + if (c.Truncated) Truncated = true; + if (MaxHighPoint < c.MaxHighPoint) MaxHighPoint = c.MaxHighPoint; + } + + void PrintStatus(AString &s) const + { + s.Empty(); + + // s.Add_OptSpaced("MaxByte="); + // s.Add_UInt32(MaxByte); + + if (NonUtf) s.Add_OptSpaced("non-UTF8"); + if (ZeroChar) s.Add_OptSpaced("ZeroChar"); + if (SingleSurrogate) s.Add_OptSpaced("SingleSurrogate"); + if (Escape) s.Add_OptSpaced("Escape"); + if (Truncated) s.Add_OptSpaced("Truncated"); + + if (MaxHighPoint != 0) + { + s.Add_OptSpaced("MaxUnicode="); + s.Add_UInt32(MaxHighPoint); + } + } + + + bool IsOK(bool allowReduced = false) const + { + if (NonUtf || SingleSurrogate || ZeroChar) + return false; + if (MaxHighPoint >= 0x110000) + return false; + if (Truncated && !allowReduced) + return false; + return true; + } + + // it checks full buffer as specified in (size) and it doesn't stop on zero char + void Check_Buf(const char *src, size_t size) throw(); + + void Check_AString(const AString &s) throw() + { + Check_Buf(s.Ptr(), s.Len()); + } +}; + +/* +if (allowReduced == false) - all UTF-8 character sequences must be finished. +if (allowReduced == true) - it allows truncated last character-Utf8-sequence +*/ + +bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw(); +bool CheckUTF8_AString(const AString &s) throw(); + +#define UTF_FLAG__FROM_UTF8__SURROGATE_ERROR (1 << 0) +#define UTF_FLAG__FROM_UTF8__USE_ESCAPE (1 << 1) +#define UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT (1 << 2) + +/* +UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + + if (flag is NOT set) + { + it processes SINGLE-SURROGATE-8 as valid Unicode point. + it converts SINGLE-SURROGATE-8 to SINGLE-SURROGATE-16 + Note: some sequencies of two SINGLE-SURROGATE-8 points + will generate correct SURROGATE-16-PAIR, and + that SURROGATE-16-PAIR later will be converted to correct + UTF8-SURROGATE-21 point. So we don't restore original + STR-8 sequence in that case. + } + + if (flag is set) + { + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is defined) + it generates ESCAPE for SINGLE-SURROGATE-8, + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is not defined) + it generates U+fffd for SINGLE-SURROGATE-8, + } + + +UTF_FLAG__FROM_UTF8__USE_ESCAPE + + if (flag is NOT set) + it generates (U+fffd) code for non-UTF-8 (invalid) characters + + if (flag is set) + { + It generates (ESCAPE) codes for NON-UTF-8 (invalid) characters. + And later we can restore original UTF-8-RAW characters from (ESCAPE-16-21) codes. + } + +UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + + if (flag is NOT set) + { + it process ESCAPE-8 points as another Unicode points. + In Linux: ESCAPE-16 will mean two different ESCAPE-8 seqences, + so we need HIGH-ESCAPE-PLANE-21 to restore UTF-8-RAW -> UTF-16 -> UTF-8-RAW + } + + if (flag is set) + { + it generates ESCAPE-16-21 for ESCAPE-8 points + so we can restore UTF-8-RAW -> UTF-16 -> UTF-8-RAW without HIGH-ESCAPE-PLANE-21. + } + + +Main USE CASES with UTF-8 <-> UTF-16 conversions: + + WIN32: UTF-16-RAW -> UTF-8 (Archive) -> UTF-16-RAW + { + set UTF_FLAG__FROM_UTF8__USE_ESCAPE + Do NOT set UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + Do NOT set UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + + So we restore original SINGLE-SURROGATE-16 from single SINGLE-SURROGATE-8. + } + + Linux: UTF-8-RAW -> UTF-16 (Intermediate / Archive) -> UTF-8-RAW + { + we want restore original UTF-8-RAW sequence later from that ESCAPE-16. + Set the flags: + UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + UTF_FLAG__FROM_UTF8__USE_ESCAPE + UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + } + + MacOS: UTF-8-RAW -> UTF-16 (Intermediate / Archive) -> UTF-8-RAW + { + we want to restore correct UTF-8 without any BMP processing: + Set the flags: + UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + UTF_FLAG__FROM_UTF8__USE_ESCAPE + } + +*/ + +// zero char is not allowed in (src) buf +bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags = 0); + +bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags = 0); +bool ConvertUTF8ToUnicode(const AString &src, UString &dest); + +#define UTF_FLAG__TO_UTF8__SURROGATE_ERROR (1 << 8) +#define UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE (1 << 9) +// #define UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE (1 << 10) + +/* +UTF_FLAG__TO_UTF8__SURROGATE_ERROR + + if (flag is NOT set) + { + we extract SINGLE-SURROGATE as normal UTF-8 + + In Windows : for UTF-16-RAW <-> UTF-8 (archive) <-> UTF-16-RAW in . + + In Linux : + use-case-1: UTF-8 -> UTF-16 -> UTF-8 doesn't generate UTF-16 SINGLE-SURROGATE, + if (UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) is used. + use-case 2: UTF-16-7z (with SINGLE-SURROGATE from Windows) -> UTF-8 (Linux) + will generate SINGLE-SURROGATE-UTF-8 here. + } + + if (flag is set) + { + we generate UTF_REPLACEMENT_CHAR (0xfffd) for SINGLE_SURROGATE + it can be used for compatibility mode with WIN32 UTF function + or if we want UTF-8 stream without any errors + } + + +UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE + + if (flag is NOT set) it doesn't extract raw 8-bit symbol from Escape-Plane-16 + if (flag is set) it extracts raw 8-bit symbol from Escape-Plane-16 + + in Linux we need some way to extract NON-UTF8 RAW 8-bits from BMP (UTF-16 7z archive): + if (we use High-Escape-Plane), we can transfer BMP escapes to High-Escape-Plane. + if (we don't use High-Escape-Plane), we must use UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE. + + +UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE + // that flag affects the code only if (wchar_t is 32-bit) + // that mode with high-escape can be disabled now in UTFConvert.cpp + if (flag is NOT set) + it doesn't extract raw 8-bit symbol from High-Escape-Plane + if (flag is set) + it extracts raw 8-bit symbol from High-Escape-Plane + +Main use cases: + +WIN32 : UTF-16-RAW -> UTF-8 (archive) -> UTF-16-RAW + { + Do NOT set UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE. + Do NOT set UTF_FLAG__TO_UTF8__SURROGATE_ERROR. + So we restore original UTF-16-RAW. + } + +Linix : UTF-8 with Escapes -> UTF-16 (7z archive) -> UTF-8 with Escapes + set UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE to extract non-UTF from 7z archive + set UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE for intermediate UTF-16. + Note: high esacape mode can be ignored now in UTFConvert.cpp + +macOS: + the system doesn't support incorrect UTF-8 in file names. + set UTF_FLAG__TO_UTF8__SURROGATE_ERROR +*/ + +extern unsigned g_Unicode_To_UTF8_Flags; + +void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags = 0); +void ConvertUnicodeToUTF8(const UString &src, AString &dest); + +void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest); + +/* +#ifndef _WIN32 +void Convert_UTF16_To_UTF32(const UString &src, UString &dest); +void Convert_UTF32_To_UTF16(const UString &src, UString &dest); +bool UTF32_IsThere_BigPoint(const UString &src); +bool Unicode_IsThere_BmpEscape(const UString &src); +#endif + +bool Unicode_IsThere_Utf16SurrogateError(const UString &src); +*/ + +#ifdef _WCHART_IS_16BIT +#define Convert_UnicodeEsc16_To_UnicodeEscHigh(s) +#else +void Convert_UnicodeEsc16_To_UnicodeEscHigh(UString &s); +#endif + +/* +// #include "../../C/CpuArch.h" + +// ---------- Utf16 Little endian functions ---------- + +// We store 16-bit surrogates even in 32-bit WCHARs in Linux. +// So now we don't use the following code: + +#if WCHAR_MAX > 0xffff + +// void *p : pointer to src bytes stream +// size_t len : num Utf16 characters : it can include or not include NULL character + +inline size_t Utf16LE__Get_Num_WCHARs(const void *p, size_t len) +{ + #if WCHAR_MAX > 0xffff + size_t num_wchars = 0; + for (size_t i = 0; i < len; i++) + { + wchar_t c = GetUi16(p); + p = (const void *)((const Byte *)p + 2); + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = GetUi16(p); + if (c2 >= 0xdc00 && c2 < 0xe000) + { + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + p = (const void *)((const Byte *)p + 2); + i++; + } + } + num_wchars++; + } + return num_wchars; + #else + UNUSED_VAR(p) + return len; + #endif +} + +// #include <stdio.h> + +inline wchar_t *Utf16LE__To_WCHARs_Sep(const void *p, size_t len, wchar_t *dest) +{ + for (size_t i = 0; i < len; i++) + { + wchar_t c = GetUi16(p); + p = (const void *)((const Byte *)p + 2); + + #if WCHAR_PATH_SEPARATOR != L'/' + if (c == L'/') + c = WCHAR_PATH_SEPARATOR; + #endif + + #if WCHAR_MAX > 0xffff + + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = GetUi16(p); + if (c2 >= 0xdc00 && c2 < 0xe000) + { + // printf("\nSurragate : %4x %4x -> ", (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + p = (const void *)((const Byte *)p + 2); + i++; + // printf("%4x\n", (int)c); + } + } + + #endif + + *dest++ = c; + } + return dest; +} + + +inline size_t Get_Num_Utf16_chars_from_wchar_string(const wchar_t *p) +{ + size_t num = 0; + for (;;) + { + wchar_t c = *p++; + if (c == 0) + return num; + num += ((c >= 0x10000 && c < 0x110000) ? 2 : 1); + } + return num; +} + +inline Byte *wchars_to_Utf16LE(const wchar_t *p, Byte *dest) +{ + for (;;) + { + wchar_t c = *p++; + if (c == 0) + return dest; + if (c >= 0x10000 && c < 0x110000) + { + SetUi16(dest , (UInt16)(0xd800 + ((c >> 10) & 0x3FF))); + SetUi16(dest + 2, (UInt16)(0xdc00 + ( c & 0x3FF))); + dest += 4; + } + else + { + SetUi16(dest, c); + dest += 2; + } + } +} + +#endif +*/ #endif diff --git a/CPP/Common/Wildcard.cpp b/CPP/Common/Wildcard.cpp index a7199170..e2040120 100644 --- a/CPP/Common/Wildcard.cpp +++ b/CPP/Common/Wildcard.cpp @@ -4,9 +4,17 @@ #include "Wildcard.h" +extern +bool g_CaseSensitive; bool g_CaseSensitive = #ifdef _WIN32 false; + #elif defined (__APPLE__) + #ifdef TARGET_OS_IPHONE + true; + #else + false; + #endif #else true; #endif @@ -19,8 +27,16 @@ bool IsPath1PrefixedByPath2(const wchar_t *s1, const wchar_t *s2) return IsString1PrefixedByString2_NoCase(s1, s2); } +// #include <stdio.h> + int CompareFileNames(const wchar_t *s1, const wchar_t *s2) STRING_UNICODE_THROW { + /* + printf("\nCompareFileNames"); + printf("\n S1: %ls", s1); + printf("\n S2: %ls", s2); + printf("\n"); + */ if (g_CaseSensitive) return MyStringCompare(s1, s2); return MyStringCompareNoCase(s1, s2); @@ -131,7 +147,7 @@ UString ExtractDirPrefixFromPath(const UString &path) UString ExtractFileNameFromPath(const UString &path) { - return UString(path.Ptr(path.ReverseFind_PathSepar() + 1)); + return UString(path.Ptr((unsigned)(path.ReverseFind_PathSepar() + 1))); } @@ -229,12 +245,12 @@ bool CItem::CheckPath(const UStringVector &pathParts, bool isFile) const { if (WildcardMatching) { - if (!DoesWildcardMatchName(PathParts[i], pathParts[i + d])) + if (!DoesWildcardMatchName(PathParts[i], pathParts[i + (unsigned)d])) break; } else { - if (CompareFileNames(PathParts[i], pathParts[i + d]) != 0) + if (CompareFileNames(PathParts[i], pathParts[i + (unsigned)d]) != 0) break; } } @@ -258,16 +274,14 @@ int CCensorNode::FindSubNode(const UString &name) const { FOR_VECTOR (i, SubNodes) if (CompareFileNames(SubNodes[i].Name, name) == 0) - return i; + return (int)i; return -1; } void CCensorNode::AddItemSimple(bool include, CItem &item) { - if (include) - IncludeItems.Add(item); - else - ExcludeItems.Add(item); + CObjectVector<CItem> &items = include ? IncludeItems : ExcludeItems; + items.Add(item); } void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) @@ -282,6 +296,7 @@ void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) AddItemSimple(include, item); return; } + const UString &front = item.PathParts.Front(); // WIN32 doesn't support wildcards in file names @@ -292,11 +307,9 @@ void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) AddItemSimple(include, item); return; } - int index = FindSubNode(front); - if (index < 0) - index = SubNodes.Add(CCensorNode(front, this)); + CCensorNode &subNode = Find_SubNode_Or_Add_New(front); item.PathParts.Delete(0); - SubNodes[index].AddItem(include, item, ignoreWildcardIndex - 1); + subNode.AddItem(include, item, ignoreWildcardIndex - 1); } void CCensorNode::AddItem(bool include, const UString &path, bool recursive, bool forFile, bool forDir, bool wildcardMatching) @@ -347,18 +360,19 @@ bool CCensorNode::CheckPathVect(const UStringVector &pathParts, bool isFile, boo include = false; return true; } - include = true; - bool finded = CheckPathCurrent(true, pathParts, isFile); - if (pathParts.Size() <= 1) - return finded; - int index = FindSubNode(pathParts.Front()); - if (index >= 0) + if (pathParts.Size() > 1) { - UStringVector pathParts2 = pathParts; - pathParts2.Delete(0); - if (SubNodes[index].CheckPathVect(pathParts2, isFile, include)) - return true; + int index = FindSubNode(pathParts.Front()); + if (index >= 0) + { + UStringVector pathParts2 = pathParts; + pathParts2.Delete(0); + if (SubNodes[(unsigned)index].CheckPathVect(pathParts2, isFile, include)) + return true; + } } + bool finded = CheckPathCurrent(true, pathParts, isFile); + include = finded; // if (!finded), then (true) is allowed also return finded; } @@ -394,14 +408,26 @@ bool CCensorNode::CheckPath(bool isAltStream, const UString &path, bool isFile) } */ -bool CCensorNode::CheckPathToRoot(bool include, UStringVector &pathParts, bool isFile) const +bool CCensorNode::CheckPathToRoot_Change(bool include, UStringVector &pathParts, bool isFile) const { if (CheckPathCurrent(include, pathParts, isFile)) return true; - if (Parent == 0) + if (!Parent) return false; pathParts.Insert(0, Name); - return Parent->CheckPathToRoot(include, pathParts, isFile); + return Parent->CheckPathToRoot_Change(include, pathParts, isFile); +} + +bool CCensorNode::CheckPathToRoot(bool include, const UStringVector &pathParts, bool isFile) const +{ + if (CheckPathCurrent(include, pathParts, isFile)) + return true; + if (!Parent) + return false; + UStringVector pathParts2; + pathParts2.Add(Name); + pathParts2 += pathParts; + return Parent->CheckPathToRoot_Change(include, pathParts2, isFile); } /* @@ -434,18 +460,15 @@ void CCensorNode::ExtendExclude(const CCensorNode &fromNodes) FOR_VECTOR (i, fromNodes.SubNodes) { const CCensorNode &node = fromNodes.SubNodes[i]; - int subNodeIndex = FindSubNode(node.Name); - if (subNodeIndex < 0) - subNodeIndex = SubNodes.Add(CCensorNode(node.Name, this)); - SubNodes[subNodeIndex].ExtendExclude(node); + Find_SubNode_Or_Add_New(node.Name).ExtendExclude(node); } } -int CCensor::FindPrefix(const UString &prefix) const +int CCensor::FindPairForPrefix(const UString &prefix) const { FOR_VECTOR (i, Pairs) if (CompareFileNames(Pairs[i].Prefix, prefix) == 0) - return i; + return (int)i; return -1; } @@ -454,7 +477,11 @@ int CCensor::FindPrefix(const UString &prefix) const bool IsDriveColonName(const wchar_t *s) { wchar_t c = s[0]; - return c != 0 && s[1] == ':' && s[2] == 0 && (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'); + return c != 0 + && s[1] == ':' + && s[2] == 0 + && ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z')); } unsigned GetNumPrefixParts_if_DrivePath(UStringVector &pathParts) @@ -571,14 +598,16 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat { const UString &part = pathParts[i]; if (part == L".." || part == L".") - dotsIndex = i; + dotsIndex = (int)i; } if (dotsIndex >= 0) + { if (dotsIndex == (int)pathParts.Size() - 1) numSkipParts = pathParts.Size(); else numSkipParts = pathParts.Size() - 1; + } } for (unsigned i = 0; i < numSkipParts; i++) @@ -596,13 +625,16 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat } } - int index = FindPrefix(prefix); + int index = FindPairForPrefix(prefix); if (index < 0) - index = Pairs.Add(CPair(prefix)); + { + index = (int)Pairs.Size(); + Pairs.AddNew().Prefix = prefix; + } if (pathMode != k_AbsPath) { - if (pathParts.IsEmpty() || pathParts.Size() == 1 && pathParts[0].IsEmpty()) + if (pathParts.IsEmpty() || (pathParts.Size() == 1 && pathParts[0].IsEmpty())) { // we create universal item, if we skip all parts as prefix (like \ or L:\ ) pathParts.Clear(); @@ -619,7 +651,7 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat item.ForFile = forFile; item.Recursive = recursive; item.WildcardMatching = wildcardMatching; - Pairs[index].Head.AddItem(include, item, ignoreWildcardIndex); + Pairs[(unsigned)index].Head.AddItem(include, item, ignoreWildcardIndex); } /* diff --git a/CPP/Common/Wildcard.h b/CPP/Common/Wildcard.h index 93f53c0f..32b1521f 100644 --- a/CPP/Common/Wildcard.h +++ b/CPP/Common/Wildcard.h @@ -51,23 +51,34 @@ struct CItem bool CheckPath(const UStringVector &pathParts, bool isFile) const; }; -class CCensorNode + +class CCensorNode MY_UNCOPYABLE { CCensorNode *Parent; bool CheckPathCurrent(bool include, const UStringVector &pathParts, bool isFile) const; void AddItemSimple(bool include, CItem &item); public: - bool CheckPathVect(const UStringVector &pathParts, bool isFile, bool &include) const; - - CCensorNode(): Parent(0) { }; - CCensorNode(const UString &name, CCensorNode *parent): Name(name), Parent(parent) { }; + CCensorNode(): Parent(NULL) { }; + CCensorNode(const UString &name, CCensorNode *parent): Parent(parent), Name(name) { }; UString Name; // WIN32 doesn't support wildcards in file names CObjectVector<CCensorNode> SubNodes; CObjectVector<CItem> IncludeItems; CObjectVector<CItem> ExcludeItems; + CCensorNode &Find_SubNode_Or_Add_New(const UString &name) + { + int i = FindSubNode(name); + if (i >= 0) + return SubNodes[(unsigned)i]; + // return SubNodes.Add(CCensorNode(name, this)); + CCensorNode &node = SubNodes.AddNew(); + node.Parent = this; + node.Name = name; + return node; + } + bool AreAllAllowed() const; int FindSubNode(const UString &path) const; @@ -76,25 +87,41 @@ public: void AddItem(bool include, const UString &path, bool recursive, bool forFile, bool forDir, bool wildcardMatching); void AddItem2(bool include, const UString &path, bool recursive, bool wildcardMatching); + // NeedCheckSubDirs() returns true, if there are IncludeItems rules that affect items in subdirs bool NeedCheckSubDirs() const; bool AreThereIncludeItems() const; + /* + CheckPathVect() doesn't check path in Parent CCensorNode + so use CheckPathVect() for root CCensorNode + OUT: + returns (true) && (include = false) - file in exlude list + returns (true) && (include = true) - file in include list and is not in exlude list + returns (false) - file is not in (include/exlude) list + */ + bool CheckPathVect(const UStringVector &pathParts, bool isFile, bool &include) const; + // bool CheckPath2(bool isAltStream, const UString &path, bool isFile, bool &include) const; // bool CheckPath(bool isAltStream, const UString &path, bool isFile) const; - bool CheckPathToRoot(bool include, UStringVector &pathParts, bool isFile) const; + // CheckPathToRoot_Change() changes pathParts !!! + bool CheckPathToRoot_Change(bool include, UStringVector &pathParts, bool isFile) const; + bool CheckPathToRoot(bool include, const UStringVector &pathParts, bool isFile) const; + // bool CheckPathToRoot(const UString &path, bool isFile, bool include) const; void ExtendExclude(const CCensorNode &fromNodes); }; -struct CPair + +struct CPair MY_UNCOPYABLE { UString Prefix; CCensorNode Head; - CPair(const UString &prefix): Prefix(prefix) { }; + // CPair(const UString &prefix): Prefix(prefix) { }; }; + enum ECensorPathMode { k_RelatPath, // absolute prefix as Prefix, remain path in Tree @@ -102,6 +129,7 @@ enum ECensorPathMode k_AbsPath // full path in Tree }; + struct CCensorPath { UString Path; @@ -116,9 +144,10 @@ struct CCensorPath {} }; -class CCensor + +class CCensor MY_UNCOPYABLE { - int FindPrefix(const UString &prefix) const; + int FindPairForPrefix(const UString &prefix) const; public: CObjectVector<CPair> Pairs; @@ -143,7 +172,6 @@ public: } }; - } #endif diff --git a/CPP/Windows/CommonDialog.cpp b/CPP/Windows/CommonDialog.cpp index 8f4f56d3..eaaecada 100644 --- a/CPP/Windows/CommonDialog.cpp +++ b/CPP/Windows/CommonDialog.cpp @@ -67,8 +67,11 @@ bool CDoubleZeroStringListW::Add(LPCWSTR s) throw() return true; } + +#ifdef UNDER_CE #define MY__OFN_PROJECT 0x00400000 #define MY__OFN_SHOW_ALL 0x01000000 +#endif /* if (lpstrFilter == NULL && nFilterIndex == 0) MSDN : "the system doesn't show any files", @@ -91,14 +94,34 @@ So we use size of old version of structure. */ #if defined(UNDER_CE) || defined(_WIN64) || (_WIN32_WINNT < 0x0500) // || !defined(WINVER) + #ifndef _UNICODE #define my_compatib_OPENFILENAMEA_size sizeof(OPENFILENAMEA) + #endif #define my_compatib_OPENFILENAMEW_size sizeof(OPENFILENAMEW) #else + + // MinGW doesn't support some required macros. So we define them here: + #ifndef CDSIZEOF_STRUCT + #define CDSIZEOF_STRUCT(structname, member) (((int)((LPBYTE)(&((structname*)0)->member) - ((LPBYTE)((structname*)0)))) + sizeof(((structname*)0)->member)) + #endif + #ifndef _UNICODE + #ifndef OPENFILENAME_SIZE_VERSION_400A + #define OPENFILENAME_SIZE_VERSION_400A CDSIZEOF_STRUCT(OPENFILENAMEA,lpTemplateName) + #endif + #endif + #ifndef OPENFILENAME_SIZE_VERSION_400W + #define OPENFILENAME_SIZE_VERSION_400W CDSIZEOF_STRUCT(OPENFILENAMEW,lpTemplateName) + #endif + + #ifndef _UNICODE #define my_compatib_OPENFILENAMEA_size OPENFILENAME_SIZE_VERSION_400A + #endif #define my_compatib_OPENFILENAMEW_size OPENFILENAME_SIZE_VERSION_400W #endif +#ifndef _UNICODE #define CONV_U_To_A(dest, src, temp) AString temp; if (src) { temp = GetSystemString(src); dest = temp; } +#endif bool MyGetOpenFileName(HWND hwnd, LPCWSTR title, LPCWSTR initialDir, diff --git a/CPP/Windows/Control/ComboBox.cpp b/CPP/Windows/Control/ComboBox.cpp index febc61ef..f6ed8d34 100644 --- a/CPP/Windows/Control/ComboBox.cpp +++ b/CPP/Windows/Control/ComboBox.cpp @@ -43,10 +43,10 @@ LRESULT CComboBox::GetLBText(int index, UString &s) s.Empty(); if (g_IsNT) { - LRESULT len = SendMsgW(CB_GETLBTEXTLEN, index, 0); + LRESULT len = SendMsgW(CB_GETLBTEXTLEN, MY__int_TO_WPARAM(index), 0); if (len == CB_ERR) return len; - LRESULT len2 = SendMsgW(CB_GETLBTEXT, index, (LPARAM)s.GetBuf((unsigned)len)); + LRESULT len2 = SendMsgW(CB_GETLBTEXT, MY__int_TO_WPARAM(index), (LPARAM)s.GetBuf((unsigned)len)); if (len2 == CB_ERR) return len; if (len > len2) diff --git a/CPP/Windows/Control/ComboBox.h b/CPP/Windows/Control/ComboBox.h index 1d5a4821..f08b1f7c 100644 --- a/CPP/Windows/Control/ComboBox.h +++ b/CPP/Windows/Control/ComboBox.h @@ -5,13 +5,15 @@ #include "../../Common/MyWindows.h" -#include <commctrl.h> +#include <CommCtrl.h> #include "../Window.h" namespace NWindows { namespace NControl { +#define MY__int_TO_WPARAM(i) ((WPARAM)(INT_PTR)(i)) + class CComboBox: public CWindow { public: @@ -20,19 +22,25 @@ public: #ifndef _UNICODE LRESULT AddString(LPCWSTR s); #endif - LRESULT SetCurSel(int index) { return SendMsg(CB_SETCURSEL, index, 0); } + + /* If this parameter is -1, any current selection in the list is removed and the edit control is cleared.*/ + LRESULT SetCurSel(int index) { return SendMsg(CB_SETCURSEL, MY__int_TO_WPARAM(index), 0); } + + /* If no item is selected, it returns CB_ERR (-1) */ int GetCurSel() { return (int)SendMsg(CB_GETCURSEL, 0, 0); } + + /* If an error occurs, it is CB_ERR (-1) */ int GetCount() { return (int)SendMsg(CB_GETCOUNT, 0, 0); } - LRESULT GetLBTextLen(int index) { return SendMsg(CB_GETLBTEXTLEN, index, 0); } - LRESULT GetLBText(int index, LPTSTR s) { return SendMsg(CB_GETLBTEXT, index, (LPARAM)s); } + LRESULT GetLBTextLen(int index) { return SendMsg(CB_GETLBTEXTLEN, MY__int_TO_WPARAM(index), 0); } + LRESULT GetLBText(int index, LPTSTR s) { return SendMsg(CB_GETLBTEXT, MY__int_TO_WPARAM(index), (LPARAM)s); } LRESULT GetLBText(int index, CSysString &s); #ifndef _UNICODE LRESULT GetLBText(int index, UString &s); #endif - LRESULT SetItemData(int index, LPARAM lParam) { return SendMsg(CB_SETITEMDATA, index, lParam); } - LRESULT GetItemData(int index) { return SendMsg(CB_GETITEMDATA, index, 0); } + LRESULT SetItemData(int index, LPARAM lParam) { return SendMsg(CB_SETITEMDATA, MY__int_TO_WPARAM(index), lParam); } + LRESULT GetItemData(int index) { return SendMsg(CB_GETITEMDATA, MY__int_TO_WPARAM(index), 0); } LRESULT GetItemData_of_CurSel() { return GetItemData(GetCurSel()); } @@ -46,7 +54,11 @@ class CComboBoxEx: public CComboBox public: bool SetUnicodeFormat(bool fUnicode) { return LRESULTToBool(SendMsg(CBEM_SETUNICODEFORMAT, BOOLToBool(fUnicode), 0)); } - LRESULT DeleteItem(int index) { return SendMsg(CBEM_DELETEITEM, index, 0); } + /* Returns: + an INT value that represents the number of items remaining in the control. + If (index) is invalid, the message returns CB_ERR. */ + LRESULT DeleteItem(int index) { return SendMsg(CBEM_DELETEITEM, MY__int_TO_WPARAM(index), 0); } + LRESULT InsertItem(COMBOBOXEXITEM *item) { return SendMsg(CBEM_INSERTITEM, 0, (LPARAM)item); } #ifndef _UNICODE LRESULT InsertItem(COMBOBOXEXITEMW *item) { return SendMsg(CBEM_INSERTITEMW, 0, (LPARAM)item); } diff --git a/CPP/Windows/Control/ImageList.h b/CPP/Windows/Control/ImageList.h index 0d9c9313..19feb117 100644 --- a/CPP/Windows/Control/ImageList.h +++ b/CPP/Windows/Control/ImageList.h @@ -3,7 +3,7 @@ #ifndef __WINDOWS_CONTROL_IMAGE_LIST_H #define __WINDOWS_CONTROL_IMAGE_LIST_H -#include <commctrl.h> +#include <CommCtrl.h> #include "../Defs.h" diff --git a/CPP/Windows/Control/ListView.cpp b/CPP/Windows/Control/ListView.cpp index 6d916591..16cfd396 100644 --- a/CPP/Windows/Control/ListView.cpp +++ b/CPP/Windows/Control/ListView.cpp @@ -35,7 +35,7 @@ int CListView::InsertColumn(int columnIndex, LPCTSTR text, int width) { LVCOLUMN ci; ci.mask = LVCF_TEXT | LVCF_WIDTH | LVCF_SUBITEM; - ci.pszText = (LPTSTR)text; + ci.pszText = (LPTSTR)(void *)text; ci.iSubItem = columnIndex; ci.cx = width; return InsertColumn(columnIndex, &ci); @@ -47,7 +47,7 @@ int CListView::InsertItem(int index, LPCTSTR text) item.mask = LVIF_TEXT | LVIF_PARAM; item.iItem = index; item.lParam = index; - item.pszText = (LPTSTR)text; + item.pszText = (LPTSTR)(void *)text; item.iSubItem = 0; return InsertItem(&item); } @@ -57,7 +57,7 @@ int CListView::SetSubItem(int index, int subIndex, LPCTSTR text) LVITEM item; item.mask = LVIF_TEXT; item.iItem = index; - item.pszText = (LPTSTR)text; + item.pszText = (LPTSTR)(void *)text; item.iSubItem = subIndex; return SetItem(&item); } @@ -68,7 +68,7 @@ int CListView::InsertColumn(int columnIndex, LPCWSTR text, int width) { LVCOLUMNW ci; ci.mask = LVCF_TEXT | LVCF_WIDTH | LVCF_SUBITEM; - ci.pszText = (LPWSTR)text; + ci.pszText = (LPWSTR)(void *)text; ci.iSubItem = columnIndex; ci.cx = width; return InsertColumn(columnIndex, &ci); @@ -80,7 +80,7 @@ int CListView::InsertItem(int index, LPCWSTR text) item.mask = LVIF_TEXT | LVIF_PARAM; item.iItem = index; item.lParam = index; - item.pszText = (LPWSTR)text; + item.pszText = (LPWSTR)(void *)text; item.iSubItem = 0; return InsertItem(&item); } @@ -90,7 +90,7 @@ int CListView::SetSubItem(int index, int subIndex, LPCWSTR text) LVITEMW item; item.mask = LVIF_TEXT; item.iItem = index; - item.pszText = (LPWSTR)text; + item.pszText = (LPWSTR)(void *)text; item.iSubItem = subIndex; return SetItem(&item); } diff --git a/CPP/Windows/Control/ListView.h b/CPP/Windows/Control/ListView.h index 9a3abe70..a13b1041 100644 --- a/CPP/Windows/Control/ListView.h +++ b/CPP/Windows/Control/ListView.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include <commctrl.h> +#include <CommCtrl.h> #include "../Window.h" @@ -32,7 +32,8 @@ public: int InsertColumn(int columnIndex, const LVCOLUMN *columnInfo) { return ListView_InsertColumn(_window, columnIndex, columnInfo); } int InsertColumn(int columnIndex, LPCTSTR text, int width); - bool SetColumnOrderArray(int count, const int *columns) { return BOOLToBool(ListView_SetColumnOrderArray(_window, count, columns)); } + bool SetColumnOrderArray(int count, const int *columns) + { return BOOLToBool(ListView_SetColumnOrderArray(_window, count, (int *)(void *)columns)); } /* int GetNumColumns() diff --git a/CPP/Windows/Control/ProgressBar.h b/CPP/Windows/Control/ProgressBar.h index 38ebcb61..03743067 100644 --- a/CPP/Windows/Control/ProgressBar.h +++ b/CPP/Windows/Control/ProgressBar.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include <commctrl.h> +#include <CommCtrl.h> #include "../Window.h" diff --git a/CPP/Windows/Control/PropertyPage.h b/CPP/Windows/Control/PropertyPage.h index 4c4ddad9..b68fd8fe 100644 --- a/CPP/Windows/Control/PropertyPage.h +++ b/CPP/Windows/Control/PropertyPage.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include <prsht.h> +#include <PrSht.h> #include "Dialog.h" diff --git a/CPP/Windows/DLL.cpp b/CPP/Windows/DLL.cpp index d7f38375..cf5d01a3 100644 --- a/CPP/Windows/DLL.cpp +++ b/CPP/Windows/DLL.cpp @@ -4,6 +4,8 @@ #include "DLL.h" +#ifdef _WIN32 + #ifndef _UNICODE extern bool g_IsNT; #endif @@ -97,7 +99,7 @@ FString GetModuleDirPrefix() { int pos = s.ReverseFind_PathSepar(); if (pos >= 0) - s.DeleteFrom(pos + 1); + s.DeleteFrom((unsigned)(pos + 1)); } if (s.IsEmpty()) s = "." STRING_PATH_SEPARATOR; @@ -107,3 +109,83 @@ FString GetModuleDirPrefix() #endif }} + +#else + +#include <dlfcn.h> +#include <stdlib.h> + +namespace NWindows { +namespace NDLL { + +bool CLibrary::Free() throw() +{ + if (_module == NULL) + return true; + int ret = dlclose(_module); + if (ret != 0) + return false; + _module = NULL; + return true; +} + +static +// FARPROC +void * +local_GetProcAddress(HMODULE module, LPCSTR procName) +{ + void *ptr = NULL; + if (module) + { + ptr = dlsym(module, procName); + } + return ptr; +} + +bool CLibrary::Load(CFSTR path) throw() +{ + if (!Free()) + return false; + + int options = 0; + + #ifdef RTLD_LOCAL + options |= RTLD_LOCAL; + #endif + + #ifdef RTLD_NOW + options |= RTLD_NOW; + #endif + + #ifdef RTLD_GROUP + #if ! (defined(hpux) || defined(__hpux)) + options |= RTLD_GROUP; // mainly for solaris but not for HPUX + #endif + #endif + + void *handler = dlopen(path, options); + + if (handler) + { + // here we can transfer some settings to DLL + } + else + { + } + + _module = handler; + + return (_module != NULL); +} + +// FARPROC +void * CLibrary::GetProc(LPCSTR procName) const +{ + // return My_GetProcAddress(_module, procName); + return local_GetProcAddress(_module, procName); + // return NULL; +} + +}} + +#endif diff --git a/CPP/Windows/DLL.h b/CPP/Windows/DLL.h index 984a1d33..0c093eed 100644 --- a/CPP/Windows/DLL.h +++ b/CPP/Windows/DLL.h @@ -8,10 +8,12 @@ namespace NWindows { namespace NDLL { +#ifdef _WIN32 + #ifdef UNDER_CE -#define My_GetProcAddress(module, procName) ::GetProcAddressA(module, procName) +#define My_GetProcAddress(module, procName) (void *)::GetProcAddressA(module, procName) #else -#define My_GetProcAddress(module, procName) ::GetProcAddress(module, procName) +#define My_GetProcAddress(module, procName) (void *)::GetProcAddress(module, procName) #endif /* Win32: Don't call CLibrary::Free() and FreeLibrary() from another @@ -46,9 +48,33 @@ public: bool Free() throw(); bool LoadEx(CFSTR path, DWORD flags = LOAD_LIBRARY_AS_DATAFILE) throw(); bool Load(CFSTR path) throw(); - FARPROC GetProc(LPCSTR procName) const { return My_GetProcAddress(_module, procName); } + // FARPROC + void *GetProc(LPCSTR procName) const { return My_GetProcAddress(_module, procName); } }; +#else + +typedef void * HMODULE; +// typedef int (*FARPROC)(); +// typedef void *FARPROC; + +class CLibrary +{ + HMODULE _module; + + // CLASS_NO_COPY(CLibrary); +public: + CLibrary(): _module(NULL) {}; + ~CLibrary() { Free(); } + + bool Free() throw(); + bool Load(CFSTR path) throw(); + // FARPROC + void *GetProc(LPCSTR procName) const; // { return My_GetProcAddress(_module, procName); } +}; + +#endif + bool MyGetModuleFileName(FString &path); FString GetModuleDirPrefix(); diff --git a/CPP/Windows/Defs.h b/CPP/Windows/Defs.h index 281c40c3..1d96078d 100644 --- a/CPP/Windows/Defs.h +++ b/CPP/Windows/Defs.h @@ -7,10 +7,11 @@ #ifdef _WIN32 inline bool LRESULTToBool(LRESULT v) { return (v != FALSE); } -inline bool BOOLToBool(BOOL v) { return (v != FALSE); } inline BOOL BoolToBOOL(bool v) { return (v ? TRUE: FALSE); } #endif +inline bool BOOLToBool(BOOL v) { return (v != FALSE); } + inline VARIANT_BOOL BoolToVARIANT_BOOL(bool v) { return (v ? VARIANT_TRUE: VARIANT_FALSE); } inline bool VARIANT_BOOLToBool(VARIANT_BOOL v) { return (v != VARIANT_FALSE); } diff --git a/CPP/Windows/ErrorMsg.cpp b/CPP/Windows/ErrorMsg.cpp index b86c0b39..bfa21e50 100644 --- a/CPP/Windows/ErrorMsg.cpp +++ b/CPP/Windows/ErrorMsg.cpp @@ -2,21 +2,25 @@ #include "StdAfx.h" -#ifndef _UNICODE +#if !defined(_UNICODE) || !defined(_WIN32) #include "../Common/StringConvert.h" #endif #include "ErrorMsg.h" -#ifndef _UNICODE +#ifdef _WIN32 +#if !defined(_UNICODE) extern bool g_IsNT; #endif +#endif namespace NWindows { namespace NError { static bool MyFormatMessage(DWORD errorCode, UString &message) { + #ifdef _WIN32 + LPVOID msgBuf; #ifndef _UNICODE if (!g_IsNT) @@ -38,8 +42,63 @@ static bool MyFormatMessage(DWORD errorCode, UString &message) } ::LocalFree(msgBuf); return true; + + #else // _WIN32 + + AString m; + + const char *s = NULL; + + switch ((Int32)errorCode) + { + // case ERROR_NO_MORE_FILES : s = "No more files"; break; + // case ERROR_DIRECTORY : s = "Error Directory"; break; + case E_NOTIMPL : s = "E_NOTIMPL : Not implemented"; break; + case E_NOINTERFACE : s = "E_NOINTERFACE : No such interface supported"; break; + case E_ABORT : s = "E_ABORT : Operation aborted"; break; + case E_FAIL : s = "E_FAIL : Unspecified error"; break; + + case STG_E_INVALIDFUNCTION : s = "STG_E_INVALIDFUNCTION"; break; + case CLASS_E_CLASSNOTAVAILABLE : s = "CLASS_E_CLASSNOTAVAILABLE"; break; + + case E_OUTOFMEMORY : s = "E_OUTOFMEMORY : Can't allocate required memory"; break; + case E_INVALIDARG : s = "E_INVALIDARG : One or more arguments are invalid"; break; + + // case MY__E_ERROR_NEGATIVE_SEEK : s = "MY__E_ERROR_NEGATIVE_SEEK"; break; + default: + break; + } + + /* strerror() for unknow errors still shows message "Unknown error -12345678") + So we must trasfer error codes before strerror() */ + if (!s) + { + if ((errorCode & 0xFFFF0000) == (UInt32)((MY__FACILITY__WRes << 16) | 0x80000000)) + errorCode &= 0xFFFF; + else if ((errorCode & ((UInt32)1 << 31))) + return false; // we will show hex error later for that case + + s = strerror((int)errorCode); + + // if (!s) + { + m += "errno="; + m.Add_UInt32(errorCode); + if (s) + m += " : "; + } + } + + if (s) + m += s; + + MultiByteToUnicodeString2(message, m); + return true; + + #endif } + UString MyFormatMessage(DWORD errorCode) { UString m; diff --git a/CPP/Windows/ErrorMsg.h b/CPP/Windows/ErrorMsg.h index 0957c696..01204eb9 100644 --- a/CPP/Windows/ErrorMsg.h +++ b/CPP/Windows/ErrorMsg.h @@ -9,6 +9,7 @@ namespace NWindows { namespace NError { UString MyFormatMessage(DWORD errorCode); +inline UString MyFormatMessage(HRESULT errorCode) { return MyFormatMessage((DWORD)errorCode); } }} diff --git a/CPP/Windows/FileDir.cpp b/CPP/Windows/FileDir.cpp index 62b11c10..c06e7f37 100644 --- a/CPP/Windows/FileDir.cpp +++ b/CPP/Windows/FileDir.cpp @@ -2,8 +2,21 @@ #include "StdAfx.h" -#ifndef _UNICODE + +#ifndef _WIN32 +#include <stdio.h> +#include <errno.h> +#include <limits.h> +#include <unistd.h> +#include <time.h> +#include <utime.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + #include "../Common/StringConvert.h" +#include "../Common/C_FileIO.h" +#include "TimeUtils.h" #endif #include "FileDir.h" @@ -22,6 +35,8 @@ namespace NWindows { namespace NFile { namespace NDir { +#ifdef _WIN32 + #ifndef UNDER_CE bool GetWindowsDir(FString &path) @@ -67,7 +82,8 @@ bool GetSystemDir(FString &path) } return (needLength > 0 && needLength <= MAX_PATH); } -#endif +#endif // UNDER_CE + bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) { @@ -102,6 +118,8 @@ bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const return res; } + + bool SetFileAttrib(CFSTR path, DWORD attrib) { #ifndef _UNICODE @@ -131,8 +149,10 @@ bool SetFileAttrib(CFSTR path, DWORD attrib) bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) { + #ifdef _WIN32 if ((attrib & 0xF0000000) != 0) attrib &= 0x3FFF; + #endif return SetFileAttrib(path, attrib); } @@ -163,6 +183,7 @@ bool RemoveDir(CFSTR path) return false; } + bool MyMoveFile(CFSTR oldFile, CFSTR newFile) { #ifndef _UNICODE @@ -175,8 +196,10 @@ bool MyMoveFile(CFSTR oldFile, CFSTR newFile) #endif { IF_USE_MAIN_PATH_2(oldFile, newFile) + { if (::MoveFileW(fs2us(oldFile), fs2us(newFile))) return true; + } #ifdef WIN_LONG_PATH if (USE_SUPER_PATH_2) { @@ -190,7 +213,6 @@ bool MyMoveFile(CFSTR oldFile, CFSTR newFile) } #ifndef UNDER_CE - EXTERN_C_BEGIN typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPCWSTR lpFileName, @@ -198,6 +220,7 @@ typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPSECURITY_ATTRIBUTES lpSecurityAttributes ); EXTERN_C_END +#endif // UNDER_CE bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) { @@ -215,12 +238,14 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) #endif { Func_CreateHardLinkW my_CreateHardLinkW = (Func_CreateHardLinkW) - ::GetProcAddress(::GetModuleHandleW(L"kernel32.dll"), "CreateHardLinkW"); + (void *)::GetProcAddress(::GetModuleHandleW(L"kernel32.dll"), "CreateHardLinkW"); if (!my_CreateHardLinkW) return false; IF_USE_MAIN_PATH_2(newFileName, existFileName) + { if (my_CreateHardLinkW(fs2us(newFileName), fs2us(existFileName), NULL)) return true; + } #ifdef WIN_LONG_PATH if (USE_SUPER_PATH_2) { @@ -233,7 +258,6 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) return false; } -#endif /* WinXP-64 CreateDir(): @@ -333,6 +357,10 @@ static bool CreateDir2(CFSTR path) return fi.IsDir(); } +#endif // _WIN32 + +static bool CreateDir2(CFSTR path); + bool CreateComplexDir(CFSTR _path) { #ifdef _WIN32 @@ -348,11 +376,21 @@ bool CreateComplexDir(CFSTR _path) if (IsDriveRootPath_SuperAllowed(_path)) return false; - unsigned prefixSize = GetRootPrefixSize(_path); + const unsigned prefixSize = GetRootPrefixSize(_path); - #endif + #endif // UNDER_CE + + #else // _WIN32 + + // Posix + NFind::CFileInfo fi; + if (fi.Find(_path)) + { + if (fi.IsDir()) + return true; + } - #endif + #endif // _WIN32 FString path (_path); @@ -365,7 +403,7 @@ bool CreateComplexDir(CFSTR _path) } const FString path2 (path); - pos = path.Len(); + pos = (int)path.Len(); for (;;) { @@ -384,17 +422,17 @@ bool CreateComplexDir(CFSTR _path) return false; #endif - path.DeleteFrom(pos); + path.DeleteFrom((unsigned)pos); } while (pos < (int)path2.Len()) { - int pos2 = NName::FindSepar(path2.Ptr(pos + 1)); + int pos2 = NName::FindSepar(path2.Ptr((unsigned)pos + 1)); if (pos2 < 0) - pos = path2.Len(); + pos = (int)path2.Len(); else pos += 1 + pos2; - path.SetFrom(path2, pos); + path.SetFrom(path2, (unsigned)pos); if (!CreateDir(path)) return false; } @@ -402,6 +440,9 @@ bool CreateComplexDir(CFSTR _path) return true; } + +#ifdef _WIN32 + bool DeleteFileAlways(CFSTR path) { /* If alt stream, we also need to clear READ-ONLY attribute of main file before delete. @@ -412,7 +453,7 @@ bool DeleteFileAlways(CFSTR path) && (attrib & FILE_ATTRIBUTE_DIRECTORY) == 0 && (attrib & FILE_ATTRIBUTE_READONLY) != 0) { - if (!SetFileAttrib(path, attrib & ~FILE_ATTRIBUTE_READONLY)) + if (!SetFileAttrib(path, attrib & ~(DWORD)FILE_ATTRIBUTE_READONLY)) return false; } } @@ -443,6 +484,8 @@ bool DeleteFileAlways(CFSTR path) return false; } + + bool RemoveDirWithSubItems(const FString &path) { bool needRemoveSubItems = true; @@ -466,7 +509,9 @@ bool RemoveDirWithSubItems(const FString &path) const unsigned prefixSize = s.Len(); NFind::CEnumerator enumerator; enumerator.SetDirPrefix(s); - NFind::CFileInfo fi; + NFind::CDirEntry fi; + bool isError = false; + DWORD lastError = 0; while (enumerator.Next(fi)) { s.DeleteFrom(prefixSize); @@ -474,18 +519,32 @@ bool RemoveDirWithSubItems(const FString &path) if (fi.IsDir()) { if (!RemoveDirWithSubItems(s)) - return false; + { + lastError = GetLastError(); + isError = true; + } } else if (!DeleteFileAlways(s)) - return false; + { + lastError = GetLastError(); + isError = false; + } + } + if (isError) + { + SetLastError(lastError); + return false; } } + // we clear read-only attrib to remove read-only dir if (!SetFileAttrib(path, 0)) return false; return RemoveDir(path); } +#endif // _WIN32 + #ifdef UNDER_CE bool MyGetFullPathName(CFSTR path, FString &resFullPath) @@ -501,6 +560,8 @@ bool MyGetFullPathName(CFSTR path, FString &resFullPath) return GetFullPath(path, resFullPath); } +#ifdef _WIN32 + bool SetCurrentDir(CFSTR path) { // SetCurrentDirectory doesn't support \\?\ prefix @@ -516,9 +577,11 @@ bool SetCurrentDir(CFSTR path) } } + bool GetCurrentDir(FString &path) { path.Empty(); + DWORD needLength; #ifndef _UNICODE if (!g_IsNT) @@ -539,7 +602,9 @@ bool GetCurrentDir(FString &path) return (needLength > 0 && needLength <= MAX_PATH); } -#endif +#endif // _WIN32 +#endif // UNDER_CE + bool GetFullPathAndSplit(CFSTR path, FString &resDirPrefix, FString &resFileName) { @@ -547,8 +612,9 @@ bool GetFullPathAndSplit(CFSTR path, FString &resDirPrefix, FString &resFileName if (!res) resDirPrefix = path; int pos = resDirPrefix.ReverseFind_PathSepar(); - resFileName = resDirPrefix.Ptr(pos + 1); - resDirPrefix.DeleteFrom(pos + 1); + pos++; + resFileName = resDirPrefix.Ptr((unsigned)pos); + resDirPrefix.DeleteFrom((unsigned)pos); return res; } @@ -560,6 +626,7 @@ bool GetOnlyDirPrefix(CFSTR path, FString &resDirPrefix) bool MyGetTempPath(FString &path) { + #ifdef _WIN32 path.Empty(); DWORD needLength; #ifndef _UNICODE @@ -579,11 +646,27 @@ bool MyGetTempPath(FString &path) path = us2fs(s); } return (needLength > 0 && needLength <= MAX_PATH); + + #else + + // FIXME: improve that code + path = "/tmp/"; + if (!NFind::DoesDirExist_FollowLink(path)) + path = "./"; + return true; + #endif } + static bool CreateTempFile(CFSTR prefix, bool addRandom, FString &path, NIO::COutFile *outFile) { - UInt32 d = (GetTickCount() << 12) ^ (GetCurrentThreadId() << 14) ^ GetCurrentProcessId(); + UInt32 d = + #ifdef _WIN32 + (GetTickCount() << 12) ^ (GetCurrentThreadId() << 14) ^ GetCurrentProcessId(); + #else + (UInt32)(time(NULL) << 12) ^ ((UInt32)getppid() << 14) ^ (UInt32)(getpid()); + #endif + for (unsigned i = 0; i < 100; i++) { path = prefix; @@ -670,7 +753,7 @@ bool CTempFile::MoveTo(CFSTR name, bool deleteDestBefore) // DWORD attrib = 0; if (deleteDestBefore) { - if (NFind::DoesFileExist(name)) + if (NFind::DoesFileExist_Raw(name)) { // attrib = NFind::GetFileAttrib(name); if (!DeleteFileAlways(name)) @@ -690,6 +773,7 @@ bool CTempFile::MoveTo(CFSTR name, bool deleteDestBefore) */ } +#ifdef _WIN32 bool CTempDir::Create(CFSTR prefix) { if (!Remove()) @@ -710,5 +794,285 @@ bool CTempDir::Remove() _mustBeDeleted = !RemoveDirWithSubItems(_path); return !_mustBeDeleted; } +#endif + + + +#ifndef _WIN32 + +bool RemoveDir(CFSTR path) +{ + return (rmdir(path) == 0); +} + + +static BOOL My__CopyFile(CFSTR oldFile, CFSTR newFile) +{ + NWindows::NFile::NIO::COutFile outFile; + if (!outFile.Create(newFile, false)) + return FALSE; + + NWindows::NFile::NIO::CInFile inFile; + if (!inFile.Open(oldFile)) + return FALSE; + + char buf[1 << 14]; + + for (;;) + { + const ssize_t num = inFile.read_part(buf, sizeof(buf)); + if (num == 0) + return TRUE; + if (num < 0) + return FALSE; + size_t processed; + const ssize_t num2 = outFile.write_full(buf, (size_t)num, processed); + if (num2 != num || processed != (size_t)num) + return FALSE; + } +} + + +bool MyMoveFile(CFSTR oldFile, CFSTR newFile) +{ + int res = rename(oldFile, newFile); + if (res == 0) + return true; + if (errno != EXDEV) // (oldFile and newFile are not on the same mounted filesystem) + return false; + + if (My__CopyFile(oldFile, newFile) == FALSE) + return false; + + struct stat info_file; + res = stat(oldFile, &info_file); + if (res != 0) + return false; + + /* + ret = chmod(dst,info_file.st_mode & g_umask.mask); + */ + return (unlink(oldFile) == 0); +} + + +bool CreateDir(CFSTR path) +{ + return (mkdir(path, 0777) == 0); // change it +} + +static bool CreateDir2(CFSTR path) +{ + return (mkdir(path, 0777) == 0); // change it +} + + +bool DeleteFileAlways(CFSTR path) +{ + return (remove(path) == 0); +} + +bool SetCurrentDir(CFSTR path) +{ + return (chdir(path) == 0); +} + + +bool GetCurrentDir(FString &path) +{ + path.Empty(); + + #define MY__PATH_MAX PATH_MAX + // #define MY__PATH_MAX 1024 + + char s[MY__PATH_MAX + 1]; + char *res = getcwd(s, MY__PATH_MAX); + if (!res) + { + // if (errno != ERANGE) + return false; + } + path = fas2fs(s); + return true; +} + + +static void FILETME_To_timespec(const FILETIME *ft, timespec &ts) +{ + if (ft) + { + const Int64 sec = NTime::FileTimeToUnixTime64(*ft); + // time_t is long + const time_t sec2 = (time_t)sec; + if (sec2 == sec) + { + ts.tv_sec = sec2; + const UInt64 winTime = (((UInt64)ft->dwHighDateTime) << 32) + ft->dwLowDateTime; + ts.tv_nsec = (long)((winTime % 10000000) * 100); + return; + } + } + // else + { + ts.tv_sec = 0; + // ts.tv_nsec = UTIME_NOW; // set to the current time + ts.tv_nsec = UTIME_OMIT; // keep old timesptamp + } +} + + + + +bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) +{ + // need testing + /* + struct utimbuf buf; + struct stat st; + UNUSED_VAR(cTime) + + printf("\nstat = %s\n", path); + int ret = stat(path, &st); + + if (ret == 0) + { + buf.actime = st.st_atime; + buf.modtime = st.st_mtime; + } + else + { + time_t cur_time = time(0); + buf.actime = cur_time; + buf.modtime = cur_time; + } + + if (aTime) + { + UInt32 ut; + if (NTime::FileTimeToUnixTime(*aTime, ut)) + buf.actime = ut; + } + + if (mTime) + { + UInt32 ut; + if (NTime::FileTimeToUnixTime(*mTime, ut)) + buf.modtime = ut; + } + + return utime(path, &buf) == 0; + */ + + // if (!aTime && !mTime) return true; + + struct timespec times[2]; + UNUSED_VAR(cTime) + + FILETME_To_timespec(aTime, times[0]); + FILETME_To_timespec(mTime, times[1]); + + const int flags = 0; // follow link + // = AT_SYMLINK_NOFOLLOW; // don't follow link + return utimensat(AT_FDCWD, path, times, flags) == 0; +} + + + +struct C_umask +{ + mode_t mask; + + C_umask() + { + /* by security reasons we restrict attributes according + with process's file mode creation mask (umask) */ + const mode_t um = umask(0); // octal :0022 is expected + mask = 0777 & (~um); // octal: 0755 is expected + umask(um); // restore the umask + // printf("\n umask = 0%03o mask = 0%03o\n", um, mask); + + // mask = 0777; // debug we can disable the restriction: + } +}; + +static C_umask g_umask; + +// #define PRF(x) x; +#define PRF(x) + +#define TRACE_SetFileAttrib(msg) \ + PRF(printf("\nSetFileAttrib(%s, %x) : %s\n", (const char *)path, attrib, msg)); + +#define TRACE_chmod(s, mode) \ + PRF(printf("\n chmod(%s, %o)\n", (const char *)path, (unsigned)(mode))); + + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) +{ + TRACE_SetFileAttrib(""); + + struct stat st; + + bool use_lstat = true; + if (use_lstat) + { + if (lstat(path, &st) != 0) + { + TRACE_SetFileAttrib("bad lstat()"); + return false; + } + } + else + { + if (stat(path, &st) != 0) + { + TRACE_SetFileAttrib("bad stat()"); + return false; + } + } + + if (attrib & FILE_ATTRIBUTE_UNIX_EXTENSION) + { + st.st_mode = attrib >> 16; + if (S_ISDIR(st.st_mode)) + { + // user/7z must be able to create files in this directory + st.st_mode |= (S_IRUSR | S_IWUSR | S_IXUSR); + } + else if (!S_ISREG(st.st_mode)) + return true; + } + else if (S_ISLNK(st.st_mode)) + { + // change it + SetLastError(ENOSYS); + return false; + } + else + { + // Only Windows Attributes + if (S_ISDIR(st.st_mode) + || (attrib & FILE_ATTRIBUTE_READONLY) == 0) + return true; + st.st_mode &= ~(mode_t)(S_IWUSR | S_IWGRP | S_IWOTH); // octal: ~0222; // disable write permissions + } + + TRACE_chmod(path, (st.st_mode) & g_umask.mask); + int res = chmod(path, (st.st_mode) & g_umask.mask); + + // TRACE_SetFileAttrib("OK") + return (res == 0); +} + + +bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) +{ + PRF(printf("\nhard link() %s -> %s\n", newFileName, existFileName)); + return (link(existFileName, newFileName) == 0); +} + +#endif // !_WIN32 + +// #endif }}} diff --git a/CPP/Windows/FileDir.h b/CPP/Windows/FileDir.h index 154ed973..6d6ddeaf 100644 --- a/CPP/Windows/FileDir.h +++ b/CPP/Windows/FileDir.h @@ -17,6 +17,8 @@ bool GetSystemDir(FString &path); bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime); +#ifdef _WIN32 + bool SetFileAttrib(CFSTR path, DWORD attrib); /* @@ -26,6 +28,8 @@ bool SetFileAttrib(CFSTR path, DWORD attrib); bits that are related to current system only. */ +#endif + bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib); @@ -61,7 +65,7 @@ bool GetCurrentDir(FString &resultPath); bool MyGetTempPath(FString &resultPath); -class CTempFile +class CTempFile MY_UNCOPYABLE { bool _mustBeDeleted; FString _path; @@ -76,7 +80,9 @@ public: bool MoveTo(CFSTR name, bool deleteDestBefore); }; -class CTempDir + +#ifdef _WIN32 +class CTempDir MY_UNCOPYABLE { bool _mustBeDeleted; FString _path; @@ -88,9 +94,11 @@ public: bool Create(CFSTR namePrefix) ; bool Remove(); }; +#endif + #if !defined(UNDER_CE) -class CCurrentDirRestorer +class CCurrentDirRestorer MY_UNCOPYABLE { FString _path; public: diff --git a/CPP/Windows/FileFind.cpp b/CPP/Windows/FileFind.cpp index b9692b81..8ef0f7b6 100644 --- a/CPP/Windows/FileFind.cpp +++ b/CPP/Windows/FileFind.cpp @@ -2,8 +2,11 @@ #include "StdAfx.h" -#ifndef _UNICODE -#include "../Common/StringConvert.h" +// #include <stdio.h> + +#ifndef _WIN32 +#include <fcntl.h> /* Definition of AT_* constants */ +#include "TimeUtils.h" #endif #include "FileFind.h" @@ -34,27 +37,51 @@ typedef struct WCHAR cStreamName[MAX_PATH + 36]; } MY_WIN32_FIND_STREAM_DATA, *MY_PWIN32_FIND_STREAM_DATA; -typedef WINBASEAPI HANDLE (WINAPI *FindFirstStreamW_Ptr)(LPCWSTR fileName, MY_STREAM_INFO_LEVELS infoLevel, +typedef HANDLE (WINAPI *FindFirstStreamW_Ptr)(LPCWSTR fileName, MY_STREAM_INFO_LEVELS infoLevel, LPVOID findStreamData, DWORD flags); -typedef WINBASEAPI BOOL (APIENTRY *FindNextStreamW_Ptr)(HANDLE findStream, LPVOID findStreamData); +typedef BOOL (APIENTRY *FindNextStreamW_Ptr)(HANDLE findStream, LPVOID findStreamData); EXTERN_C_END -#endif +#endif // defined(_WIN32) && !defined(UNDER_CE) + namespace NWindows { namespace NFile { + +#ifdef _WIN32 #ifdef SUPPORT_DEVICE_FILE namespace NSystem { bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); } #endif +#endif namespace NFind { +#define MY_CLEAR_FILETIME(ft) ft.dwLowDateTime = ft.dwHighDateTime = 0; + +void CFileInfoBase::ClearBase() throw() +{ + Size = 0; + MY_CLEAR_FILETIME(CTime); + MY_CLEAR_FILETIME(ATime); + MY_CLEAR_FILETIME(MTime); + Attrib = 0; + // ReparseTag = 0; + IsAltStream = false; + IsDevice = false; + + #ifndef _WIN32 + ino = 0; + nlink = 0; + mode = 0; + #endif +} + bool CFileInfo::IsDots() const throw() { if (!IsDir() || Name.IsEmpty()) @@ -64,12 +91,17 @@ bool CFileInfo::IsDots() const throw() return Name.Len() == 1 || (Name.Len() == 2 && Name[1] == '.'); } + +#ifdef _WIN32 + + #define WIN_FD_TO_MY_FI(fi, fd) \ fi.Attrib = fd.dwFileAttributes; \ fi.CTime = fd.ftCreationTime; \ fi.ATime = fd.ftLastAccessTime; \ fi.MTime = fd.ftLastWriteTime; \ fi.Size = (((UInt64)fd.nFileSizeHigh) << 32) + fd.nFileSizeLow; \ + /* fi.ReparseTag = fd.dwReserved0; */ \ fi.IsAltStream = false; \ fi.IsDevice = false; @@ -91,7 +123,6 @@ static void Convert_WIN32_FIND_DATA_to_FileInfo(const WIN32_FIND_DATAW &fd, CFil } #ifndef _UNICODE - static void Convert_WIN32_FIND_DATA_to_FileInfo(const WIN32_FIND_DATA &fd, CFileInfo &fi) { WIN_FD_TO_MY_FI(fi, fd); @@ -143,7 +174,8 @@ WinXP-64 FindFirstFile(): \\Server\Share_RootDrive - ERROR_INVALID_NAME \\Server\Share_RootDrive\ - ERROR_INVALID_NAME - c:\* - ERROR_FILE_NOT_FOUND, if thare are no item in that folder + e:\* - ERROR_FILE_NOT_FOUND, if there are no items in that root folder + w:\* - ERROR_PATH_NOT_FOUND, if there is no such drive w: */ bool CFindFile::FindFirst(CFSTR path, CFileInfo &fi) @@ -210,12 +242,13 @@ bool CFindFile::FindNext(CFileInfo &fi) static FindFirstStreamW_Ptr g_FindFirstStreamW; static FindNextStreamW_Ptr g_FindNextStreamW; -struct CFindStreamLoader +static struct CFindStreamLoader { CFindStreamLoader() { - g_FindFirstStreamW = (FindFirstStreamW_Ptr)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "FindFirstStreamW"); - g_FindNextStreamW = (FindNextStreamW_Ptr)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "FindNextStreamW"); + HMODULE hm = ::GetModuleHandleA("kernel32.dll"); + g_FindFirstStreamW = (FindFirstStreamW_Ptr)(void *)::GetProcAddress(hm, "FindFirstStreamW"); + g_FindNextStreamW = (FindNextStreamW_Ptr)(void *)::GetProcAddress(hm, "FindNextStreamW"); } } g_FindStreamLoader; @@ -245,7 +278,7 @@ UString CStreamInfo::GetReducedName2() const static void Convert_WIN32_FIND_STREAM_DATA_to_StreamInfo(const MY_WIN32_FIND_STREAM_DATA &sd, CStreamInfo &si) { - si.Size = sd.StreamSize.QuadPart; + si.Size = (UInt64)sd.StreamSize.QuadPart; si.Name = sd.cStreamName; } @@ -336,19 +369,6 @@ bool CStreamEnumerator::Next(CStreamInfo &si, bool &found) #endif -#define MY_CLEAR_FILETIME(ft) ft.dwLowDateTime = ft.dwHighDateTime = 0; - -void CFileInfoBase::ClearBase() throw() -{ - Size = 0; - MY_CLEAR_FILETIME(CTime); - MY_CLEAR_FILETIME(ATime); - MY_CLEAR_FILETIME(MTime); - Attrib = 0; - IsAltStream = false; - IsDevice = false; -} - /* WinXP-64 GetFileAttributes(): If the function fails, it returns INVALID_FILE_ATTRIBUTES and use GetLastError() to get error code @@ -416,7 +436,7 @@ also we support paths that are not supported by FindFirstFile: c::stream - Name = c::stream */ -bool CFileInfo::Find(CFSTR path) +bool CFileInfo::Find(CFSTR path, bool followLink) { #ifdef SUPPORT_DEVICE_FILE if (IsDevicePath(path)) @@ -454,7 +474,7 @@ bool CFileInfo::Find(CFSTR path) { UString streamName = fs2us(path + (unsigned)colonPos); FString filePath (path); - filePath.DeleteFrom(colonPos); + filePath.DeleteFrom((unsigned)colonPos); /* we allow both cases: name:stream name:stream:$DATA @@ -467,7 +487,7 @@ bool CFileInfo::Find(CFSTR path) bool isOk = true; if (IsDrivePath2(filePath) && - (colonPos == 2 || colonPos == 3 && filePath[2] == '\\')) + (colonPos == 2 || (colonPos == 3 && filePath[2] == '\\'))) { // FindFirstFile doesn't work for "c:\" and for "c:" (if current dir is ROOT) ClearBase(); @@ -476,11 +496,11 @@ bool CFileInfo::Find(CFSTR path) Name = filePath; } else - isOk = Find(filePath); + isOk = Find(filePath, followLink); // check it (followLink) if (isOk) { - Attrib &= ~(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); + Attrib &= ~(DWORD)(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); Size = 0; CStreamEnumerator enumerator(filePath); for (;;) @@ -536,11 +556,15 @@ bool CFileInfo::Find(CFSTR path) ClearBase(); Attrib = attrib; Name = path + rootSize; - Name.DeleteFrom(2); // we don't need backslash (C:) + Name.DeleteFrom(2); + if (!Fill_From_ByHandleFileInfo(path)) + { + } return true; } } else if (IS_PATH_SEPAR(path[0])) + { if (path[1] == 0) { DWORD attrib = GetFileAttrib(path); @@ -559,10 +583,15 @@ bool CFileInfo::Find(CFSTR path) { if (NName::FindSepar(path + prefixSize) < 0) { + if (Fill_From_ByHandleFileInfo(path)) + { + Name = path + prefixSize; + return true; + } + FString s (path); s.Add_PathSepar(); s += '*'; // CHAR_ANY_MASK - bool isOK = false; if (finder.FindFirst(s, *this)) { @@ -577,7 +606,7 @@ bool CFileInfo::Find(CFSTR path) } { DWORD attrib = GetFileAttrib(path); - if (isOK || attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY) != 0) + if (isOK || (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY) != 0)) { ClearBase(); if (attrib != INVALID_FILE_ATTRIBUTES) @@ -592,23 +621,105 @@ bool CFileInfo::Find(CFSTR path) } } } + } } #endif - return finder.FindFirst(path, *this); + bool res = finder.FindFirst(path, *this); + if (!followLink + || !res + || !HasReparsePoint()) + return res; + + // return FollowReparse(path, IsDir()); + return Fill_From_ByHandleFileInfo(path); +} + +bool CFileInfo::Fill_From_ByHandleFileInfo(CFSTR path) +{ + BY_HANDLE_FILE_INFORMATION info; + if (!NIO::CFileBase::GetFileInformation(path, &info)) + return false; + { + Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + CTime = info.ftCreationTime; + ATime = info.ftLastAccessTime; + MTime = info.ftLastWriteTime; + Attrib = info.dwFileAttributes; + return true; + } } +/* +bool CFileInfo::FollowReparse(CFSTR path, bool isDir) +{ + if (isDir) + { + FString prefix = path; + prefix.Add_PathSepar(); + + // "folder/." refers to folder itself. So we can't use that path + // we must use enumerator and search "." item + CEnumerator enumerator; + enumerator.SetDirPrefix(prefix); + for (;;) + { + CFileInfo fi; + if (!enumerator.NextAny(fi)) + break; + if (fi.Name.IsEqualTo_Ascii_NoCase(".")) + { + // we can copy preperies; + CTime = fi.CTime; + ATime = fi.ATime; + MTime = fi.MTime; + Attrib = fi.Attrib; + Size = fi.Size; + return true; + } + break; + } + // LastError(lastError); + return false; + } + + { + NIO::CInFile inFile; + if (inFile.Open(path)) + { + BY_HANDLE_FILE_INFORMATION info; + if (inFile.GetFileInformation(&info)) + { + ClearBase(); + Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + CTime = info.ftCreationTime; + ATime = info.ftLastAccessTime; + MTime = info.ftLastWriteTime; + Attrib = info.dwFileAttributes; + return true; + } + } + return false; + } +} +*/ -bool DoesFileExist(CFSTR name) +bool DoesFileExist_Raw(CFSTR name) { CFileInfo fi; return fi.Find(name) && !fi.IsDir(); } -bool DoesDirExist(CFSTR name) +bool DoesFileExist_FollowLink(CFSTR name) +{ + CFileInfo fi; + return fi.Find_FollowLink(name) && !fi.IsDir(); +} + +bool DoesDirExist(CFSTR name, bool followLink) { CFileInfo fi; - return fi.Find(name) && fi.IsDir(); + return fi.Find(name, followLink) && fi.IsDir(); } bool DoesFileOrDirExist(CFSTR name) @@ -645,15 +756,46 @@ bool CEnumerator::Next(CFileInfo &fi) bool CEnumerator::Next(CFileInfo &fi, bool &found) { + /* + for (;;) + { + if (!NextAny(fi)) + break; + if (!fi.IsDots()) + { + found = true; + return true; + } + } + */ + if (Next(fi)) { found = true; return true; } + found = false; - return (::GetLastError() == ERROR_NO_MORE_FILES); + DWORD lastError = ::GetLastError(); + if (_findFile.IsHandleAllocated()) + return (lastError == ERROR_NO_MORE_FILES); + // we support the case for empty root folder: FindFirstFile("c:\\*") returns ERROR_FILE_NOT_FOUND + if (lastError == ERROR_FILE_NOT_FOUND) + return true; + if (lastError == ERROR_ACCESS_DENIED) + { + // here we show inaccessible root system folder as empty folder to eliminate redundant user warnings + const char *s = "System Volume Information" STRING_PATH_SEPARATOR "*"; + const int len = (int)strlen(s); + const int delta = (int)_wildcard.Len() - len; + if (delta == 0 || (delta > 0 && IS_PATH_SEPAR(_wildcard[(unsigned)delta - 1]))) + if (StringsAreEqual_Ascii(_wildcard.Ptr((unsigned)delta), s)) + return true; + } + return false; } + //////////////////////////////// // CFindChangeNotification // FindFirstChangeNotification can return 0. MSDN doesn't tell about it. @@ -744,6 +886,331 @@ bool MyGetLogicalDriveStrings(CObjectVector<FString> &driveStrings) } } -#endif +#endif // UNDER_CE + + + +#else // _WIN32 + +// ---------- POSIX ---------- + +static int MY__lstat(CFSTR path, struct stat *st, bool followLink) +{ + memset(st, 0, sizeof(*st)); + int res; + // #ifdef ENV_HAVE_LSTAT + if (/* global_use_lstat && */ !followLink) + { + // printf("\nlstat\n"); + res = lstat(path, st); + } + else + // #endif + { + // printf("\nstat\n"); + res = stat(path, st); + } + /* + printf("\nres = %d\n", res); + printf("\n st_dev = %lld \n", (long long)(st->st_dev)); + printf("\n st_ino = %lld \n", (long long)(st->st_ino)); + printf("\n st_mode = %lld \n", (long long)(st->st_mode)); + printf("\n st_nlink = %lld \n", (long long)(st->st_nlink)); + printf("\n st_uid = %lld \n", (long long)(st->st_uid)); + printf("\n st_gid = %lld \n", (long long)(st->st_gid)); + printf("\n st_size = %lld \n", (long long)(st->st_size)); + printf("\n st_blksize = %lld \n", (long long)(st->st_blksize)); + printf("\n st_blocks = %lld \n", (long long)(st->st_blocks)); + */ + + return res; +} + + +static const char *Get_Name_from_Path(CFSTR path) throw() +{ + size_t len = strlen(path); + if (len == 0) + return path; + const char *p = path + len - 1; + { + if (p == path) + return path; + p--; + } + for (;;) + { + char c = *p; + if (IS_PATH_SEPAR(c)) + return p + 1; + if (p == path) + return path; + p--; + } +} + + +void timespec_To_FILETIME(const MY_ST_TIMESPEC &ts, FILETIME &ft) +{ + UInt64 v = NTime::UnixTime64ToFileTime64(ts.tv_sec) + ((UInt64)ts.tv_nsec / 100); + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} + +UInt32 Get_WinAttribPosix_From_PosixMode(UInt32 mode) +{ + UInt32 attrib = S_ISDIR(mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((mode & 0222) == 0) // S_IWUSR in p7zip + attrib |= FILE_ATTRIBUTE_READONLY; + return attrib | FILE_ATTRIBUTE_UNIX_EXTENSION | ((mode & 0xFFFF) << 16); +} + +/* +UInt32 Get_WinAttrib_From_stat(const struct stat &st) +{ + UInt32 attrib = S_ISDIR(st.st_mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + + if ((st.st_mode & 0222) == 0) // check it !!! + attrib |= FILE_ATTRIBUTE_READONLY; + + attrib |= FILE_ATTRIBUTE_UNIX_EXTENSION + ((st.st_mode & 0xFFFF) << 16); + return attrib; +} +*/ + +void CFileInfo::SetFrom_stat(const struct stat &st) +{ + IsDevice = false; + + if (S_ISDIR(st.st_mode)) + { + Size = 0; + } + else + { + Size = (UInt64)st.st_size; // for a symbolic link, size = size of filename + } + + Attrib = Get_WinAttribPosix_From_PosixMode(st.st_mode); + + // NTime::UnixTimeToFileTime(st.st_ctime, CTime); + // NTime::UnixTimeToFileTime(st.st_mtime, MTime); + // NTime::UnixTimeToFileTime(st.st_atime, ATime); + #ifdef __APPLE__ + timespec_To_FILETIME(st.st_ctimespec, CTime); + timespec_To_FILETIME(st.st_mtimespec, MTime); + timespec_To_FILETIME(st.st_atimespec, ATime); + #else + timespec_To_FILETIME(st.st_ctim, CTime); + timespec_To_FILETIME(st.st_mtim, MTime); + timespec_To_FILETIME(st.st_atim, ATime); + #endif + + dev = st.st_dev; + ino = st.st_ino; + nlink = st.st_nlink; + mode = st.st_mode; +} + +bool CFileInfo::Find_DontFill_Name(CFSTR path, bool followLink) +{ + struct stat st; + if (MY__lstat(path, &st, followLink) != 0) + return false; + SetFrom_stat(st); + return true; +} + + +bool CFileInfo::Find(CFSTR path, bool followLink) +{ + // printf("\nCEnumerator::Find() name = %s\n", path); + if (!Find_DontFill_Name(path, followLink)) + return false; + + // printf("\nOK\n"); + + Name = Get_Name_from_Path(path); + if (!Name.IsEmpty()) + { + char c = Name.Back(); + if (IS_PATH_SEPAR(c)) + Name.DeleteBack(); + } + return true; +} + + +bool DoesFileExist_Raw(CFSTR name) +{ + // FIXME for symbolic links. + struct stat st; + if (MY__lstat(name, &st, false) != 0) + return false; + return !S_ISDIR(st.st_mode); +} + +bool DoesFileExist_FollowLink(CFSTR name) +{ + // FIXME for symbolic links. + struct stat st; + if (MY__lstat(name, &st, true) != 0) + return false; + return !S_ISDIR(st.st_mode); +} + +bool DoesDirExist(CFSTR name, bool followLink) +{ + struct stat st; + if (MY__lstat(name, &st, followLink) != 0) + return false; + return S_ISDIR(st.st_mode); +} + +bool DoesFileOrDirExist(CFSTR name) +{ + struct stat st; + if (MY__lstat(name, &st, false) != 0) + return false; + return true; +} + + +CEnumerator::~CEnumerator() +{ + if (_dir) + closedir(_dir); +} + +void CEnumerator::SetDirPrefix(const FString &dirPrefix) +{ + _wildcard = dirPrefix; +} + +bool CDirEntry::IsDots() const throw() +{ + if ( + #if !defined(_AIX) + !IsDir() || + #endif + Name.IsEmpty()) + return false; + if (Name[0] != '.') + return false; + return Name.Len() == 1 || (Name.Len() == 2 && Name[1] == '.'); +} + + +bool CEnumerator::NextAny(CDirEntry &fi, bool &found) +{ + found = false; + + if (!_dir) + { + const char *w = "./"; + if (!_wildcard.IsEmpty()) + w = _wildcard.Ptr(); + _dir = ::opendir((const char *)w); + if (_dir == NULL) + return false; + } + + // To distinguish end of stream from an error, we must set errno to zero before readdir() + errno = 0; + + struct dirent *de = readdir(_dir); + if (!de) + { + if (errno == 0) + return true; // it's end of stream, and we report it with (found = false) + // it's real error + return false; + } + + fi.iNode = de->d_ino; + + #if !defined(_AIX) + fi.Type = de->d_type; + #endif + + /* + if (de->d_type == DT_DIR) + fi.Attrib = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_UNIX_EXTENSION | ((UInt32)(S_IFDIR) << 16); + else if (de->d_type < 16) + fi.Attrib = FILE_ATTRIBUTE_UNIX_EXTENSION | ((UInt32)(de->d_type) << (16 + 12)); + */ + fi.Name = de->d_name; + + /* + printf("\nCEnumerator::NextAny; len = %d %s \n", (int)fi.Name.Len(), fi.Name.Ptr()); + for (unsigned i = 0; i < fi.Name.Len(); i++) + printf (" %02x", (unsigned)(Byte)de->d_name[i]); + printf("\n"); + */ + + found = true; + return true; +} + + +bool CEnumerator::Next(CDirEntry &fi, bool &found) +{ + // printf("\nCEnumerator::Next()\n"); + // PrintName("Next", ""); + for (;;) + { + if (!NextAny(fi, found)) + return false; + if (!found) + return true; + if (!fi.IsDots()) + { + /* + if (!NeedFullStat) + return true; + // we silently skip error file here - it can be wrong link item + if (fi.Find_DontFill_Name(path)) + return true; + */ + return true; + } + } +} + +/* +bool CEnumerator::Next(CDirEntry &fileInfo, bool &found) +{ + bool found; + if (!Next(fi, found)) + return false; + return found; +} +*/ + +bool CEnumerator::Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) +{ + // printf("\nCEnumerator::Fill_FileInfo()\n"); + struct stat st; + // probably it's OK to use fstatat() even if it changes file position dirfd(_dir) + int res = fstatat(dirfd(_dir), de.Name, &st, followLink ? 0 : AT_SYMLINK_NOFOLLOW); + // if fstatat() is not supported, we can use stat() / lstat() + + /* + const FString path = _wildcard + s; + int res = MY__lstat(path, &st, followLink); + */ + + if (res != 0) + return false; + fileInfo.SetFrom_stat(st); + fileInfo.Name = de.Name; + return true; +} + +#endif // _WIN32 }}} diff --git a/CPP/Windows/FileFind.h b/CPP/Windows/FileFind.h index bfb29206..009e84e7 100644 --- a/CPP/Windows/FileFind.h +++ b/CPP/Windows/FileFind.h @@ -3,13 +3,36 @@ #ifndef __WINDOWS_FILE_FIND_H #define __WINDOWS_FILE_FIND_H +#ifndef _WIN32 +#include <sys/stat.h> +#include <sys/types.h> +#include <dirent.h> +#endif + #include "../Common/MyString.h" +#include "../Common/MyWindows.h" #include "Defs.h" namespace NWindows { namespace NFile { namespace NFind { +// bool DoesFileExist(CFSTR name, bool followLink); +bool DoesFileExist_Raw(CFSTR name); +bool DoesFileExist_FollowLink(CFSTR name); +bool DoesDirExist(CFSTR name, bool followLink); + +inline bool DoesDirExist(CFSTR name) + { return DoesDirExist(name, false); } +inline bool DoesDirExist_FollowLink(CFSTR name) + { return DoesDirExist(name, true); } + +// it's always _Raw +bool DoesFileOrDirExist(CFSTR name); + +DWORD GetFileAttrib(CFSTR path); + + namespace NAttributes { inline bool IsReadOnly(DWORD attrib) { return (attrib & FILE_ATTRIBUTE_READONLY) != 0; } @@ -33,6 +56,7 @@ public: bool IsAltStream; bool IsDevice; + #ifdef _WIN32 /* #ifdef UNDER_CE DWORD ObjectID; @@ -40,11 +64,24 @@ public: UINT32 ReparseTag; #endif */ + #else + dev_t dev; + ino_t ino; + nlink_t nlink; + mode_t mode; + // bool Use_lstat; + #endif CFileInfoBase() { ClearBase(); } void ClearBase() throw(); - - void SetAsDir() { Attrib = FILE_ATTRIBUTE_DIRECTORY; } + + void SetAsDir() + { + Attrib = FILE_ATTRIBUTE_DIRECTORY; + #ifndef _WIN32 + Attrib |= (FILE_ATTRIBUTE_UNIX_EXTENSION + (S_IFDIR << 16)); + #endif + } bool IsArchived() const { return MatchesMask(FILE_ATTRIBUTE_ARCHIVE); } bool IsCompressed() const { return MatchesMask(FILE_ATTRIBUTE_COMPRESSED); } @@ -58,6 +95,14 @@ public: bool IsSparse() const { return MatchesMask(FILE_ATTRIBUTE_SPARSE_FILE); } bool IsSystem() const { return MatchesMask(FILE_ATTRIBUTE_SYSTEM); } bool IsTemporary() const { return MatchesMask(FILE_ATTRIBUTE_TEMPORARY); } + + #ifndef _WIN32 + bool IsPosixLink() const + { + const UInt32 mod = Attrib >> 16; + return S_ISLNK(mod); + } + #endif }; struct CFileInfo: public CFileInfoBase @@ -68,10 +113,22 @@ struct CFileInfo: public CFileInfoBase #endif bool IsDots() const throw(); - bool Find(CFSTR path); + bool Find(CFSTR path, bool followLink = false); + bool Find_FollowLink(CFSTR path) { return Find(path, true); } + + #ifdef _WIN32 + bool Fill_From_ByHandleFileInfo(CFSTR path); + // bool FollowReparse(CFSTR path, bool isDir); + #else + bool Find_DontFill_Name(CFSTR path, bool followLink = false); + void SetFrom_stat(const struct stat &st); + #endif }; -class CFindFileBase + +#ifdef _WIN32 + +class CFindFileBase MY_UNCOPYABLE { protected: HANDLE _handle; @@ -108,26 +165,21 @@ public: bool FindNext(CStreamInfo &streamInfo); }; -class CStreamEnumerator +class CStreamEnumerator MY_UNCOPYABLE { CFindStream _find; FString _filePath; - bool NextAny(CFileInfo &fileInfo); + bool NextAny(CFileInfo &fileInfo, bool &found); public: CStreamEnumerator(const FString &filePath): _filePath(filePath) {} bool Next(CStreamInfo &streamInfo, bool &found); }; -#endif +#endif // defined(_WIN32) && !defined(UNDER_CE) -bool DoesFileExist(CFSTR name); -bool DoesDirExist(CFSTR name); -bool DoesFileOrDirExist(CFSTR name); - -DWORD GetFileAttrib(CFSTR path); -class CEnumerator +class CEnumerator MY_UNCOPYABLE { CFindFile _findFile; FString _wildcard; @@ -139,7 +191,8 @@ public: bool Next(CFileInfo &fileInfo, bool &found); }; -class CFindChangeNotification + +class CFindChangeNotification MY_UNCOPYABLE { HANDLE _handle; public: @@ -156,6 +209,71 @@ public: bool MyGetLogicalDriveStrings(CObjectVector<FString> &driveStrings); #endif +typedef CFileInfo CDirEntry; + + +#else // WIN32 + + +struct CDirEntry +{ + ino_t iNode; + #if !defined(_AIX) + Byte Type; + #endif + FString Name; + + #if !defined(_AIX) + bool IsDir() const + { + // DT_DIR is + return Type == DT_DIR; + } + #endif + + bool IsDots() const throw(); +}; + +class CEnumerator MY_UNCOPYABLE +{ + DIR *_dir; + FString _wildcard; + + bool NextAny(CDirEntry &fileInfo, bool &found); +public: + CEnumerator(): _dir(NULL) {} + ~CEnumerator(); + void SetDirPrefix(const FString &dirPrefix); + + bool Next(CDirEntry &fileInfo, bool &found); + bool Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink); +}; + +/* +inline UInt32 Get_WinAttrib_From_PosixMode(UInt32 mode) +{ + UInt32 attrib = S_ISDIR(mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((st.st_mode & 0222) == 0) // check it !!! + attrib |= FILE_ATTRIBUTE_READONLY; + return attrib; +} +*/ + +UInt32 Get_WinAttribPosix_From_PosixMode(UInt32 mode); + +// UInt32 Get_WinAttrib_From_stat(const struct stat &st); +#if defined(_AIX) + #define MY_ST_TIMESPEC st_timespec +#else + #define MY_ST_TIMESPEC timespec +#endif + +void timespec_To_FILETIME(const MY_ST_TIMESPEC &ts, FILETIME &ft); + +#endif // WIN32 + }}} #endif diff --git a/CPP/Windows/FileIO.cpp b/CPP/Windows/FileIO.cpp index 56e6ca45..b6f2f26e 100644 --- a/CPP/Windows/FileIO.cpp +++ b/CPP/Windows/FileIO.cpp @@ -6,9 +6,21 @@ #include "../../C/Alloc.h" #endif +// #include <stdio.h> + #include "FileIO.h" #include "FileName.h" +HRESULT GetLastError_noZero_HRESULT() +{ + DWORD res = ::GetLastError(); + if (res == 0) + return E_FAIL; + return HRESULT_FROM_WIN32(res); +} + +#ifdef _WIN32 + #ifndef _UNICODE extern bool g_IsNT; #endif @@ -78,6 +90,42 @@ bool CFileBase::Create(CFSTR path, DWORD desiredAccess, } #endif } + + /* + #ifndef UNDER_CE + #ifndef _SFX + if (_handle == INVALID_HANDLE_VALUE) + { + // it's debug hack to open symbolic links in Windows XP and WSL links in Windows 10 + DWORD lastError = GetLastError(); + if (lastError == ERROR_CANT_ACCESS_FILE) + { + CByteBuffer buf; + if (NIO::GetReparseData(path, buf, NULL)) + { + CReparseAttr attr; + if (attr.Parse(buf, buf.Size())) + { + FString dirPrefix, fileName; + if (NDir::GetFullPathAndSplit(path, dirPrefix, fileName)) + { + FString fullPath; + if (GetFullPath(dirPrefix, us2fs(attr.GetPath()), fullPath)) + { + // FIX IT: recursion levels must be restricted + return Create(fullPath, desiredAccess, + shareMode, creationDisposition, flagsAndAttributes); + } + } + } + } + SetLastError(lastError); + } + } + #endif + #endif + */ + return (_handle != INVALID_HANDLE_VALUE); } @@ -136,7 +184,7 @@ bool CFileBase::Seek(Int64 distanceToMove, DWORD moveMethod, UInt64 &newPosition bool CFileBase::Seek(UInt64 position, UInt64 &newPosition) const throw() { - return Seek(position, FILE_BEGIN, newPosition); + return Seek((Int64)position, FILE_BEGIN, newPosition); } bool CFileBase::SeekToBegin() const throw() @@ -258,7 +306,7 @@ void CInFile::CalcDeviceSize(CFSTR s) if (GetPartitionInfo(&partInfo)) { - Size = partInfo.PartitionLength.QuadPart; + Size = (UInt64)partInfo.PartitionLength.QuadPart; SizeDefined = true; needCorrectSize = false; if ((s)[0] == '\\' && (s)[1] == '\\' && (s)[2] == '.' && (s)[3] == '\\' && (s)[5] == ':' && (s)[6] == 0) @@ -277,7 +325,7 @@ void CInFile::CalcDeviceSize(CFSTR s) my_DISK_GEOMETRY_EX geomEx; SizeDefined = GetGeometryEx(&geomEx); if (SizeDefined) - Size = geomEx.DiskSize.QuadPart; + Size = (UInt64)geomEx.DiskSize.QuadPart; else { DISK_GEOMETRY geom; @@ -285,7 +333,7 @@ void CInFile::CalcDeviceSize(CFSTR s) if (!SizeDefined) SizeDefined = GetCdRomGeometry(&geom); if (SizeDefined) - Size = geom.Cylinders.QuadPart * geom.TracksPerCylinder * geom.SectorsPerTrack * geom.BytesPerSector; + Size = (UInt64)geom.Cylinders.QuadPart * geom.TracksPerCylinder * geom.SectorsPerTrack * geom.BytesPerSector; } } @@ -310,7 +358,24 @@ void CInFile::CalcDeviceSize(CFSTR s) bool CInFile::Open(CFSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) { - bool res = Create(fileName, GENERIC_READ, shareMode, creationDisposition, flagsAndAttributes); + DWORD desiredAccess = GENERIC_READ; + + #ifdef _WIN32 + if (PreserveATime) + desiredAccess |= FILE_WRITE_ATTRIBUTES; + #endif + + bool res = Create(fileName, desiredAccess, shareMode, creationDisposition, flagsAndAttributes); + + #ifdef _WIN32 + if (res && PreserveATime) + { + FILETIME ft; + ft.dwHighDateTime = ft.dwLowDateTime = 0xFFFFFFFF; + ::SetFileTime(_handle, NULL, &ft, NULL); + } + #endif + MY_DEVICE_EXTRA_CODE return res; } @@ -330,7 +395,7 @@ bool CInFile::Open(CFSTR fileName) // for 32 MB (maybe also for 16 MB). // And message can be "Network connection was lost" -static UInt32 kChunkSizeMax = (1 << 22); +static const UInt32 kChunkSizeMax = (1 << 22); bool CInFile::Read1(void *data, UInt32 size, UInt32 &processedSize) throw() { @@ -366,6 +431,26 @@ bool CInFile::Read(void *data, UInt32 size, UInt32 &processedSize) throw() return true; } +bool CInFile::ReadFull(void *data, size_t size, size_t &processedSize) throw() +{ + processedSize = 0; + do + { + UInt32 processedLoc = 0; + const UInt32 sizeLoc = (size > kChunkSizeMax ? (UInt32)kChunkSizeMax : (UInt32)size); + const bool res = Read1(data, sizeLoc, processedLoc); + processedSize += processedLoc; + if (!res) + return false; + if (processedLoc == 0) + return true; + data = (void *)((unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size > 0); + return true; +} + // ---------- COutFile --------- static inline DWORD GetCreationDisposition(bool createAlways) @@ -430,3 +515,216 @@ bool COutFile::SetLength(UInt64 length) throw() } }}} + +#else // _WIN32 + + +// POSIX + +#include <fcntl.h> +#include <unistd.h> + +namespace NWindows { +namespace NFile { + +namespace NDir { +bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime); +} + +namespace NIO { + +bool CFileBase::OpenBinary(const char *name, int flags) +{ + #ifdef O_BINARY + flags |= O_BINARY; + #endif + + Close(); + _handle = ::open(name, flags, 0666); + return _handle != -1; +} + +bool CFileBase::Close() +{ + if (_handle == -1) + return true; + if (close(_handle) != 0) + return false; + _handle = -1; + return true; +} + +bool CFileBase::GetLength(UInt64 &length) const +{ + const off_t curPos = seek(0, SEEK_CUR); + if (curPos == -1) + return false; + const off_t lengthTemp = seek(0, SEEK_END); + seek(curPos, SEEK_SET); + length = (UInt64)lengthTemp; + return (lengthTemp != -1); +} + +off_t CFileBase::seek(off_t distanceToMove, int moveMethod) const +{ + // printf("\nCFileBase::seek() moveMethod = %d, distanceToMove = %lld", moveMethod, (long long)distanceToMove); + // off_t res = ::lseek(_handle, distanceToMove, moveMethod); + return ::lseek(_handle, distanceToMove, moveMethod); + // printf(" res = %lld", (long long)res); + // return res; +} + +off_t CFileBase::seekToBegin() const throw() +{ + return seek(0, SEEK_SET); +} + +/* +bool CFileBase::SeekToBegin() const throw() +{ + return (::seek(0, SEEK_SET) != -1); +} +*/ + + +///////////////////////// +// CInFile + +bool CInFile::Open(const char *name) +{ + return CFileBase::OpenBinary(name, O_RDONLY); +} + +bool CInFile::OpenShared(const char *name, bool) +{ + return Open(name); +} + +/* +On Linux (32-bit and 64-bit): +read(), write() (and similar system calls) will transfer at most +0x7ffff000 = (2GiB - 4 KiB) bytes, returning the number of bytes actually transferred. +*/ + +static const size_t kChunkSizeMax = ((size_t)1 << 22); + +ssize_t CInFile::read_part(void *data, size_t size) throw() +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + return ::read(_handle, data, size); +} + +bool CInFile::ReadFull(void *data, size_t size, size_t &processed) throw() +{ + processed = 0; + do + { + const ssize_t res = read_part(data, size); + if (res < 0) + return false; + if (res == 0) + break; + data = (void *)((unsigned char *)data + (size_t)res); + size -= (size_t)res; + processed += (size_t)res; + } + while (size > 0); + return true; +} + + +///////////////////////// +// COutFile + +bool COutFile::Create(const char *name, bool createAlways) +{ + Path = name; // change it : set it only if open is success. + if (createAlways) + { + Close(); + _handle = ::creat(name, 0666); + return _handle != -1; + } + return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY); +} + +bool COutFile::Open(const char *name, DWORD creationDisposition) +{ + UNUSED_VAR(creationDisposition) // FIXME + return Create(name, false); +} + +ssize_t COutFile::write_part(const void *data, size_t size) throw() +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + return ::write(_handle, data, size); +} + +ssize_t COutFile::write_full(const void *data, size_t size, size_t &processed) throw() +{ + processed = 0; + do + { + const ssize_t res = write_part(data, size); + if (res < 0) + return res; + if (res == 0) + break; + data = (const void *)((const unsigned char *)data + (size_t)res); + size -= (size_t)res; + processed += (size_t)res; + } + while (size > 0); + return (ssize_t)processed; +} + +bool COutFile::SetLength(UInt64 length) throw() +{ + const off_t len2 = (off_t)length; + if ((Int64)length != len2) + { + SetLastError(EFBIG); + return false; + } + int iret = ftruncate(_handle, len2); + return (iret == 0); +} + +bool COutFile::Close() +{ + bool res = CFileBase::Close(); + if (!res) + return res; + if (CTime_defined || ATime_defined || MTime_defined) + { + /* bool res2 = */ NWindows::NFile::NDir::SetDirTime(Path, + CTime_defined ? &CTime : NULL, + ATime_defined ? &ATime : NULL, + MTime_defined ? &MTime : NULL); + } + return res; +} + +bool COutFile::SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) throw() +{ + // On some OS (cygwin, MacOSX ...), you must close the file before updating times + // return true; + + if (cTime) { CTime = *cTime; CTime_defined = true; } else CTime_defined = false; + if (aTime) { ATime = *aTime; ATime_defined = true; } else ATime_defined = false; + if (mTime) { MTime = *mTime; MTime_defined = true; } else MTime_defined = false; + return true; +} + +bool COutFile::SetMTime(const FILETIME *mTime) throw() +{ + if (mTime) { MTime = *mTime; MTime_defined = true; } else MTime_defined = false; + return true; +} + +}}} + + +#endif diff --git a/CPP/Windows/FileIO.h b/CPP/Windows/FileIO.h index 5ca5448b..276e595e 100644 --- a/CPP/Windows/FileIO.h +++ b/CPP/Windows/FileIO.h @@ -5,8 +5,26 @@ #include "../Common/MyWindows.h" +#define _my_IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) +#define _my_IO_REPARSE_TAG_SYMLINK (0xA000000CL) +#define _my_IO_REPARSE_TAG_LX_SYMLINK (0xA000001DL) + +#define _my_SYMLINK_FLAG_RELATIVE 1 + +// what the meaning of that FLAG or field (2)? +#define _my_LX_SYMLINK_FLAG 2 + +#ifdef _WIN32 + #if defined(_WIN32) && !defined(UNDER_CE) -#include <winioctl.h> +#include <WinIoCtl.h> +#endif + +#else + +#include <sys/types.h> +#include <sys/stat.h> + #endif #include "../Common/MyString.h" @@ -14,19 +32,17 @@ #include "Defs.h" -#define _my_IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) -#define _my_IO_REPARSE_TAG_SYMLINK (0xA000000CL) - -#define _my_SYMLINK_FLAG_RELATIVE 1 +HRESULT GetLastError_noZero_HRESULT(); -#define my_FSCTL_SET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 41, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER -#define my_FSCTL_GET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_SET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 41, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_GET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_DELETE_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 43, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER namespace NWindows { namespace NFile { #if defined(_WIN32) && !defined(UNDER_CE) -bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink); +bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink, bool isWSL); #endif struct CReparseShortInfo @@ -44,28 +60,50 @@ struct CReparseAttr UString SubsName; UString PrintName; + AString WslName; + + bool HeaderError; + bool TagIsUnknown; + bool MinorError; + DWORD ErrorCode; + CReparseAttr(): Tag(0), Flags(0) {} // Parse() - // returns true and (errorCode = 0), if (correct MOUNT_POINT or SYMLINK) - // returns false and (errorCode = ERROR_REPARSE_TAG_MISMATCH), if not (MOUNT_POINT or SYMLINK) - bool Parse(const Byte *p, size_t size, DWORD &errorCode); + // returns (true) and (ErrorCode = 0), if (it'a correct known link) + // returns (false) and (ErrorCode = ERROR_REPARSE_TAG_INVALID), if unknown tag + bool Parse(const Byte *p, size_t size); + + bool IsMountPoint() const { return Tag == _my_IO_REPARSE_TAG_MOUNT_POINT; } // it's Junction + bool IsSymLink_Win() const { return Tag == _my_IO_REPARSE_TAG_SYMLINK; } + bool IsSymLink_WSL() const { return Tag == _my_IO_REPARSE_TAG_LX_SYMLINK; } + + bool IsRelative_Win() const { return Flags == _my_SYMLINK_FLAG_RELATIVE; } + + bool IsRelative_WSL() const + { + if (WslName.IsEmpty()) + return true; + char c = WslName[0]; + return !IS_PATH_SEPAR(c); + } - bool IsMountPoint() const { return Tag == _my_IO_REPARSE_TAG_MOUNT_POINT; } // it's Junction - bool IsSymLink() const { return Tag == _my_IO_REPARSE_TAG_SYMLINK; } - bool IsRelative() const { return Flags == _my_SYMLINK_FLAG_RELATIVE; } // bool IsVolume() const; bool IsOkNamePair() const; UString GetPath() const; }; + +#ifdef _WIN32 + namespace NIO { bool GetReparseData(CFSTR path, CByteBuffer &reparseData, BY_HANDLE_FILE_INFORMATION *fileInfo = NULL); bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size); +bool DeleteReparseData(CFSTR path); -class CFileBase +class CFileBase MY_UNCOPYABLE { protected: HANDLE _handle; @@ -94,13 +132,14 @@ public: } public: + bool PreserveATime; #ifdef SUPPORT_DEVICE_FILE bool IsDeviceFile; bool SizeDefined; UInt64 Size; // it can be larger than real available size #endif - CFileBase(): _handle(INVALID_HANDLE_VALUE) {}; + CFileBase(): _handle(INVALID_HANDLE_VALUE), PreserveATime(false) {}; ~CFileBase() { Close(); } bool Close() throw(); @@ -118,6 +157,7 @@ public: static bool GetFileInformation(CFSTR path, BY_HANDLE_FILE_INFORMATION *info) { + // probably it can work for complex paths: unsupported by another things NIO::CFileBase file; if (!file.Create(path, 0, FILE_SHARE_READ, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS)) return false; @@ -189,6 +229,7 @@ public: bool Read1(void *data, UInt32 size, UInt32 &processedSize) throw(); bool ReadPart(void *data, UInt32 size, UInt32 &processedSize) throw(); bool Read(void *data, UInt32 size, UInt32 &processedSize) throw(); + bool ReadFull(void *data, size_t size, size_t &processedSize) throw(); }; class COutFile: public CFileBase @@ -207,6 +248,83 @@ public: bool SetLength(UInt64 length) throw(); }; -}}} +} + + +#else // _WIN32 + +namespace NIO { + +bool GetReparseData(CFSTR path, CByteBuffer &reparseData); +// bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size); + +// parameters are in reverse order of symlink() function !!! +bool SetSymLink(CFSTR from, CFSTR to); +bool SetSymLink_UString(CFSTR from, const UString &to); + + +class CFileBase +{ +protected: + int _handle; + + bool OpenBinary(const char *name, int flags); +public: + bool PreserveATime; + + CFileBase(): _handle(-1), PreserveATime(false) {}; + ~CFileBase() { Close(); } + bool Close(); + bool GetLength(UInt64 &length) const; + off_t seek(off_t distanceToMove, int moveMethod) const; + off_t seekToBegin() const throw(); + // bool SeekToBegin() throw(); + int my_fstat(struct stat *st) const { return fstat(_handle, st); } +}; + +class CInFile: public CFileBase +{ +public: + bool Open(const char *name); + bool OpenShared(const char *name, bool shareForWrite); + ssize_t read_part(void *data, size_t size) throw(); + // ssize_t read_full(void *data, size_t size, size_t &processed); + bool ReadFull(void *data, size_t size, size_t &processedSize) throw(); +}; + +class COutFile: public CFileBase +{ + bool CTime_defined; + bool ATime_defined; + bool MTime_defined; + + FILETIME CTime; + FILETIME ATime; + FILETIME MTime; + + AString Path; + ssize_t write_part(const void *data, size_t size) throw(); +public: + COutFile(): + CTime_defined(false), + ATime_defined(false), + MTime_defined(false) + {} + + bool Close(); + bool Create(const char *name, bool createAlways); + bool Open(const char *name, DWORD creationDisposition); + ssize_t write_full(const void *data, size_t size, size_t &processed) throw(); + bool SetLength(UInt64 length) throw(); + bool SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) throw(); + bool SetMTime(const FILETIME *mTime) throw(); +}; + +} + +#endif // _WIN32 + +}} + #endif diff --git a/CPP/Windows/FileLink.cpp b/CPP/Windows/FileLink.cpp index 3e2f6431..8ce634fd 100644 --- a/CPP/Windows/FileLink.cpp +++ b/CPP/Windows/FileLink.cpp @@ -4,10 +4,17 @@ #include "../../C/CpuArch.h" +#ifndef _WIN32 +#include <unistd.h> +#endif + #ifdef SUPPORT_DEVICE_FILE #include "../../C/Alloc.h" #endif +#include "../Common/UTFConvert.h" +#include "../Common/StringConvert.h" + #include "FileDir.h" #include "FileFind.h" #include "FileIO.h" @@ -55,6 +62,12 @@ using namespace NName; */ /* +Win10 WSL2: +admin rights + sudo: it creates normal windows symbolic link. +in another cases : it creates IO_REPARSE_TAG_LX_SYMLINK repare point. +*/ + +/* static const UInt32 kReparseFlags_Alias = (1 << 29); static const UInt32 kReparseFlags_HighLatency = (1 << 30); static const UInt32 kReparseFlags_Microsoft = ((UInt32)1 << 31); @@ -71,13 +84,10 @@ static const UInt32 kReparseFlags_Microsoft = ((UInt32)1 << 31); #define Get16(p) GetUi16(p) #define Get32(p) GetUi32(p) -#define Set16(p, v) SetUi16(p, v) -#define Set32(p, v) SetUi32(p, v) - static const wchar_t * const k_LinkPrefix = L"\\??\\"; static const unsigned k_LinkPrefix_Size = 4; -static const bool IsLinkPrefix(const wchar_t *s) +static bool IsLinkPrefix(const wchar_t *s) { return IsString1PrefixedByString2(s, k_LinkPrefix); } @@ -90,7 +100,12 @@ static const bool IsVolumeName(const wchar_t *s) } */ -void WriteString(Byte *dest, const wchar_t *path) +#if defined(_WIN32) && !defined(UNDER_CE) + +#define Set16(p, v) SetUi16(p, v) +#define Set32(p, v) SetUi32(p, v) + +static void WriteString(Byte *dest, const wchar_t *path) { for (;;) { @@ -102,14 +117,32 @@ void WriteString(Byte *dest, const wchar_t *path) } } -#if defined(_WIN32) && !defined(UNDER_CE) - -bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) +bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink, bool isWSL) { bool isAbs = IsAbsolutePath(path); if (!isAbs && !isSymLink) return false; + if (isWSL) + { + // unsupported characters probably use Replacement Character UTF-16 0xFFFD + AString utf; + ConvertUnicodeToUTF8(path, utf); + const size_t size = 4 + utf.Len(); + if (size != (UInt16)size) + return false; + dest.Alloc(8 + size); + Byte *p = dest; + Set32(p, _my_IO_REPARSE_TAG_LX_SYMLINK); + Set16(p + 4, (UInt16)(size)); + Set16(p + 6, 0); + Set32(p + 8, _my_LX_SYMLINK_FLAG); + memcpy(p + 12, utf.Ptr(), utf.Len()); + return true; + } + + // usual symbolic LINK (NOT WSL) + bool needPrintName = true; if (IsSuperPath(path)) @@ -121,12 +154,12 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) const unsigned add_Prefix_Len = isAbs ? k_LinkPrefix_Size : 0; - unsigned len2 = MyStringLen(path) * 2; - const unsigned len1 = len2 + add_Prefix_Len * 2; + size_t len2 = (size_t)MyStringLen(path) * 2; + const size_t len1 = len2 + add_Prefix_Len * 2; if (!needPrintName) len2 = 0; - unsigned totalNamesSize = (len1 + len2); + size_t totalNamesSize = (len1 + len2); /* some WIM imagex software uses old scheme for symbolic links. so we can old scheme for byte to byte compatibility */ @@ -138,6 +171,8 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) totalNamesSize += 2 * 2; const size_t size = 8 + 8 + (isSymLink ? 4 : 0) + totalNamesSize; + if (size != (UInt16)size) + return false; dest.Alloc(size); memset(dest, 0, size); const UInt32 tag = isSymLink ? @@ -152,9 +187,9 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) unsigned subOffs = 0; unsigned printOffs = 0; if (newOrderScheme) - subOffs = len2; + subOffs = (unsigned)len2; else - printOffs = len1 + 2; + printOffs = (unsigned)len1 + 2; Set16(p + 0, (UInt16)subOffs); Set16(p + 2, (UInt16)len1); @@ -177,7 +212,8 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) return true; } -#endif +#endif // defined(_WIN32) && !defined(UNDER_CE) + static void GetString(const Byte *p, unsigned len, UString &res) { @@ -194,35 +230,69 @@ static void GetString(const Byte *p, unsigned len, UString &res) res.ReleaseBuf_SetLen(i); } -bool CReparseAttr::Parse(const Byte *p, size_t size, DWORD &errorCode) +bool CReparseAttr::Parse(const Byte *p, size_t size) { - errorCode = ERROR_INVALID_REPARSE_DATA; + ErrorCode = (DWORD)ERROR_INVALID_REPARSE_DATA; + HeaderError = true; + TagIsUnknown = true; + MinorError = false; + if (size < 8) return false; Tag = Get32(p); UInt32 len = Get16(p + 4); - if (len + 8 > size) + if (len + 8 != size) + // if (len + 8 > size) return false; /* if ((type & kReparseFlags_Alias) == 0 || (type & kReparseFlags_Microsoft) == 0 || (type & 0xFFFF) != 3) */ - if (Tag != _my_IO_REPARSE_TAG_MOUNT_POINT && - Tag != _my_IO_REPARSE_TAG_SYMLINK) + + if (Get16(p + 6) != 0) // padding + return false; + + HeaderError = false; + + if ( Tag != _my_IO_REPARSE_TAG_MOUNT_POINT + && Tag != _my_IO_REPARSE_TAG_SYMLINK + && Tag != _my_IO_REPARSE_TAG_LX_SYMLINK) { - errorCode = ERROR_REPARSE_TAG_MISMATCH; // ERROR_REPARSE_TAG_INVALID + // for unsupported reparse points + ErrorCode = (DWORD)ERROR_REPARSE_TAG_INVALID; // ERROR_REPARSE_TAG_MISMATCH + // errorCode = ERROR_REPARSE_TAG_MISMATCH; // ERROR_REPARSE_TAG_INVALID return false; } - if (Get16(p + 6) != 0) // padding - return false; - + TagIsUnknown = false; + p += 8; size -= 8; - if (len != size) // do we need that check? - return false; + if (Tag == _my_IO_REPARSE_TAG_LX_SYMLINK) + { + if (len < 4) + return false; + Flags = Get32(p); // maybe it's not Flags + if (Flags != _my_LX_SYMLINK_FLAG) + return false; + len -= 4; + p += 4; + char *s = WslName.GetBuf(len); + unsigned i; + for (i = 0; i < len; i++) + { + char c = (char)p[i]; + s[i] = c; + if (c == 0) + break; + } + WslName.ReleaseBuf_SetEnd(i); + MinorError = (i != len); + ErrorCode = 0; + return true; + } if (len < 8) return false; @@ -250,10 +320,11 @@ bool CReparseAttr::Parse(const Byte *p, size_t size, DWORD &errorCode) GetString(p + subOffs, subLen >> 1, SubsName); GetString(p + printOffs, printLen >> 1, PrintName); - errorCode = 0; + ErrorCode = 0; return true; } + bool CReparseShortInfo::Parse(const Byte *p, size_t size) { const Byte *start = p; @@ -336,26 +407,34 @@ bool CReparseAttr::IsVolume() const UString CReparseAttr::GetPath() const { + if (IsSymLink_WSL()) + { + UString u; + // if (CheckUTF8(attr.WslName) + if (!ConvertUTF8ToUnicode(WslName, u)) + MultiByteToUnicodeString2(u, WslName); + return u; + } + UString s (SubsName); if (IsLinkPrefix(s)) { - s.ReplaceOneCharAtPos(1, '\\'); + s.ReplaceOneCharAtPos(1, '\\'); // we normalize prefix from "\??\" to "\\?\" if (IsDrivePath(s.Ptr(k_LinkPrefix_Size))) s.DeleteFrontal(k_LinkPrefix_Size); } return s; } - #ifdef SUPPORT_DEVICE_FILE namespace NSystem { bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); } -#endif +#endif // SUPPORT_DEVICE_FILE -#ifndef UNDER_CE +#if defined(_WIN32) && !defined(UNDER_CE) namespace NIO { @@ -388,11 +467,26 @@ static bool CreatePrefixDirOfFile(CFSTR path) if (pos == 2 && path2[1] == L':') return true; // we don't create Disk folder; #endif - path2.DeleteFrom(pos); + path2.DeleteFrom((unsigned)pos); return NDir::CreateComplexDir(path2); } -// If there is Reprase data already, it still writes new Reparse data + +static bool OutIoReparseData(DWORD controlCode, CFSTR path, void *data, DWORD size) +{ + COutFile file; + if (!file.Open(path, + FILE_SHARE_WRITE, + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)) + return false; + + DWORD returnedSize; + return file.DeviceIoControl(controlCode, data, size, NULL, 0, &returnedSize); +} + + +// If there is Reparse data already, it still writes new Reparse data bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) { NFile::NFind::CFileInfo fi; @@ -420,21 +514,100 @@ bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) } } - COutFile file; - if (!file.Open(path, - FILE_SHARE_WRITE, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)) + return OutIoReparseData(my_FSCTL_SET_REPARSE_POINT, path, (void *)(const Byte *)(data), size); +} + + +bool DeleteReparseData(CFSTR path) +{ + CByteBuffer reparseData; + if (!GetReparseData(path, reparseData, NULL)) return false; + /* MSDN: The tag specified in the ReparseTag member of this structure + must match the tag of the reparse point to be deleted, + and the ReparseDataLength member must be zero */ + #define my_REPARSE_DATA_BUFFER_HEADER_SIZE 8 + if (reparseData.Size() < my_REPARSE_DATA_BUFFER_HEADER_SIZE) + { + SetLastError(ERROR_INVALID_REPARSE_DATA); + return false; + } + BYTE buf[my_REPARSE_DATA_BUFFER_HEADER_SIZE]; + memset(buf, 0, sizeof(buf)); + memcpy(buf, reparseData, 4); // tag + return OutIoReparseData(my_FSCTL_DELETE_REPARSE_POINT, path, buf, sizeof(buf)); +} - DWORD returnedSize; - if (!file.DeviceIoControl(my_FSCTL_SET_REPARSE_POINT, (void *)data, size, NULL, 0, &returnedSize)) +} + +#endif // defined(_WIN32) && !defined(UNDER_CE) + + +#ifndef _WIN32 + +namespace NIO { + +bool GetReparseData(CFSTR path, CByteBuffer &reparseData) +{ + reparseData.Free(); + + #define MAX_PATHNAME_LEN 1024 + char buf[MAX_PATHNAME_LEN + 2]; + const size_t request = sizeof(buf) - 1; + + // printf("\nreadlink() path = %s \n", path); + const ssize_t size = readlink(path, buf, request); + // there is no tail zero + + if (size < 0) return false; + if ((size_t)size >= request) + { + SetLastError(EINVAL); // check it: ENAMETOOLONG + return false; + } + + // printf("\nreadlink() res = %s size = %d \n", buf, (int)size); + reparseData.CopyFrom((const Byte *)buf, (size_t)size); return true; } + +/* +// If there is Reparse data already, it still writes new Reparse data +bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) +{ + // AString s; + // s.SetFrom_CalcLen(data, size); + // return (symlink(s, path) == 0); + UNUSED_VAR(path) + UNUSED_VAR(isDir) + UNUSED_VAR(data) + UNUSED_VAR(size) + SetLastError(ENOSYS); + return false; } +*/ -#endif +bool SetSymLink(CFSTR from, CFSTR to) +{ + // printf("\nsymlink() %s -> %s\n", from, to); + int ir; + // ir = unlink(path); + // if (ir == 0) + ir = symlink(to, from); + return (ir == 0); +} + +bool SetSymLink_UString(CFSTR from, const UString &to) +{ + AString utf; + ConvertUnicodeToUTF8(to, utf); + return SetSymLink(from, utf); +} + +} + +#endif // !_WIN32 }} diff --git a/CPP/Windows/FileName.cpp b/CPP/Windows/FileName.cpp index 2d0b50d5..4ee94575 100644 --- a/CPP/Windows/FileName.cpp +++ b/CPP/Windows/FileName.cpp @@ -2,6 +2,12 @@ #include "StdAfx.h" +#ifndef _WIN32 +#include <limits.h> +#include <unistd.h> +#include "../Common/StringConvert.h" +#endif + #include "FileName.h" #ifndef _UNICODE @@ -58,7 +64,18 @@ void NormalizeDirPathPrefix(UString &dirPath) dirPath.Add_PathSepar(); } -#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z') +#ifdef _WIN32 +void NormalizeDirSeparators(FString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + if (s[i] == '/') + s.ReplaceOneCharAtPos(i, FCHAR_PATH_SEPARATOR); +} +#endif + + +#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) bool IsDrivePath(const wchar_t *s) throw() { return IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2]); } @@ -88,7 +105,9 @@ bool IsAltPathPrefix(CFSTR s) throw() #if defined(_WIN32) && !defined(UNDER_CE) const char * const kSuperPathPrefix = "\\\\?\\"; +#ifdef WIN_LONG_PATH static const char * const kSuperUncPrefix = "\\\\?\\UNC\\"; +#endif #define IS_DEVICE_PATH(s) (IS_SEPAR((s)[0]) && IS_SEPAR((s)[1]) && (s)[2] == '.' && IS_SEPAR((s)[3])) #define IS_SUPER_PREFIX(s) (IS_SEPAR((s)[0]) && IS_SEPAR((s)[1]) && (s)[2] == '?' && IS_SEPAR((s)[3])) @@ -160,7 +179,7 @@ unsigned GetNetworkServerPrefixSize(CFSTR s) throw() int pos = FindSepar(s + prefixSize); if (pos < 0) return 0; - return prefixSize + pos + 1; + return prefixSize + (unsigned)(pos + 1); } bool IsNetworkShareRootPath(CFSTR s) throw() @@ -224,7 +243,7 @@ int FindAltStreamColon(CFSTR path) throw() if (c == ':') { if (colonPos < 0) - colonPos = i; + colonPos = (int)i; continue; } if (IS_SEPAR(c)) @@ -292,7 +311,7 @@ static unsigned GetRootPrefixSize_Of_NetworkPath(const wchar_t *s) throw() int pos2 = FindSepar(s + (unsigned)pos + 1); if (pos2 < 0) return 0; - return pos + pos2 + 2; + return (unsigned)(pos + pos2 + 2); } static unsigned GetRootPrefixSize_Of_SimplePath(const wchar_t *s) throw() @@ -318,7 +337,7 @@ static unsigned GetRootPrefixSize_Of_SuperPath(const wchar_t *s) throw() int pos = FindSepar(s + kSuperPathPrefixSize); if (pos < 0) return 0; - return kSuperPathPrefixSize + pos + 1; + return kSuperPathPrefixSize + (unsigned)(pos + 1); } unsigned GetRootPrefixSize(const wchar_t *s) throw() @@ -332,12 +351,13 @@ unsigned GetRootPrefixSize(const wchar_t *s) throw() #else // _WIN32 -bool IsAbsolutePath(const wchar_t *s) { return IS_SEPAR(s[0]); } +bool IsAbsolutePath(const wchar_t *s) throw() { return IS_SEPAR(s[0]); } #ifndef USE_UNICODE_FSTRING -unsigned GetRootPrefixSize(CFSTR s) { return IS_SEPAR(s[0]) ? 1 : 0; } +unsigned GetRootPrefixSize(CFSTR s) throw(); +unsigned GetRootPrefixSize(CFSTR s) throw() { return IS_SEPAR(s[0]) ? 1 : 0; } #endif -unsigned GetRootPrefixSize(const wchar_t *s) { return IS_SEPAR(s[0]) ? 1 : 0; } +unsigned GetRootPrefixSize(const wchar_t *s) throw() { return IS_SEPAR(s[0]) ? 1 : 0; } #endif // _WIN32 @@ -347,6 +367,9 @@ unsigned GetRootPrefixSize(const wchar_t *s) { return IS_SEPAR(s[0]) ? 1 : 0; } static bool GetCurDir(UString &path) { path.Empty(); + + #ifdef _WIN32 + DWORD needLength; #ifndef _UNICODE if (!g_IsNT) @@ -365,6 +388,23 @@ static bool GetCurDir(UString &path) path = s; } return (needLength > 0 && needLength <= MAX_PATH); + + #else + + #define MY__PATH_MAX PATH_MAX + // #define MY__PATH_MAX 1024 + + char s[MY__PATH_MAX + 1]; + char *res = getcwd(s, MY__PATH_MAX); + if (!res) + { + // if (errno != ERANGE) + return false; + } + path = GetUnicodeString(s); + return true; + + #endif } static bool ResolveDotsFolders(UString &s) @@ -388,7 +428,7 @@ static bool ResolveDotsFolders(UString &s) { if (i == 0) return false; - int k = i - 2; + int k = (int)i - 2; i += 2; for (;; k--) @@ -407,8 +447,8 @@ static bool ResolveDotsFolders(UString &s) if (k >= 0) { - num = i - k; - i = k; + num = i - (unsigned)k; + i = (unsigned)k; } else { @@ -528,6 +568,7 @@ int GetUseSuperPathType(CFSTR s) throw() } + /* returns false in two cases: - if GetCurDir was used, and GetCurDir returned error. @@ -538,7 +579,6 @@ int GetUseSuperPathType(CFSTR s) throw() for absolute paths, returns true, res is Super path. */ - static bool GetSuperPathBase(CFSTR s, UString &res) { res.Empty(); @@ -702,6 +742,8 @@ bool GetSuperPath(CFSTR path, UString &superPath, bool onlyIfNew) return false; superPath = fs2us(path); } + + NormalizeDirSeparators(superPath); return true; } return false; @@ -712,6 +754,10 @@ bool GetSuperPaths(CFSTR s1, CFSTR s2, UString &d1, UString &d2, bool onlyIfNew) if (!GetSuperPathBase(s1, d1) || !GetSuperPathBase(s2, d2)) return false; + + NormalizeDirSeparators(d1); + NormalizeDirSeparators(d2); + if (d1.IsEmpty() && d2.IsEmpty() && onlyIfNew) return false; if (d1.IsEmpty()) d1 = fs2us(s1); diff --git a/CPP/Windows/FileName.h b/CPP/Windows/FileName.h index 2c9c56db..de8bd134 100644 --- a/CPP/Windows/FileName.h +++ b/CPP/Windows/FileName.h @@ -17,6 +17,10 @@ int FindSepar(const FChar *s) throw(); void NormalizeDirPathPrefix(FString &dirPath); // ensures that it ended with '\\', if dirPath is not epmty void NormalizeDirPathPrefix(UString &dirPath); +#ifdef _WIN32 +void NormalizeDirSeparators(FString &s); +#endif + bool IsDrivePath(const wchar_t *s) throw(); // first 3 chars are drive chars like "a:\\" bool IsAltPathPrefix(CFSTR s) throw(); /* name: */ diff --git a/CPP/Windows/FileSystem.cpp b/CPP/Windows/FileSystem.cpp index 6c1f48a2..62594532 100644 --- a/CPP/Windows/FileSystem.cpp +++ b/CPP/Windows/FileSystem.cpp @@ -19,6 +19,8 @@ namespace NWindows { namespace NFile { namespace NSystem { +#ifdef _WIN32 + bool MyGetVolumeInformation( CFSTR rootPath, UString &volumeName, @@ -90,7 +92,7 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, #ifndef _UNICODE if (!g_IsNT) { - GetDiskFreeSpaceExA_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExA_Pointer)GetProcAddress( + GetDiskFreeSpaceExA_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExA_Pointer)(void *)GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExA"); if (pGetDiskFreeSpaceEx) { @@ -105,7 +107,7 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, else #endif { - GetDiskFreeSpaceExW_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExW_Pointer)GetProcAddress( + GetDiskFreeSpaceExW_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExW_Pointer)(void *)GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExW"); if (pGetDiskFreeSpaceEx) { @@ -126,6 +128,8 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, return true; } +#endif + }}} #endif diff --git a/CPP/Windows/FileSystem.h b/CPP/Windows/FileSystem.h index 9076ea13..9b49a025 100644 --- a/CPP/Windows/FileSystem.h +++ b/CPP/Windows/FileSystem.h @@ -10,6 +10,8 @@ namespace NWindows { namespace NFile { namespace NSystem { +#ifdef _WIN32 + bool MyGetVolumeInformation( CFSTR rootPath , UString &volumeName, @@ -22,6 +24,8 @@ UINT MyGetDriveType(CFSTR pathName); bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); +#endif + }}} #endif diff --git a/CPP/Windows/Handle.h b/CPP/Windows/Handle.h index bb7cb705..5878c830 100644 --- a/CPP/Windows/Handle.h +++ b/CPP/Windows/Handle.h @@ -3,9 +3,11 @@ #ifndef __WINDOWS_HANDLE_H #define __WINDOWS_HANDLE_H +#include "../Common/MyTypes.h" + namespace NWindows { -class CHandle +class CHandle MY_UNCOPYABLE { protected: HANDLE _handle; diff --git a/CPP/Windows/MemoryLock.cpp b/CPP/Windows/MemoryLock.cpp index f9d08a6e..fdfbeb9d 100644 --- a/CPP/Windows/MemoryLock.cpp +++ b/CPP/Windows/MemoryLock.cpp @@ -75,11 +75,11 @@ typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); /* We suppose that Window 10 works incorrectly with "Large Pages" at: - - Windows 10 1703 (15063) - - Windows 10 1709 (16299) - - - Windows 10 1809 (17763) on some CPUs that have no 1 GB page support. - We need more information about that new BUG in Windows. + - Windows 10 1703 (15063) : incorrect allocating after VirtualFree() + - Windows 10 1709 (16299) : incorrect allocating after VirtualFree() + - Windows 10 1809 (17763) : the failures for blocks of 1 GiB and larger, + if CPU doesn't support 1 GB pages. + Windows 10 1903 (18362) probably works correctly. */ unsigned Get_LargePages_RiskLevel() @@ -88,7 +88,7 @@ unsigned Get_LargePages_RiskLevel() HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); if (!ntdll) return 0; - Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); + Func_RtlGetVersion func = (Func_RtlGetVersion)(void *)GetProcAddress(ntdll, "RtlGetVersion"); if (!func) return 0; func(&vi); @@ -100,7 +100,7 @@ unsigned Get_LargePages_RiskLevel() return 1; #ifdef MY_CPU_X86_OR_AMD64 - if (!CPU_IsSupported_PageGB()) + if (vi.dwBuildNumber < 18362 && !CPU_IsSupported_PageGB()) return 1; #endif diff --git a/CPP/Windows/Menu.cpp b/CPP/Windows/Menu.cpp index 3834881a..3ad69530 100644 --- a/CPP/Windows/Menu.cpp +++ b/CPP/Windows/Menu.cpp @@ -26,11 +26,15 @@ will not work at NT 4.0, if cbSize is set as sizeof(MENUITEMINFO*). So we use size of old version of structure. */ #if defined(UNDER_CE) || defined(_WIN64) || (WINVER < 0x0500) + #ifndef _UNICODE #define my_compatib_MENUITEMINFOA_size sizeof(MENUITEMINFOA) + #endif #define my_compatib_MENUITEMINFOW_size sizeof(MENUITEMINFOW) #else #define MY_STRUCT_SIZE_BEFORE(structname, member) ((UINT)(UINT_PTR)((LPBYTE)(&((structname*)0)->member) - (LPBYTE)(structname*)0)) + #ifndef _UNICODE #define my_compatib_MENUITEMINFOA_size MY_STRUCT_SIZE_BEFORE(MENUITEMINFOA, hbmpItem) + #endif #define my_compatib_MENUITEMINFOW_size MY_STRUCT_SIZE_BEFORE(MENUITEMINFOW, hbmpItem) #endif @@ -145,7 +149,7 @@ bool CMenu::SetItem(UINT itemIndex, bool byPosition, const CMenuItem &item) if (item.IsString()) { s = GetSystemString(item.StringValue); - si.dwTypeData = (LPTSTR)(LPCTSTR)s; + si.dwTypeData = s.Ptr_non_const(); } return SetItemInfo(itemIndex, byPosition, &si); } @@ -155,7 +159,7 @@ bool CMenu::SetItem(UINT itemIndex, bool byPosition, const CMenuItem &item) MENUITEMINFOW si; ConvertItemToSysForm(item, si); if (item.IsString()) - si.dwTypeData = (LPWSTR)(LPCWSTR)item.StringValue; + si.dwTypeData = item.StringValue.Ptr_non_const(); return SetItemInfo(itemIndex, byPosition, &si); } } @@ -171,7 +175,7 @@ bool CMenu::InsertItem(UINT itemIndex, bool byPosition, const CMenuItem &item) if (item.IsString()) { s = GetSystemString(item.StringValue); - si.dwTypeData = (LPTSTR)(LPCTSTR)s; + si.dwTypeData = s.Ptr_non_const(); } return InsertItem(itemIndex, byPosition, &si); } @@ -181,7 +185,7 @@ bool CMenu::InsertItem(UINT itemIndex, bool byPosition, const CMenuItem &item) MENUITEMINFOW si; ConvertItemToSysForm(item, si); if (item.IsString()) - si.dwTypeData = (LPWSTR)(LPCWSTR)item.StringValue; + si.dwTypeData = item.StringValue.Ptr_non_const(); #ifdef UNDER_CE UINT flags = (item.fType & MFT_SEPARATOR) ? MF_SEPARATOR : MF_STRING; UINT id = item.wID; diff --git a/CPP/Windows/Net.cpp b/CPP/Windows/Net.cpp index 14d06d6e..2a3952a1 100644 --- a/CPP/Windows/Net.cpp +++ b/CPP/Windows/Net.cpp @@ -35,11 +35,11 @@ DWORD CEnum::Open(DWORD scope, DWORD type, DWORD usage, LPNETRESOURCEW netResour } #endif -static void SetComplexString(bool &defined, CSysString &destString, LPCTSTR srsString) +static void SetComplexString(bool &defined, CSysString &destString, LPCTSTR srcString) { - defined = (srsString != 0); + defined = (srcString != 0); if (defined) - destString = srsString; + destString = srcString; else destString.Empty(); } @@ -59,9 +59,9 @@ static void ConvertNETRESOURCEToCResource(const NETRESOURCE &netResource, CResou static void SetComplexString2(LPTSTR *destString, bool defined, const CSysString &srcString) { if (defined) - *destString = (TCHAR *)(const TCHAR *)srcString; + *destString = srcString.Ptr_non_const(); else - *destString = 0; + *destString = NULL; } static void ConvertCResourceToNETRESOURCE(const CResource &resource, NETRESOURCE &netResource) @@ -78,11 +78,11 @@ static void ConvertCResourceToNETRESOURCE(const CResource &resource, NETRESOURCE #ifndef _UNICODE -static void SetComplexString(bool &defined, UString &destString, LPCWSTR srsString) +static void SetComplexString(bool &defined, UString &destString, LPCWSTR src) { - defined = (srsString != 0); + defined = (src != NULL); if (defined) - destString = srsString; + destString = src; else destString.Empty(); } @@ -102,9 +102,9 @@ static void ConvertNETRESOURCEToCResource(const NETRESOURCEW &netResource, CReso static void SetComplexString2(LPWSTR *destString, bool defined, const UString &srcString) { if (defined) - *destString = (WCHAR *)(const WCHAR *)srcString; + *destString = srcString.Ptr_non_const(); else - *destString = 0; + *destString = NULL; } static void ConvertCResourceToNETRESOURCE(const CResourceW &resource, NETRESOURCEW &netResource) @@ -141,10 +141,8 @@ static void ConvertResourceToResourceW(const CResource &resource, CResourceW &re DWORD CEnum::Open(DWORD scope, DWORD type, DWORD usage, const CResource *resource) { NETRESOURCE netResource; - LPNETRESOURCE pointer; - if (resource == 0) - pointer = 0; - else + LPNETRESOURCE pointer = NULL; + if (resource) { ConvertCResourceToNETRESOURCE(*resource, netResource); pointer = &netResource; @@ -158,21 +156,17 @@ DWORD CEnum::Open(DWORD scope, DWORD type, DWORD usage, const CResourceW *resour if (g_IsNT) { NETRESOURCEW netResource; - LPNETRESOURCEW pointer; - if (resource == 0) - pointer = 0; - else + LPNETRESOURCEW pointer = NULL; + if (resource) { ConvertCResourceToNETRESOURCE(*resource, netResource); pointer = &netResource; } return Open(scope, type, usage, pointer); } - CResource *pointer; CResource resourceA; - if (resource == 0) - pointer = 0; - else + CResource *pointer = NULL; + if (resource) { ConvertResourceWToResource(*resource, resourceA); pointer = &resourceA; @@ -206,7 +200,7 @@ DWORD CEnum::Next(CResource &resource) { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (BYTE *)(byteBuffer); + LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; DWORD numEntries = 1; @@ -226,7 +220,7 @@ DWORD CEnum::Next(CResourceW &resource) { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (BYTE *)(byteBuffer); + LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; DWORD numEntries = 1; @@ -250,7 +244,7 @@ DWORD GetResourceParent(const CResource &resource, CResource &parentResource) { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (BYTE *)(byteBuffer); + LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; NETRESOURCE netResource; @@ -269,7 +263,7 @@ DWORD GetResourceParent(const CResourceW &resource, CResourceW &parentResource) { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (BYTE *)(byteBuffer); + LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; NETRESOURCEW netResource; @@ -293,7 +287,7 @@ DWORD GetResourceInformation(const CResource &resource, { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (BYTE *)(byteBuffer); + LPNETRESOURCE lpnrLocal = (LPNETRESOURCE) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; NETRESOURCE netResource; @@ -317,7 +311,7 @@ DWORD GetResourceInformation(const CResourceW &resource, { const DWORD kBufferSize = 16384; CByteArr byteBuffer(kBufferSize); - LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (BYTE *)(byteBuffer); + LPNETRESOURCEW lpnrLocal = (LPNETRESOURCEW) (void *) (BYTE *)(byteBuffer); ZeroMemory(lpnrLocal, kBufferSize); DWORD bufferSize = kBufferSize; NETRESOURCEW netResource; diff --git a/CPP/Windows/NtCheck.h b/CPP/Windows/NtCheck.h index a1b89ef2..0af32911 100644 --- a/CPP/Windows/NtCheck.h +++ b/CPP/Windows/NtCheck.h @@ -17,6 +17,8 @@ static inline bool IsItWindowsNT() #endif #ifndef _UNICODE + extern + bool g_IsNT; #if defined(_WIN64) || defined(UNDER_CE) bool g_IsNT = true; #define SET_IS_NT diff --git a/CPP/Windows/ProcessUtils.cpp b/CPP/Windows/ProcessUtils.cpp index f7878d51..9bf05383 100644 --- a/CPP/Windows/ProcessUtils.cpp +++ b/CPP/Windows/ProcessUtils.cpp @@ -24,6 +24,21 @@ static UString GetQuotedString(const UString &s) WRes CProcess::Create(LPCWSTR imageName, const UString ¶ms, LPCWSTR curDir) { + /* + OutputDebugStringW(L"CProcess::Create"); + OutputDebugStringW(imageName); + if (params) + { + OutputDebugStringW(L"params:"); + OutputDebugStringW(params); + } + if (curDir) + { + OutputDebugStringW(L"cur dir:"); + OutputDebugStringW(curDir); + } + */ + Close(); const UString params2 = #ifndef UNDER_CE @@ -52,7 +67,8 @@ WRes CProcess::Create(LPCWSTR imageName, const UString ¶ms, LPCWSTR curDir) CSysString curDirA; if (curDir != 0) curDirA = GetSystemString(curDir); - result = ::CreateProcessA(NULL, (LPSTR)(LPCSTR)GetSystemString(params2), + const AString s = GetSystemString(params2); + result = ::CreateProcessA(NULL, s.Ptr_non_const(), NULL, NULL, FALSE, 0, NULL, ((curDir != 0) ? (LPCSTR)curDirA: 0), &si, &pi); } else @@ -67,8 +83,8 @@ WRes CProcess::Create(LPCWSTR imageName, const UString ¶ms, LPCWSTR curDir) si.cbReserved2 = 0; si.lpReserved2 = 0; - result = CreateProcessW(imageName, (LPWSTR)(LPCWSTR)params2, - NULL, NULL, FALSE, 0, NULL, (LPWSTR)curDir, &si, &pi); + result = CreateProcessW(imageName, params2.Ptr_non_const(), + NULL, NULL, FALSE, 0, NULL, curDir, &si, &pi); } if (result == 0) return ::GetLastError(); diff --git a/CPP/Windows/ProcessUtils.h b/CPP/Windows/ProcessUtils.h index a50bb5fc..e46f9ab2 100644 --- a/CPP/Windows/ProcessUtils.h +++ b/CPP/Windows/ProcessUtils.h @@ -3,7 +3,7 @@ #ifndef __WINDOWS_PROCESS_UTILS_H #define __WINDOWS_PROCESS_UTILS_H -#include <psapi.h> +#include <Psapi.h> #include "../Common/MyString.h" diff --git a/CPP/Windows/PropVariant.cpp b/CPP/Windows/PropVariant.cpp index c4ad3acb..8cc89a3a 100644 --- a/CPP/Windows/PropVariant.cpp +++ b/CPP/Windows/PropVariant.cpp @@ -278,7 +278,8 @@ HRESULT CPropVariant::Attach(PROPVARIANT *pSrc) throw() HRESULT hr = Clear(); if (FAILED(hr)) return hr; - memcpy(this, pSrc, sizeof(PROPVARIANT)); + // memcpy((PROPVARIANT *)this, pSrc, sizeof(PROPVARIANT)); + *(PROPVARIANT *)this = *pSrc; pSrc->vt = VT_EMPTY; return S_OK; } @@ -291,7 +292,8 @@ HRESULT CPropVariant::Detach(PROPVARIANT *pDest) throw() if (FAILED(hr)) return hr; } - memcpy(pDest, this, sizeof(PROPVARIANT)); + // memcpy(pDest, this, sizeof(PROPVARIANT)); + *pDest = *(PROPVARIANT *)this; vt = VT_EMPTY; return S_OK; } diff --git a/CPP/Windows/PropVariantConv.cpp b/CPP/Windows/PropVariantConv.cpp index 65aa9f7e..b58d37e6 100644 --- a/CPP/Windows/PropVariantConv.cpp +++ b/CPP/Windows/PropVariantConv.cpp @@ -1,4 +1,4 @@ -// PropVariantConvert.cpp +// PropVariantConv.cpp #include "StdAfx.h" @@ -86,7 +86,7 @@ bool ConvertUtcFileTimeToString(const FILETIME &ft, wchar_t *dest, int level) th bool res = ConvertUtcFileTimeToString(ft, s, level); for (unsigned i = 0;; i++) { - unsigned char c = s[i]; + Byte c = (Byte)s[i]; dest[i] = c; if (c == 0) break; diff --git a/CPP/Windows/PropVariantUtils.cpp b/CPP/Windows/PropVariantUtils.cpp index fab556a5..6daee839 100644 --- a/CPP/Windows/PropVariantUtils.cpp +++ b/CPP/Windows/PropVariantUtils.cpp @@ -128,7 +128,7 @@ void FlagsToProp(const CUInt32PCharPair *pairs, unsigned num, UInt32 flags, NCOM } -AString Flags64ToString(const CUInt32PCharPair *pairs, unsigned num, UInt64 flags) +static AString Flags64ToString(const CUInt32PCharPair *pairs, unsigned num, UInt64 flags) { AString s; for (unsigned i = 0; i < num; i++) diff --git a/CPP/Windows/Registry.cpp b/CPP/Windows/Registry.cpp index a2893131..2c4643bc 100644 --- a/CPP/Windows/Registry.cpp +++ b/CPP/Windows/Registry.cpp @@ -119,7 +119,7 @@ LONG CKey::SetValue(LPCTSTR name, UInt32 value) throw() { MYASSERT(_object != NULL); return RegSetValueEx(_object, name, 0, REG_DWORD, - (BYTE * const)&value, sizeof(UInt32)); + (const BYTE *)&value, sizeof(UInt32)); } LONG CKey::SetValue(LPCTSTR name, bool value) throw() @@ -132,7 +132,7 @@ LONG CKey::SetValue(LPCTSTR name, LPCTSTR value) throw() MYASSERT(value != NULL); MYASSERT(_object != NULL); return RegSetValueEx(_object, name, 0, REG_SZ, - (const BYTE * )value, (lstrlen(value) + 1) * sizeof(TCHAR)); + (const BYTE *)value, ((DWORD)lstrlen(value) + 1) * sizeof(TCHAR)); } /* @@ -193,7 +193,7 @@ LONG CKey::QueryValue(LPCTSTR name, UInt32 &value) throw() { DWORD type = 0; DWORD count = sizeof(DWORD); - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)&value, &count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_DWORD)); MYASSERT((res != ERROR_SUCCESS) || (count == sizeof(UInt32))); @@ -219,7 +219,7 @@ LONG CKey::GetValue_IfOk(LPCTSTR name, UInt32 &value) throw() LONG CKey::GetValue_IfOk(LPCTSTR name, bool &value) throw() { - bool newVal; + bool newVal = false; LONG res = QueryValue(name, newVal); if (res == ERROR_SUCCESS) value = newVal; @@ -229,7 +229,7 @@ LONG CKey::GetValue_IfOk(LPCTSTR name, bool &value) throw() LONG CKey::QueryValue(LPCTSTR name, LPTSTR value, UInt32 &count) throw() { DWORD type = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, (LPBYTE)value, (DWORD *)&count); + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)value, (DWORD *)&count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_SZ) || (type == REG_MULTI_SZ) || (type == REG_EXPAND_SZ)); return res; } @@ -239,7 +239,7 @@ LONG CKey::QueryValue(LPCTSTR name, CSysString &value) value.Empty(); DWORD type = 0; UInt32 curSize = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, NULL, (DWORD *)&curSize); + LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, (DWORD *)&curSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; UInt32 curSize2 = curSize; @@ -296,7 +296,7 @@ LONG CKey::QueryValue(LPCWSTR name, UString &value) LONG CKey::QueryValue(LPCTSTR name, void *value, UInt32 &count) throw() { DWORD type = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, (LPBYTE)value, (DWORD *)&count); + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)value, (DWORD *)&count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_BINARY)); return res; } @@ -306,7 +306,7 @@ LONG CKey::QueryValue(LPCTSTR name, CByteBuffer &value, UInt32 &dataSize) { DWORD type = 0; dataSize = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, NULL, (DWORD *)&dataSize); + LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, (DWORD *)&dataSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; value.Alloc(dataSize); @@ -369,7 +369,7 @@ LONG CKey::GetValue_Strings(LPCTSTR valueName, UStringVector &strings) if (dataSize % sizeof(wchar_t) != 0) return E_FAIL; - const wchar_t *data = (const wchar_t *)(const Byte *)buffer; + const wchar_t *data = (const wchar_t *)(const void *)(const Byte *)buffer; size_t numChars = dataSize / sizeof(wchar_t); size_t prev = 0; UString s; diff --git a/CPP/Windows/ResourceString.cpp b/CPP/Windows/ResourceString.cpp index cc8b964a..ae8182ed 100644 --- a/CPP/Windows/ResourceString.cpp +++ b/CPP/Windows/ResourceString.cpp @@ -25,10 +25,10 @@ static CSysString MyLoadStringA(HINSTANCE hInstance, UINT resourceID) do { size <<= 1; - len = ::LoadString(hInstance, resourceID, s.GetBuf(size - 1), size); + len = ::LoadString(hInstance, resourceID, s.GetBuf((unsigned)size - 1), size); } while (size - len <= 1); - s.ReleaseBuf_CalcLen(len); + s.ReleaseBuf_CalcLen((unsigned)len); return s; } @@ -43,10 +43,10 @@ static void MyLoadString2(HINSTANCE hInstance, UINT resourceID, UString &s) do { size <<= 1; - len = ::LoadStringW(hInstance, resourceID, s.GetBuf(size - 1), size); + len = ::LoadStringW(hInstance, resourceID, s.GetBuf((unsigned)size - 1), size); } while (size - len <= 1); - s.ReleaseBuf_CalcLen(len); + s.ReleaseBuf_CalcLen((unsigned)len); } // NT4 doesn't support LoadStringW(,,, 0) to get pointer to resource string. So we don't use it. diff --git a/CPP/Windows/SecurityUtils.cpp b/CPP/Windows/SecurityUtils.cpp index 67a9d7fd..640c90dc 100644 --- a/CPP/Windows/SecurityUtils.cpp +++ b/CPP/Windows/SecurityUtils.cpp @@ -34,7 +34,7 @@ bool MyLookupAccountSid(LPCTSTR systemName, PSID sid, static void SetLsaString(LPWSTR src, PLSA_UNICODE_STRING dest) { - int len = (int)wcslen(src); + size_t len = (size_t)wcslen(src); dest->Length = (USHORT)(len * sizeof(WCHAR)); dest->MaximumLength = (USHORT)((len + 1) * sizeof(WCHAR)); dest->Buffer = src; diff --git a/CPP/Windows/Shell.cpp b/CPP/Windows/Shell.cpp index b424e67c..d0f9032c 100644 --- a/CPP/Windows/Shell.cpp +++ b/CPP/Windows/Shell.cpp @@ -192,7 +192,7 @@ bool BrowseForFolder(LPBROWSEINFO browseInfo, CSysString &resultPath) } -int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) +static int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) { #ifndef UNDER_CE switch (uMsg) @@ -221,7 +221,7 @@ int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM da } -bool BrowseForFolder(HWND owner, LPCTSTR title, UINT ulFlags, +static bool BrowseForFolder(HWND owner, LPCTSTR title, UINT ulFlags, LPCTSTR initialFolder, CSysString &resultPath) { CSysString displayName; @@ -275,7 +275,7 @@ bool GetPathFromIDList(LPCITEMIDLIST itemIDList, UString &path) typedef LPITEMIDLIST (WINAPI * SHBrowseForFolderWP)(LPBROWSEINFOW lpbi); -bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) +static bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) { NWindows::NCOM::CComInitializer comInitializer; SHBrowseForFolderWP shBrowseForFolderW = (SHBrowseForFolderWP) @@ -290,7 +290,7 @@ bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) return GetPathFromIDList(itemIDList, resultPath); } - +static int CALLBACK BrowseCallbackProc2(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) { switch (uMsg) diff --git a/CPP/Windows/Shell.h b/CPP/Windows/Shell.h index 4bff18cf..de91d3f1 100644 --- a/CPP/Windows/Shell.h +++ b/CPP/Windows/Shell.h @@ -3,8 +3,8 @@ #ifndef __WINDOWS_SHELL_H #define __WINDOWS_SHELL_H -#include <windows.h> -#include <shlobj.h> +#include "../Common/MyWindows.h" +#include <ShlObj.h> #include "../Common/MyString.h" diff --git a/CPP/Windows/Synchronization.cpp b/CPP/Windows/Synchronization.cpp index 5f86d1eb..fbf919dc 100644 --- a/CPP/Windows/Synchronization.cpp +++ b/CPP/Windows/Synchronization.cpp @@ -2,9 +2,62 @@ #include "StdAfx.h" +#ifndef _WIN32 + #include "Synchronization.h" namespace NWindows { namespace NSynchronization { +/* +#define INFINITE 0xFFFFFFFF +#define MAXIMUM_WAIT_OBJECTS 64 +#define STATUS_ABANDONED_WAIT_0 ((NTSTATUS)0x00000080L) +#define WAIT_ABANDONED ((STATUS_ABANDONED_WAIT_0 ) + 0 ) +#define WAIT_ABANDONED_0 ((STATUS_ABANDONED_WAIT_0 ) + 0 ) +// WINAPI +DWORD WaitForMultipleObjects(DWORD count, const HANDLE *handles, BOOL wait_all, DWORD timeout); +*/ + +DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles) +{ + if (count < 1) + { + // abort(); + SetLastError(EINVAL); + return WAIT_FAILED; + } + + CSynchro *synchro = handles[0]->_sync; + synchro->Enter(); + + // #ifdef DEBUG_SYNCHRO + for (DWORD i = 1; i < count; i++) + { + if (synchro != handles[i]->_sync) + { + // abort(); + synchro->Leave(); + SetLastError(EINVAL); + return WAIT_FAILED; + } + } + // #endif + + for (;;) + { + for (DWORD i = 0; i < count; i++) + { + if (handles[i]->IsSignaledAndUpdate()) + { + synchro->Leave(); + return WAIT_OBJECT_0 + i; + } + } + synchro->WaitCond(); + } +} + }} + +#endif diff --git a/CPP/Windows/Synchronization.h b/CPP/Windows/Synchronization.h index dc695f6f..98ea0b69 100644 --- a/CPP/Windows/Synchronization.h +++ b/CPP/Windows/Synchronization.h @@ -5,6 +5,8 @@ #include "../../C/Threads.h" +#include "../Common/MyTypes.h" + #include "Defs.h" #ifdef _WIN32 @@ -14,17 +16,19 @@ namespace NWindows { namespace NSynchronization { -class CBaseEvent +class CBaseEvent MY_UNCOPYABLE { protected: ::CEvent _object; public: bool IsCreated() { return Event_IsCreated(&_object) != 0; } - operator HANDLE() { return _object; } + CBaseEvent() { Event_Construct(&_object); } ~CBaseEvent() { Close(); } WRes Close() { return Event_Close(&_object); } + #ifdef _WIN32 + operator HANDLE() { return _object; } WRes Create(bool manualReset, bool initiallyOwn, LPCTSTR name = NULL, LPSECURITY_ATTRIBUTES sa = NULL) { _object = ::CreateEvent(sa, BoolToBOOL(manualReset), BoolToBOOL(initiallyOwn), name); @@ -54,10 +58,10 @@ public: { return ManualResetEvent_Create(&_object, initiallyOwn ? 1: 0); } - WRes CreateIfNotCreated() + WRes CreateIfNotCreated_Reset() { if (IsCreated()) - return 0; + return Reset(); return ManualResetEvent_CreateNotSignaled(&_object); } #ifdef _WIN32 @@ -75,21 +79,25 @@ public: { return AutoResetEvent_CreateNotSignaled(&_object); } - WRes CreateIfNotCreated() + WRes CreateIfNotCreated_Reset() { if (IsCreated()) - return 0; + return Reset(); return AutoResetEvent_CreateNotSignaled(&_object); } }; + +/* #ifdef _WIN32 + class CObject: public CHandle { public: WRes Lock(DWORD timeoutInterval = INFINITE) { return (::WaitForSingleObject(_handle, timeoutInterval) == WAIT_OBJECT_0 ? 0 : ::GetLastError()); } }; + class CMutex: public CObject { public: @@ -114,33 +122,43 @@ public: return ::ReleaseMutex(_handle) ? 0 : ::GetLastError(); } }; -class CMutexLock + +class CMutexLock MY_UNCOPYABLE { CMutex *_object; public: CMutexLock(CMutex &object): _object(&object) { _object->Lock(); } ~CMutexLock() { _object->Release(); } }; -#endif -class CSemaphore +#endif // _WIN32 +*/ + + +class CSemaphore MY_UNCOPYABLE { ::CSemaphore _object; public: CSemaphore() { Semaphore_Construct(&_object); } ~CSemaphore() { Close(); } - WRes Close() { return Semaphore_Close(&_object); } + WRes Close() { return Semaphore_Close(&_object); } + + #ifdef _WIN32 operator HANDLE() { return _object; } - WRes Create(UInt32 initiallyCount, UInt32 maxCount) + #endif + + // bool IsCreated() const { return Semaphore_IsCreated(&_object) != 0; } + + WRes Create(UInt32 initCount, UInt32 maxCount) { - return Semaphore_Create(&_object, initiallyCount, maxCount); + return Semaphore_Create(&_object, initCount, maxCount); } WRes Release() { return Semaphore_Release1(&_object); } WRes Release(UInt32 releaseCount) { return Semaphore_ReleaseN(&_object, releaseCount); } WRes Lock() { return Semaphore_Wait(&_object); } }; -class CCriticalSection +class CCriticalSection MY_UNCOPYABLE { ::CCriticalSection _object; public: @@ -150,7 +168,7 @@ public: void Leave() { CriticalSection_Leave(&_object); } }; -class CCriticalSectionLock +class CCriticalSectionLock MY_UNCOPYABLE { CCriticalSection *_object; void Unlock() { _object->Leave(); } @@ -159,6 +177,213 @@ public: ~CCriticalSectionLock() { Unlock(); } }; + +#ifdef _WIN32 + +typedef HANDLE CHandle_WFMO; +typedef CSemaphore CSemaphore_WFMO; +typedef CAutoResetEvent CAutoResetEvent_WFMO; +typedef CManualResetEvent CManualResetEvent_WFMO; + +inline DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles) +{ + return ::WaitForMultipleObjects(count, handles, FALSE, INFINITE); +} + +#define SYNC_OBJ_DECL(obj) +#define SYNC_WFMO(x) +#define SYNC_PARAM(x) +#define SYNC_PARAM_DECL(x) + +#else // _WIN32 + +// POSIX sync objects for WaitForMultipleObjects + +#define SYNC_WFMO(x) x +#define SYNC_PARAM(x) x, +#define SYNC_PARAM_DECL(x) NWindows::NSynchronization::CSynchro *x +#define SYNC_OBJ_DECL(x) NWindows::NSynchronization::CSynchro x; + +class CSynchro MY_UNCOPYABLE +{ + pthread_mutex_t _mutex; + pthread_cond_t _cond; + bool _isValid; + +public: + CSynchro() { _isValid = false; } + ~CSynchro() + { + if (_isValid) + { + ::pthread_mutex_destroy(&_mutex); + ::pthread_cond_destroy(&_cond); + } + _isValid = false; + } + WRes Create() + { + RINOK(::pthread_mutex_init(&_mutex, 0)); + WRes ret = ::pthread_cond_init(&_cond, 0); + _isValid = 1; + return ret; + } + WRes Enter() + { + return ::pthread_mutex_lock(&_mutex); + } + WRes Leave() + { + return ::pthread_mutex_unlock(&_mutex); + } + WRes WaitCond() + { + return ::pthread_cond_wait(&_cond, &_mutex); + } + WRes LeaveAndSignal() + { + WRes res1 = ::pthread_cond_broadcast(&_cond); + WRes res2 = ::pthread_mutex_unlock(&_mutex); + return (res2 ? res2 : res1); + } +}; + + +struct CBaseHandle_WFMO; +typedef NWindows::NSynchronization::CBaseHandle_WFMO *CHandle_WFMO; + +// these constants are from Windows +#define WAIT_OBJECT_0 0 +#define WAIT_FAILED ((DWORD)0xFFFFFFFF) + +DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles); + + +struct CBaseHandle_WFMO MY_UNCOPYABLE +{ + CSynchro *_sync; + + CBaseHandle_WFMO(): _sync(NULL) {} + + operator CHandle_WFMO() { return this; } + virtual bool IsSignaledAndUpdate() = 0; +}; + + +class CBaseEvent_WFMO : public CBaseHandle_WFMO +{ + bool _manual_reset; + bool _state; + +public: + + // bool IsCreated() { return (this->_sync != NULL); } + // CBaseEvent_WFMO() { ; } + ~CBaseEvent_WFMO() { Close(); } + + WRes Close() { this->_sync = NULL; return 0; } + + WRes Create( + CSynchro *sync, + bool manualReset, bool initiallyOwn) + { + this->_sync = sync; + this->_manual_reset = manualReset; + this->_state = initiallyOwn; + return 0; + } + + WRes Set() + { + RINOK(this->_sync->Enter()); + this->_state = true; + return this->_sync->LeaveAndSignal(); + } + + WRes Reset() + { + RINOK(this->_sync->Enter()); + this->_state = false; + return this->_sync->Leave(); + } + + virtual bool IsSignaledAndUpdate() + { + if (this->_state == false) + return false; + if (this->_manual_reset == false) + this->_state = false; + return true; + } +}; + + +class CManualResetEvent_WFMO: public CBaseEvent_WFMO +{ +public: + WRes Create(CSynchro *sync, bool initiallyOwn = false) { return CBaseEvent_WFMO::Create(sync, true, initiallyOwn); } +}; + + +class CAutoResetEvent_WFMO: public CBaseEvent_WFMO +{ +public: + WRes Create(CSynchro *sync) { return CBaseEvent_WFMO::Create(sync, false, false); } + WRes CreateIfNotCreated_Reset(CSynchro *sync) + { + return Create(sync); + } +}; + + +class CSemaphore_WFMO : public CBaseHandle_WFMO +{ + UInt32 _count; + UInt32 _maxCount; + +public: + CSemaphore_WFMO() : _count(0), _maxCount(0) {} + + WRes Close() { this->_sync = NULL; return 0; } + + WRes Create(CSynchro *sync, UInt32 initCount, UInt32 maxCount) + { + if (initCount > maxCount || maxCount < 1) + return EINVAL; + this->_sync = sync; + this->_count = initCount; + this->_maxCount = maxCount; + return 0; + } + + WRes Release(UInt32 releaseCount = 1) + { + if (releaseCount < 1) + return EINVAL; + + RINOK(this->_sync->Enter()); + UInt32 newCount = this->_count + releaseCount; + if (newCount > this->_maxCount) + { + RINOK(this->_sync->Leave()); + return ERROR_TOO_MANY_POSTS; // EINVAL + } + this->_count = newCount; + + return this->_sync->LeaveAndSignal(); + } + + virtual bool IsSignaledAndUpdate() + { + if (this->_count == 0) + return false; + this->_count--; + return true; + } +}; + +#endif // _WIN32 + }} #endif diff --git a/CPP/Windows/System.cpp b/CPP/Windows/System.cpp index cc33169a..099407ec 100644 --- a/CPP/Windows/System.cpp +++ b/CPP/Windows/System.cpp @@ -2,15 +2,27 @@ #include "StdAfx.h" -#include "../Common/MyWindows.h" +#ifndef _WIN32 +#include <unistd.h> +#ifdef __APPLE__ +#include <sys/sysctl.h> +#else +#include <sys/sysinfo.h> +#endif +#endif #include "../Common/Defs.h" +// #include "../Common/MyWindows.h" + +// #include "../../C/CpuArch.h" #include "System.h" namespace NWindows { namespace NSystem { +#ifdef _WIN32 + UInt32 CountAffinity(DWORD_PTR mask) { UInt32 num = 0; @@ -19,8 +31,6 @@ UInt32 CountAffinity(DWORD_PTR mask) return num; } -#ifdef _WIN32 - BOOL CProcessAffinity::Get() { #ifndef UNDER_CE @@ -52,9 +62,45 @@ UInt32 GetNumberOfProcessors() #else + +BOOL CProcessAffinity::Get() +{ + numSysThreads = GetNumberOfProcessors(); + + /* + numSysThreads = 8; + for (unsigned i = 0; i < numSysThreads; i++) + CpuSet_Set(&cpu_set, i); + return TRUE; + */ + + #ifdef _7ZIP_AFFINITY_SUPPORTED + + // numSysThreads = sysconf(_SC_NPROCESSORS_ONLN); // The number of processors currently online + if (sched_getaffinity(0, sizeof(cpu_set), &cpu_set) != 0) + return FALSE; + return TRUE; + + #else + + // cpu_set = ((CCpuSet)1 << (numSysThreads)) - 1; + return TRUE; + // errno = ENOSYS; + // return FALSE; + + #endif +} + UInt32 GetNumberOfProcessors() { + #ifndef _7ZIP_ST + long n = sysconf(_SC_NPROCESSORS_CONF); // The number of processors configured + if (n < 1) + n = 1; + return (UInt32)n; + #else return 1; + #endif } #endif @@ -87,17 +133,13 @@ typedef struct _MY_MEMORYSTATUSEX { typedef BOOL (WINAPI *GlobalMemoryStatusExP)(MY_LPMEMORYSTATUSEX lpBuffer); -#endif - -#endif - +#endif // !UNDER_CE + bool GetRamSize(UInt64 &size) { size = (UInt64)(sizeof(size_t)) << 29; - #ifdef _WIN32 - #ifndef UNDER_CE MY_MEMORYSTATUSEX stat; stat.dwLength = sizeof(stat); @@ -114,7 +156,7 @@ bool GetRamSize(UInt64 &size) #ifndef UNDER_CE GlobalMemoryStatusExP globalMemoryStatusEx = (GlobalMemoryStatusExP) - ::GetProcAddress(::GetModuleHandle(TEXT("kernel32.dll")), "GlobalMemoryStatusEx"); + (void *)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "GlobalMemoryStatusEx"); if (globalMemoryStatusEx && globalMemoryStatusEx(&stat)) { size = MyMin(stat.ullTotalVirtual, stat.ullTotalPhys); @@ -129,14 +171,61 @@ bool GetRamSize(UInt64 &size) size = MyMin(stat2.dwTotalVirtual, stat2.dwTotalPhys); return true; } - #endif +} + +#else + +// POSIX +// #include <stdio.h> + +bool GetRamSize(UInt64 &size) +{ + size = (UInt64)(sizeof(size_t)) << 29; + + #ifdef __APPLE__ + + #ifdef HW_MEMSIZE + uint64_t val = 0; // support 2Gb+ RAM + int mib[2] = { CTL_HW, HW_MEMSIZE }; + #elif defined(HW_PHYSMEM64) + uint64_t val = 0; // support 2Gb+ RAM + int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + #else + unsigned int val = 0; // For old system + int mib[2] = { CTL_HW, HW_PHYSMEM }; + #endif // HW_MEMSIZE + size_t size_sys = sizeof(val); + + sysctl(mib, 2, &val, &size_sys, NULL, 0); + if (val) + size = val; + + #elif defined(_AIX) + + // fixme #else - return false; + struct sysinfo info; + if (::sysinfo(&info) != 0) + return false; + size = (UInt64)info.mem_unit * info.totalram; + const UInt64 kLimit = (UInt64)1 << (sizeof(size_t) * 8 - 1); + if (size > kLimit) + size = kLimit; + + /* + printf("\n mem_unit = %lld", (UInt64)info.mem_unit); + printf("\n totalram = %lld", (UInt64)info.totalram); + printf("\n freeram = %lld", (UInt64)info.freeram); + */ #endif + + return true; } +#endif + }} diff --git a/CPP/Windows/System.h b/CPP/Windows/System.h index 519e0444..23cb0dab 100644 --- a/CPP/Windows/System.h +++ b/CPP/Windows/System.h @@ -3,11 +3,19 @@ #ifndef __WINDOWS_SYSTEM_H #define __WINDOWS_SYSTEM_H +#ifndef _WIN32 +// #include <sched.h> +#include "../../C/Threads.h" +#endif + #include "../Common/MyTypes.h" namespace NWindows { namespace NSystem { + +#ifdef _WIN32 + UInt32 CountAffinity(DWORD_PTR mask); struct CProcessAffinity @@ -25,12 +33,93 @@ struct CProcessAffinity systemAffinityMask = 1; } + void CpuZero() + { + processAffinityMask = 0; + } + + void CpuSet(unsigned cpuIndex) + { + processAffinityMask |= ((DWORD_PTR)1 << cpuIndex); + } + UInt32 GetNumProcessThreads() const { return CountAffinity(processAffinityMask); } UInt32 GetNumSystemThreads() const { return CountAffinity(systemAffinityMask); } BOOL Get(); + + BOOL SetProcAffinity() const + { + return SetProcessAffinityMask(GetCurrentProcess(), processAffinityMask); + } +}; + + +#else // WIN32 + +struct CProcessAffinity +{ + UInt32 numSysThreads; + + UInt32 GetNumSystemThreads() const { return (UInt32)numSysThreads; } + BOOL Get(); + + #ifdef _7ZIP_AFFINITY_SUPPORTED + + CCpuSet cpu_set; + + void InitST() + { + numSysThreads = 1; + CpuSet_Zero(&cpu_set); + CpuSet_Set(&cpu_set, 0); + } + + UInt32 GetNumProcessThreads() const { return (UInt32)CPU_COUNT(&cpu_set); } + void CpuZero() { CpuSet_Zero(&cpu_set); } + void CpuSet(unsigned cpuIndex) { CpuSet_Set(&cpu_set, cpuIndex); } + int IsCpuSet(unsigned cpuIndex) const { return CpuSet_IsSet(&cpu_set, cpuIndex); } + // void CpuClr(int cpuIndex) { CPU_CLR(cpuIndex, &cpu_set); } + + BOOL SetProcAffinity() const + { + return sched_setaffinity(0, sizeof(cpu_set), &cpu_set) == 0; + } + + #else + + void InitST() + { + numSysThreads = 1; + } + + UInt32 GetNumProcessThreads() const + { + return numSysThreads; + /* + UInt32 num = 0; + for (unsigned i = 0; i < sizeof(cpu_set) * 8; i++) + num += (UInt32)((cpu_set >> i) & 1); + return num; + */ + } + + void CpuZero() { } + void CpuSet(unsigned cpuIndex) { UNUSED_VAR(cpuIndex); } + int IsCpuSet(unsigned cpuIndex) const { return (cpuIndex < numSysThreads) ? 1 : 0; } + + BOOL SetProcAffinity() const + { + errno = ENOSYS; + return FALSE; + } + + #endif }; +#endif + + UInt32 GetNumberOfProcessors(); bool GetRamSize(UInt64 &size); // returns false, if unknown ram size diff --git a/CPP/Windows/SystemInfo.cpp b/CPP/Windows/SystemInfo.cpp new file mode 100644 index 00000000..55403efc --- /dev/null +++ b/CPP/Windows/SystemInfo.cpp @@ -0,0 +1,716 @@ +// Windows/SystemInfo.cpp + +#include "StdAfx.h" + +#include "../../C/CpuArch.h" + +#include "../Common/IntToString.h" + +#ifdef _WIN32 + +#include "Registry.h" + +#else + +#include <sys/utsname.h> +#ifdef __APPLE__ +#include <sys/sysctl.h> +#elif !defined(_AIX) + + +#include <sys/auxv.h> + +#ifdef MY_CPU_ARM_OR_ARM64 +#include <asm/hwcap.h> +#endif +#endif + +#endif + +#include "SystemInfo.h" +#include "System.h" + +using namespace NWindows; + +#ifndef __APPLE__ +static void PrintHex(AString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + +static void PrintCpuChars(AString &s, UInt32 v) +{ + for (int j = 0; j < 4; j++) + { + Byte b = (Byte)(v & 0xFF); + v >>= 8; + if (b == 0) + break; + s += (char)b; + } +} + + +static void x86cpuid_to_String(const Cx86cpuid &c, AString &s) +{ + s.Empty(); + + UInt32 maxFunc2 = 0; + UInt32 t[3]; + + MyCPUID(0x80000000, &maxFunc2, &t[0], &t[1], &t[2]); + + bool fullNameIsAvail = (maxFunc2 >= 0x80000004); + + if (fullNameIsAvail) + { + for (unsigned i = 0; i < 3; i++) + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000002 + i, &d[0], &d[1], &d[2], &d[3]); + for (unsigned j = 0; j < 4; j++) + PrintCpuChars(s, d[j]); + } + } + + s.Trim(); + + if (s.IsEmpty()) + { + for (int i = 0; i < 3; i++) + PrintCpuChars(s, c.vendor[i]); + s.Trim(); + } + + s.Add_Space_if_NotEmpty(); + { + char temp[32]; + ConvertUInt32ToHex(c.ver, temp); + s += '('; + s += temp; + s += ')'; + } +} + +/* +static void x86cpuid_all_to_String(AString &s) +{ + Cx86cpuid p; + if (!x86cpuid_CheckAndRead(&p)) + return; + s += "x86cpuid maxFunc = "; + s.Add_UInt32(p.maxFunc); + for (unsigned j = 0; j <= p.maxFunc; j++) + { + s.Add_LF(); + // s.Add_UInt32(j); // align + { + char temp[32]; + ConvertUInt32ToString(j, temp); + unsigned len = (unsigned)strlen(temp); + while (len < 8) + { + len++; + s.Add_Space(); + } + s += temp; + } + + s += ":"; + UInt32 d[4] = { 0 }; + MyCPUID(j, &d[0], &d[1], &d[2], &d[3]); + for (unsigned i = 0; i < 4; i++) + { + char temp[32]; + ConvertUInt32ToHex8Digits(d[i], temp); + s += " "; + s += temp; + } + } +} +*/ + +#endif + + + +#ifdef _WIN32 + +static const char * const k_PROCESSOR_ARCHITECTURE[] = +{ + "x86" // "INTEL" + , "MIPS" + , "ALPHA" + , "PPC" + , "SHX" + , "ARM" + , "IA64" + , "ALPHA64" + , "MSIL" + , "x64" // "AMD64" + , "IA32_ON_WIN64" + , "NEUTRAL" + , "ARM64" + , "ARM32_ON_WIN64" +}; + +#define MY__PROCESSOR_ARCHITECTURE_INTEL 0 +#define MY__PROCESSOR_ARCHITECTURE_AMD64 9 + + +#define MY__PROCESSOR_INTEL_PENTIUM 586 +#define MY__PROCESSOR_AMD_X8664 8664 + +/* +static const CUInt32PCharPair k_PROCESSOR[] = +{ + { 2200, "IA64" }, + { 8664, "x64" } +}; + +#define PROCESSOR_INTEL_386 386 +#define PROCESSOR_INTEL_486 486 +#define PROCESSOR_INTEL_PENTIUM 586 +#define PROCESSOR_INTEL_860 860 +#define PROCESSOR_INTEL_IA64 2200 +#define PROCESSOR_AMD_X8664 8664 +#define PROCESSOR_MIPS_R2000 2000 +#define PROCESSOR_MIPS_R3000 3000 +#define PROCESSOR_MIPS_R4000 4000 +#define PROCESSOR_ALPHA_21064 21064 +#define PROCESSOR_PPC_601 601 +#define PROCESSOR_PPC_603 603 +#define PROCESSOR_PPC_604 604 +#define PROCESSOR_PPC_620 620 +#define PROCESSOR_HITACHI_SH3 10003 +#define PROCESSOR_HITACHI_SH3E 10004 +#define PROCESSOR_HITACHI_SH4 10005 +#define PROCESSOR_MOTOROLA_821 821 +#define PROCESSOR_SHx_SH3 103 +#define PROCESSOR_SHx_SH4 104 +#define PROCESSOR_STRONGARM 2577 // 0xA11 +#define PROCESSOR_ARM720 1824 // 0x720 +#define PROCESSOR_ARM820 2080 // 0x820 +#define PROCESSOR_ARM920 2336 // 0x920 +#define PROCESSOR_ARM_7TDMI 70001 +#define PROCESSOR_OPTIL 18767 // 0x494f +*/ + + +/* +static const char * const k_PF[] = +{ + "FP_ERRATA" + , "FP_EMU" + , "CMPXCHG" + , "MMX" + , "PPC_MOVEMEM_64BIT" + , "ALPHA_BYTE" + , "SSE" + , "3DNOW" + , "RDTSC" + , "PAE" + , "SSE2" + , "SSE_DAZ" + , "NX" + , "SSE3" + , "CMPXCHG16B" + , "CMP8XCHG16" + , "CHANNELS" + , "XSAVE" + , "ARM_VFP_32" + , "ARM_NEON" + , "L2AT" + , "VIRT_FIRMWARE" + , "RDWRFSGSBASE" + , "FASTFAIL" + , "ARM_DIVIDE" + , "ARM_64BIT_LOADSTORE_ATOMIC" + , "ARM_EXTERNAL_CACHE" + , "ARM_FMAC" + , "RDRAND" + , "ARM_V8" + , "ARM_V8_CRYPTO" + , "ARM_V8_CRC32" + , "RDTSCP" + , "RDPID" + , "ARM_V81_ATOMIC" + , "MONITORX" +}; +*/ + +#endif + + +#ifdef _WIN32 + +static void PrintPage(AString &s, UInt32 v) +{ + if ((v & 0x3FF) == 0) + { + s.Add_UInt32(v >> 10); + s += "K"; + } + else + s.Add_UInt32(v >> 10); +} + +static AString TypeToString2(const char * const table[], unsigned num, UInt32 value) +{ + char sz[16]; + const char *p = NULL; + if (value < num) + p = table[value]; + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + return (AString)p; +} + +// #if defined(_7ZIP_LARGE_PAGES) || defined(_WIN32) +// #ifdef _WIN32 +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v) +{ + char c = 0; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + PrintHex(s, v); + return; + } + char temp[32]; + ConvertUInt64ToString(v, temp); + s += temp; + if (c) + s += c; +} +// #endif +// #endif + +static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) +{ + s += TypeToString2(k_PROCESSOR_ARCHITECTURE, ARRAY_SIZE(k_PROCESSOR_ARCHITECTURE), si.wProcessorArchitecture); + + if (!( (si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_INTEL && si.dwProcessorType == MY__PROCESSOR_INTEL_PENTIUM) + || (si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_AMD64 && si.dwProcessorType == MY__PROCESSOR_AMD_X8664))) + { + s += " "; + // s += TypePairToString(k_PROCESSOR, ARRAY_SIZE(k_PROCESSOR), si.dwProcessorType); + s.Add_UInt32(si.dwProcessorType); + } + s += " "; + PrintHex(s, si.wProcessorLevel); + s += "."; + PrintHex(s, si.wProcessorRevision); + if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) + { + s += " act:"; + PrintHex(s, si.dwActiveProcessorMask); + } + s += " cpus:"; + s.Add_UInt32(si.dwNumberOfProcessors); + if (si.dwPageSize != 1 << 12) + { + s += " page:"; + PrintPage(s, si.dwPageSize); + } + if (si.dwAllocationGranularity != 1 << 16) + { + s += " gran:"; + PrintPage(s, si.dwAllocationGranularity); + } + s += " "; + + DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress; + UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1; + const UInt32 kReserveSize = ((UInt32)1 << 16); + if (minAdd != kReserveSize) + { + PrintSize_KMGT_Or_Hex(s, minAdd); + s += "-"; + } + else + { + if ((maxSize & (kReserveSize - 1)) == 0) + maxSize += kReserveSize; + } + PrintSize_KMGT_Or_Hex(s, maxSize); +} + +#ifndef _WIN64 +EXTERN_C_BEGIN +typedef VOID (WINAPI *Func_GetNativeSystemInfo)(LPSYSTEM_INFO lpSystemInfo); +EXTERN_C_END +#endif + +#endif + +#ifdef __APPLE__ +#ifndef MY_CPU_X86_OR_AMD64 +static void Add_sysctlbyname_to_String(const char *name, AString &s) +{ + size_t bufSize = 256; + char buf[256]; + if (My_sysctlbyname_Get(name, &buf, &bufSize) == 0) + s += buf; +} +#endif +#endif + +void GetSysInfo(AString &s1, AString &s2); +void GetSysInfo(AString &s1, AString &s2) +{ + s1.Empty(); + s2.Empty(); + + #ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + { + SysInfo_To_String(s1, si); + // s += " : "; + } + + #if !defined(_WIN64) && !defined(UNDER_CE) + Func_GetNativeSystemInfo fn_GetNativeSystemInfo = (Func_GetNativeSystemInfo)(void *)GetProcAddress( + GetModuleHandleA("kernel32.dll"), "GetNativeSystemInfo"); + if (fn_GetNativeSystemInfo) + { + SYSTEM_INFO si2; + fn_GetNativeSystemInfo(&si2); + // if (memcmp(&si, &si2, sizeof(si)) != 0) + { + // s += " - "; + SysInfo_To_String(s2, si2); + } + } + #endif + #endif +} + + +void GetCpuName(AString &s); +void GetCpuName(AString &s) +{ + s.Empty(); + + #ifdef MY_CPU_X86_OR_AMD64 + { + Cx86cpuid cpuid; + if (x86cpuid_CheckAndRead(&cpuid)) + { + AString s2; + x86cpuid_to_String(cpuid, s2); + s += s2; + } + else + { + #ifdef MY_CPU_AMD64 + s += "x64"; + #else + s += "x86"; + #endif + } + } + #elif defined(__APPLE__) + { + Add_sysctlbyname_to_String("machdep.cpu.brand_string", s); + } + #endif + + + if (s.IsEmpty()) + { + #ifdef MY_CPU_LE + s += "LE"; + #elif defined(MY_CPU_BE) + s += "BE"; + #endif + } + + #ifdef __APPLE__ + { + AString s2; + UInt32 v = 0; + if (My_sysctlbyname_Get_UInt32("machdep.cpu.core_count", &v) == 0) + { + s2.Add_UInt32(v); + s2 += 'C'; + } + if (My_sysctlbyname_Get_UInt32("machdep.cpu.thread_count", &v) == 0) + { + s2.Add_UInt32(v); + s2 += 'T'; + } + if (!s2.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += s2; + } + } + #endif + + + #ifdef _WIN32 + { + NRegistry::CKey key; + if (key.Open(HKEY_LOCAL_MACHINE, TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"), KEY_READ) == ERROR_SUCCESS) + { + LONG res[2]; + CByteBuffer bufs[2]; + { + for (int i = 0; i < 2; i++) + { + UInt32 size = 0; + res[i] = key.QueryValue(i == 0 ? + TEXT("Previous Update Revision") : + TEXT("Update Revision"), bufs[i], size); + if (res[i] == ERROR_SUCCESS) + if (size != bufs[i].Size()) + res[i] = ERROR_SUCCESS + 1; + } + } + if (res[0] == ERROR_SUCCESS || res[1] == ERROR_SUCCESS) + { + s.Add_OptSpaced("("); + for (int i = 0; i < 2; i++) + { + if (i == 1) + s += "->"; + if (res[i] != ERROR_SUCCESS) + continue; + const CByteBuffer &buf = bufs[i]; + if (buf.Size() == 8) + { + UInt32 high = GetUi32(buf); + if (high != 0) + { + PrintHex(s, high); + s += "."; + } + PrintHex(s, GetUi32(buf + 4)); + } + } + s += ")"; + } + } + } + #endif + + + #ifdef _7ZIP_LARGE_PAGES + Add_LargePages_String(s); + #endif +} + +void AddCpuFeatures(AString &s); +void AddCpuFeatures(AString &s) +{ + #ifdef _WIN32 + // const unsigned kNumFeatures_Extra = 32; // we check also for unknown features + // const unsigned kNumFeatures = ARRAY_SIZE(k_PF) + kNumFeatures_Extra; + const unsigned kNumFeatures = 64; + UInt64 flags = 0; + for (unsigned i = 0; i < kNumFeatures; i++) + { + if (IsProcessorFeaturePresent(i)) + { + flags += (UInt64)1 << i; + // s.Add_Space_if_NotEmpty(); + // s += TypeToString2(k_PF, ARRAY_SIZE(k_PF), i); + } + } + s.Add_Space_if_NotEmpty(); + s += "f:"; + PrintHex(s, flags); + + #else // _WIN32 + + #ifdef __APPLE__ + { + UInt32 v = 0; + if (My_sysctlbyname_Get_UInt32("hw.pagesize", &v) == 0) + { + s += "PageSize:"; + s.Add_UInt32(v >> 10); + s += "KB"; + } + } + + #elif !defined(_AIX) + + s.Add_Space_if_NotEmpty(); + s += "hwcap:"; + { + unsigned long h = getauxval(AT_HWCAP); + PrintHex(s, h); + #ifdef MY_CPU_ARM64 + if (h & HWCAP_CRC32) s += ":CRC32"; + if (h & HWCAP_SHA1) s += ":SHA1"; + if (h & HWCAP_SHA2) s += ":SHA2"; + if (h & HWCAP_AES) s += ":AES"; + #endif + } + + { + unsigned long h = getauxval(AT_HWCAP2); + #ifndef MY_CPU_ARM + if (h != 0) + #endif + { + s += " hwcap2:"; + PrintHex(s, h); + #ifdef MY_CPU_ARM + if (h & HWCAP2_CRC32) s += ":CRC32"; + if (h & HWCAP2_SHA1) s += ":SHA1"; + if (h & HWCAP2_SHA2) s += ":SHA2"; + if (h & HWCAP2_AES) s += ":AES"; + #endif + } + } + + #endif + #endif // _WIN32 +} + + +#ifdef _WIN32 +#ifndef UNDER_CE + +EXTERN_C_BEGIN +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); +EXTERN_C_END + +static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) +{ + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return FALSE; + Func_RtlGetVersion func = (Func_RtlGetVersion)(void *)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return FALSE; + func(vi); + return TRUE; +} + +#endif +#endif + + +void GetSystemInfoText(AString &sRes) +{ + { + { + AString s; + #ifdef _WIN32 + #ifndef UNDER_CE + // OSVERSIONINFO vi; + OSVERSIONINFOEXW vi; + vi.dwOSVersionInfoSize = sizeof(vi); + // if (::GetVersionEx(&vi)) + if (My_RtlGetVersion(&vi)) + { + s += "Windows"; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + s.Add_UInt32(vi.dwPlatformId); + s += " "; s.Add_UInt32(vi.dwMajorVersion); + s += "."; s.Add_UInt32(vi.dwMinorVersion); + s += " "; s.Add_UInt32(vi.dwBuildNumber); + + if (vi.wServicePackMajor != 0 || vi.wServicePackMinor != 0) + { + s += " SP:"; s.Add_UInt32(vi.wServicePackMajor); + s += "."; s.Add_UInt32(vi.wServicePackMinor); + } + s += " Suite:"; PrintHex(s, vi.wSuiteMask); + s += " Type:"; s.Add_UInt32(vi.wProductType); + // s += " "; s += GetOemString(vi.szCSDVersion); + } + { + s += " OEMCP:"; + s.Add_UInt32(GetOEMCP()); + s += " ACP:"; + s.Add_UInt32(GetACP()); + } + #endif + #else // _WIN32 + + if (!s.IsEmpty()) + s.Add_LF(); + struct utsname un; + if (uname(&un) == 0) + { + s += un.sysname; + // s += " : "; s += un.nodename; // we don't want to show name of computer + s += " : "; s += un.release; + s += " : "; s += un.version; + s += " : "; s += un.machine; + + #ifdef __APPLE__ + // Add_sysctlbyname_to_String("kern.version", s); + // it's same as "utsname.version" + #endif + } + #endif // _WIN32 + + sRes += s; + sRes.Add_LF(); + } + + { + AString s, s1, s2; + GetSysInfo(s1, s2); + if (!s1.IsEmpty() || !s2.IsEmpty()) + { + s = s1; + if (s1 != s2 && !s2.IsEmpty()) + { + s += " - "; + s += s2; + } + } + { + AddCpuFeatures(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + } + { + AString s; + GetCpuName(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + /* + #ifdef MY_CPU_X86_OR_AMD64 + { + AString s; + x86cpuid_all_to_String(s); + if (!s.IsEmpty()) + { + printCallback->Print(s); + printCallback->NewLine(); + } + } + #endif + */ + } +} diff --git a/CPP/Windows/SystemInfo.h b/CPP/Windows/SystemInfo.h new file mode 100644 index 00000000..856bb2ba --- /dev/null +++ b/CPP/Windows/SystemInfo.h @@ -0,0 +1,12 @@ +// Windows/SystemInfo.h + +#ifndef __WINDOWS_SYSTEM_INFO_H +#define __WINDOWS_SYSTEM_INFO_H + +#include "../Common/MyString.h" + +void GetSystemInfoText(AString &s); +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v); +void Add_LargePages_String(AString &s); + +#endif diff --git a/CPP/Windows/Thread.h b/CPP/Windows/Thread.h index 16a509d4..c9571469 100644 --- a/CPP/Windows/Thread.h +++ b/CPP/Windows/Thread.h @@ -9,7 +9,7 @@ namespace NWindows { -class CThread +class CThread MY_UNCOPYABLE { ::CThread thread; public: @@ -17,9 +17,15 @@ public: ~CThread() { Close(); } bool IsCreated() { return Thread_WasCreated(&thread) != 0; } WRes Close() { return Thread_Close(&thread); } - WRes Create(THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID parameter) - { return Thread_Create(&thread, startAddress, parameter); } - WRes Wait() { return Thread_Wait(&thread); } + // WRes Wait() { return Thread_Wait(&thread); } + WRes Wait_Close() { return Thread_Wait_Close(&thread); } + + WRes Create(THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID param) + { return Thread_Create(&thread, startAddress, param); } + WRes Create_With_Affinity(THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID param, CAffinityMask affinity) + { return Thread_Create_With_Affinity(&thread, startAddress, param, affinity); } + WRes Create_With_CpuSet(THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID param, const CCpuSet *cpuSet) + { return Thread_Create_With_CpuSet(&thread, startAddress, param, cpuSet); } #ifdef _WIN32 operator HANDLE() { return thread; } diff --git a/CPP/Windows/TimeUtils.cpp b/CPP/Windows/TimeUtils.cpp index d288f121..1f1335f9 100644 --- a/CPP/Windows/TimeUtils.cpp +++ b/CPP/Windows/TimeUtils.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifndef _WIN32 +#include <sys/time.h> +#endif + #include "Defs.h" #include "TimeUtils.h" @@ -10,7 +14,9 @@ namespace NTime { static const UInt32 kNumTimeQuantumsInSecond = 10000000; static const UInt32 kFileTimeStartYear = 1601; +#if !defined(_WIN32) || defined(UNDER_CE) static const UInt32 kDosTimeStartYear = 1980; +#endif static const UInt32 kUnixTimeStartYear = 1970; static const UInt64 kUnixTimeOffset = (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); @@ -37,10 +43,6 @@ bool DosTimeToFileTime(UInt32 dosTime, FILETIME &ft) throw() static const UInt32 kHighDosTime = 0xFF9FBF7D; static const UInt32 kLowDosTime = 0x210000; -#define PERIOD_4 (4 * 365 + 1) -#define PERIOD_100 (PERIOD_4 * 25 - 1) -#define PERIOD_400 (PERIOD_100 * 4 + 1) - bool FileTimeToDosTime(const FILETIME &ft, UInt32 &dosTime) throw() { #if defined(_WIN32) && !defined(UNDER_CE) @@ -55,6 +57,10 @@ bool FileTimeToDosTime(const FILETIME &ft, UInt32 &dosTime) throw() #else +#define PERIOD_4 (4 * 365 + 1) +#define PERIOD_100 (PERIOD_4 * 25 - 1) +#define PERIOD_400 (PERIOD_100 * 4 + 1) + unsigned year, mon, day, hour, min, sec; UInt64 v64 = ft.dwLowDateTime | ((UInt64)ft.dwHighDateTime << 32); Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; @@ -129,7 +135,7 @@ void UnixTimeToFileTime(UInt32 unixTime, FILETIME &ft) throw() UInt64 UnixTime64ToFileTime64(Int64 unixTime) throw() { - return (UInt64)(kUnixTimeOffset + unixTime) * kNumTimeQuantumsInSecond; + return (UInt64)((Int64)kUnixTimeOffset + unixTime) * kNumTimeQuantumsInSecond; } bool UnixTime64ToFileTime(Int64 unixTime, FILETIME &ft) throw() @@ -200,6 +206,7 @@ void GetCurUtcFileTime(FILETIME &ft) throw() { // Both variants provide same low resolution on WinXP: about 15 ms. // But GetSystemTimeAsFileTime is much faster. + #ifdef _WIN32 #ifdef UNDER_CE SYSTEMTIME st; @@ -208,6 +215,20 @@ void GetCurUtcFileTime(FILETIME &ft) throw() #else GetSystemTimeAsFileTime(&ft); #endif + + #else + + UInt64 v = 0; + struct timeval now; + if (gettimeofday(&now, 0 ) == 0) + { + v = ((UInt64)now.tv_sec + kUnixTimeOffset) * + kNumTimeQuantumsInSecond + (UInt64)now.tv_usec * 10; + } + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); + + #endif } }} diff --git a/CPP/Windows/Window.cpp b/CPP/Windows/Window.cpp index 36585022..32af4aab 100644 --- a/CPP/Windows/Window.cpp +++ b/CPP/Windows/Window.cpp @@ -114,12 +114,12 @@ bool MySetWindowText(HWND wnd, LPCWSTR s) bool CWindow::GetText(CSysString &s) { s.Empty(); - int len = GetTextLength(); + unsigned len = (unsigned)GetTextLength(); if (len == 0) return (::GetLastError() == ERROR_SUCCESS); TCHAR *p = s.GetBuf(len); { - int len2 = GetText(p, len + 1); + unsigned len2 = (unsigned)GetText(p, (int)(len + 1)); if (len > len2) len = len2; } @@ -135,12 +135,12 @@ bool CWindow::GetText(UString &s) if (g_IsNT) { s.Empty(); - int len = GetWindowTextLengthW(_window); + unsigned len = (unsigned)GetWindowTextLengthW(_window); if (len == 0) return (::GetLastError() == ERROR_SUCCESS); wchar_t *p = s.GetBuf(len); { - int len2 = GetWindowTextW(_window, p, len + 1); + unsigned len2 = (unsigned)GetWindowTextW(_window, p, (int)(len + 1)); if (len > len2) len = len2; } diff --git a/CPP/Windows/Window.h b/CPP/Windows/Window.h index 3bda6795..83726c7a 100644 --- a/CPP/Windows/Window.h +++ b/CPP/Windows/Window.h @@ -171,7 +171,7 @@ public: bool Update() { return BOOLToBool(::UpdateWindow(_window)); } bool InvalidateRect(LPCRECT rect, bool backgroundErase = true) { return BOOLToBool(::InvalidateRect(_window, rect, BoolToBOOL(backgroundErase))); } - void SetRedraw(bool redraw = true) { SendMsg(WM_SETREDRAW, BoolToBOOL(redraw), 0); } + void SetRedraw(bool redraw = true) { SendMsg(WM_SETREDRAW, (WPARAM)BoolToBOOL(redraw), 0); } LONG_PTR SetStyle(LONG_PTR style) { return SetLongPtr(GWL_STYLE, style); } LONG_PTR GetStyle() const { return GetLongPtr(GWL_STYLE); } @@ -244,7 +244,7 @@ public: int GetTextLength() const { return GetWindowTextLength(_window); } - UINT GetText(LPTSTR string, int maxCount) const + int GetText(LPTSTR string, int maxCount) const { return GetWindowText(_window, string, maxCount); } bool GetText(CSysString &s); #ifndef _UNICODE diff --git a/DOC/7zip.inf b/DOC/7zip.inf deleted file mode 100644 index b4e18e83..00000000 --- a/DOC/7zip.inf +++ /dev/null @@ -1,55 +0,0 @@ -[CODE] - -[Version] -Signature = "$Windows NT$" -Provider = "7-zip.org" -CESignature = "$Windows CE$" - -[CEStrings] -AppName = "7-Zip" -InstallDir = %CE1%\%AppName% - -[Strings] -AppVer = "19.00" -AppDate = "2019-01-21" - -[CEDevice] -; ProcessorType = 2577 ; ARM -VersionMin = 3.0 -BuildMin = 0.0 -VersionMax = 1000.0 -BuildMax = 0xE0000000 - -[DefaultInstall] -CopyFiles = CopyFilesSection,CopyFilesSection.Lang -AddReg = RegSettings -CEShortcuts = Shortcuts - -[SourceDisksNames] -1 = ,"Common files",,"." -2 = ,"Lang files",,"Lang" - -[SourceDisksFiles] -7zFM.exe = 1 -7z.sfx = 1 -7zS2.sfx = 1 -ru.txt = 2 - -[DestinationDirs] -DefaultDestDir = ,%InstallDir% -CopyFilesSection = ,%InstallDir% -CopyFilesSection.Lang = ,"%InstallDir%\Lang" -Shortcuts = ,%CE11% - -[CopyFilesSection] -7zFM.exe -7z.sfx -7zS2.sfx - -[CopyFilesSection.Lang] -ru.txt - -[RegSettings] - -[Shortcuts] -7-Zip,0,7zFM.exe diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi deleted file mode 100644 index 5a806ae7..00000000 --- a/DOC/7zip.nsi +++ /dev/null @@ -1,559 +0,0 @@ -;-------------------------------- -;Defines - -!define VERSION_MAJOR 19 -!define VERSION_MINOR 00 -!define VERSION_POSTFIX_FULL "" -!ifdef WIN64 -!ifdef IA64 -!define VERSION_SYS_POSTFIX_FULL " for Windows IA-64" -!else -!define VERSION_SYS_POSTFIX_FULL " for Windows x64" -!endif -!else -!define VERSION_SYS_POSTFIX_FULL "" -!endif -!define NAME_FULL "7-Zip ${VERSION_MAJOR}.${VERSION_MINOR}${VERSION_POSTFIX_FULL}${VERSION_SYS_POSTFIX_FULL}" -!define VERSION_POSTFIX "" -!ifdef WIN64 -!ifdef IA64 -!define VERSION_SYS_POSTFIX "-ia64" -!else -!define VERSION_SYS_POSTFIX "-x64" -!endif -!else -!define VERSION_SYS_POSTFIX "" -!endif - - - -!define FM_LINK "7-Zip File Manager.lnk" -!define HELP_LINK "7-Zip Help.lnk" - -!define CLSID_CONTEXT_MENU {23170F69-40C1-278A-1000-000100020000} - -#!define NO_COMPRESSION - - !include "Library.nsh" - !include "MUI.nsh" - -;-------------------------------- -;Configuration - - ;General - Name "${NAME_FULL}" - BrandingText "www.7-zip.org" - OutFile "..\7z${VERSION_MAJOR}${VERSION_MINOR}${VERSION_POSTFIX}${VERSION_SYS_POSTFIX}.exe" - - ;Folder selection page - InstallDir "$PROGRAMFILES\7-Zip" - - ;Get install folder from registry if available - InstallDirRegKey HKCU "Software\7-Zip" "Path32" - - ;Compressor -!ifndef NO_COMPRESSION - SetCompressor /solid lzma - ; SetCompressorFilter 1 -!ifdef IA64 - SetCompressorDictSize 8 -!else - SetCompressorDictSize 4 -!endif -!else - SetCompress off -!endif - - -;-------------------------------- -;Variables - -;-------------------------------- -;Interface Settings - - !define MUI_ABORTWARNING - -;-------------------------------- -;Pages - - #!insertmacro MUI_PAGE_LICENSE "License.txt" - !insertmacro MUI_PAGE_DIRECTORY - !insertmacro MUI_PAGE_INSTFILES - !insertmacro MUI_PAGE_FINISH - - !insertmacro MUI_UNPAGE_CONFIRM - !insertmacro MUI_UNPAGE_INSTFILES - !insertmacro MUI_UNPAGE_FINISH - -;-------------------------------- -;Languages - - !insertmacro MUI_LANGUAGE "English" - -;-------------------------------- -;Reserve Files - - ;These files should be inserted before other files in the data block - ;Keep these lines before any File command - ;Only for solid compression (by default, solid compression is enabled for BZIP2 and LZMA) - - !insertmacro MUI_RESERVEFILE_LANGDLL - -;-------------------------------- -;Installer Sections - -Section - !ifndef WIN64 - DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\App Paths\7zFMn.exe" - !endif - - # delete old unwanted files - - Delete $INSTDIR\7zFMn.exe - Delete $INSTDIR\7zgn.exe - Delete $INSTDIR\7zn.exe - Delete $INSTDIR\7zan.exe - Delete $INSTDIR\7zC.sfx - Delete $INSTDIR\7-zipn.dll - Delete $INSTDIR\7zipDoc.txt - Delete $INSTDIR\file_id.diz - Delete $INSTDIR\7zip_pad.xml - Delete $INSTDIR\copying.txt - - Delete $INSTDIR\Codecs\7zAES.dll - Delete $INSTDIR\Codecs\AES.dll - Delete $INSTDIR\Codecs\Branch.dll - Delete $INSTDIR\Codecs\BZip2.dll - Delete $INSTDIR\Codecs\Copy.dll - Delete $INSTDIR\Codecs\Deflate.dll - Delete $INSTDIR\Codecs\Implode.dll - Delete $INSTDIR\Codecs\LZMA.dll - Delete $INSTDIR\Codecs\PPMD.dll - Delete $INSTDIR\Codecs\Rar29.dll - Delete $INSTDIR\Codecs\Swap.dll - - RMDir $INSTDIR\Codecs - - Delete $INSTDIR\Formats\7z.dll - Delete $INSTDIR\Formats\arj.dll - Delete $INSTDIR\Formats\bz2.dll - Delete $INSTDIR\Formats\cab.dll - Delete $INSTDIR\Formats\chm.dll - Delete $INSTDIR\Formats\cpio.dll - Delete $INSTDIR\Formats\deb.dll - Delete $INSTDIR\Formats\gz.dll - Delete $INSTDIR\Formats\iso.dll - Delete $INSTDIR\Formats\lzh.dll - Delete $INSTDIR\Formats\nsis.dll - Delete $INSTDIR\Formats\rar.dll - Delete $INSTDIR\Formats\rpm.dll - Delete $INSTDIR\Formats\split.dll - Delete $INSTDIR\Formats\tar.dll - Delete $INSTDIR\Formats\z.dll - Delete $INSTDIR\Formats\zip.dll - - RMDir $INSTDIR\Formats - - Delete $INSTDIR\Lang\no.txt - - # install files - - SetOutPath $INSTDIR - - File descript.ion - File History.txt - File License.txt - File readme.txt - File 7-zip.chm - - # File 7-zip.dll - - File 7z.dll - File 7zFM.exe - File 7zG.exe - File 7z.exe - File 7z.sfx - File 7zCon.sfx - - SetOutPath $INSTDIR\Lang - - File en.ttt - File af.txt - File an.txt - File ar.txt - File ast.txt - File az.txt - File ba.txt - File be.txt - File bg.txt - File bn.txt - File br.txt - File ca.txt - File co.txt - File cs.txt - File cy.txt - File da.txt - File de.txt - File el.txt - File eo.txt - File es.txt - File et.txt - File eu.txt - File ext.txt - File fa.txt - File fi.txt - File fr.txt - File fur.txt - File fy.txt - File ga.txt - File gl.txt - File gu.txt - File he.txt - File hi.txt - File hr.txt - File hu.txt - File hy.txt - File id.txt - File io.txt - File is.txt - File it.txt - File ja.txt - File ka.txt - File kaa.txt - File kab.txt - File kk.txt - File ko.txt - File ku.txt - File ku-ckb.txt - File ky.txt - File lij.txt - File lt.txt - File lv.txt - File mk.txt - File mn.txt - File mng.txt - File mng2.txt - File mr.txt - File ms.txt - File ne.txt - File nl.txt - File nb.txt - File nn.txt - File pa-in.txt - File pl.txt - File ps.txt - File pt.txt - File pt-br.txt - File ro.txt - File ru.txt - File sa.txt - File si.txt - File sk.txt - File sl.txt - File sq.txt - File sr-spc.txt - File sr-spl.txt - File sv.txt - File ta.txt - File th.txt - File tr.txt - File tt.txt - File ug.txt - File uk.txt - File uz.txt - File va.txt - File vi.txt - File yo.txt - File zh-cn.txt - File zh-tw.txt - - SetOutPath $INSTDIR - - # delete "current user" menu items - - Delete "$SMPROGRAMS\7-Zip\${FM_LINK}" - Delete "$SMPROGRAMS\7-Zip\${HELP_LINK}" - RMDir $SMPROGRAMS\7-Zip - - # set "all users" mode - - SetShellVarContext all - - !insertmacro InstallLib DLL NOTSHARED REBOOT_NOTPROTECTED 7-zip.dll $INSTDIR\7-zip.dll $INSTDIR - - ClearErrors - - # create start menu icons - - SetOutPath $INSTDIR - - CreateDirectory $SMPROGRAMS\7-Zip - CreateShortCut "$SMPROGRAMS\7-Zip\${FM_LINK}" $INSTDIR\7zFM.exe - CreateShortCut "$SMPROGRAMS\7-Zip\${HELP_LINK}" $INSTDIR\7-zip.chm - - IfErrors 0 noScErrors - - SetShellVarContext current - - CreateDirectory $SMPROGRAMS\7-Zip - CreateShortCut "$SMPROGRAMS\7-Zip\${FM_LINK}" $INSTDIR\7zFM.exe - CreateShortCut "$SMPROGRAMS\7-Zip\${HELP_LINK}" $INSTDIR\7-zip.chm - -noScErrors: - - # store install folder - - WriteRegStr HKLM Software\7-Zip Path32 $INSTDIR - WriteRegStr HKLM Software\7-Zip Path $INSTDIR - WriteRegStr HKCU Software\7-Zip Path32 $INSTDIR - WriteRegStr HKCU Software\7-Zip Path $INSTDIR - - # write reg entries - - WriteRegStr HKCR "CLSID\${CLSID_CONTEXT_MENU}" "" "7-Zip Shell Extension" - WriteRegStr HKCR "CLSID\${CLSID_CONTEXT_MENU}\InprocServer32" "" $INSTDIR\7-zip.dll - WriteRegStr HKCR "CLSID\${CLSID_CONTEXT_MENU}\InprocServer32" ThreadingModel Apartment - DeleteRegValue HKCR "CLSID\${CLSID_CONTEXT_MENU}\InprocServer32" "InprocServer32" - - WriteRegStr HKCR "*\shellex\ContextMenuHandlers\7-Zip" "" "${CLSID_CONTEXT_MENU}" - WriteRegStr HKCR "Directory\shellex\ContextMenuHandlers\7-Zip" "" "${CLSID_CONTEXT_MENU}" - WriteRegStr HKCR "Folder\shellex\ContextMenuHandlers\7-Zip" "" "${CLSID_CONTEXT_MENU}" - - WriteRegStr HKCR "Directory\shellex\DragDropHandlers\7-Zip" "" "${CLSID_CONTEXT_MENU}" - WriteRegStr HKCR "Drive\shellex\DragDropHandlers\7-Zip" "" "${CLSID_CONTEXT_MENU}" - - WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Shell Extensions\Approved" "${CLSID_CONTEXT_MENU}" "7-Zip Shell Extension" - - WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\App Paths\7zFM.exe" "" $INSTDIR\7zFM.exe - WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\App Paths\7zFM.exe" Path $INSTDIR - - # create uninstaller - - WriteRegStr HKLM Software\Microsoft\Windows\CurrentVersion\Uninstall\7-Zip DisplayName "${NAME_FULL}" - WriteRegStr HKLM Software\Microsoft\Windows\CurrentVersion\Uninstall\7-Zip UninstallString '"$INSTDIR\Uninstall.exe"' - WriteRegDWORD HKLM Software\Microsoft\Windows\CurrentVersion\Uninstall\7-Zip NoModify 1 - WriteRegDWORD HKLM Software\Microsoft\Windows\CurrentVersion\Uninstall\7-Zip NoRepair 1 - - WriteUninstaller $INSTDIR\Uninstall.exe - - !ifdef WIN64 - ExecWait 'regsvr32 /s "$INSTDIR\7-zip.dll"' - !endif - -SectionEnd - - -;-------------------------------- -;Installer Functions - -/* -Function .onInit - - !insertmacro MUI_LANGDLL_DISPLAY - -FunctionEnd -*/ - - - - -;-------------------------------- -;Uninstaller Section - -Section Uninstall - - ExecWait 'regsvr32 /u /s "$INSTDIR\7-zip.dll"' - - # delete files - - Delete $INSTDIR\descript.ion - Delete $INSTDIR\History.txt - Delete $INSTDIR\License.txt - Delete $INSTDIR\readme.txt - Delete $INSTDIR\7-zip.chm - - Delete $INSTDIR\7z.dll - Delete $INSTDIR\7zFM.exe - Delete $INSTDIR\7zG.exe - Delete $INSTDIR\7z.exe - Delete $INSTDIR\7z.sfx - Delete $INSTDIR\7zCon.sfx - - Delete $INSTDIR\Lang\en.ttt - Delete $INSTDIR\Lang\af.txt - Delete $INSTDIR\Lang\an.txt - Delete $INSTDIR\Lang\ar.txt - Delete $INSTDIR\Lang\ast.txt - Delete $INSTDIR\Lang\az.txt - Delete $INSTDIR\Lang\ba.txt - Delete $INSTDIR\Lang\be.txt - Delete $INSTDIR\Lang\bg.txt - Delete $INSTDIR\Lang\bn.txt - Delete $INSTDIR\Lang\br.txt - Delete $INSTDIR\Lang\ca.txt - Delete $INSTDIR\Lang\co.txt - Delete $INSTDIR\Lang\cs.txt - Delete $INSTDIR\Lang\cy.txt - Delete $INSTDIR\Lang\da.txt - Delete $INSTDIR\Lang\de.txt - Delete $INSTDIR\Lang\el.txt - Delete $INSTDIR\Lang\eo.txt - Delete $INSTDIR\Lang\es.txt - Delete $INSTDIR\Lang\et.txt - Delete $INSTDIR\Lang\eu.txt - Delete $INSTDIR\Lang\ext.txt - Delete $INSTDIR\Lang\fa.txt - Delete $INSTDIR\Lang\fi.txt - Delete $INSTDIR\Lang\fr.txt - Delete $INSTDIR\Lang\fur.txt - Delete $INSTDIR\Lang\fy.txt - Delete $INSTDIR\Lang\ga.txt - Delete $INSTDIR\Lang\gl.txt - Delete $INSTDIR\Lang\gu.txt - Delete $INSTDIR\Lang\he.txt - Delete $INSTDIR\Lang\hi.txt - Delete $INSTDIR\Lang\hr.txt - Delete $INSTDIR\Lang\hu.txt - Delete $INSTDIR\Lang\hy.txt - Delete $INSTDIR\Lang\id.txt - Delete $INSTDIR\Lang\io.txt - Delete $INSTDIR\Lang\is.txt - Delete $INSTDIR\Lang\it.txt - Delete $INSTDIR\Lang\ja.txt - Delete $INSTDIR\Lang\ka.txt - Delete $INSTDIR\Lang\kaa.txt - Delete $INSTDIR\Lang\kab.txt - Delete $INSTDIR\Lang\kk.txt - Delete $INSTDIR\Lang\ko.txt - Delete $INSTDIR\Lang\ku.txt - Delete $INSTDIR\Lang\ku-ckb.txt - Delete $INSTDIR\Lang\ky.txt - Delete $INSTDIR\Lang\lij.txt - Delete $INSTDIR\Lang\lt.txt - Delete $INSTDIR\Lang\lv.txt - Delete $INSTDIR\Lang\mk.txt - Delete $INSTDIR\Lang\mn.txt - Delete $INSTDIR\Lang\mng.txt - Delete $INSTDIR\Lang\mng2.txt - Delete $INSTDIR\Lang\mr.txt - Delete $INSTDIR\Lang\ms.txt - Delete $INSTDIR\Lang\ne.txt - Delete $INSTDIR\Lang\nl.txt - Delete $INSTDIR\Lang\nb.txt - Delete $INSTDIR\Lang\nn.txt - Delete $INSTDIR\Lang\pa-in.txt - Delete $INSTDIR\Lang\pl.txt - Delete $INSTDIR\Lang\ps.txt - Delete $INSTDIR\Lang\pt.txt - Delete $INSTDIR\Lang\pt-br.txt - Delete $INSTDIR\Lang\ro.txt - Delete $INSTDIR\Lang\ru.txt - Delete $INSTDIR\Lang\sa.txt - Delete $INSTDIR\Lang\si.txt - Delete $INSTDIR\Lang\sk.txt - Delete $INSTDIR\Lang\sl.txt - Delete $INSTDIR\Lang\sq.txt - Delete $INSTDIR\Lang\sr.txt - Delete $INSTDIR\Lang\sr-spc.txt - Delete $INSTDIR\Lang\sr-spl.txt - Delete $INSTDIR\Lang\sv.txt - Delete $INSTDIR\Lang\ta.txt - Delete $INSTDIR\Lang\th.txt - Delete $INSTDIR\Lang\tr.txt - Delete $INSTDIR\Lang\tt.txt - Delete $INSTDIR\Lang\ug.txt - Delete $INSTDIR\Lang\uk.txt - Delete $INSTDIR\Lang\uz.txt - Delete $INSTDIR\Lang\va.txt - Delete $INSTDIR\Lang\vi.txt - Delete $INSTDIR\Lang\vr.txt - Delete $INSTDIR\Lang\yo.txt - Delete $INSTDIR\Lang\zh-cn.txt - Delete $INSTDIR\Lang\zh-tw.txt - - RMDir $INSTDIR\Lang - - Delete /REBOOTOK $INSTDIR\7-zip.dll - Delete $INSTDIR\Uninstall.exe - - RMDir $INSTDIR - - # delete start menu entires - - SetShellVarContext all - - # ClearErrors - - Delete "$SMPROGRAMS\7-Zip\${FM_LINK}" - Delete "$SMPROGRAMS\7-Zip\${HELP_LINK}" - RMDir $SMPROGRAMS\7-Zip - - # IfErrors 0 noScErrors - - SetShellVarContext current - - Delete "$SMPROGRAMS\7-Zip\${FM_LINK}" - Delete "$SMPROGRAMS\7-Zip\${HELP_LINK}" - RMDir $SMPROGRAMS\7-Zip - - # noScErrors: - - - # delete registry entries - - DeleteRegKey HKLM Software\Microsoft\Windows\CurrentVersion\Uninstall\7-Zip - DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\App Paths\7zFM.exe" - - DeleteRegKey HKLM Software\7-Zip - DeleteRegKey HKCU Software\7-Zip - - DeleteRegKey HKCR CLSID\${CLSID_CONTEXT_MENU} - - DeleteRegKey HKCR *\shellex\ContextMenuHandlers\7-Zip - DeleteRegKey HKCR Directory\shellex\ContextMenuHandlers\7-Zip - DeleteRegKey HKCR Folder\shellex\ContextMenuHandlers\7-Zip - - DeleteRegKey HKCR Drive\shellex\DragDropHandlers\7-Zip - DeleteRegKey HKCR Directory\shellex\DragDropHandlers\7-Zip - DeleteRegKey HKCR Folder\shellex\DragDropHandlers\7-Zip - - DeleteRegValue HKLM "Software\Microsoft\Windows\CurrentVersion\Shell Extensions\Approved" "${CLSID_CONTEXT_MENU}" - - DeleteRegKey HKCR 7-Zip.001 - DeleteRegKey HKCR 7-Zip.7z - DeleteRegKey HKCR 7-Zip.arj - DeleteRegKey HKCR 7-Zip.bz2 - DeleteRegKey HKCR 7-Zip.bzip2 - DeleteRegKey HKCR 7-Zip.tbz - DeleteRegKey HKCR 7-Zip.tbz2 - DeleteRegKey HKCR 7-Zip.cab - DeleteRegKey HKCR 7-Zip.cpio - DeleteRegKey HKCR 7-Zip.deb - DeleteRegKey HKCR 7-Zip.dmg - DeleteRegKey HKCR 7-Zip.fat - DeleteRegKey HKCR 7-Zip.gz - DeleteRegKey HKCR 7-Zip.gzip - DeleteRegKey HKCR 7-Zip.hfs - DeleteRegKey HKCR 7-Zip.iso - DeleteRegKey HKCR 7-Zip.lha - DeleteRegKey HKCR 7-Zip.lzh - DeleteRegKey HKCR 7-Zip.lzma - DeleteRegKey HKCR 7-Zip.ntfs - DeleteRegKey HKCR 7-Zip.rar - DeleteRegKey HKCR 7-Zip.rpm - DeleteRegKey HKCR 7-Zip.split - DeleteRegKey HKCR 7-Zip.squashfs - DeleteRegKey HKCR 7-Zip.swm - DeleteRegKey HKCR 7-Zip.tar - DeleteRegKey HKCR 7-Zip.taz - DeleteRegKey HKCR 7-Zip.tgz - DeleteRegKey HKCR 7-Zip.tpz - DeleteRegKey HKCR 7-Zip.txz - DeleteRegKey HKCR 7-Zip.vhd - DeleteRegKey HKCR 7-Zip.wim - DeleteRegKey HKCR 7-Zip.xar - DeleteRegKey HKCR 7-Zip.xz - DeleteRegKey HKCR 7-Zip.z - DeleteRegKey HKCR 7-Zip.zip - -SectionEnd diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index 338d5829..14793a67 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ <?xml version="1.0"?> -<?define VerMajor = "19" ?> -<?define VerMinor = "00" ?> +<?define VerMajor = "21" ?> +<?define VerMinor = "02" ?> <?define VerBuild = "00" ?> <?define MmVer = "$(var.VerMajor).$(var.VerMinor)" ?> <?define MmHex = "$(var.VerMajor)$(var.VerMinor)" ?> @@ -330,13 +330,17 @@ <File Id="sr_spl.txt" Name="sr-spl.txt" /> <File Id="sr_spc.txt" Name="sr-spc.txt" /> <File Id="sv.txt" Name="sv.txt" /> + <File Id="sw.txt" Name="sw.txt" /> <File Id="ta.txt" Name="ta.txt" /> + <File Id="tg.txt" Name="tg.txt" /> <File Id="th.txt" Name="th.txt" /> + <File Id="tk.txt" Name="tk.txt" /> <File Id="tr.txt" Name="tr.txt" /> <File Id="tt.txt" Name="tt.txt" /> <File Id="ug.txt" Name="ug.txt" /> <File Id="uk.txt" Name="uk.txt" /> <File Id="uz.txt" Name="uz.txt" /> + <File Id="uz_cyrl.txt" Name="uz-cyrl.txt" /> <File Id="va.txt" Name="va.txt" /> <File Id="vi.txt" Name="vi.txt" /> <File Id="yo.txt" Name="yo.txt" /> diff --git a/DOC/License.txt b/DOC/License.txt index d6b672bb..c9e858fe 100644 --- a/DOC/License.txt +++ b/DOC/License.txt @@ -3,7 +3,7 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2019 Igor Pavlov. + 7-Zip Copyright (C) 1999-2020 Igor Pavlov. The licenses for files are: diff --git a/DOC/readme.txt b/DOC/readme.txt index 0d1aebf7..18e5e387 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 19.00 Sources +7-Zip 21.02 Sources ------------------- 7-Zip is a file archiver for Windows. -7-Zip Copyright (C) 1999-2019 Igor Pavlov. +7-Zip Copyright (C) 1999-2021 Igor Pavlov. License Info diff --git a/DOC/src-history.txt b/DOC/src-history.txt index 51921cd0..b802c69a 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,56 @@ HISTORY of the 7-Zip source code -------------------------------- +21.02 alpha 2021-05-06 +------------------------- +- 7-Zip now writes additional field for filename in UTF-8 encoding to zip archives. + It allows to extract correct file name from zip archives on different systems. +- The command line version of 7-Zip for macOS was released. +- The speed for LZMA and LZMA2 decompression in arm64 versions for macOS and Linux + was increased by 20%-60%. +- Some changes and improvements in ZIP, TAR and NSIS code. + + +21.01 alpha 2021-03-09 +------------------------- +- The command line version of 7-Zip for Linux was released. +- The improvements for speed of ARM64 version using hardware CPU instructions + for AES, CRC-32, SHA-1 and SHA-256. +- The bug in versions 18.02 - 21.00 was fixed: + 7-Zip could not correctly extract some ZIP archives created with xz compression method. +- Some bugs were fixed. + + +20.02 alpha 2020-08-08 +------------------------- +- The default number of LZMA2 chunks per solid block in 7z archive was increased to 64. + It allows to increase the compression speed for big 7z archives, if there is a big number + of CPU cores and threads. +- The speed of PPMd compressing/decompressing was increased for 7z/ZIP/RAR archives. +- The new -ssp switch. If the switch -ssp is specified, 7-Zip doesn't allow the system + to modify "Last Access Time" property of source files for archiving and hashing operations. +- Some bugs were fixed. + + +20.00 alpha 2020-02-06 +------------------------- +- 7-Zip now supports new optional match finders for LZMA/LZMA2 compression: bt5 and hc5, + that can work faster than bt4 and hc4 match finders for the data with big redundancy. +- The compression ratio was improved for Fast and Fastest compression levels with the + following default settings: + - Fastest level (-mx1) : hc5 match finder with 256 KB dictionary. + - Fast level (-mx3) : hc5 match finder with 4 MB dictionary. +- Minor speed optimizations in multithreaded LZMA/LZMA2 compression for Normal/Maximum/Ultra + compression levels. +- bzip2 decoding code was updated to support bzip2 archives, created by lbzip2 program. + + +19.02 2019-09-05 +------------------------- +- Support for SHA-1/SHA-256 optimized code in + Sha1Opt.c, Sha256Opt.c, Sha256Opt.asm, Sha1Opt.asm. + + 19.00 2019-02-21 ------------------------- - Encryption strength for 7z archives was increased: |