diff options
| author | Igor Pavlov <ipavlov@users.sourceforge.net> | 2018-03-12 14:19:17 +0300 |
|---|---|---|
| committer | Kornel <kornel@geekhood.net> | 2018-03-12 14:19:46 +0300 |
| commit | f19b649c73cd8b74c4e0b8a3a728a82c6bda47b4 (patch) | |
| tree | 3725026df472886583a019239ed169e356ce1f63 | |
| parent | 866a06f5a0c0ede11979504a318e150030aa7a11 (diff) | |
18.0318.03
81 files changed, 8214 insertions, 1267 deletions
diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm index 3707e90c..067db822 100644 --- a/Asm/x86/7zAsm.asm +++ b/Asm/x86/7zAsm.asm @@ -1,5 +1,5 @@ ; 7zAsm.asm -- ASM macros -; 2012-12-30 : Igor Pavlov : Public domain +; 2018-02-03 : Igor Pavlov : Public domain MY_ASM_START macro ifdef x64 @@ -52,6 +52,15 @@ endif x6 equ ESI x7 equ EDI + x0_W equ AX + x1_W equ CX + x2_W equ DX + x3_W equ BX + + x5_W equ BP + x6_W equ SI + x7_W equ DI + x0_L equ AL x1_L equ CL x2_L equ DL @@ -63,6 +72,10 @@ endif x3_H equ BH ifdef x64 + x5_L equ BPL + x6_L equ SIL + x7_L equ DIL + r0 equ RAX r1 equ RCX r2 equ RDX @@ -103,3 +116,32 @@ MY_POP_4_REGS macro pop r5 pop r3 endm + + +ifdef x64 + +; for WIN64-x64 ABI: + +REG_PARAM_0 equ r1 +REG_PARAM_1 equ r2 +REG_PARAM_2 equ r8 +REG_PARAM_3 equ r9 + +MY_PUSH_PRESERVED_REGS macro + MY_PUSH_4_REGS + push r12 + push r13 + push r14 + push r15 +endm + + +MY_POP_PRESERVED_REGS macro + pop r15 + pop r14 + pop r13 + pop r12 + MY_POP_4_REGS +endm + +endif diff --git a/Asm/x86/LzmaDecOpt.asm b/Asm/x86/LzmaDecOpt.asm new file mode 100644 index 00000000..8ebbc5f2 --- /dev/null +++ b/Asm/x86/LzmaDecOpt.asm @@ -0,0 +1,1258 @@ +; LzmaDecOpt.asm -- ASM version of LzmaDec_DecodeReal_3() function +; 2018-02-06: Igor Pavlov : Public domain +; +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). + +ifndef x64 +; x64=1 +; .err <x64_IS_REQUIRED> +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZMADECOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_16 macro + MY_ALIGN 16 +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +; _LZMA_SIZE_OPT equ 1 + +; _LZMA_PROB32 equ 1 + +ifdef _LZMA_PROB32 + PSHIFT equ 2 + PLOAD macro dest, mem + mov dest, dword ptr [mem] + endm + PSTORE macro src, mem + mov dword ptr [mem], src + endm +else + PSHIFT equ 1 + PLOAD macro dest, mem + movzx dest, word ptr [mem] + endm + PSTORE macro src, mem + mov word ptr [mem], @CatStr(src, _W) + endm +endif + +PMULT equ (1 SHL PSHIFT) +PMULT_HALF equ (1 SHL (PSHIFT - 1)) +PMULT_2 equ (1 SHL (PSHIFT + 1)) + + +; x0 range +; x1 pbPos / (prob) TREE +; x2 probBranch / prm (MATCHED) / pbPos / cnt +; x3 sym +;====== r4 === RSP +; x5 cod +; x6 t1 NORM_CALC / probs_state / dist +; x7 t0 NORM_CALC / prob2 IF_BIT_1 +; x8 state +; x9 match (MATCHED) / sym2 / dist2 / lpMask_reg +; x10 kBitModelTotal_reg +; r11 probs +; x12 offs (MATCHED) / dic / len_temp +; x13 processedPos +; x14 bit (MATCHED) / dicPos +; r15 buf + + +cod equ x5 +cod_L equ x5_L +range equ x0 +state equ x8 +state_R equ r8 +buf equ r15 +processedPos equ x13 +kBitModelTotal_reg equ x10 + +probBranch equ x2 +probBranch_R equ r2 +probBranch_W equ x2_W + +pbPos equ x1 +pbPos_R equ r1 + +cnt equ x2 +cnt_R equ r2 + +lpMask_reg equ x9 +dicPos equ r14 + +sym equ x3 +sym_R equ r3 +sym_L equ x3_L + +probs equ r11 +dic equ r12 + +t0 equ x7 +t0_W equ x7_W +t0_R equ r7 + +prob2 equ t0 +prob2_W equ t0_W + +t1 equ x6 +t1_R equ r6 + +probs_state equ t1 +probs_state_R equ t1_R + +prm equ r2 +match equ x9 +match_R equ r9 +offs equ x12 +offs_R equ r12 +bit equ x14 +bit_R equ r14 + +sym2 equ x9 +sym2_R equ r9 + +len_temp equ x12 + +dist equ sym +dist2 equ x9 + + + +kNumBitModelTotalBits equ 11 +kBitModelTotal equ (1 SHL kNumBitModelTotalBits) +kNumMoveBits equ 5 +kBitModelOffset equ ((1 SHL kNumMoveBits) - 1) +kTopValue equ (1 SHL 24) + +NORM_2 macro + ; movzx t0, BYTE PTR [buf] + shl cod, 8 + mov cod_L, BYTE PTR [buf] + shl range, 8 + ; or cod, t0 + inc buf +endm + + +NORM macro + cmp range, kTopValue + jae SHORT @F + NORM_2 +@@: +endm + + +; ---------- Branch MACROS ---------- + +UPDATE_0 macro probsArray:req, probOffset:req, probDisp:req + mov prob2, kBitModelTotal_reg + sub prob2, probBranch + shr prob2, kNumMoveBits + add probBranch, prob2 + PSTORE probBranch, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +UPDATE_1 macro probsArray:req, probOffset:req, probDisp:req + sub prob2, range + sub cod, range + mov range, prob2 + mov prob2, probBranch + shr probBranch, kNumMoveBits + sub prob2, probBranch + PSTORE prob2, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +CMP_COD macro probsArray:req, probOffset:req, probDisp:req + PLOAD probBranch, probOffset * 1 + probsArray + probDisp * PMULT + NORM + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +endm + + +IF_BIT_1_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jae toLabel +endm + + +IF_BIT_1 macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP probsArray, probOffset, probDisp, toLabel + UPDATE_0 probsArray, probOffset, probDisp +endm + + +IF_BIT_0_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jb toLabel +endm + + +; ---------- CMOV MACROS ---------- + +NORM_CALC macro prob:req + NORM + mov t0, range + shr range, kNumBitModelTotalBits + imul range, prob + sub t0, range + mov t1, cod + sub cod, range +endm + + +PUP macro prob:req, probPtr:req + sub t0, prob + ; only sar works for both 16/32 bit prob modes + sar t0, kNumMoveBits + add t0, prob + PSTORE t0, probPtr +endm + + +PUP_SUB macro prob:req, probPtr:req, symSub:req + sbb sym, symSub + PUP prob, probPtr +endm + + +PUP_COD macro prob:req, probPtr:req, symSub:req + mov t0, kBitModelOffset + cmovb cod, t1 + mov t1, sym + cmovb t0, kBitModelTotal_reg + PUP_SUB prob, probPtr, symSub +endm + + +BIT_0 macro prob:req, probNext:req + PLOAD prob, probs + 1 * PMULT + PLOAD probNext, probs + 1 * PMULT_2 + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 1 * PMULT_2 + PMULT + cmovae probNext, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 2 + PUP_SUB prob, probs + 1 * PMULT, 0 - 1 +endm + + +BIT_1 macro prob:req, probNext:req + PLOAD probNext, probs + sym_R * PMULT_2 + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + sym_R * PMULT + PMULT + cmovae probNext, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, 0 - 1 +endm + + +BIT_2 macro prob:req, symSub:req + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, symSub +endm + + +; ---------- MATCHED LITERAL ---------- + +LITM_0 macro + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, offs + and bit, match + PLOAD x1, probs + 256 * PMULT + bit_R * 1 + 1 * PMULT + lea prm, [probs + 256 * PMULT + bit_R * 1 + 1 * PMULT] + ; lea prm, [probs + 256 * PMULT + 1 * PMULT] + ; add prm, bit_R + xor offs, bit + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 0 + PUP_SUB x1, prm, -2-1 +endm + + +LITM macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + xor offs, bit + add sym, sym + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, - 1 +endm + + +LITM_2 macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + add sym, sym + + NORM_CALC x1 + + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, 256 - 1 +endm + + +; ---------- REVERSE BITS ---------- + +REV_0 macro prob:req, probNext:req + ; PLOAD prob, probs + 1 * PMULT + ; lea sym2_R, [probs + 2 * PMULT] + ; PLOAD probNext, probs + 2 * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 3 * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [probs + 3 * PMULT] + cmovae sym2_R, t1_R + PUP prob, probs + 1 * PMULT +endm + + +REV_1 macro prob:req, probNext:req, step:req + add sym2_R, step * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, sym2_R + step * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [sym2_R + step * PMULT] + cmovae sym2_R, t1_R + PUP prob, t1_R - step * PMULT_2 +endm + + +REV_2 macro prob:req, step:req + sub sym2_R, probs + shr sym2, PSHIFT + or sym, sym2 + + NORM_CALC prob + + cmovae range, t0 + lea t0, [sym - step] + cmovb sym, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + PUP prob, probs + sym2_R * PMULT +endm + + +REV_1_VAR macro prob:req + PLOAD prob, sym_R + mov probs, sym_R + add sym_R, sym2_R + + NORM_CALC prob + + cmovae range, t0 + lea t0_R, [sym_R + sym2_R] + cmovae sym_R, t0_R + mov t0, kBitModelOffset + cmovb cod, t1 + ; mov t1, kBitModelTotal + ; cmovb t0, t1 + cmovb t0, kBitModelTotal_reg + add sym2, sym2 + PUP prob, probs +endm + + + + +LIT_PROBS macro lpMaskParam:req + ; prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + mov t0, processedPos + shl t0, 8 + add sym, t0 + and sym, lpMaskParam + add probs_state_R, pbPos_R + mov x1, LOC lc2 + lea sym, dword ptr[sym_R + 2 * sym_R] + add probs, Literal * PMULT + shl sym, x1_L + add probs, sym_R + UPDATE_0 probs_state_R, 0, IsMatch + inc processedPos +endm + + + +kNumPosBitsMax equ 4 +kNumPosStatesMax equ (1 SHL kNumPosBitsMax) + +kLenNumLowBits equ 3 +kLenNumLowSymbols equ (1 SHL kLenNumLowBits) +kLenNumHighBits equ 8 +kLenNumHighSymbols equ (1 SHL kLenNumHighBits) +kNumLenProbs equ (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +LenLow equ 0 +LenChoice equ LenLow +LenChoice2 equ (LenLow + kLenNumLowSymbols) +LenHigh equ (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +kNumStates equ 12 +kNumStates2 equ 16 +kNumLitStates equ 7 + +kStartPosModelIndex equ 4 +kEndPosModelIndex equ 14 +kNumFullDistances equ (1 SHL (kEndPosModelIndex SHR 1)) + +kNumPosSlotBits equ 6 +kNumLenToPosStates equ 4 + +kNumAlignBits equ 4 +kAlignTableSize equ (1 SHL kNumAlignBits) + +kMatchMinLen equ 2 +kMatchSpecLenStart equ (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +kStartOffset equ 1664 +SpecPos equ (-kStartOffset) +IsRep0Long equ (SpecPos + kNumFullDistances) +RepLenCoder equ (IsRep0Long + (kNumStates2 SHL kNumPosBitsMax)) +LenCoder equ (RepLenCoder + kNumLenProbs) +IsMatch equ (LenCoder + kNumLenProbs) +kAlign equ (IsMatch + (kNumStates2 SHL kNumPosBitsMax)) +IsRep equ (kAlign + kAlignTableSize) +IsRepG0 equ (IsRep + kNumStates) +IsRepG1 equ (IsRepG0 + kNumStates) +IsRepG2 equ (IsRepG1 + kNumStates) +PosSlot equ (IsRepG2 + kNumStates) +Literal equ (PosSlot + (kNumLenToPosStates SHL kNumPosSlotBits)) +NUM_BASE_PROBS equ (Literal + kStartOffset) + +if kAlign ne 0 + .err <Stop_Compiling_Bad_LZMA_kAlign> +endif + +if NUM_BASE_PROBS ne 1984 + .err <Stop_Compiling_Bad_LZMA_PROBS> +endif + + +PTR_FIELD equ dq ? + +CLzmaDec_Asm struct + lc db ? + lp db ? + pb db ? + _pad_ db ? + dicSize dd ? + + probs_Spec PTR_FIELD + probs_1664 PTR_FIELD + dic_Spec PTR_FIELD + dicBufSize PTR_FIELD + dicPos_Spec PTR_FIELD + buf_Spec PTR_FIELD + + range_Spec dd ? + code_Spec dd ? + processedPos_Spec dd ? + checkDicSize dd ? + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? + state_Spec dd ? + remainLen dd ? +CLzmaDec_Asm ends + + +CLzmaDec_Asm_Loc struct + OLD_RSP PTR_FIELD + lzmaPtr PTR_FIELD + _pad0_ PTR_FIELD + _pad1_ PTR_FIELD + _pad2_ PTR_FIELD + dicBufSize PTR_FIELD + probs_Spec PTR_FIELD + dic_Spec PTR_FIELD + + limit PTR_FIELD + bufLimit PTR_FIELD + lc2 dd ? + lpMask dd ? + pbMask dd ? + checkDicSize dd ? + + _pad_ dd ? + remainLen dd ? + dicPos_Spec PTR_FIELD + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? +CLzmaDec_Asm_Loc ends + + +GLOB_2 equ [sym_R].CLzmaDec_Asm. +GLOB equ [r1].CLzmaDec_Asm. +LOC_0 equ [r0].CLzmaDec_Asm_Loc. +LOC equ [RSP].CLzmaDec_Asm_Loc. + + +COPY_VAR macro name + mov t0, GLOB_2 name + mov LOC_0 name, t0 +endm + + +RESTORE_VAR macro name + mov t0, LOC name + mov GLOB name, t0 +endm + + + +IsMatchBranch_Pre macro reg + ; prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + mov pbPos, LOC pbMask + and pbPos, processedPos + shl pbPos, (kLenNumLowBits + 1 + PSHIFT) + lea probs_state_R, [probs + state_R] +endm + + +IsMatchBranch macro reg + IsMatchBranch_Pre + IF_BIT_1 probs_state_R, pbPos_R, IsMatch, IsMatch_label +endm + + +CheckLimits macro reg + cmp buf, LOC bufLimit + jae fin_OK + cmp dicPos, LOC limit + jae fin_OK +endm + + + +; RSP is (16x + 8) bytes aligned in WIN64-x64 +; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) + +PARAM_lzma equ REG_PARAM_0 +PARAM_limit equ REG_PARAM_1 +PARAM_bufLimit equ REG_PARAM_2 + +; MY_ALIGN_64 +MY_PROC LzmaDec_DecodeReal_3, 3 +MY_PUSH_PRESERVED_REGS + + lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] + and r0, -128 + mov r5, RSP + mov RSP, r0 + mov LOC_0 Old_RSP, r5 + mov LOC_0 lzmaPtr, PARAM_lzma + + mov LOC_0 remainLen, 0 ; remainLen must be ZERO + + mov LOC_0 bufLimit, PARAM_bufLimit + mov sym_R, PARAM_lzma ; CLzmaDec_Asm_Loc pointer for GLOB_2 + mov dic, GLOB_2 dic_Spec + add PARAM_limit, dic + mov LOC_0 limit, PARAM_limit + + COPY_VAR(rep0) + COPY_VAR(rep1) + COPY_VAR(rep2) + COPY_VAR(rep3) + + mov dicPos, GLOB_2 dicPos_Spec + add dicPos, dic + mov LOC_0 dicPos_Spec, dicPos + mov LOC_0 dic_Spec, dic + + mov x1_L, GLOB_2 pb + mov t0, 1 + shl t0, x1_L + dec t0 + mov LOC_0 pbMask, t0 + + ; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + ; unsigned lc = p->prop.lc; + ; unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); + + mov x1_L, GLOB_2 lc + mov x2, 100h + mov t0, x2 + shr x2, x1_L + ; inc x1 + add x1_L, PSHIFT + mov LOC_0 lc2, x1 + mov x1_L, GLOB_2 lp + shl t0, x1_L + sub t0, x2 + mov LOC_0 lpMask, t0 + mov lpMask_reg, t0 + + ; mov probs, GLOB_2 probs_Spec + ; add probs, kStartOffset SHL PSHIFT + mov probs, GLOB_2 probs_1664 + mov LOC_0 probs_Spec, probs + + mov t0_R, GLOB_2 dicBufSize + mov LOC_0 dicBufSize, t0_R + + mov x1, GLOB_2 checkDicSize + mov LOC_0 checkDicSize, x1 + + mov processedPos, GLOB_2 processedPos_Spec + + mov state, GLOB_2 state_Spec + shl state, PSHIFT + + mov buf, GLOB_2 buf_Spec + mov range, GLOB_2 range_Spec + mov cod, GLOB_2 code_Spec + mov kBitModelTotal_reg, kBitModelTotal + xor sym, sym + + ; if (processedPos != 0 || checkDicSize != 0) + or x1, processedPos + jz @f + + add t0_R, dic + cmp dicPos, dic + cmovnz t0_R, dicPos + movzx sym, byte ptr[t0_R - 1] + +@@: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + + +; ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + xor state, state +lit_start_2: + LIT_PROBS lpMask_reg + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +lit_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 127 + jbe lit_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + BIT_2 x2, 256 - 1 + + ; mov dic, LOC dic_Spec + mov probs, LOC probs_Spec + IsMatchBranch_Pre + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_end: + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, lit_start + + ; jmp IsMatch_label + +; ---------- MATCHES ---------- +; MY_ALIGN_32 +IsMatch_label: + UPDATE_1 probs_state_R, pbPos_R, IsMatch + IF_BIT_1 probs_state_R, 0, IsRep, IsRep_label + + add probs, LenCoder * PMULT + add state, kNumStates * PMULT + +; ---------- LEN DECODE ---------- +len_decode: + mov len_temp, 8 - 1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, (1 SHL (kLenNumLowBits + PSHIFT)) + mov len_temp, -1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, LenHigh * PMULT - (1 SHL (kLenNumLowBits + PSHIFT)) + mov sym, 1 + PLOAD x1, probs + 1 * PMULT + +MY_ALIGN_32 +len8_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 64 + jb len8_loop + + mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen + jmp len_mid_2 + +MY_ALIGN_32 +len_mid_0: + UPDATE_0 probs, 0, 0 + add probs, pbPos_R + BIT_0 x2, x1 +len_mid_2: + BIT_1 x1, x2 + BIT_2 x2, len_temp + mov probs, LOC probs_Spec + cmp state, kNumStates * PMULT + jb copy_match + + +; ---------- DECODE DISTANCE ---------- + ; probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp sym, 3 + kMatchMinLen + cmovb t0, sym + add probs, PosSlot * PMULT - (kMatchMinLen SHL (kNumPosSlotBits + PSHIFT)) + shl t0, (kNumPosSlotBits + PSHIFT) + add probs, t0_R + + ; sym = Len + ; mov LOC remainLen, sym + mov len_temp, sym + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +slot_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 32 + jb slot_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + mov x1, sym + BIT_2 x2, 64-1 + + and sym, 3 + mov probs, LOC probs_Spec + cmp x1, 32 + kEndPosModelIndex / 2 + jb short_dist + + ; unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + sub x1, (32 + 1 + kNumAlignBits) + ; distance = (2 | (distance & 1)); + or sym, 2 + PLOAD x2, probs + 1 * PMULT + shl sym, kNumAlignBits + 1 + lea sym2_R, [probs + 2 * PMULT] + + jmp direct_norm + ; lea t1, [sym_R + (1 SHL kNumAlignBits)] + ; cmp range, kTopValue + ; jb direct_norm + +; ---------- DIRECT DISTANCE ---------- +MY_ALIGN_32 +direct_loop: + shr range, 1 + mov t0, cod + sub cod, range + cmovs cod, t0 + cmovns sym, t1 + + comment ~ + sub cod, range + mov x2, cod + sar x2, 31 + lea sym, dword ptr [r2 + sym_R * 2 + 1] + and x2, range + add cod, x2 + ~ + dec x1 + je direct_end + + add sym, sym +direct_norm: + lea t1, [sym_R + (1 SHL kNumAlignBits)] + cmp range, kTopValue + jae near ptr direct_loop + ; we align for 32 here with "near ptr" command above + NORM_2 + jmp direct_loop + +MY_ALIGN_32 +direct_end: + ; prob = + kAlign; + ; distance <<= kNumAlignBits; + REV_0 x2, x1 + REV_1 x1, x2, 2 + REV_1 x2, x1, 4 + REV_2 x1, 8 + +decode_dist_end: + + ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + mov t0, LOC checkDicSize + test t0, t0 + cmove t0, processedPos + cmp sym, t0 + jae end_of_payload + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + inc sym + mov t0, LOC rep0 + mov t1, LOC rep1 + mov x1, LOC rep2 + mov LOC rep0, sym + ; mov sym, LOC remainLen + mov sym, len_temp + mov LOC rep1, t0 + mov LOC rep2, t1 + mov LOC rep3, x1 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + +; ---------- COPY MATCH ---------- +copy_match: + + ; len += kMatchMinLen; + ; add sym, kMatchMinLen + + ; if ((rem = limit - dicPos) == 0) + ; { + ; p->dicPos = dicPos; + ; return SZ_ERROR_DATA; + ; } + mov cnt_R, LOC limit + sub cnt_R, dicPos + jz fin_ERROR + + ; curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, sym_R + ; cmovae cnt_R, sym_R ; 64-bit + cmovae cnt, sym ; 32-bit + + mov dic, LOC dic_Spec + mov x1, LOC rep0 + + mov t0_R, dicPos + add dicPos, cnt_R + ; processedPos += curLen; + add processedPos, cnt + ; len -= curLen; + sub sym, cnt + mov LOC remainLen, sym + + sub t0_R, dic + + ; pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + sub t0_R, r1 + jae @f + + mov r1, LOC dicBufSize + add t0_R, r1 + sub r1, t0_R + cmp cnt_R, r1 + ja copy_match_cross +@@: + ; if (curLen <= dicBufSize - pos) + +; ---------- COPY MATCH FAST ---------- + ; Byte *dest = dic + dicPos; + ; mov r1, dic + ; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + ; sub t0_R, dicPos + ; dicPos += curLen; + + ; const Byte *lim = dest + curLen; + add t0_R, dic + movzx sym, byte ptr[t0_R] + add t0_R, cnt_R + neg cnt_R + ; lea r1, [dicPos - 1] +copy_common: + dec dicPos + ; cmp LOC rep0, 1 + ; je rep0Label + + ; t0_R - src_lim + ; r1 - dest_lim - 1 + ; cnt_R - (-cnt) + + IsMatchBranch_Pre + inc cnt_R + jz copy_end +MY_ALIGN_16 +@@: + mov byte ptr[cnt_R * 1 + dicPos], sym_L + movzx sym, byte ptr[cnt_R * 1 + t0_R] + inc cnt_R + jnz @b + +copy_end: +lz_end_match: + mov byte ptr[dicPos], sym_L + inc dicPos + + ; IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + + + +; ---------- LITERAL MATCHED ---------- + + LIT_PROBS LOC lpMask + + ; matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov x1, LOC rep0 + ; mov dic, LOC dic_Spec + mov LOC dicPos_Spec, dicPos + + ; state -= (state < 10) ? 3 : 6; + lea t0, [state_R - 6 * PMULT] + sub state, 3 * PMULT + cmp state, 7 * PMULT + cmovae state, t0 + + sub dicPos, dic + sub dicPos, r1 + jae @f + add dicPos, LOC dicBufSize +@@: + comment ~ + xor t0, t0 + sub dicPos, r1 + cmovb t0_R, LOC dicBufSize + ~ + + movzx match, byte ptr[dic + dicPos * 1] + + ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, match + mov sym, 1 +MY_ALIGN_16 +litm_loop: + LITM + cmp sym, 256 + jb litm_loop + sub sym, 256 + + else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + endif + + mov probs, LOC probs_Spec + IsMatchBranch_Pre + ; mov dic, LOC dic_Spec + mov dicPos, LOC dicPos_Spec + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_matched_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + ; IsMatchBranch + mov lpMask_reg, LOC lpMask + sub state, 3 * PMULT + jmp lit_start_2 + + + +; ---------- REP 0 LITERAL ---------- +MY_ALIGN_32 +IsRep0Short_label: + UPDATE_0 probs_state_R, pbPos_R, IsRep0Long + + ; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov dic, LOC dic_Spec + mov t0_R, dicPos + mov probBranch, LOC rep0 + sub t0_R, dic + + sub probs, RepLenCoder * PMULT + inc processedPos + ; state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + IsMatchBranch_Pre + + sub t0_R, probBranch_R + jae @f + add t0_R, LOC dicBufSize +@@: + movzx sym, byte ptr[dic + t0_R * 1] + jmp lz_end_match + + +MY_ALIGN_32 +IsRep_label: + UPDATE_1 probs_state_R, 0, IsRep + + ; The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + ; So we don't check it here. + + ; mov t0, processedPos + ; or t0, LOC checkDicSize + ; jz fin_ERROR_2 + + ; state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + ; prob = probs + RepLenCoder; + add probs, RepLenCoder * PMULT + + IF_BIT_1 probs_state_R, 0, IsRepG0, IsRepG0_label + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, IsRep0Short_label + UPDATE_1 probs_state_R, pbPos_R, IsRep0Long + jmp len_decode + +MY_ALIGN_32 +IsRepG0_label: + UPDATE_1 probs_state_R, 0, IsRepG0 + mov dist2, LOC rep0 + mov dist, LOC rep1 + mov LOC rep1, dist2 + + IF_BIT_1 probs_state_R, 0, IsRepG1, IsRepG1_label + mov LOC rep0, dist + jmp len_decode + +; MY_ALIGN_32 +IsRepG1_label: + UPDATE_1 probs_state_R, 0, IsRepG1 + mov dist2, LOC rep2 + mov LOC rep2, dist + + IF_BIT_1 probs_state_R, 0, IsRepG2, IsRepG2_label + mov LOC rep0, dist2 + jmp len_decode + +; MY_ALIGN_32 +IsRepG2_label: + UPDATE_1 probs_state_R, 0, IsRepG2 + mov dist, LOC rep3 + mov LOC rep3, dist2 + mov LOC rep0, dist + jmp len_decode + + + +; ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_32 +short_dist: + sub x1, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, x1_L + lea sym_R, [probs + sym_R * PMULT + SpecPos * PMULT + 1 * PMULT] + mov sym2, PMULT ; step +MY_ALIGN_32 +spec_loop: + REV_1_VAR x2 + dec x1 + jnz spec_loop + + mov probs, LOC probs_Spec + sub sym, sym2 + sub sym, SpecPos * PMULT + sub sym_R, probs + shr sym, PSHIFT + + jmp decode_dist_end + + +; ---------- COPY MATCH CROSS ---------- +copy_match_cross: + ; t0_R - src pos + ; r1 - len to dicBufSize + ; cnt_R - total copy len + + mov t1_R, t0_R ; srcPos + mov t0_R, dic + mov r1, LOC dicBufSize ; + neg cnt_R +@@: + movzx sym, byte ptr[t1_R * 1 + t0_R] + inc t1_R + mov byte ptr[cnt_R * 1 + dicPos], sym_L + inc cnt_R + cmp t1_R, r1 + jne @b + + movzx sym, byte ptr[t0_R] + sub t0_R, cnt_R + jmp copy_common + + + + +fin_ERROR: + mov LOC remainLen, len_temp +; fin_ERROR_2: + mov sym, 1 + jmp fin + +end_of_payload: + cmp sym, 0FFFFFFFFh ; -1 + jne fin_ERROR + + mov LOC remainLen, kMatchSpecLenStart + sub state, kNumStates * PMULT + +fin_OK: + xor sym, sym + +fin: + NORM + + mov r1, LOC lzmaPtr + + sub dicPos, LOC dic_Spec + mov GLOB dicPos_Spec, dicPos + mov GLOB buf_Spec, buf + mov GLOB range_Spec, range + mov GLOB code_Spec, cod + shr state, PSHIFT + mov GLOB state_Spec, state + mov GLOB processedPos_Spec, processedPos + + RESTORE_VAR(remainLen) + RESTORE_VAR(rep0) + RESTORE_VAR(rep1) + RESTORE_VAR(rep2) + RESTORE_VAR(rep3) + + mov x0, sym + + mov RSP, LOC Old_RSP + +MY_POP_PRESERVED_REGS +MY_ENDP + +_TEXT$LZMADECOPT ENDS + +end diff --git a/C/7zVersion.h b/C/7zVersion.h index 69ef6985..42b2fb64 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 18 -#define MY_VER_MINOR 01 +#define MY_VER_MINOR 03 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "18.01" +#define MY_VERSION_NUMBERS "18.03 beta" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2018-01-28" +#define MY_DATE "2018-03-04" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" @@ -1,8 +1,10 @@ /* Alloc.c -- Memory allocation functions -2017-06-15 : Igor Pavlov : Public domain */ +2018-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include <stdio.h> + #ifdef _WIN32 #include <windows.h> #endif @@ -14,12 +16,119 @@ /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include <stdio.h> int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) @@ -27,7 +136,7 @@ void *MyAlloc(size_t size) #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -37,10 +146,8 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); } @@ -50,18 +157,16 @@ void *MidAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif + PRINT_FREE("Free-Mid", g_allocCountMid, address); + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); @@ -96,9 +201,8 @@ void *BigAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++); - #endif + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES { @@ -123,10 +227,7 @@ void *BigAlloc(size_t size) void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); if (!address) return; @@ -138,8 +239,217 @@ void BigFree(void *address) static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc const g_Alloc = { SzAlloc, SzFree }; +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc const g_BigAlloc = { SzBigAlloc, SzBigFree }; +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : <stdint.h> C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -31,6 +31,20 @@ void BigFree(void *address); extern const ISzAlloc g_Alloc; extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END diff --git a/C/CpuArch.h b/C/CpuArch.h index 51dd607b..056116dd 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2017-06-30 : Igor Pavlov : Public domain */ +2017-09-04 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -174,7 +174,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #ifndef MY_CPU_NAME #ifdef MY_CPU_LE #define MY_CPU_NAME "LE" - #elif MY_CPU_BE + #elif defined(MY_CPU_BE) #define MY_CPU_NAME "BE" #else /* diff --git a/C/Lzma2Dec.c b/C/Lzma2Dec.c index 63853c6d..ecde5508 100644 --- a/C/Lzma2Dec.c +++ b/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -14,28 +14,22 @@ #include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) @@ -91,9 +85,11 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); } @@ -102,19 +98,26 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (b > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0: @@ -124,8 +127,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8; @@ -134,9 +137,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: { @@ -145,13 +148,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } } @@ -231,11 +233,6 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - break; - p->needInitDic = False; LzmaDec_InitDicAndState(&p->decoder, initDic, False); } @@ -257,23 +254,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - break; - + Bool initDic = (p->control >= 0xE0); + Bool initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } if (inCur > p->packSize) inCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + src += inCur; *srcLen += inCur; p->packSize -= (UInt32)inCur; @@ -310,6 +301,129 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, } + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return LZMA_STATUS_NOT_SPECIFIED; +} + + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; diff --git a/C/Lzma2Dec.h b/C/Lzma2Dec.h index 917af990..b8ddeac8 100644 --- a/C/Lzma2Dec.h +++ b/C/Lzma2Dec.h @@ -1,5 +1,5 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H @@ -12,25 +12,24 @@ EXTERN_C_BEGIN typedef struct { - CLzmaDec decoder; - UInt32 packSize; - UInt32 unpackSize; unsigned state; Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode: It has meaning only if the decoding reaches output limit (*destLen or dicLimit). @@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /* diff --git a/C/Lzma2DecMt.c b/C/Lzma2DecMt.c new file mode 100644 index 00000000..c0a9d495 --- /dev/null +++ b/C/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2018-03-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + Bool finishMode; + Bool outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + Bool readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + Bool mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + Bool overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + Bool blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + Bool *needContinue, Bool *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + Bool needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , Bool tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + Bool outFinished; + Bool needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + Bool tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + Bool needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/C/Lzma2DecMt.h b/C/Lzma2DecMt.h new file mode 100644 index 00000000..7791c310 --- /dev/null +++ b/C/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/C/Lzma2Enc.c b/C/Lzma2Enc.c index e970944c..3aa89d78 100644 --- a/C/Lzma2Enc.c +++ b/C/Lzma2Enc.c @@ -1,5 +1,5 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2017-08-28 : Igor Pavlov : Public domain */ +2018-02-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -666,7 +666,7 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -674,7 +674,8 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + progressThunk.inSize = 0; + progressThunk.outSize = 0; res = Lzma2Enc_EncodeMt1(me, &me->coders[coderIndex], @@ -720,10 +721,10 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, CLzma2Enc *p = (CLzma2Enc *)pp; if (inStream && inData) - return E_INVALIDARG; + return SZ_ERROR_PARAM; if (outStream && outBuf) - return E_INVALIDARG; + return SZ_ERROR_PARAM; { unsigned i; diff --git a/C/LzmaDec.c b/C/LzmaDec.c index e96fa975..cccb9326 100644 --- a/C/LzmaDec.c +++ b/C/LzmaDec.c @@ -1,8 +1,9 @@ /* LzmaDec.c -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-28 : Igor Pavlov : Public domain */ #include "Precomp.h" +/* #include "CpuArch.h" */ #include "LzmaDec.h" #include <string.h> @@ -24,9 +25,16 @@ #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } @@ -46,12 +54,15 @@ i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } @@ -66,25 +77,28 @@ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -98,54 +112,117 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 0x300 +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + Out: + RangeCoder is normalized Result: SZ_OK - OK SZ_ERROR_DATA - Error p->remainLen: < kMatchSpecLenStart : normal remain = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - unsigned state = p->state; +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; @@ -164,17 +241,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates) @@ -240,13 +316,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else { UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode if (checkDicSize == 0 && processedPos == 0) return SZ_ERROR_DATA; + */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob); @@ -299,7 +378,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; lim = (1 << kLenNumLowBits); } @@ -310,15 +389,15 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - lim = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; lim = (1 << kLenNumHighBits); } } @@ -331,7 +410,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -345,7 +424,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -356,7 +435,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } } @@ -376,16 +455,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else @@ -412,19 +491,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; } @@ -435,20 +515,12 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { p->dicPos = dicPos; return SZ_ERROR_DATA; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen; @@ -511,6 +583,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte return SZ_OK; } +#endif static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { @@ -519,7 +592,7 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) Byte *dic = p->dic; SizeT dicPos = p->dicPos; SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; + unsigned len = (unsigned)p->remainLen; SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ SizeT rem = limit - dicPos; if (rem < len) @@ -540,6 +613,14 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) } } + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { do @@ -550,9 +631,13 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte UInt32 rem = p->prop.dicSize - p->processedPos; if (limit - p->dicPos > rem) limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; @@ -561,9 +646,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; - return 0; } @@ -580,17 +662,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS UInt32 range = p->range; UInt32 code = p->code; const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK @@ -618,10 +700,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); } @@ -648,7 +731,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; @@ -691,7 +774,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } @@ -702,15 +785,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } @@ -722,7 +805,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex) @@ -733,7 +816,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else { @@ -745,17 +828,18 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } } @@ -768,18 +852,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) @@ -788,53 +871,54 @@ void LzmaDec_Init(CLzmaDec *p) LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->remainLen != kMatchSpecLenStart) + if (p->remainLen > kMatchSpecLenStart) { - int checkEndMarkNow; + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow = 0; - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0) @@ -855,9 +939,6 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - if (p->tempBufSize == 0) { SizeT processed; @@ -930,11 +1011,14 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->tempBufSize = 0; } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; @@ -1011,10 +1095,10 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } @@ -1026,9 +1110,10 @@ static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAl { LzmaDec_FreeProbs(p, alloc); p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } diff --git a/C/LzmaDec.h b/C/LzmaDec.h index d6af9220..2567aaaa 100644 --- a/C/LzmaDec.h +++ b/C/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-06 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -25,7 +25,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -47,26 +50,28 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); @@ -132,8 +137,8 @@ LzmaDec_Allocate* can return: SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAllocPtr alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ diff --git a/C/MtCoder.c b/C/MtCoder.c index 32bd0fae..389a70a5 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,85 +1,28 @@ /* MtCoder.c -- Multi-thread Coder -2017-07-17 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; +#ifndef _7ZIP_ST - p->progress = progress; - p->res = SZ_OK; - p->totalInSize = 0; - p->totalOutSize = 0; - - for (i = 0; i < MTCODER__THREADS_MAX; i++) +SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) { - CMtProgressSizes *pair = &p->sizes[i]; - pair->inSize = 0; - pair->outSize = 0; + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; } -} - - -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - CMtProgressSizes *pair = &p->sizes[index]; - pair->inSize = 0; - pair->outSize = 0; -} - - -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } - -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) -{ - SRes res; - CMtProgressSizes *pair; - - CriticalSection_Enter(&p->cs); - - pair = &p->sizes[index]; - UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize) - UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize) - if (p->res == SZ_OK && p->progress) + if (outSize != (UInt64)(Int64)-1) { - if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) - p->res = SZ_ERROR_PROGRESS; + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; } - res = p->res; - - CriticalSection_Leave(&p->cs); - - return res; -} - - -static SRes MtProgress_GetError(CMtProgress *p) -{ - SRes res; - CriticalSection_Enter(&p->cs); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; -} - - -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} - - -static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) -{ - CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); - return MtProgress_Set(p->mtProgress, p->index, inSize, outSize); + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } @@ -298,7 +241,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, mtc->inStream ? t->inBuf : inData, size, finished); - MtProgress_Reinit(&mtc->mtProgress, t->index); + // MtProgress_Reinit(&mtc->mtProgress, t->index); if (res != SZ_OK) MtProgress_SetError(&mtc->mtProgress, res); @@ -654,3 +597,5 @@ SRes MtCoder_Code(CMtCoder *p) MtCoder_Free(p); return res; } + +#endif diff --git a/C/MtCoder.h b/C/MtCoder.h index 624aff24..ed584e12 100644 --- a/C/MtCoder.h +++ b/C/MtCoder.h @@ -1,10 +1,10 @@ /* MtCoder.h -- Multi-thread Coder -2017-06-18 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN @@ -24,33 +24,20 @@ EXTERN_C_BEGIN #endif -typedef struct -{ - UInt64 inSize; - UInt64 outSize; -} CMtProgressSizes; - - -typedef struct -{ - ICompressProgress *progress; - SRes res; - UInt64 totalInSize; - UInt64 totalOutSize; - CCriticalSection cs; - CMtProgressSizes sizes[MTCODER__THREADS_MAX]; -} CMtProgress; +#ifndef _7ZIP_ST typedef struct { ICompressProgress vt; CMtProgress *mtProgress; - unsigned index; + UInt64 inSize; + UInt64 outSize; } CMtProgressThunk; void MtProgressThunk_CreateVTable(CMtProgressThunk *p); +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } struct _CMtCoder; @@ -146,6 +133,9 @@ void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); +#endif + + EXTERN_C_END #endif diff --git a/C/MtDec.c b/C/MtDec.c new file mode 100644 index 00000000..f8ab6b38 --- /dev/null +++ b/C/MtDec.c @@ -0,0 +1,1137 @@ +/* MtDec.c -- Multi-thread Decoder +2018-03-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include <stdio.h> + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + + +typedef struct +{ + void *next; + void *pad[3]; +} CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, Bool *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + Bool wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + Bool needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + + Bool canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + // PRF(printf("\nParse size = %d\n", (unsigned)size)) + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + p->crossStart = 0; + p->crossEnd = 0; + + if (crossSize != 0) + memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data + afterEndData_Size = size - parse.srcSize; + afterEndData = parseData + parse.srcSize; + + // we reduce data size to required bytes (parsed only) + inDataSize -= (size - parse.srcSize); + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + Bool isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + Bool isErrorMode = False; + Bool canRecode = True; + Bool needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, + &needContinue, + &canRecode); + + // res= E_INVALIDARG; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include <malloc.h> +#else +#include <stdlib.h> +#endif +#endif + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return res; +} + +static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtDecThread *t = (CMtDecThread *)pp; + + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + #ifdef USE_ALLOCA + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + WRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return E_FAIL; + } +} + +#endif diff --git a/C/MtDec.h b/C/MtDec.h new file mode 100644 index 00000000..760bbc84 --- /dev/null +++ b/C/MtDec.h @@ -0,0 +1,201 @@ +/* MtDec.h -- Multi-thread Decoder +2018-03-02 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + Bool canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + Bool *needContinue, + Bool *canRecode); +} IMtDecCallback; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + Bool exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + Bool isAllocError; + Bool overflow; + SRes threadingErrorSRes; + + Bool needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + Bool wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + Bool readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + Bool needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index 0bf27cc2..2effeed6 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,5 +1,5 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2017-04-04 : Igor Pavlov : Public domain */ +2018-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -546,7 +546,7 @@ static BOOL RemoveDir() #define k_Lang L"Lang" // NUM_LANG_TXT_FILES files are placed before en.ttt -#define NUM_LANG_TXT_FILES 87 +#define NUM_LANG_TXT_FILES 88 #ifdef _64BIT_INSTALLER #define NUM_EXTRA_FILES_64BIT 1 @@ -558,7 +558,7 @@ static BOOL RemoveDir() static const char * const k_Names = "af an ar ast az ba be bg bn br ca co cs cy da de el eo es et eu ext" - " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kk ko ku ku-ckb ky" + " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kab kk ko ku ku-ckb ky" " lij lt lv mk mn mng mng2 mr ms nb ne nl nn pa-in pl ps pt pt-br ro ru" " sa si sk sl sq sr-spc sr-spl sv ta th tr tt ug uk uz va vi yo zh-cn zh-tw" " en.ttt" @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2017-07-27 : Igor Pavlov : Public domain */ +2018-02-28 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -139,6 +139,9 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICom UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ @@ -147,22 +150,31 @@ typedef enum CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; void (*Free)(void *p, ISzAllocPtr alloc); SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct @@ -170,20 +182,23 @@ typedef struct ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + Bool wasFinished; + SRes res; + ECoderStatus status; + // Bool SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum { @@ -197,6 +212,7 @@ typedef enum XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state; @@ -210,7 +226,7 @@ typedef struct UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize; @@ -225,17 +241,65 @@ typedef struct CXzCheck check; CSha256 sha; + Bool parseMode; + Bool headerParsedOk; + Bool decodeToStreamSignature; unsigned decodeOnlyOneBlock; + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; +/* alloc : aligned for cache line allocation is better */ void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); /* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + +/* finishMode: It has meaning only if the decoding reaches output limit (*destLen). CODER_FINISH_ANY - use smallest number of input bytes @@ -264,15 +328,23 @@ Returns: SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ @@ -297,6 +369,92 @@ Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p); #define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + Bool ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); + EXTERN_C_END #endif @@ -1,14 +1,33 @@ /* XzDec.c -- Xz Decode -2018-01-21 : Igor Pavlov : Public domain */ +2018-02-28 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include <stdio.h> + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include <stdio.h> #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include <stdlib.h> #include <string.h> @@ -19,9 +38,11 @@ #include "Delta.h" #include "Lzma2Dec.h" +// #define USE_SUBBLOCK + #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" @@ -56,8 +77,9 @@ typedef struct size_t bufConv; size_t bufTotal; - UInt32 methodId; int encodeMode; + + UInt32 methodId; UInt32 delta; UInt32 ip; UInt32 x86State; @@ -121,69 +143,84 @@ static void BraState_Init(void *pp) Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -192,36 +229,42 @@ static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } + SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + decoder = p->p; if (!decoder) - return SZ_ERROR_MEM; + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK @@ -246,8 +289,10 @@ static void SbState_Init(void *pp) SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res; @@ -260,33 +305,53 @@ static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + CSbDec *decoder = (CSbDec *)p->p; if (!decoder) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ + + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + CLzma2Dec decoder; + Bool outBufMode; +} CLzma2Dec_Spec; + static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); ISzAlloc_Free(alloc, pp); } @@ -294,60 +359,123 @@ static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, IS { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) { - CLzma2Dec *decoder = (CLzma2Dec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (!decoder) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { ISzAlloc_Free(p->alloc, p->buf); @@ -355,7 +483,7 @@ void MixCoder_Free(CMixCoder *p) } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) @@ -368,16 +496,22 @@ void MixCoder_Init(CMixCoder *p) { IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif @@ -387,32 +521,128 @@ SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { Bool processed = False; Bool allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break; @@ -422,78 +652,101 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { size_t k = i - 1; - srcCur = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; - srcLenCur = p->size[k] - p->pos[k]; - srcFinishedCur = p->finished[k]; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[(size_t)i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } if (!processed) { if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; } } } @@ -522,6 +775,33 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } +static Bool XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + SRes XzBlock_Parse(CXzBlock *p, const Byte *header) { unsigned pos; @@ -580,31 +860,47 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; Bool needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; @@ -616,6 +912,8 @@ SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER; @@ -625,14 +923,32 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numTotalBlocks = 0; p->padSize = 0; p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; } + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; +} + + void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); @@ -650,14 +966,28 @@ void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) } +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { SizeT srcRem; @@ -669,12 +999,17 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; + Bool srcFinished2 = srcFinished; + Bool destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) { UInt64 rem = p->block.packSize - p->packSize; - if (srcLen2 > rem) + if (srcLen2 >= rem) + { + srcFinished2 = True; srcLen2 = (SizeT)rem; + } if (rem == 0 && p->block.unpackSize == p->unpackSize) return SZ_ERROR_DATA; } @@ -684,6 +1019,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt64 rem = p->block.unpackSize - p->unpackSize; if (destLen2 >= rem) { + destFinish = True; finishMode2 = CODER_FINISH_END; destLen2 = (SizeT)rem; } @@ -697,15 +1033,23 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; RINOK(res); @@ -715,27 +1059,26 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, if (p->block.packSize == p->packSize && *status == CODER_STATUS_NEEDS_MORE_INPUT) { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); *status = CODER_STATUS_NOT_SPECIFIED; return SZ_ERROR_DATA; } - // if (srcLen2 == 0 && destLen2 == 0) return SZ_OK; } { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; - - if (p->block.unpackSize != (UInt64)(Int64)-1) - if (p->block.unpackSize != p->unpackSize) - return SZ_ERROR_DATA; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } // continue; } @@ -757,6 +1100,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; } @@ -809,12 +1154,19 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } @@ -980,6 +1332,20 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); @@ -994,8 +1360,1414 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + Bool skipMode; + // Bool finishedWithMark; + EMtDecParseState parseState; + Bool parsing_Truncated; + Bool atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + Bool dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + Bool finishMode; + Bool outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + Bool readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + Bool dec_created; + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + Bool mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + Bool isBlockHeaderState_Parse; + Bool isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + Bool parsing_Truncated; + + Bool mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + if (p->dec_created) + { + XzUnpacker_Free(&p->dec); + p->dec_created = False; + } + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + Bool *needContinue, + Bool *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , Bool tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + Bool finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + Bool decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + Bool tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + if (!p->dec_created) + { + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = True; + } + XzUnpacker_Init(&p->dec); + + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + Bool needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + Bool truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2017-08-25 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -136,7 +136,7 @@ static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) { - Byte *blocks = ISzAlloc_Alloc(alloc, newSize); + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); if (!blocks) return SZ_ERROR_MEM; if (p->size != 0) @@ -329,7 +329,7 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPt { if (!p->buf) { - p->buf = ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); if (!p->buf) return SZ_ERROR_MEM; } @@ -362,13 +362,16 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } } @@ -1061,7 +1064,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -1069,7 +1072,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + MtProgressThunk_Init(&progressThunk); { CXzEncBlockInfo blockSizes; @@ -1230,7 +1233,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr if (!p->outBufs[0] || t2 != p->outBufSize) { XzEnc_FreeOutBufs(p); - p->outBufs[0] = ISzAlloc_Alloc(p->alloc, t2); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); if (!p->outBufs[0]) return SZ_ERROR_MEM; p->outBufSize = t2; @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2017-05-11 : Igor Pavlov : Public domain */ +2018-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -103,7 +103,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt { size_t i; p->numBlocks = numBlocks; - p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) @@ -131,7 +131,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = ISzAlloc_Alloc(alloc, size); + buf = (Byte *)ISzAlloc_Alloc(alloc, size); if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index 8105ff04..608293d6 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -13,7 +13,9 @@ struct CMethodFull: public CMethodProps { CMethodId Id; UInt32 NumStreams; + int CodecIndex; + CMethodFull(): CodecIndex(-1) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; diff --git a/CPP/7zip/Archive/7z/7zDecode.cpp b/CPP/7zip/Archive/7z/7zDecode.cpp index d2687479..fcaef8ca 100644 --- a/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/CPP/7zip/Archive/7z/7zDecode.cpp @@ -236,8 +236,8 @@ HRESULT CDecoder::Decode( _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ) { @@ -312,7 +312,7 @@ HRESULT CDecoder::Decode( #endif CCreatedCoder cod; - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS coderInfo.MethodID, false, cod)); @@ -355,11 +355,39 @@ HRESULT CDecoder::Decode( unsigned i; + bool mt_wasUsed = false; + for (i = 0; i < folderInfo.Coders.Size(); i++) { const CCoderInfo &coderInfo = folderInfo.Coders[i]; IUnknown *decoder = _mixer->GetCoder(i).GetUnknown(); + #if !defined(_7ZIP_ST) + if (!mt_wasUsed) + { + if (mtMode) + { + CMyComPtr<ICompressSetCoderMt> setCoderMt; + decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); + if (setCoderMt) + { + mt_wasUsed = true; + RINOK(setCoderMt->SetNumberOfThreads(numThreads)); + } + } + // if (memUsage != 0) + { + CMyComPtr<ICompressSetMemLimit> setMemLimit; + decoder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + mt_wasUsed = true; + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } + } + #endif + { CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties; decoder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); @@ -376,18 +404,6 @@ HRESULT CDecoder::Decode( } } - #if !defined(_7ZIP_ST) && !defined(_SFX) - if (mtMode) - { - CMyComPtr<ICompressSetCoderMt> setCoderMt; - decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); - if (setCoderMt) - { - RINOK(setCoderMt->SetNumberOfThreads(numThreads)); - } - } - #endif - #ifndef _NO_CRYPTO { CMyComPtr<ICryptoSetPassword> cryptoSetPassword; diff --git a/CPP/7zip/Archive/7z/7zDecode.h b/CPP/7zip/Archive/7z/7zDecode.h index 62a38038..eeb146e3 100644 --- a/CPP/7zip/Archive/7z/7zDecode.h +++ b/CPP/7zip/Archive/7z/7zDecode.h @@ -59,8 +59,8 @@ public: _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ); }; diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 8700d721..7d8270f9 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -154,9 +154,18 @@ HRESULT CEncoder::CreateMixerCoder( CCreatedCoder cod; - RINOK(CreateCoder( + if (methodFull.CodecIndex >= 0) + { + RINOK(CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + methodFull.CodecIndex, true, cod)); + } + else + { + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodFull.Id, true, cod)); + } if (cod.NumStreams != methodFull.NumStreams) return E_FAIL; diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index 8fea5aa5..9ffe2fdc 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -152,6 +152,12 @@ STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc if (_fileIsOpen) { UInt32 cur = (size < _rem ? size : (UInt32)_rem); + if (_calcCrc) + { + const UInt32 k_Step = (UInt32)1 << 20; + if (cur > k_Step) + cur = k_Step; + } HRESULT result = S_OK; if (_stream) result = _stream->Write(data, cur, &cur); @@ -363,8 +369,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) - , true, _numThreads + #if !defined(_7ZIP_ST) + , true, _numThreads, _memUsage #endif ); diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index 2642e691..988b35f7 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -40,7 +40,6 @@ CHandler::CHandler() _crcSize = 4; #ifdef __7Z_SET_PROPERTIES - _numThreads = NSystem::GetNumberOfProcessors(); _useMultiThreadMixer = true; #endif @@ -714,8 +713,8 @@ STDMETHODIMP CHandler::Close() STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN - const UInt32 numProcessors = NSystem::GetNumberOfProcessors(); - _numThreads = numProcessors; + + InitCommon(); _useMultiThreadMixer = true; for (UInt32 i = 0; i < numProps; i++) @@ -734,13 +733,15 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(PROPVARIANT_to_bool(value, _useMultiThreadMixer)); continue; } - if (name.IsPrefixedBy_Ascii_NoCase("mt")) { - RINOK(ParseMtProp(name.Ptr(2), value, numProcessors, _numThreads)); - continue; + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres); + continue; + } } - else - return E_INVALIDARG; + return E_INVALIDARG; } } return S_OK; diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index 89e3275f..99942eb0 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -8,16 +8,6 @@ #include "../../Common/CreateCoder.h" -#ifndef EXTRACT_ONLY -#include "../Common/HandlerOut.h" -#endif - -#include "7zCompressionMode.h" -#include "7zIn.h" - -namespace NArchive { -namespace N7z { - #ifndef __7Z_SET_PROPERTIES #ifdef EXTRACT_ONLY @@ -30,6 +20,16 @@ namespace N7z { #endif +// #ifdef __7Z_SET_PROPERTIES +#include "../Common/HandlerOut.h" +// #endif + +#include "7zCompressionMode.h" +#include "7zIn.h" + +namespace NArchive { +namespace N7z { + #ifndef EXTRACT_ONLY @@ -38,8 +38,6 @@ class COutHandler: public CMultiMethodProps HRESULT SetSolidFromString(const UString &s); HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); public: - bool _removeSfxBlock; - UInt64 _numSolidFiles; UInt64 _numSolidBytes; bool _numSolidBytesDefined; @@ -58,6 +56,8 @@ public: bool _useMultiThreadMixer; + bool _removeSfxBlock; + // bool _volumeMode; void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); } @@ -70,9 +70,10 @@ public: _numSolidBytesDefined = false; } + void InitProps7z(); void InitProps(); - COutHandler() { InitProps(); } + COutHandler() { InitProps7z(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; @@ -82,16 +83,23 @@ public: class CHandler: public IInArchive, public IArchiveGetRawProps, + #ifdef __7Z_SET_PROPERTIES public ISetProperties, #endif + #ifndef EXTRACT_ONLY public IOutArchive, #endif + PUBLIC_ISetCompressCodecsInfo - public CMyUnknownImp + + public CMyUnknownImp, + #ifndef EXTRACT_ONLY - , public COutHandler + public COutHandler + #else + public CCommonMethodProps #endif { public: @@ -135,7 +143,6 @@ private: #ifdef EXTRACT_ONLY #ifdef __7Z_SET_PROPERTIES - UInt32 _numThreads; bool _useMultiThreadMixer; #endif diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index c4fabed7..79f83bac 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -13,6 +13,8 @@ #include "7zOut.h" #include "7zUpdate.h" +#ifndef EXTRACT_ONLY + using namespace NWindows; namespace NArchive { @@ -41,9 +43,11 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) HRESULT CHandler::PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m) { - if (!FindMethod( + dest.CodecIndex = FindMethod_Index( EXTERNAL_CODECS_VARS - m.MethodName, dest.Id, dest.NumStreams)) + m.MethodName, true, + dest.Id, dest.NumStreams); + if (dest.CodecIndex < 0) return E_INVALIDARG; (CProps &)dest = (CProps &)m; return S_OK; @@ -699,10 +703,8 @@ static HRESULT ParseBond(UString &srcString, UInt32 &coder, UInt32 &stream) return S_OK; } -void COutHandler::InitProps() +void COutHandler::InitProps7z() { - CMultiMethodProps::Init(); - _removeSfxBlock = false; _compressHeaders = true; _encryptHeadersSpecified = false; @@ -722,6 +724,14 @@ void COutHandler::InitProps() _useTypeSorting = false; } +void COutHandler::InitProps() +{ + CMultiMethodProps::Init(); + InitProps7z(); +} + + + HRESULT COutHandler::SetSolidFromString(const UString &s) { UString s2 = s; @@ -762,6 +772,10 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) } _numSolidBytes = (v << numBits); _numSolidBytesDefined = true; + /* + if (_numSolidBytes == 0) + _numSolidFiles = 1; + */ } } return S_OK; @@ -810,7 +824,7 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val return E_INVALIDARG; return SetSolidFromString(name); } - + UInt32 number; int index = ParseStringToUInt32(name, number); // UString realName = name.Ptr(index); @@ -921,3 +935,5 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR } }} + +#endif diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp index 3db5f515..b7b71d13 100644 --- a/CPP/7zip/Archive/7z/7zIn.cpp +++ b/CPP/7zip/Archive/7z/7zIn.cpp @@ -1115,11 +1115,11 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) + #if !defined(_7ZIP_ST) , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(result); diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 53ea3a55..5450c8ba 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -1507,7 +1507,8 @@ void CThreadDecoder::Execute() _7Z_DECODER_CRYPRO_VARS #ifndef _7ZIP_ST - , MtMode, NumThreads + , MtMode, NumThreads, + 0 // MemUsage #endif ); @@ -1696,13 +1697,14 @@ HRESULT Update( UInt64 inSizeForReduce = 0; { + bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); FOR_VECTOR (i, updateItems) { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) { complexity += ui.Size; - if (numSolidFiles != 1) + if (isSolid) inSizeForReduce += ui.Size; else if (inSizeForReduce < ui.Size) inSizeForReduce = ui.Size; @@ -2142,8 +2144,8 @@ HRESULT Update( #ifndef _7ZIP_ST , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(res); @@ -2293,7 +2295,8 @@ HRESULT Update( continue; CRecordVector<CRefItem> refItems; refItems.ClearAndSetSize(numFiles); - bool sortByType = (options.UseTypeSorting && numSolidFiles > 1); + // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1 + bool sortByType = options.UseTypeSorting; unsigned i; diff --git a/CPP/7zip/Archive/Common/HandlerOut.cpp b/CPP/7zip/Archive/Common/HandlerOut.cpp index ea320e66..77a35c74 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -2,18 +2,92 @@ #include "StdAfx.h" -#ifndef _7ZIP_ST -#include "../../../Windows/System.h" -#endif +#include "../../../Common/StringToInt.h" #include "../Common/ParseProperties.h" #include "HandlerOut.h" -using namespace NWindows; - namespace NArchive { +bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res) +{ + if (*s == 0) + { + switch (prop.vt) + { + case VT_UI4: res = prop.ulVal; return true; + case VT_UI8: res = prop.uhVal.QuadPart; return true; + case VT_BSTR: + s = prop.bstrVal; + break; + default: return false; + } + } + else if (prop.vt != VT_EMPTY) + return false; + + const wchar_t *end; + UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + wchar_t c = *end; + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + if (c == '%') + { + res = percentsBase / 100 * v; + return true; + } + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; +} + +bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres) +{ + hres = S_OK; + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + #ifndef _7ZIP_ST + hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads); + #endif + return true; + } + + if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage)) + hres = E_INVALIDARG; + return true; + } + + return false; +} + + +#ifndef EXTRACT_ONLY + static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) { if (m.FindProp(propID) < 0) @@ -34,21 +108,23 @@ void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 } #endif -void CMultiMethodProps::Init() +void CMultiMethodProps::InitMulti() { - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NSystem::GetNumberOfProcessors(); - #endif - _level = (UInt32)(Int32)-1; _analysisLevel = -1; - - _autoFilter = true; _crcSize = 4; - _filterMethod.Clear(); + _autoFilter = true; +} + +void CMultiMethodProps::Init() +{ + InitCommon(); + InitMulti(); _methods.Clear(); + _filterMethod.Clear(); } + HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { UString name = nameSpec; @@ -78,20 +154,18 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN _crcSize = 4; return ParsePropToUInt32(name, value, _crcSize); } + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } UInt32 number; unsigned index = ParseStringToUInt32(name, number); UString realName = name.Ptr(index); if (index == 0) { - if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - #endif - - return S_OK; - } if (name.IsEqualTo("f")) { HRESULT res = PROPVARIANT_to_bool(value, _autoFilter); @@ -110,20 +184,20 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN return _methods[number].ParseMethodFromPROPVARIANT(realName, value); } + + void CSingleMethodProps::Init() { + InitCommon(); + InitSingle(); Clear(); - _level = (UInt32)(Int32)-1; - - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); - AddProp_NumThreads(_numThreads); - #endif } + HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { Init(); + for (UInt32 i = 0; i < numProps; i++) { UString name = names[i]; @@ -137,20 +211,22 @@ HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PR RINOK(ParsePropToUInt32(name.Ptr(1), value, a)); _level = a; AddProp_Level(a); + continue; } - else if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - AddProp_NumThreads(_numThreads); - #endif - } - else { - RINOK(ParseMethodFromPROPVARIANT(names[i], value)); + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres) + continue; + } } + RINOK(ParseMethodFromPROPVARIANT(names[i], value)); } + return S_OK; } +#endif + } diff --git a/CPP/7zip/Archive/Common/HandlerOut.h b/CPP/7zip/Archive/Common/HandlerOut.h index e24686da..bbb4336e 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.h +++ b/CPP/7zip/Archive/Common/HandlerOut.h @@ -3,20 +3,57 @@ #ifndef __HANDLER_OUT_H #define __HANDLER_OUT_H +#include "../../../Windows/System.h" + #include "../../Common/MethodProps.h" namespace NArchive { -class CMultiMethodProps +bool ParseSizeString(const wchar_t *name, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res); + +class CCommonMethodProps { - UInt32 _level; - int _analysisLevel; +protected: + void InitCommon() + { + #ifndef _7ZIP_ST + _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + #endif + + UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; + _memAvail = memAvail; + _memUsage = memAvail; + if (NWindows::NSystem::GetRamSize(memAvail)) + { + _memAvail = memAvail; + _memUsage = memAvail / 32 * 17; + } + } + public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; #endif + UInt64 _memUsage; + UInt64 _memAvail; + + bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); + + CCommonMethodProps() { InitCommon(); } +}; + + +#ifndef EXTRACT_ONLY + +class CMultiMethodProps: public CCommonMethodProps +{ + UInt32 _level; + int _analysisLevel; + + void InitMulti(); +public: UInt32 _crcSize; CObjectVector<COneMethodInfo> _methods; COneMethodInfo _filterMethod; @@ -43,27 +80,31 @@ public: int GetAnalysisLevel() const { return _analysisLevel; } void Init(); + CMultiMethodProps() { InitMulti(); } - CMultiMethodProps() { Init(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; -class CSingleMethodProps: public COneMethodInfo + +class CSingleMethodProps: public COneMethodInfo, public CCommonMethodProps { UInt32 _level; - -public: - #ifndef _7ZIP_ST - UInt32 _numThreads; - UInt32 _numProcessors; - #endif + void InitSingle() + { + _level = (UInt32)(Int32)-1; + } + +public: void Init(); - CSingleMethodProps() { Init(); } + CSingleMethodProps() { InitSingle(); } + int GetLevel() const { return _level == (UInt32)(Int32)-1 ? 5 : (int)_level; } HRESULT SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); }; +#endif + } #endif diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 0028d762..7d7256c9 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -488,6 +488,16 @@ ARCHIVE_INTERFACE(IOutArchive, 0xA0) }; +/* +ISetProperties::SetProperties() + PROPVARIANT values[i].vt: + VT_EMPTY + VT_BOOL + VT_UI4 - if 32-bit number + VT_UI8 - if 64-bit number + VT_BSTR +*/ + ARCHIVE_INTERFACE(ISetProperties, 0x03) { STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) PURE; diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index c4225426..ab8c70e3 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -1086,7 +1086,7 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool if (!lzCoder) { const UInt32 methodID = 0x40305; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); if (!lzCoder) return E_NOTIMPL; } diff --git a/CPP/7zip/Archive/Rar/RarHandler.cpp b/CPP/7zip/Archive/Rar/RarHandler.cpp index c097b15c..fb5ef756 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.cpp +++ b/CPP/7zip/Archive/Rar/RarHandler.cpp @@ -1691,7 +1691,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, methodID += 2; else methodID += 3; - RINOK(CreateCoder(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); } if (mi.Coder == 0) diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index bc9ff8b0..5c024328 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -2,7 +2,6 @@ #include "StdAfx.h" -#include "../../../C/7zCrc.h" #include "../../../C/Alloc.h" #include "../../../C/CpuArch.h" #include "../../../C/Xz.h" @@ -1209,8 +1208,10 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool else { ECoderStatus status; - XzUnpacker_Init(&_xz); - SRes res = XzUnpacker_Code(&_xz, dest, &destLen, _inputBuffer, &srcLen, CODER_FINISH_END, &status); + SRes res = XzUnpacker_CodeFull(&_xz, + dest, &destLen, + _inputBuffer, &srcLen, + CODER_FINISH_END, &status); if (res != 0) return SResToHRESULT(res); if (status != CODER_STATUS_NEEDS_MORE_INPUT || !XzUnpacker_IsStreamWasFinished(&_xz)) diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index e3779a41..ddaddcd5 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -24,9 +24,7 @@ #include "IArchive.h" -#ifndef EXTRACT_ONLY #include "Common/HandlerOut.h" -#endif using namespace NWindows; @@ -49,14 +47,22 @@ class CHandler: public IInArchive, public IArchiveOpenSeq, public IInArchiveGetStream, + public ISetProperties, + #ifndef EXTRACT_ONLY public IOutArchive, - public ISetProperties, - public CMultiMethodProps, #endif - public CMyUnknownImp + + public CMyUnknownImp, + + #ifndef EXTRACT_ONLY + public CMultiMethodProps + #else + public CCommonMethodProps + #endif { - NCompress::NXz::CStatInfo _stat; + CXzStatInfo _stat; + SRes MainDecodeSRes; bool _isArc; bool _needSeekToStart; @@ -71,34 +77,48 @@ class CHandler: UInt64 _numSolidBytes; - HRESULT SetSolidFromString(const UString &s); - HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); - HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); - - void InitSolid() + void InitXz() { + _filterId = 0; _numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO; } + #endif + void Init() { - InitSolid(); - _filterId = 0; - CMultiMethodProps::Init(); + #ifndef EXTRACT_ONLY + InitXz(); + CMultiMethodProps::Init(); + #else + CCommonMethodProps::InitCommon(); + #endif } - #endif + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); HRESULT Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCallback *callback); - HRESULT Decode2(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, - NCompress::NXz::CDecoder &decoder, ICompressProgressInfo *progress) + HRESULT Decode(NCompress::NXz::CDecoder &decoder, + ISequentialInStream *seqInStream, + ISequentialOutStream *outStream, + ICompressProgressInfo *progress) { + #ifndef _7ZIP_ST + decoder._numThreads = _numThreads; + #endif + decoder._memUsage = _memUsage; + + MainDecodeSRes = SZ_OK; + RINOK(decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit true, // finishStream progress)); - _stat = decoder; + + _stat = decoder.Stat; + MainDecodeSRes = decoder.MainDecodeSRes; + _phySize_Defined = true; return S_OK; } @@ -107,9 +127,9 @@ public: MY_QUERYINTERFACE_BEGIN2(IInArchive) MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq) MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream) + MY_QUERYINTERFACE_ENTRY(ISetProperties) #ifndef EXTRACT_ONLY MY_QUERYINTERFACE_ENTRY(IOutArchive) - MY_QUERYINTERFACE_ENTRY(ISetProperties) #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -117,10 +137,10 @@ public: INTERFACE_IInArchive(;) STDMETHOD(OpenSeq)(ISequentialInStream *stream); STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #ifndef EXTRACT_ONLY INTERFACE_IOutArchive(;) - STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #endif size_t _blocksArraySize; @@ -146,7 +166,7 @@ CHandler::CHandler(): _blocksArraySize(0) { #ifndef EXTRACT_ONLY - Init(); + InitXz(); #endif } @@ -307,7 +327,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) NCOM::CPropVariant prop; switch (propID) { - case kpidPhySize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPhySize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; @@ -330,19 +350,22 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidErrorFlags: { UInt32 v = 0; + SRes sres = MainDecodeSRes; // _stat.DecodeRes2; // if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; - if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; + if (/*_stat.UnexpectedEnd */ sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; - if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError; - if (_stat.Unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; - if (_stat.DataError) v |= kpv_ErrorFlags_DataError; - if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError; - prop = v; + if (/* _stat.HeadersError */ sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; + if (/* _stat.Unsupported */ sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; + if (/* _stat.DataError */ sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; + if (/* _stat.CrcError */ sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; + if (v != 0) + prop = v; break; } case kpidMainSubfile: { + // debug only, comment it: // if (_blocks) prop = (UInt32)0; break; } @@ -365,7 +388,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) switch (propID) { case kpidSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidPackSize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPackSize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; } prop.Detach(value); @@ -483,10 +506,10 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal } } - RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.PhySize)); + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.InSize)); if (callback) { - RINOK(callback->SetTotal(NULL, &_stat.PhySize)); + RINOK(callback->SetTotal(NULL, &_stat.InSize)); } CSeekInStreamWrap inStreamImp; @@ -621,7 +644,7 @@ STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) STDMETHODIMP CHandler::Close() { - _stat.Clear(); + XzStatInfo_Clear(&_stat); _isArc = false; _needSeekToStart = false; @@ -637,6 +660,8 @@ STDMETHODIMP CHandler::Close() _blocksArraySize = 0; _maxBlocksSize = 0; + MainDecodeSRes = SZ_OK; + return S_OK; } @@ -743,6 +768,8 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, xzu.p.streamFlags = (UInt16)streamFlags; XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p); + XzUnpacker_SetOutBuf(&xzu.p, dest, unpackSize); + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); UInt64 packRem = packSizeAligned; @@ -771,8 +798,11 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, ECoderStatus status; SRes res = XzUnpacker_Code(&xzu.p, - dest + outPos, &outLen, + // dest + outPos, + NULL, + &outLen, xzu.InBuf + inPos, &inLen, + (inLen == 0), // srcFinished CODER_FINISH_END, &status); // return E_OUTOFMEMORY; @@ -890,15 +920,16 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) return E_INVALIDARG; if (!_stat.UnpackSize_Defined + || _maxBlocksSize == 0 // 18.02 || _maxBlocksSize > kMaxBlockSize_for_GetStream || _maxBlocksSize != (size_t)_maxBlocksSize) return S_FALSE; - UInt64 physSize = (UInt64)(sizeof(size_t)) << 29; - bool ramSize_Defined = NSystem::GetRamSize(physSize); - if (ramSize_Defined) + UInt64 memSize; + if (!NSystem::GetRamSize(memSize)) + memSize = (UInt64)(sizeof(size_t)) << 28; { - if (_maxBlocksSize > physSize / 4) + if (_maxBlocksSize > memSize / 4) return S_FALSE; } @@ -917,6 +948,31 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } +static Int32 Get_Extract_OperationResult(const NCompress::NXz::CDecoder &decoder) +{ + Int32 opRes; + SRes sres = decoder.MainDecodeSRes; // decoder.Stat.DecodeRes2; + if (sres == SZ_ERROR_NO_ARCHIVE) // (!IsArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (sres == SZ_ERROR_INPUT_EOF) // (UnexpectedEnd) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (decoder.Stat.DataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (sres == SZ_ERROR_CRC) // (CrcError) + opRes = NExtract::NOperationResult::kCRCError; + else if (sres == SZ_ERROR_UNSUPPORTED) // (Unsupported) + opRes = NExtract::NOperationResult::kUnsupportedMethod; + else if (sres == SZ_ERROR_ARCHIVE) // (HeadersError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres == SZ_ERROR_DATA) // (DataError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres != SZ_OK) + opRes = NExtract::NOperationResult::kDataError; + else + opRes = NExtract::NOperationResult::kOK; + return opRes; +} + @@ -930,7 +986,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_phySize_Defined) - extractCallback->SetTotal(_stat.PhySize); + extractCallback->SetTotal(_stat.InSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)); @@ -959,9 +1015,18 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, else _needSeekToStart = true; + NCompress::NXz::CDecoder decoder; - RINOK(Decode2(_seqStream, realOutStream, decoder, lpsRef)); - Int32 opRes = decoder.Get_Extract_OperationResult(); + + HRESULT hres = Decode(decoder, _seqStream, realOutStream, lpsRef); + + if (!decoder.MainDecodeSRes_wasUsed) + return hres == S_OK ? E_FAIL : hres; + + Int32 opRes = Get_Extract_OperationResult(decoder); + if (opRes == NExtract::NOperationResult::kOK + && hres != S_OK) + opRes = NExtract::NOperationResult::kDataError; realOutStream.Release(); return extractCallback->SetOperationResult(opRes); @@ -1112,7 +1177,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) { if (_phySize_Defined) - RINOK(updateCallback->SetTotal(_stat.PhySize)); + RINOK(updateCallback->SetTotal(_stat.InSize)); RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); } @@ -1125,78 +1190,63 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt COM_TRY_END } +#endif + -HRESULT CHandler::SetSolidFromString(const UString &s) +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { - UString s2 = s; - s2.MakeLower_Ascii(); + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + #ifndef EXTRACT_ONLY + if (name[0] == L's') { - const wchar_t *start = ((const wchar_t *)s2); - const wchar_t *end; - UInt64 v = ConvertStringToUInt64(start, &end); - if (start == end) - return E_INVALIDARG; - if ((unsigned)(end - start) + 1 != s2.Len()) - return E_INVALIDARG; - wchar_t c = *end; + const wchar_t *s = name.Ptr(1); + if (*s == 0) { - unsigned numBits; - switch (c) + bool useStr = false; + bool isSolid; + switch (value.vt) { - case 'b': numBits = 0; break; - case 'k': numBits = 10; break; - case 'm': numBits = 20; break; - case 'g': numBits = 30; break; - case 't': numBits = 40; break; + case VT_EMPTY: isSolid = true; break; + case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; + case VT_BSTR: + if (!StringToBool(value.bstrVal, isSolid)) + useStr = true; + break; default: return E_INVALIDARG; } - _numSolidBytes = (v << numBits); + if (!useStr) + { + _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); + return S_OK; + } } + return ParseSizeString(s, value, + 0, // percentsBase + _numSolidBytes) ? S_OK: E_INVALIDARG; } - return S_OK; -} - -HRESULT CHandler::SetSolidFromPROPVARIANT(const PROPVARIANT &value) -{ - bool isSolid; - switch (value.vt) - { - case VT_EMPTY: isSolid = true; break; - case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; - case VT_BSTR: - if (StringToBool(value.bstrVal, isSolid)) - break; - return SetSolidFromString(value.bstrVal); - default: return E_INVALIDARG; - } - _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); - return S_OK; -} + return CMultiMethodProps::SetProperty(name, value); + #else -HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) -{ - UString name = nameSpec; - name.MakeLower_Ascii(); - if (name.IsEmpty()) - return E_INVALIDARG; - - if (name[0] == L's') { - name.Delete(0); - if (name.IsEmpty()) - return SetSolidFromPROPVARIANT(value); - if (value.vt != VT_EMPTY) - return E_INVALIDARG; - return SetSolidFromString(name); + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; } + + return E_INVALIDARG; - return CMultiMethodProps::SetProperty(name, value); + #endif } + STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN @@ -1208,6 +1258,8 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(SetProperty(names[i], values[i])); } + #ifndef EXTRACT_ONLY + if (!_filterMethod.MethodName.IsEmpty()) { unsigned k; @@ -1238,12 +1290,13 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR return E_INVALIDARG; } + #endif + return S_OK; COM_TRY_END } -#endif REGISTER_ARC_IO( "xz", "xz txz", "* .tar", 0xC, diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 0baa254d..1ee7e22f 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -384,7 +384,7 @@ HRESULT CAddCommon::Compress( methodId = kMethodId_ZipBase + method; break; } - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, true, _compressEncoder)); if (!_compressEncoder) diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index 927b3749..494b9d70 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -112,7 +112,8 @@ static const CUInt32PCharPair g_HeaderCharacts[] = { 3, "Descriptor" }, // { 5, "Patched" }, { 6, kMethod_StrongCrypto }, - { 11, "UTF8" } + { 11, "UTF8" }, + { 14, "Alt" } }; struct CIdToNamePair @@ -169,6 +170,7 @@ static const Byte kProps[] = kpidUnpackVer, kpidVolumeIndex, kpidOffset + // kpidIsAltStream }; static const Byte kArcProps[] = @@ -307,6 +309,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = true; break; } + + // case kpidIsAltStream: prop = true; break; } prop.Detach(value); COM_TRY_END @@ -333,6 +337,21 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val UString res; item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); NItemName::ReplaceToOsSlashes_Remove_TailSlash(res); + /* + if (item.ParentOfAltStream >= 0) + { + const CItemEx &prevItem = m_Items[item.ParentOfAltStream]; + UString prevName; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + if (res.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(res.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + { + res.Delete(prevName.Len(), (unsigned)strlen(k_SpecName_NTFS_STREAM)); + res.Insert(prevName.Len(), L":"); + } + } + */ prop = res; break; } @@ -596,6 +615,19 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidOffset: prop = item.LocalHeaderPos; break; + + /* + case kpidIsAltStream: + prop = (bool)(item.ParentOfAltStream >= 0); // item.IsAltStream(); + break; + + case kpidName: + if (item.ParentOfAltStream >= 0) + { + // extract name of stream here + } + break; + */ } prop.Detach(value); @@ -604,6 +636,85 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } + +/* +STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + UNUSED_VAR(index); + *propID = 0; + *name = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + if (index >= m_Items.Size()) + return S_OK; + const CItemEx &item = m_Items[index]; + + if (item.ParentOfAltStream >= 0) + { + *parentType = NParentType::kAltStream; + *parent = item.ParentOfAltStream; + } + return S_OK; +} + +STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + UNUSED_VAR(index); + UNUSED_VAR(propID); + *data = NULL; + *dataSize = 0; + *propType = 0; + return S_OK; +} + + +void CHandler::MarkAltStreams(CObjectVector<CItemEx> &items) +{ + int prevIndex = -1; + UString prevName; + UString name; + + for (unsigned i = 0; i < items.Size(); i++) + { + CItemEx &item = m_Items[i]; + if (item.IsAltStream()) + { + if (prevIndex == -1) + continue; + if (prevName.IsEmpty()) + { + const CItemEx &prevItem = m_Items[prevIndex]; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + } + name.Empty(); + item.GetUnicodeString(name, item.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(name); + + if (name.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(name.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + item.ParentOfAltStream = prevIndex; + } + else + { + prevIndex = i; + prevName.Empty(); + } + } +} +*/ + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback) { @@ -617,6 +728,7 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, m_Items.Clear(); m_Archive.ClearRefs(); // we don't want to clear error flags } + // MarkAltStreams(m_Items); return res; } catch(...) { Close(); throw; } @@ -738,7 +850,7 @@ public: IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res); }; @@ -767,7 +879,7 @@ HRESULT CZipDecoder::Decode( IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res) { @@ -962,7 +1074,7 @@ HRESULT CZipDecoder::Decode( szMethodID = kMethodId_ZipBase + (Byte)id; } - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); if (!mi.Coder) { @@ -974,16 +1086,7 @@ HRESULT CZipDecoder::Decode( } ICompressCoder *coder = methodItems[m].Coder; - - { - CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties; - coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); - if (setDecoderProperties) - { - Byte properties = (Byte)item.Flags; - RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); - } - } + #ifndef _7ZIP_ST { @@ -994,7 +1097,27 @@ HRESULT CZipDecoder::Decode( RINOK(setCoderMt->SetNumberOfThreads(numThreads)); } } + // if (memUsage != 0) + { + CMyComPtr<ICompressSetMemLimit> setMemLimit; + coder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } #endif + + { + CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties; + coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); + if (setDecoderProperties) + { + Byte properties = (Byte)item.Flags; + RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); + } + } + CMyComPtr<ISequentialInStream> inStreamNew; @@ -1319,7 +1442,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Archive, item, realOutStream, extractCallback, progress, #ifndef _7ZIP_ST - _props._numThreads, + _props._numThreads, _props._memUsage, #endif res); diff --git a/CPP/7zip/Archive/Zip/ZipHandler.h b/CPP/7zip/Archive/Zip/ZipHandler.h index 53e6a460..bee57c00 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.h +++ b/CPP/7zip/Archive/Zip/ZipHandler.h @@ -25,6 +25,7 @@ extern const char * const kMethodNames2[kNumMethodNames2]; class CHandler: public IInArchive, + // public IArchiveGetRawProps, public IOutArchive, public ISetProperties, PUBLIC_ISetCompressCodecsInfo @@ -32,6 +33,7 @@ class CHandler: { public: MY_QUERYINTERFACE_BEGIN2(IInArchive) + // MY_QUERYINTERFACE_ENTRY(IArchiveGetRawProps) MY_QUERYINTERFACE_ENTRY(IOutArchive) MY_QUERYINTERFACE_ENTRY(ISetProperties) QUERY_ENTRY_ISetCompressCodecsInfo @@ -39,6 +41,7 @@ public: MY_ADDREF_RELEASE INTERFACE_IInArchive(;) + // INTERFACE_IArchiveGetRawProps(;) INTERFACE_IOutArchive(;) STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); @@ -75,6 +78,10 @@ private: _forceCodePage = false; _specifiedCodePage = CP_OEMCP; } + + // void MarkAltStreams(CObjectVector<CItemEx> &items); + + HRESULT GetOutProperty(IArchiveUpdateCallback *callback, UInt32 callbackIndex, Int32 arcIndex, PROPID propID, PROPVARIANT *value); }; }} diff --git a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index 5acbb6d4..c21b5605 100644 --- a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -376,7 +376,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt { CMethodId methodId; UInt32 numStreams; - if (!FindMethod(EXTERNAL_CODECS_VARS methodName, methodId, numStreams)) + if (FindMethod_Index(EXTERNAL_CODECS_VARS methodName, true, + methodId, numStreams) < 0) return E_NOTIMPL; if (numStreams != 1) return E_NOTIMPL; diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index f959e4a1..71715aae 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -2632,6 +2632,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2744,6 +2772,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index 8e9d59a6..ba68fe54 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -198,10 +198,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -222,5 +224,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index 334b6635..65308db4 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -1696,6 +1696,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1789,6 +1817,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index d8335818..98600154 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -133,10 +133,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ @@ -150,5 +152,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index 7ebf37ef..82ac7fa4 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -1004,6 +1004,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1040,6 +1049,15 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sha256.c !IF "$(CFG)" == "FM - Win32 Release" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 1fb1d451..1233d4fc 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -123,10 +123,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -136,5 +138,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zExtract/makefile b/CPP/7zip/Bundles/Format7zExtract/makefile index 82265b08..e22faf36 100644 --- a/CPP/7zip/Bundles/Format7zExtract/makefile +++ b/CPP/7zip/Bundles/Format7zExtract/makefile @@ -42,6 +42,7 @@ AR_OBJS = \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ + $O\HandlerOut.obj \ $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ $O\ParseProperties.obj \ @@ -98,7 +99,9 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -106,5 +109,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zExtractR/makefile b/CPP/7zip/Bundles/Format7zExtractR/makefile index 724477e9..756c8ae6 100644 --- a/CPP/7zip/Bundles/Format7zExtractR/makefile +++ b/CPP/7zip/Bundles/Format7zExtractR/makefile @@ -85,9 +85,12 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index 28dce218..fd24d5a8 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -263,10 +263,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -285,3 +287,4 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/CPP/7zip/Bundles/Format7zF/Format7z.dsp index 7b1b233e..3e202fe7 100644 --- a/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -1805,6 +1805,26 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "7z - Win32 Release" @@ -1874,6 +1894,26 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Format7zR/makefile b/CPP/7zip/Bundles/Format7zR/makefile index 916dc568..5c05abd1 100644 --- a/CPP/7zip/Bundles/Format7zR/makefile +++ b/CPP/7zip/Bundles/Format7zR/makefile @@ -102,12 +102,15 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index b21b9e84..c5a0ceff 100644 --- a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -117,6 +117,14 @@ SOURCE=..\..\Archive\Common\CoderMixer2.h # End Source File # Begin Source File +SOURCE=..\..\Archive\Common\HandlerOut.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Archive\Common\HandlerOut.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\ItemNameUtils.cpp # End Source File # Begin Source File @@ -293,6 +301,10 @@ SOURCE=..\..\Compress\Lzma2Decoder.cpp # End Source File # Begin Source File +SOURCE=..\..\Compress\Lzma2Decoder.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\Lzma2Register.cpp # End Source File # Begin Source File @@ -411,6 +423,14 @@ SOURCE=..\..\..\Windows\Synchronization.cpp SOURCE=..\..\..\Windows\Synchronization.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File # End Group # Begin Group "Common" @@ -822,6 +842,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -831,6 +860,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index 4c201c00..01d1bf60 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -41,6 +41,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\Synchronization.obj \ + $O\System.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ @@ -117,7 +118,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -125,5 +128,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp index 787c1bde..754cc0e0 100644 --- a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp +++ b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp @@ -449,6 +449,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -469,6 +477,14 @@ SOURCE=..\..\Common\CreateCoder.h # End Source File # Begin Source File +SOURCE=..\..\Common\CWrappers.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\CWrappers.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\FileStreams.cpp # End Source File # Begin Source File @@ -724,6 +740,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -733,6 +758,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXSetup/makefile b/CPP/7zip/Bundles/SFXSetup/makefile index 5849cf36..19afc3b3 100644 --- a/CPP/7zip/Bundles/SFXSetup/makefile +++ b/CPP/7zip/Bundles/SFXSetup/makefile @@ -33,6 +33,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\ResourceString.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -40,6 +41,7 @@ WIN_CTRL_OBJS = \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\InBuffer.obj \ $O\FilterCoder.obj \ @@ -102,9 +104,12 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 301a6c5c..14492ca1 100644 --- a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -633,6 +633,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -906,6 +914,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -915,6 +932,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index dc48ae88..d6556961 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -41,6 +41,7 @@ WIN_OBJS = \ $O\ResourceString.obj \ $O\Shell.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -135,7 +136,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -143,5 +146,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Common/CreateCoder.cpp b/CPP/7zip/Common/CreateCoder.cpp index 75074ad8..bf525dc8 100644 --- a/CPP/7zip/Common/CreateCoder.cpp +++ b/CPP/7zip/Common/CreateCoder.cpp @@ -148,20 +148,23 @@ HRESULT CExternalCodecs::Load() #endif -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams) + bool encode, + CMethodId &methodId, + UInt32 &numStreams) { unsigned i; for (i = 0; i < g_NumCodecs; i++) { const CCodecInfo &codec = *g_Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.CreateEncoder : codec.CreateDecoder) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return i; } } @@ -173,19 +176,51 @@ bool FindMethod( for (i = 0; i < __externalCodecs->Codecs.Size(); i++) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return g_NumCodecs + i; } } #endif - return false; + return -1; +} + + +static int FindMethod_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode) +{ + unsigned i; + for (i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &codec = *g_Codecs[i]; + if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder)) + return i; + } + + #ifdef EXTERNAL_CODECS + + CHECK_GLOBAL_CODECS + + if (__externalCodecs) + for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + { + const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; + if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)) + return g_NumCodecs + i; + } + + #endif + + return -1; } + bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, @@ -280,9 +315,11 @@ void GetHashMethods( #endif } -HRESULT CreateCoder( + + +HRESULT CreateCoder_Index( DECL_EXTERNAL_CODECS_LOC_VARS - CMethodId methodId, bool encode, + unsigned i, bool encode, CMyComPtr<ICompressFilter> &filter, CCreatedCoder &cod) { @@ -290,11 +327,10 @@ HRESULT CreateCoder( cod.IsFilter = false; cod.NumStreams = 1; - unsigned i; - for (i = 0; i < g_NumCodecs; i++) + if (i < g_NumCodecs) { const CCodecInfo &codec = *g_Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -325,11 +361,12 @@ HRESULT CreateCoder( if (__externalCodecs) { + i -= g_NumCodecs; cod.IsExternal = true; - for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + if (i < __externalCodecs->Codecs.Size()) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -371,13 +408,50 @@ HRESULT CreateCoder( return S_OK; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod) +{ + CMyComPtr<ICompressFilter> filter; + HRESULT res = CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + index, encode, + filter, cod); + + if (filter) + { + cod.IsFilter = true; + CFilterCoder *coderSpec = new CFilterCoder(encode); + cod.Coder = coderSpec; + coderSpec->Filter = filter; + } + + return res; +} + + +HRESULT CreateCoder_Id( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode, + CMyComPtr<ICompressFilter> &filter, + CCreatedCoder &cod) +{ + int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode); + if (index < 0) + return S_OK; + return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS index, encode, filter, cod); +} + + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod) { CMyComPtr<ICompressFilter> filter; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); @@ -393,13 +467,14 @@ HRESULT CreateCoder( return res; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr<ICompressCoder> &coder) { CCreatedCoder cod; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, cod); @@ -413,7 +488,7 @@ HRESULT CreateFilter( CMyComPtr<ICompressFilter> &filter) { CCreatedCoder cod; - return CreateCoder( + return CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); diff --git a/CPP/7zip/Common/CreateCoder.h b/CPP/7zip/Common/CreateCoder.h index f06064b6..20d0ef33 100644 --- a/CPP/7zip/Common/CreateCoder.h +++ b/CPP/7zip/Common/CreateCoder.h @@ -116,13 +116,12 @@ extern CExternalCodecs g_ExternalCodecs; #endif - - - -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams); + bool encode, + CMethodId &methodId, + UInt32 &numStreams); bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS @@ -152,18 +151,29 @@ struct CCreatedCoder }; -HRESULT CreateCoder( +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned codecIndex, bool encode, + CMyComPtr<ICompressFilter> &filter, + CCreatedCoder &cod); + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod); + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr<ICompressFilter> &filter, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr<ICompressCoder> &coder); diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index 9e82a115..8a6ebcaf 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -253,6 +253,9 @@ struct CNameToPropID const char *Name; }; + +// the following are related to NCoderPropID::EEnum values + static const CNameToPropID g_NameToPropID[] = { { VT_UI4, "" }, @@ -275,7 +278,8 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI8, "expect" }, { VT_UI4, "b" }, { VT_UI4, "check" }, - { VT_BSTR, "filter" } + { VT_BSTR, "filter" }, + { VT_UI8, "memuse" } }; static int FindPropIdExact(const UString &name) @@ -293,6 +297,13 @@ static bool ConvertProperty(const PROPVARIANT &srcProp, VARTYPE varType, NCOM::C destProp = srcProp; return true; } + + if (varType == VT_UI8 && srcProp.vt == VT_UI4) + { + destProp = (UInt64)srcProp.ulVal; + return true; + } + if (varType == VT_BOOL) { bool res; diff --git a/CPP/7zip/Compress/DeflateEncoder.cpp b/CPP/7zip/Compress/DeflateEncoder.cpp index 7cf59cc0..233edb5f 100644 --- a/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/CPP/7zip/Compress/DeflateEncoder.cpp @@ -969,6 +969,10 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou } } while (Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) != 0); + + if (_seqInStream.Res != S_OK) + return _seqInStream.Res; + if (_lzInWindow.result != SZ_OK) return SResToHRESULT(_lzInWindow.result); return m_OutStream.Flush(); diff --git a/CPP/7zip/Compress/Lzma2Decoder.cpp b/CPP/7zip/Compress/Lzma2Decoder.cpp index 98af203d..653fe2de 100644 --- a/CPP/7zip/Compress/Lzma2Decoder.cpp +++ b/CPP/7zip/Compress/Lzma2Decoder.cpp @@ -2,81 +2,48 @@ #include "StdAfx.h" +// #include <stdio.h> + #include "../../../C/Alloc.h" +// #include "../../../C/CpuTicks.h" #include "../Common/StreamUtils.h" #include "Lzma2Decoder.h" -static HRESULT SResToHRESULT(SRes res) -{ - switch (res) - { - case SZ_OK: return S_OK; - case SZ_ERROR_MEM: return E_OUTOFMEMORY; - case SZ_ERROR_PARAM: return E_INVALIDARG; - case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - case SZ_ERROR_DATA: return S_FALSE; - } - return E_FAIL; -} - namespace NCompress { namespace NLzma2 { CDecoder::CDecoder(): - _inBuf(NULL), - _finishMode(false), - _outSizeDefined(false), - _outStep(1 << 22), - _inBufSize(0), - _inBufSizeNew(1 << 20) -{ - Lzma2Dec_Construct(&_state); -} + _dec(NULL) + , _inProcessed(0) + , _prop(0xFF) + , _finishMode(false) + , _inBufSize(1 << 20) + , _outStep(1 << 20) + #ifndef _7ZIP_ST + , _tryMt(1) + , _numThreads(1) + , _memUsage((UInt64)(sizeof(size_t)) << 28) + #endif +{} CDecoder::~CDecoder() { - Lzma2Dec_Free(&_state, &g_Alloc); - MidFree(_inBuf); + if (_dec) + Lzma2DecMt_Destroy(_dec); } -STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSizeNew = size; return S_OK; } +STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSize = size; return S_OK; } STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStep = size; return S_OK; } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { if (size != 1) return E_NOTIMPL; - - RINOK(SResToHRESULT(Lzma2Dec_Allocate(&_state, prop[0], &g_Alloc))); - - if (!_inBuf || _inBufSize != _inBufSizeNew) - { - MidFree(_inBuf); - _inBufSize = 0; - _inBuf = (Byte *)MidAlloc(_inBufSizeNew); - if (!_inBuf) - return E_OUTOFMEMORY; - _inBufSize = _inBufSizeNew; - } - - return S_OK; -} - - -STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) -{ - _outSizeDefined = (outSize != NULL); - _outSize = 0; - if (_outSizeDefined) - _outSize = *outSize; - _inPos = _inLim = 0; - _inProcessed = 0; - _outProcessed = 0; - - Lzma2Dec_Init(&_state); - + if (prop[0] > 40) + return E_NOTIMPL; + _prop = prop[0]; return S_OK; } @@ -88,116 +55,189 @@ STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) } -STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) + +#ifndef _7ZIP_ST + +static UInt64 Get_ExpectedBlockSize_From_Dict(UInt32 dictSize) { - *value = _inProcessed; - return S_OK; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + return blockSize; } +#define LZMA2_DIC_SIZE_FROM_PROP_FULL(p) ((p) == 40 ? 0xFFFFFFFF : (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))) + +#endif + +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (!_inBuf) - return S_FALSE; + _inProcessed = 0; - SetOutStreamSize(outSize); + if (!_dec) + { + _dec = Lzma2DecMt_Create( + // &g_AlignedAlloc, + &g_Alloc, + &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } - SizeT wrPos = _state.decoder.dicPos; - HRESULT readRes = S_OK; + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = inStream->Read(_inBuf, _inBufSize, &_inLim); - } + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; - const SizeT dicPos = _state.decoder.dicPos; - SizeT size; - { - SizeT next = _state.decoder.dicBufSize; - if (next - wrPos > _outStep) - next = wrPos + _outStep; - size = next - dicPos; - } + #ifndef _7ZIP_ST + { + props.numThreads = 1; + UInt32 numThreads = _numThreads; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) + if (_tryMt && numThreads >= 1) { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) + UInt64 useLimit = _memUsage; + UInt32 dictSize = LZMA2_DIC_SIZE_FROM_PROP_FULL(_prop); + UInt64 expectedBlockSize64 = Get_ExpectedBlockSize_From_Dict(dictSize); + size_t expectedBlockSize = (size_t)expectedBlockSize64; + size_t inBlockMax = expectedBlockSize + expectedBlockSize / 16; + if (expectedBlockSize == expectedBlockSize64 && inBlockMax >= expectedBlockSize) { - size = (SizeT)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; + props.outBlockMax = expectedBlockSize; + props.inBlockMax = inBlockMax; + const size_t kOverheadSize = props.inBufSize_MT + (1 << 16); + UInt64 okThreads = useLimit / (props.outBlockMax + props.inBlockMax + kOverheadSize); + if (numThreads > okThreads) + numThreads = (UInt32)okThreads; + if (numThreads == 0) + numThreads = 1; + props.numThreads = numThreads; } } + } + #endif - SizeT inProcessed = _inLim - _inPos; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToDic(&_state, dicPos + size, _inBuf + _inPos, &inProcessed, finishMode, &status); + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - const SizeT outProcessed = _state.decoder.dicPos - dicPos; - _outProcessed += outProcessed; + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); - - bool outFinished = (_outSizeDefined && _outProcessed >= _outSize); + SRes res; - bool needStop = (res != 0 - || (inProcessed == 0 && outProcessed == 0) - || status == LZMA_STATUS_FINISHED_WITH_MARK - || (!_finishMode && outFinished)); + UInt64 inProcessed = 0; + int isMT = False; - if (needStop || outProcessed >= size) - { - HRESULT res2 = WriteStream(outStream, _state.decoder.dic + wrPos, _state.decoder.dicPos - wrPos); + #ifndef _7ZIP_ST + isMT = _tryMt; + #endif - if (_state.decoder.dicPos == _state.decoder.dicBufSize) - _state.decoder.dicPos = 0; - wrPos = _state.decoder.dicPos; + // UInt64 cpuTicks = GetCpuTicks(); - RINOK(res2); + res = Lzma2DecMt_Decode(_dec, _prop, &props, + &outWrap.vt, outSize, _finishMode, + &inWrap.vt, + &inProcessed, + &isMT, + progress ? &progressWrap.vt : NULL); - if (needStop) - { - if (res != 0) - return S_FALSE; - - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (_finishMode) - { - if (inSize && *inSize != _inProcessed) - return S_FALSE; - if (_outSizeDefined && _outSize != _outProcessed) - return S_FALSE; - } - return readRes; - } - - if (!_finishMode && outFinished) - return readRes; - - return S_FALSE; - } - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&_inProcessed, &_outProcessed)); - } + /* + cpuTicks = GetCpuTicks() - cpuTicks; + printf("\n ticks = %10I64u\n", cpuTicks / 1000000); + */ + + + #ifndef _7ZIP_ST + /* we reset _tryMt, only if p->props.numThreads was changed */ + if (props.numThreads > 1) + _tryMt = isMT; + #endif + + _inProcessed = inProcessed; + + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + if (res == SZ_OK && _finishMode) + { + if (inSize && *inSize != inProcessed) + res = SZ_ERROR_DATA; + if (outSize && *outSize != outWrap.Processed) + res = SZ_ERROR_DATA; } + + return SResToHRESULT(res); +} + + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _inProcessed; + return S_OK; +} + + +#ifndef _7ZIP_ST + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; + return S_OK; } +STDMETHODIMP CDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + #ifndef NO_READ_FROM_CODER +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) +{ + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; + + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create(&g_AlignedAlloc, &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + + _inWrap.Init(_inStream); + + SRes res = Lzma2DecMt_Init(_dec, _prop, &props, outSize, _finishMode, &_inWrap.vt); + + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; +} + + STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } @@ -207,62 +247,17 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) if (processedSize) *processedSize = 0; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) - { - size = (UInt32)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; - } - } + size_t size2 = size; + UInt64 inProcessed = 0; - HRESULT readRes = S_OK; + SRes res = Lzma2DecMt_Read(_dec, (Byte *)data, &size2, &inProcessed); - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = _inStream->Read(_inBuf, _inBufSize, &_inLim); - } - - SizeT inProcessed = _inLim - _inPos; - SizeT outProcessed = size; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, - _inBuf + _inPos, &inProcessed, finishMode, &status); - - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - _outProcessed += outProcessed; - size -= (UInt32)outProcessed; - data = (Byte *)data + outProcessed; - if (processedSize) - *processedSize += (UInt32)outProcessed; - - if (res != 0) - return S_FALSE; - - /* - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - return readRes; - - if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) - { - if (_finishMode && _outSizeDefined && _outProcessed >= _outSize) - return S_FALSE; - return readRes; - } - */ - - if (inProcessed == 0 && outProcessed == 0) - return readRes; - } + _inProcessed += inProcessed; + if (processedSize) + *processedSize = (UInt32)size2; + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; } #endif diff --git a/CPP/7zip/Compress/Lzma2Decoder.h b/CPP/7zip/Compress/Lzma2Decoder.h index 440914e7..e1414884 100644 --- a/CPP/7zip/Compress/Lzma2Decoder.h +++ b/CPP/7zip/Compress/Lzma2Decoder.h @@ -3,10 +3,9 @@ #ifndef __LZMA2_DECODER_H #define __LZMA2_DECODER_H -#include "../../../C/Lzma2Dec.h" +#include "../../../C/Lzma2DecMt.h" -#include "../../Common/MyCom.h" -#include "../ICoder.h" +#include "../Common/CWrappers.h" namespace NCompress { namespace NLzma2 { @@ -17,28 +16,26 @@ class CDecoder: public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, public ICompressSetBufSize, + #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, public ISequentialInStream, #endif + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + public CMyUnknownImp { - Byte *_inBuf; - UInt32 _inPos; - UInt32 _inLim; - - bool _finishMode; - bool _outSizeDefined; - UInt64 _outSize; + CLzma2DecMtHandle _dec; UInt64 _inProcessed; - UInt64 _outProcessed; - - UInt32 _outStep; + Byte _prop; + int _finishMode; UInt32 _inBufSize; - UInt32 _inBufSizeNew; - - CLzma2Dec _state; + UInt32 _outStep; public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) @@ -46,11 +43,18 @@ public: MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) MY_QUERYINTERFACE_ENTRY(ICompressSetBufSize) + #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) MY_QUERYINTERFACE_ENTRY(ISequentialInStream) #endif + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -59,20 +63,28 @@ public: STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); - STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInBufSize)(UInt32 streamIndex, UInt32 size); STDMETHOD(SetOutBufSize)(UInt32 streamIndex, UInt32 size); - #ifndef NO_READ_FROM_CODER + #ifndef _7ZIP_ST +private: + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; +public: + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + #ifndef NO_READ_FROM_CODER private: CMyComPtr<ISequentialInStream> _inStream; + CSeqInStreamWrap _inWrap; public: - + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); - #endif CDecoder(); diff --git a/CPP/7zip/Compress/LzmaDecoder.cpp b/CPP/7zip/Compress/LzmaDecoder.cpp index 2fbe0589..83c24f1c 100644 --- a/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/CPP/7zip/Compress/LzmaDecoder.cpp @@ -30,19 +30,24 @@ CDecoder::CDecoder(): FinishStream(false), _propsWereSet(false), _outSizeDefined(false), - _outStep(1 << 22), + _outStep(1 << 20), _inBufSize(0), _inBufSizeNew(1 << 20) { _inProcessed = 0; _inPos = _inLim = 0; + /* + AlignOffsetAlloc_CreateVTable(&_alloc); + _alloc.numAlignBits = 7; + _alloc.offset = 0; + */ LzmaDec_Construct(&_state); } CDecoder::~CDecoder() { - LzmaDec_Free(&_state, &g_Alloc); + LzmaDec_Free(&_state, &g_AlignedAlloc); // &_alloc.vt MyFree(_inBuf); } @@ -66,7 +71,7 @@ HRESULT CDecoder::CreateInputBuffer() STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { - RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_Alloc))); + RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_AlignedAlloc))) // &_alloc.vt _propsWereSet = true; return CreateInputBuffer(); } diff --git a/CPP/7zip/Compress/LzmaDecoder.h b/CPP/7zip/Compress/LzmaDecoder.h index e73925ed..37dec025 100644 --- a/CPP/7zip/Compress/LzmaDecoder.h +++ b/CPP/7zip/Compress/LzmaDecoder.h @@ -3,6 +3,7 @@ #ifndef __LZMA_DECODER_H #define __LZMA_DECODER_H +// #include "../../../C/Alloc.h" #include "../../../C/LzmaDec.h" #include "../../Common/MyCom.h" @@ -28,7 +29,6 @@ class CDecoder: UInt32 _inPos; UInt32 _inLim; - CLzmaDec _state; ELzmaStatus _lzmaStatus; public: @@ -45,6 +45,10 @@ private: UInt32 _inBufSize; UInt32 _inBufSizeNew; + // CAlignOffsetAlloc _alloc; + + CLzmaDec _state; + HRESULT CreateInputBuffer(); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); void SetOutStreamSizeResume(const UInt64 *outSize); diff --git a/CPP/7zip/Compress/XzDecoder.cpp b/CPP/7zip/Compress/XzDecoder.cpp index c28ac900..7a974311 100644 --- a/CPP/7zip/Compress/XzDecoder.cpp +++ b/CPP/7zip/Compress/XzDecoder.cpp @@ -4,248 +4,119 @@ #include "../../../C/Alloc.h" -#include "../Common/StreamUtils.h" - -#include "../Archive/IArchive.h" +#include "../Common/CWrappers.h" #include "XzDecoder.h" -using namespace NArchive; - namespace NCompress { namespace NXz { +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; -CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(NULL), OutBuf(NULL) -{ - XzUnpacker_Construct(&p, &g_Alloc); -} - -CXzUnpackerCPP::~CXzUnpackerCPP() -{ - XzUnpacker_Free(&p); - MidFree(InBuf); - MidFree(OutBuf); -} - +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; -void CStatInfo::Clear() +static HRESULT SResToHRESULT_Code(SRes res) throw() { - InSize = 0; - OutSize = 0; - PhySize = 0; - - NumStreams = 0; - NumBlocks = 0; - - UnpackSize_Defined = false; - - NumStreams_Defined = false; - NumBlocks_Defined = false; - - IsArc = false; - UnexpectedEnd = false; - DataAfterEnd = false; - Unsupported = false; - HeadersError = false; - DataError = false; - CrcError = false; + if (res < 0) + return res; + switch (res) + { + case SZ_OK: return S_OK; + case SZ_ERROR_MEM: return E_OUTOFMEMORY; + case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + } + return S_FALSE; } HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) { - const size_t kInBufSize = (size_t)1 << 20; - const size_t kOutBufSize = (size_t)1 << 21; - - Clear(); - DecodeRes = SZ_OK; - - XzUnpacker_Init(&xzu.p); + MainDecodeSRes = S_OK; + MainDecodeSRes_wasUsed = false; + XzStatInfo_Clear(&Stat); - if (!xzu.InBuf) + if (!xz) { - xzu.InBuf = (Byte *)MidAlloc(kInBufSize); - if (!xzu.InBuf) + xz = XzDecMt_Create(&g_Alloc, &g_MidAlloc); + if (!xz) return E_OUTOFMEMORY; } - if (!xzu.OutBuf) - { - xzu.OutBuf = (Byte *)MidAlloc(kOutBufSize); - if (!xzu.OutBuf) - return E_OUTOFMEMORY; - } - - UInt32 inSize = 0; - UInt32 inPos = 0; - SizeT outPos = 0; - HRESULT readRes = S_OK; + CXzDecMtProps props; + XzDecMtProps_Init(&props); - for (;;) + int isMT = False; + + #ifndef _7ZIP_ST { - if (inPos == inSize && readRes == S_OK) + props.numThreads = 1; + UInt32 numThreads = _numThreads; + + if (_tryMt && numThreads > 1) { - inPos = inSize = 0; - readRes = seqInStream->Read(xzu.InBuf, kInBufSize, &inSize); + size_t memUsage = (size_t)_memUsage; + if (memUsage != _memUsage) + memUsage = (size_t)0 - 1; + props.memUseMax = memUsage; + isMT = (numThreads > 1); } - SizeT inLen = inSize - inPos; - SizeT outLen = kOutBufSize - outPos; - ECoderFinishMode finishMode = CODER_FINISH_ANY; + props.numThreads = numThreads; + } + #endif - /* - // 17.01 : the code was disabled: - if (inSize == 0) - finishMode = CODER_FINISH_END; - */ + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; - if (outSizeLimit) - { - const UInt64 rem = *outSizeLimit - OutSize; - if (outLen >= rem) - { - outLen = (SizeT)rem; - if (finishStream) - finishMode = CODER_FINISH_END; - } - } - - ECoderStatus status; + inWrap.Init(seqInStream); + outWrap.Init(outStream); + progressWrap.Init(progress); - const SizeT outLenRequested = outLen; + SRes res = XzDecMt_Decode(xz, + &props, + outSizeLimit, finishStream, + &outWrap.vt, + &inWrap.vt, + &Stat, + &isMT, + progress ? &progressWrap.vt : NULL); - SRes res = XzUnpacker_Code(&xzu.p, - xzu.OutBuf + outPos, &outLen, - xzu.InBuf + inPos, &inLen, - finishMode, &status); + MainDecodeSRes = res; - DecodeRes = res; + #ifndef _7ZIP_ST + // _tryMt = isMT; + #endif - inPos += (UInt32)inLen; - outPos += outLen; + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) - InSize += inLen; - OutSize += outLen; + // return E_OUTOFMEMORY; - bool finished = ((inLen == 0 && outLen == 0) || res != SZ_OK); + MainDecodeSRes_wasUsed = true; - if (outLen >= outLenRequested || finished) - { - if (outStream && outPos != 0) - { - RINOK(WriteStream(outStream, xzu.OutBuf, outPos)); - } - outPos = 0; - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&InSize, &OutSize)); - } - - if (!finished) - continue; - - { - PhySize = InSize; - NumStreams = xzu.p.numStartedStreams; - if (NumStreams > 0) - IsArc = true; - NumBlocks = xzu.p.numTotalBlocks; - - UnpackSize_Defined = true; - NumStreams_Defined = true; - NumBlocks_Defined = true; - - UInt64 extraSize = XzUnpacker_GetExtraSize(&xzu.p); - - if (res == SZ_OK) - { - if (status == CODER_STATUS_NEEDS_MORE_INPUT) - { - extraSize = 0; - if (!XzUnpacker_IsStreamWasFinished(&xzu.p)) - { - // finished at padding bytes, but padding is not aligned for 4 - UnexpectedEnd = true; - res = SZ_ERROR_DATA; - } - } - else // status == CODER_STATUS_NOT_FINISHED - res = SZ_ERROR_DATA; - } - else if (res == SZ_ERROR_NO_ARCHIVE) - { - if (InSize == extraSize) - IsArc = false; - else - { - if (extraSize != 0 || inPos != inSize) - { - DataAfterEnd = true; - res = SZ_OK; - } - } - } - - DecodeRes = res; - PhySize -= extraSize; - - switch (res) - { - case SZ_OK: break; - case SZ_ERROR_NO_ARCHIVE: IsArc = false; break; - case SZ_ERROR_ARCHIVE: HeadersError = true; break; - case SZ_ERROR_UNSUPPORTED: Unsupported = true; break; - case SZ_ERROR_CRC: CrcError = true; break; - case SZ_ERROR_DATA: DataError = true; break; - default: DataError = true; break; - } - - return readRes; - } + if (res == SZ_OK && finishStream) + { + /* + if (inSize && *inSize != Stat.PhySize) + res = SZ_ERROR_DATA; + */ + if (outSizeLimit && *outSizeLimit != outWrap.Processed) + res = SZ_ERROR_DATA; } -} - -Int32 CDecoder::Get_Extract_OperationResult() const -{ - Int32 opRes; - if (!IsArc) - opRes = NExtract::NOperationResult::kIsNotArc; - else if (UnexpectedEnd) - opRes = NExtract::NOperationResult::kUnexpectedEnd; - else if (DataAfterEnd) - opRes = NExtract::NOperationResult::kDataAfterEnd; - else if (CrcError) - opRes = NExtract::NOperationResult::kCRCError; - else if (Unsupported) - opRes = NExtract::NOperationResult::kUnsupportedMethod; - else if (HeadersError) - opRes = NExtract::NOperationResult::kDataError; - else if (DataError) - opRes = NExtract::NOperationResult::kDataError; - else if (DecodeRes != SZ_OK) - opRes = NExtract::NOperationResult::kDataError; - else - opRes = NExtract::NOperationResult::kOK; - return opRes; + return SResToHRESULT_Code(res); } - HRESULT CComDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) { - RINOK(_decoder.Decode(inStream, outStream, outSize, _finishStream, progress)); - Int32 opRes = _decoder.Get_Extract_OperationResult(); - if (opRes == NArchive::NExtract::NOperationResult::kUnsupportedMethod) - return E_NOTIMPL; - if (opRes != NArchive::NExtract::NOperationResult::kOK) - return S_FALSE; - return S_OK; + return Decode(inStream, outStream, outSize, _finishStream, progress); } STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) @@ -256,8 +127,24 @@ STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) STDMETHODIMP CComDecoder::GetInStreamProcessedSize(UInt64 *value) { - *value = _decoder.InSize; + *value = Stat.InSize; + return S_OK; +} + +#ifndef _7ZIP_ST + +STDMETHODIMP CComDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; return S_OK; } +STDMETHODIMP CComDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + }} diff --git a/CPP/7zip/Compress/XzDecoder.h b/CPP/7zip/Compress/XzDecoder.h index 040ed21c..b65b46b8 100644 --- a/CPP/7zip/Compress/XzDecoder.h +++ b/CPP/7zip/Compress/XzDecoder.h @@ -12,57 +12,36 @@ namespace NCompress { namespace NXz { -struct CXzUnpackerCPP +struct CDecoder { - Byte *InBuf; - Byte *OutBuf; - CXzUnpacker p; + CXzDecMtHandle xz; + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; + + SRes MainDecodeSRes; // it's not HRESULT + bool MainDecodeSRes_wasUsed; + CXzStatInfo Stat; + + CDecoder(): + xz(NULL), + _tryMt(True), + _numThreads(1), + _memUsage((UInt64)(sizeof(size_t)) << 28), + MainDecodeSRes(SZ_OK), + MainDecodeSRes_wasUsed(false) + {} - CXzUnpackerCPP(); - ~CXzUnpackerCPP(); -}; - - -struct CStatInfo -{ - UInt64 InSize; - UInt64 OutSize; - UInt64 PhySize; - - UInt64 NumStreams; - UInt64 NumBlocks; - - bool UnpackSize_Defined; - - bool NumStreams_Defined; - bool NumBlocks_Defined; - - bool IsArc; - bool UnexpectedEnd; - bool DataAfterEnd; - bool Unsupported; - bool HeadersError; - bool DataError; - bool CrcError; - - CStatInfo() { Clear(); } - - void Clear(); -}; - - -struct CDecoder: public CStatInfo -{ - CXzUnpackerCPP xzu; - SRes DecodeRes; // it's not HRESULT - - CDecoder(): DecodeRes(SZ_OK) {} + ~CDecoder() + { + if (xz) + XzDecMt_Destroy(xz); + } /* Decode() can return ERROR code only if there is progress or stream error. Decode() returns S_OK in case of xz decoding error, but DecodeRes and CStatInfo contain error information */ HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *compressProgress); - Int32 Get_Extract_OperationResult() const; }; @@ -70,21 +49,41 @@ class CComDecoder: public ICompressCoder, public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, - public CMyUnknownImp + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + + public CMyUnknownImp, + public CDecoder { - CDecoder _decoder; bool _finishStream; public: - MY_UNKNOWN_IMP2( - ICompressSetFinishMode, - ICompressGetInStreamProcessedSize) + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + #ifndef _7ZIP_ST + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + CComDecoder(): _finishStream(false) {} }; diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index 47da1a39..c9da7ed7 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -49,6 +49,7 @@ 25 ICompressSetCoderMt 26 ICompressSetFinishMode 27 ICompressGetInStreamProcessedSize2 + 28 ICompressSetMemLimit 30 ICompressGetSubStreamSize 31 ICompressSetInStream @@ -165,6 +166,7 @@ Handler GUIDs: 0C xz 0D ppmd + C6 COFF C7 Ext C8 VMDK C9 VDI diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index dc9acd83..677d8cfb 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -43,6 +43,7 @@ CODER_INTERFACE(ICompressCoder2, 0x18) S_OK : OK S_FALSE : data error (for decoders) E_OUTOFMEMORY : memory allocation error + E_NOTIMPL : unsupported encoding method (for decoders) another error code : some error. For example, it can be error code received from inStream or outStream function. Parameters: @@ -129,7 +130,8 @@ namespace NCoderPropID kBlockSize2, // VT_UI4 or VT_UI8 kCheckSize, // VT_UI4 : size of digest in bytes - kFilter // VT_BSTR + kFilter, // VT_BSTR + kMemUse // VT_UI8 }; } @@ -190,6 +192,12 @@ CODER_INTERFACE(ICompressGetInStreamProcessedSize2, 0x27) STDMETHOD(GetInStreamProcessedSize2)(UInt32 streamIndex, UInt64 *value) PURE; }; +CODER_INTERFACE(ICompressSetMemLimit, 0x28) +{ + STDMETHOD(SetMemLimit)(UInt64 memUsage) PURE; +}; + + CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) { diff --git a/CPP/7zip/LzmaDec.mak b/CPP/7zip/LzmaDec.mak new file mode 100644 index 00000000..02e449d4 --- /dev/null +++ b/CPP/7zip/LzmaDec.mak @@ -0,0 +1,5 @@ +!IF "$(CPU)" == "AMD64" +CFLAGS_C_SPEC = -D_LZMA_DEC_OPT +ASM_OBJS = $(ASM_OBJS) \ + $O\LzmaDecOpt.obj +!ENDIF diff --git a/CPP/7zip/UI/Agent/Agent.h b/CPP/7zip/UI/Agent/Agent.h index 7ba96f4a..f7d7b5e4 100644 --- a/CPP/7zip/UI/Agent/Agent.h +++ b/CPP/7zip/UI/Agent/Agent.h @@ -253,8 +253,8 @@ public: IInArchive *GetArchive() const { if ( _archiveLink.Arcs.IsEmpty()) return 0; return GetArc().Archive; } bool CanUpdate() const; - bool Is_Attrib_ReadOnly() const - { + bool Is_Attrib_ReadOnly() const + { return _attrib != INVALID_FILE_ATTRIBUTES && (_attrib & FILE_ATTRIBUTE_READONLY); } diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index d0fead76..e4ac64ec 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -1183,9 +1183,11 @@ static HRESULT MethodBench( COneMethodInfo method = method2; UInt64 methodId; UInt32 numStreams; - if (!FindMethod( + int codecIndex = FindMethod_Index( EXTERNAL_CODECS_LOC_VARS - method.MethodName, methodId, numStreams)) + method.MethodName, true, + methodId, numStreams); + if (codecIndex < 0) return E_NOTIMPL; if (numStreams != 1) return E_INVALIDARG; @@ -1222,7 +1224,7 @@ static HRESULT MethodBench( { CCreatedCoder cod; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, true, encoder._encoderFilter, cod)); + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS codecIndex, true, encoder._encoderFilter, cod)); encoder._encoder = cod.Coder; if (!encoder._encoder && !encoder._encoderFilter) return E_NOTIMPL; @@ -1239,7 +1241,7 @@ static HRESULT MethodBench( { CCreatedCoder cod; CMyComPtr<ICompressCoder> &decoder = encoder._decoders[j]; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); decoder = cod.Coder; if (!encoder._decoderFilter && !decoder) return E_NOTIMPL; diff --git a/CPP/Build.mak b/CPP/Build.mak index 5bcd1b8b..d59578a5 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -115,10 +115,11 @@ COMPLB = $(CC) $(CFLAGS_O1) -Yu"StdAfx.h" -Fp$O/a.pch $< # COMPLB_O2 = $(CC) $(CFLAGS_O2) -Yu"StdAfx.h" -Fp$O/a.pch $< COMPLB_O2 = $(CC) $(CFLAGS_O2) $< -CCOMPL_PCH = $(CC) $(CFLAGS_O2) -Yc"Precomp.h" -Fp$O/a.pch $** -CCOMPL_USE = $(CC) $(CFLAGS_O2) -Yu"Precomp.h" -Fp$O/a.pch $** -CCOMPL = $(CC) $(CFLAGS_O2) $** -CCOMPLB = $(CC) $(CFLAGS_O2) $< +CFLAGS_C_ALL = $(CFLAGS_O2) $(CFLAGS_C_SPEC) +CCOMPL_PCH = $(CC) $(CFLAGS_C_ALL) -Yc"Precomp.h" -Fp$O/a.pch $** +CCOMPL_USE = $(CC) $(CFLAGS_C_ALL) -Yu"Precomp.h" -Fp$O/a.pch $** +CCOMPL = $(CC) $(CFLAGS_C_ALL) $** +CCOMPLB = $(CC) $(CFLAGS_C_ALL) $< all: $(PROGPATH) diff --git a/DOC/7zip.inf b/DOC/7zip.inf index 9bd74d52..0062a907 100644 --- a/DOC/7zip.inf +++ b/DOC/7zip.inf @@ -10,8 +10,8 @@ AppName = "7-Zip" InstallDir = %CE1%\%AppName% [Strings] -AppVer = "18.01" -AppDate = "2018-01-28" +AppVer = "18.03" +AppDate = "2018-03-04" [CEDevice] ; ProcessorType = 2577 ; ARM diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index 5483e5fb..91554ec4 100644 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -2,7 +2,7 @@ ;Defines !define VERSION_MAJOR 18 -!define VERSION_MINOR 01 +!define VERSION_MINOR 03 !define VERSION_POSTFIX_FULL "" !ifdef WIN64 !ifdef IA64 @@ -220,6 +220,7 @@ Section File ja.txt File ka.txt File kaa.txt + File kab.txt File kk.txt File ko.txt File ku.txt @@ -421,6 +422,7 @@ Section Uninstall Delete $INSTDIR\Lang\ja.txt Delete $INSTDIR\Lang\ka.txt Delete $INSTDIR\Lang\kaa.txt + Delete $INSTDIR\Lang\kab.txt Delete $INSTDIR\Lang\kk.txt Delete $INSTDIR\Lang\ko.txt Delete $INSTDIR\Lang\ku.txt diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index 96969d93..5f7896c1 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ <?xml version="1.0"?> <?define VerMajor = "18" ?> -<?define VerMinor = "01" ?> +<?define VerMinor = "03" ?> <?define VerBuild = "00" ?> <?define MmVer = "$(var.VerMajor).$(var.VerMinor)" ?> <?define MmHex = "$(var.VerMajor)$(var.VerMinor)" ?> @@ -104,6 +104,8 @@ <Media Id="1" Cabinet="product.cab" EmbedCab="yes" CompressionLevel="high" /> + <Property Id="MSIRMSHUTDOWN" Value="2"/> + <Property Id="INSTALLDIR"> <RegistrySearch Id="My7zipPathLM" Type="raw" Root="HKLM" Key="Software\7-Zip" Name="Path" /> <RegistrySearch Id="My7zipPathLM2" Type="raw" Root="HKLM" Key="Software\7-Zip" Name="Path$(var.NumBits)" /> @@ -294,6 +296,7 @@ <File Id="ja.txt" Name="ja.txt" /> <File Id="ka.txt" Name="ka.txt" /> <File Id="kaa.txt" Name="kaa.txt" /> + <File Id="kab.txt" Name="kab.txt" /> <File Id="kk.txt" Name="kk.txt" /> <File Id="ko.txt" Name="ko.txt" /> <File Id="ku.txt" Name="ku.txt" /> diff --git a/DOC/readme.txt b/DOC/readme.txt index 9907ff83..53596c11 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,4 +1,4 @@ -7-Zip 18.01 Sources +7-Zip 18.03 Sources ------------------- 7-Zip is a file archiver for Windows. diff --git a/DOC/src-history.txt b/DOC/src-history.txt index 6f637d01..f89dba04 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,17 @@ HISTORY of the 7-Zip source code -------------------------------- +18.03 beta 2018-03-04 +------------------------- +- Asm\x86\LzmaDecOpt.asm: new optimized LZMA decoder written in asm + for x64 with about 30% higher speed than main version of LZMA decoder written in C. +- The speed for single-thread LZMA/LZMA2 decoder written in C was increased by 3%. +- 7-Zip now can use multi-threading for 7z/LZMA2 decoding, + if there are multiple independent data chunks in LZMA2 stream. +- 7-Zip now can use multi-threading for xz decoding, + if there are multiple blocks in xz stream. + + 17.00 beta 2017-04-29 ------------------------- - NewHandler.h / NewHandler.cpp: |