PrusaResearch (Vojtech) homebrewed the following:

mz_zip_writer_add_staged_open(), mz_zip_writer_add_staged_data() and mz_zip_writer_add_staged_finish()
were derived from mz_zip_writer_add_read_buf_callback() by splitting it and passing a new
mz_zip_writer_staged_context between them.

3 files changed, 337 insertions, 8 deletions

diff --git a/src/miniz/README-Prusa.txt b/src/miniz/README-Prusa.txt
index 7ed99c534..494227c6b 100644
--- a/src/miniz/README-Prusa.txt
+++ b/src/miniz/README-Prusa.txt
@@ -2,6 +2,12 @@ This library is based on miniz 2.1.0 - amalgamated version.
 
 ----------------------------------------------------------------
 
+mz_zip_writer_add_staged_open(), mz_zip_writer_add_staged_data() and mz_zip_writer_add_staged_finish() 
+were derived from mz_zip_writer_add_read_buf_callback() by splitting it and passing a new
+mz_zip_writer_staged_context between them.
+
+----------------------------------------------------------------
+
 Merged with https://github.com/richgel999/miniz/pull/147
 to support writing a zipped file using a callback without
 knowing the size of the file up front.
diff --git a/src/miniz/miniz.c b/src/miniz/miniz.c
index 66afb94c2..36be26b28 100644
--- a/src/miniz/miniz.c
+++ b/src/miniz/miniz.c
@@ -5904,13 +5904,6 @@ mz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, c
     return mz_zip_writer_add_mem_ex(pZip, pArchive_name, pBuf, buf_size, NULL, 0, level_and_flags, 0, 0);
 }
 
-typedef struct
-{
-    mz_zip_archive *m_pZip;
-    mz_uint64 m_cur_archive_file_ofs;
-    mz_uint64 m_comp_size;
-} mz_zip_writer_add_state;
-
 static mz_bool mz_zip_writer_add_put_buf_callback(const void *pBuf, int len, void *pUser)
 {
     mz_zip_writer_add_state *pState = (mz_zip_writer_add_state *)pUser;
@@ -5923,7 +5916,6 @@ static mz_bool mz_zip_writer_add_put_buf_callback(const void *pBuf, int len, voi
 }
 
 #define MZ_ZIP64_MAX_LOCAL_EXTRA_FIELD_SIZE (sizeof(mz_uint16) * 2 + sizeof(mz_uint64) * 2)
-#define MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE (sizeof(mz_uint16) * 2 + sizeof(mz_uint64) * 3)
 static mz_uint32 mz_zip_writer_create_zip64_extra_data(mz_uint8 *pBuf, mz_uint64 *pUncomp_size, mz_uint64 *pComp_size, mz_uint64 *pLocal_header_ofs)
 {
     mz_uint8 *pDst = pBuf;
@@ -6683,6 +6675,285 @@ mz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pA
     return MZ_TRUE;
 }
 
+mz_bool mz_zip_writer_add_staged_open(mz_zip_archive* pZip, mz_zip_writer_staged_context* pContext, const char* pArchive_name, mz_uint64 max_size, const MZ_TIME_T* pFile_time, const void* pComment, mz_uint16 comment_size, mz_uint level_and_flags,
+    const char* user_extra_data, mz_uint user_extra_data_len, const char* user_extra_data_central, mz_uint user_extra_data_central_len)
+{
+    mz_uint level, num_alignment_padding_bytes;
+    mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE];
+    mz_zip_internal_state* pState;
+
+    memset(pContext, 0, sizeof(mz_zip_writer_staged_context));
+    pContext->gen_flags = MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR;
+    pContext->uncomp_crc32 = MZ_CRC32_INIT;
+    pContext->method = MZ_DEFLATED;
+    pContext->cur_archive_file_ofs = pZip->m_archive_size;
+    pContext->max_size = max_size;
+    pContext->pArchive_name = pArchive_name;
+    pContext->pComment = pComment;
+    pContext->comment_size = comment_size;
+    pContext->user_extra_data_central = user_extra_data_central;
+    pContext->user_extra_data_central_len = user_extra_data_central_len;
+
+    if (!(level_and_flags & MZ_ZIP_FLAG_ASCII_FILENAME))
+        pContext->gen_flags |= MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8;
+
+    if ((int)level_and_flags < 0)
+        level_and_flags = MZ_DEFAULT_LEVEL;
+    level = level_and_flags & 0xF;
+
+    /* Sanity checks */
+    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level == 0) || (level > MZ_UBER_COMPRESSION) || (max_size < 4))
+        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);
+
+    pState = pZip->m_pState;
+
+    if ((!pState->m_zip64) && (max_size > MZ_UINT32_MAX))
+    {
+        /* Source file is too large for non-zip64 */
+        /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */
+        pState->m_zip64 = MZ_TRUE;
+    }
+
+    /* We could support this, but why? */
+    if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)
+        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);
+
+    if (!mz_zip_writer_validate_archive_name(pArchive_name))
+        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);
+
+    if (pState->m_zip64)
+    {
+        if (pZip->m_total_files == MZ_UINT32_MAX)
+            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);
+    }
+    else
+    {
+        if (pZip->m_total_files == MZ_UINT16_MAX)
+        {
+            pState->m_zip64 = MZ_TRUE;
+            /*return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES); */
+        }
+    }
+
+    pContext->archive_name_size = strlen(pArchive_name);
+    if (pContext->archive_name_size > MZ_UINT16_MAX)
+        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);
+
+    num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);
+
+    /* miniz doesn't support central dirs >= MZ_UINT32_MAX bytes yet */
+    if (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + pContext->archive_name_size + MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE + comment_size) >= MZ_UINT32_MAX)
+        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);
+
+    if (!pState->m_zip64)
+    {
+        /* Bail early if the archive would obviously become too large */
+        if ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + pContext->archive_name_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE
+            + pContext->archive_name_size + comment_size + user_extra_data_len + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE + 1024
+            + MZ_ZIP_DATA_DESCRIPTER_SIZE32 + user_extra_data_central_len) > 0xFFFFFFFF)
+        {
+            pState->m_zip64 = MZ_TRUE;
+            /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */
+        }
+    }
+
+#ifndef MINIZ_NO_TIME
+    if (pFile_time)
+    {
+        mz_zip_time_t_to_dos_time(*pFile_time, &pContext->dos_time, &pContext->dos_date);
+    }
+#endif
+
+    if (!mz_zip_writer_write_zeros(pZip, pContext->cur_archive_file_ofs, num_alignment_padding_bytes))
+    {
+        return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+    }
+
+    pContext->cur_archive_file_ofs += num_alignment_padding_bytes;
+    pContext->local_dir_header_ofs = pContext->cur_archive_file_ofs;
+
+    if (pZip->m_file_offset_alignment)
+    {
+        MZ_ASSERT((cur_archive_file_ofs & (pZip->m_file_offset_alignment - 1)) == 0);
+    }
+
+    MZ_CLEAR_OBJ(local_dir_header);
+    if (pState->m_zip64)
+    {
+        if (max_size >= MZ_UINT32_MAX || pContext->local_dir_header_ofs >= MZ_UINT32_MAX)
+        {
+            pContext->pExtra_data = pContext->extra_data;
+            pContext->extra_size = mz_zip_writer_create_zip64_extra_data(pContext->extra_data, (max_size >= MZ_UINT32_MAX) ? &pContext->uncomp_size : NULL,
+                (max_size >= MZ_UINT32_MAX) ? &pContext->comp_size : NULL, (pContext->local_dir_header_ofs >= MZ_UINT32_MAX) ? &pContext->local_dir_header_ofs : NULL);
+        }
+
+        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)pContext->archive_name_size, (mz_uint16)(pContext->extra_size + user_extra_data_len), 0, 0, 0, pContext->method, pContext->gen_flags, pContext->dos_time, pContext->dos_date))
+            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+
+        pContext->cur_archive_file_ofs += sizeof(local_dir_header);
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, pArchive_name, pContext->archive_name_size) != pContext->archive_name_size)
+        {
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+        }
+
+        pContext->cur_archive_file_ofs += pContext->archive_name_size;
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, pContext->extra_data, pContext->extra_size) != pContext->extra_size)
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+
+        pContext->cur_archive_file_ofs += pContext->extra_size;
+    }
+    else
+    {
+        if ((pContext->comp_size > MZ_UINT32_MAX) || (pContext->cur_archive_file_ofs > MZ_UINT32_MAX))
+            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);
+        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)pContext->archive_name_size, (mz_uint16)user_extra_data_len, 0, 0, 0, pContext->method, pContext->gen_flags, pContext->dos_time, pContext->dos_date))
+            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+
+        pContext->cur_archive_file_ofs += sizeof(local_dir_header);
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, pArchive_name, pContext->archive_name_size) != pContext->archive_name_size)
+        {
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+        }
+
+        pContext->cur_archive_file_ofs += pContext->archive_name_size;
+    }
+
+    if (user_extra_data_len > 0)
+    {
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, user_extra_data, user_extra_data_len) != user_extra_data_len)
+            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);
+
+        pContext->cur_archive_file_ofs += user_extra_data_len;
+    }
+
+    assert(max_size);
+    assert(level);
+
+    pContext->pCompressor = (tdefl_compressor*)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor));
+    if (!pContext->pCompressor)
+    {
+        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);
+    }
+
+    pContext->add_state.m_pZip = pZip;
+    pContext->add_state.m_cur_archive_file_ofs = pContext->cur_archive_file_ofs;
+    pContext->add_state.m_comp_size = 0;
+
+    if (tdefl_init(pContext->pCompressor, mz_zip_writer_add_put_buf_callback, &pContext->add_state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY)
+    {
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pContext->pCompressor);
+        return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);
+    }
+
+    return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_add_staged_data(mz_zip_archive *pZip, mz_zip_writer_staged_context *pContext, const char *pRead_buf, size_t read_buf_size)
+{
+    if (pContext->file_ofs + read_buf_size > pContext->max_size)
+    {
+        mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);
+        pZip->m_pFree(pZip->m_pAlloc_opaque, pContext->pCompressor);
+        return MZ_FALSE;
+    }
+
+    for (;;) {
+        tdefl_status status;
+        tdefl_flush  flush = TDEFL_NO_FLUSH;
+        size_t       n = read_buf_size;
+        if (n > MZ_ZIP_MAX_IO_BUF_SIZE)
+            n = MZ_ZIP_MAX_IO_BUF_SIZE;
+
+        pContext->file_ofs += n;
+        pContext->uncomp_crc32 = (mz_uint32)mz_crc32(pContext->uncomp_crc32, (const mz_uint8 *)pRead_buf, n);
+
+        if (pZip->m_pNeeds_keepalive != NULL && pZip->m_pNeeds_keepalive(pZip->m_pIO_opaque))
+            flush = TDEFL_FULL_FLUSH;
+
+        if (n == 0)
+            flush = TDEFL_FINISH;
+
+        status = tdefl_compress_buffer(pContext->pCompressor, pRead_buf, n, flush);
+        if (status == TDEFL_STATUS_DONE || n < MZ_ZIP_MAX_IO_BUF_SIZE)
+        {
+            return MZ_TRUE;
+        }
+        else if (status != TDEFL_STATUS_OKAY)
+        {
+            mz_zip_set_error(pZip, MZ_ZIP_COMPRESSION_FAILED);
+            pZip->m_pFree(pZip->m_pAlloc_opaque, pContext->pCompressor);
+            return MZ_FALSE;
+        }
+        pRead_buf     += n;
+        read_buf_size -= n;
+    }
+    return MZ_TRUE;
+}
+
+mz_bool mz_zip_writer_add_staged_finish(mz_zip_archive *pZip, mz_zip_writer_staged_context *pContext)
+{
+    if (! mz_zip_writer_add_staged_data(pZip, pContext, NULL, 0))
+        return MZ_FALSE;
+
+    pZip->m_pFree(pZip->m_pAlloc_opaque, pContext->pCompressor);
+
+    pContext->uncomp_size = pContext->file_ofs;
+    pContext->comp_size = pContext->add_state.m_comp_size;
+    pContext->cur_archive_file_ofs = pContext->add_state.m_cur_archive_file_ofs;
+
+    {
+        mz_uint8 local_dir_footer[MZ_ZIP_DATA_DESCRIPTER_SIZE64];
+        mz_uint32 local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE32;
+
+        MZ_WRITE_LE32(local_dir_footer + 0, MZ_ZIP_DATA_DESCRIPTOR_ID);
+        MZ_WRITE_LE32(local_dir_footer + 4, pContext->uncomp_crc32);
+        if (pContext->pExtra_data == NULL)
+        {
+            if (pContext->comp_size > MZ_UINT32_MAX)
+                return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);
+
+            MZ_WRITE_LE32(local_dir_footer + 8, pContext->comp_size);
+            MZ_WRITE_LE32(local_dir_footer + 12, pContext->uncomp_size);
+        }
+        else
+        {
+            MZ_WRITE_LE64(local_dir_footer + 8, pContext->comp_size);
+            MZ_WRITE_LE64(local_dir_footer + 16, pContext->uncomp_size);
+            local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE64;
+        }
+
+        if (pZip->m_pWrite(pZip->m_pIO_opaque, pContext->cur_archive_file_ofs, local_dir_footer, local_dir_footer_size) != local_dir_footer_size)
+            return MZ_FALSE;
+
+        pContext->cur_archive_file_ofs += local_dir_footer_size;
+    }
+
+    if (pContext->pExtra_data != NULL)
+    {
+        pContext->extra_size = mz_zip_writer_create_zip64_extra_data(pContext->extra_data, (pContext->uncomp_size >= MZ_UINT32_MAX) ? &pContext->uncomp_size : NULL,
+                                                           (pContext->uncomp_size >= MZ_UINT32_MAX) ? &pContext->comp_size : NULL, (pContext->local_dir_header_ofs >= MZ_UINT32_MAX) ? &pContext->local_dir_header_ofs : NULL);
+    }
+
+    if (!mz_zip_writer_add_to_central_dir(pZip, pContext->pArchive_name, (mz_uint16)pContext->archive_name_size, pContext->pExtra_data, (mz_uint16)pContext->extra_size, pContext->pComment, pContext->comment_size,
+                                          pContext->uncomp_size, pContext->comp_size, pContext->uncomp_crc32, pContext->method, pContext->gen_flags, pContext->dos_time, pContext->dos_date, pContext->local_dir_header_ofs, pContext->ext_attributes,
+                                          pContext->user_extra_data_central, pContext->user_extra_data_central_len))
+        return MZ_FALSE;
+
+    pZip->m_total_files++;
+    pZip->m_archive_size = pContext->cur_archive_file_ofs;
+
+    return MZ_TRUE;
+}
+
 #ifndef MINIZ_NO_STDIO
 
 static size_t mz_file_read_func_stdio(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)
diff --git a/src/miniz/miniz.h b/src/miniz/miniz.h
index 63e526787..cd647b9fa 100644
--- a/src/miniz/miniz.h
+++ b/src/miniz/miniz.h
@@ -1297,6 +1297,58 @@ mz_bool mz_zip_writer_add_cfile(mz_zip_archive *pZip, const char *pArchive_name,
                                 const char *user_extra_data_central, mz_uint user_extra_data_central_len);
 #endif
 
+#define MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE (sizeof(mz_uint16) * 2 + sizeof(mz_uint64) * 3)
+
+typedef struct
+{
+    mz_zip_archive* m_pZip;
+    mz_uint64 m_cur_archive_file_ofs;
+    mz_uint64 m_comp_size;
+} mz_zip_writer_add_state;
+
+typedef struct mz_zip_writer_staged_context
+{
+    mz_uint16  gen_flags;
+    mz_uint    uncomp_crc32;
+    mz_uint16  method;
+    mz_uint16  dos_time;
+    mz_uint16  dos_date;
+    mz_uint16  ext_attributes;
+    mz_uint64  local_dir_header_ofs;
+    mz_uint64  cur_archive_file_ofs;
+    size_t     archive_name_size;
+    mz_uint64  uncomp_size;
+    mz_uint64  comp_size;
+    mz_uint64  max_size;
+    mz_uint8  *pExtra_data;
+    mz_uint32  extra_size;
+    mz_uint8   extra_data[MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE];
+
+    /*
+     * Compressor context
+     */
+    mz_zip_writer_add_state  add_state;
+    tdefl_compressor        *pCompressor;
+    mz_uint64                file_ofs;
+
+    /*
+     * The following data is passed to the "finish" stage, the referenced pointers must still be valid!
+     */
+    const char* pArchive_name;
+    const void* pComment;
+    mz_uint16    comment_size;
+    const char* user_extra_data_central;
+    mz_uint      user_extra_data_central_len;
+} mz_zip_writer_staged_context;
+
+/* Adds a file to an archive piecewise. Minimum size of the raw data is 4 bytes. */
+/* Don't call mz_zip_writer_add_staged_finish() if mz_zip_writer_add_staged_open() or mz_zip_writer_add_staged_data() fails. */
+mz_bool mz_zip_writer_add_staged_open(mz_zip_archive* pZip, mz_zip_writer_staged_context* pContext, const char* pArchive_name, 
+    mz_uint64 max_size, const MZ_TIME_T* pFile_time, const void* pComment, mz_uint16 comment_size, mz_uint level_and_flags,
+    const char* user_extra_data, mz_uint user_extra_data_len, const char* user_extra_data_central, mz_uint user_extra_data_central_len);
+mz_bool mz_zip_writer_add_staged_data(mz_zip_archive* pZip, mz_zip_writer_staged_context* pContext, const char* pRead_buf, size_t n);
+mz_bool mz_zip_writer_add_staged_finish(mz_zip_archive* pZip, mz_zip_writer_staged_context* pContext);
+
 /* Adds a file to an archive by fully cloning the data from another archive. */
 /* This function fully clones the source file's compressed data (no recompression), along with its full filename, extra data (it may add or modify the zip64 local header extra data field), and the optional descriptor following the compressed data. */
 mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint src_file_index);