#2430 7z unzip error for TDEFL_SYNC_FLUSH param

[ubuntuser]

7z unzip attachment error, but 7z version 9.2 unzip correct. There is something wrong with newest version.

[ubuntuser]

platform：windows x64

[Igor Pavlov]

I suppose deflate stream is incorrect in that archive
There is no BFINAL bit at the end of deflate stream.
What program was used to create that archive?
rfc1951:

      3.2.3. Details of block format
         Each block of compressed data begins with 3 header bits containing the following data:

            first bit       BFINAL
            next 2 bits     BTYPE

         BFINAL is set if and only if this is the last block of the data
         set.

[ubuntuser]

thanks，I have encapsulated a zip_flush method using the TDEFL_SYNC_FLUSH parameter, which is used to flush compressed content into a zip file. This method is called when a file has finished writing, and the BFINAL bit is written at the end of the file stream. However, during the zip_flush process, there could be an unexpected power outage, resulting in the BFINAL bit not being written at the end of the file stream. This could lead to the occurrence of such files. Do you have any suggestions for handling this issue?

zip_entry_flush runing on linux system.

int zip_entry_flush(struct zip_t *zip, uint64_t* tail_offset, uint64_t* tail_length, uint8_t* tail_buffer, bool flag_sync) {
  mz_zip_archive *pzip = NULL;
  mz_uint level;
  tdefl_status done;
  mz_uint16 entrylen;
  mz_uint16 dos_time = 0, dos_date = 0;
  int err = 0;
  mz_uint8 *pExtra_data = NULL;
  mz_uint32 extra_size = 0;
  mz_uint8 extra_data[MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE];
  mz_uint8 local_dir_footer[MZ_ZIP_DATA_DESCRIPTER_SIZE64];
  mz_uint32 local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE64;

  if (!zip) {
    // zip_t handler is not initialized
    err = ZIP_ENOINIT;
    return -1;
  }

  pzip = &(zip->archive);
  //-------------------------------------------------------------------------------------
  pzip->m_zip_mode = MZ_ZIP_MODE_WRITING;
  if (0 != pzip->m_pState->m_central_dir_offsets.m_size) {
    ssize_t entry_num = zip_entries_total(zip);
    mz_bool *deleted_entry_flag_array = (mz_bool *)calloc(entry_num, sizeof(mz_bool));
    ssize_t i = 0;
    while (i < entry_num) {
      deleted_entry_flag_array[i] = MZ_TRUE;
      ++i;
    }
    zip_central_dir_delete(pzip->m_pState, deleted_entry_flag_array, entry_num);
    CLEANUP(deleted_entry_flag_array);

    pzip->m_total_files--;
  }
  //-------------------------------------------------------------------------------------

  if (pzip->m_zip_mode == MZ_ZIP_MODE_READING) {
    return -1;
  }

  level = zip->level & 0xF;
  if (level) {
    done = tdefl_compress_buffer(&(zip->entry.comp), "", 0, TDEFL_SYNC_FLUSH);
    if (done != TDEFL_STATUS_DONE && done != TDEFL_STATUS_OKAY) {
      // Cannot flush compressed buffer
      err = ZIP_ETDEFLBUF;
      return -1;
    }
    zip->entry.comp_size = zip->entry.state.m_comp_size;
    zip->entry.offset = zip->entry.state.m_cur_archive_file_ofs;
    zip->entry.method = MZ_DEFLATED;
  }
  *tail_offset = zip->entry.offset;

  entrylen = (mz_uint16)strlen(zip->entry.name);
#ifndef MINIZ_NO_TIME
  mz_zip_time_t_to_dos_time(zip->entry.m_time, &dos_time, &dos_date);
#endif

  MZ_WRITE_LE32(local_dir_footer + 0, MZ_ZIP_DATA_DESCRIPTOR_ID);
  MZ_WRITE_LE32(local_dir_footer + 4, zip->entry.uncomp_crc32);
  MZ_WRITE_LE64(local_dir_footer + 8, zip->entry.comp_size);
  MZ_WRITE_LE64(local_dir_footer + 16, zip->entry.uncomp_size);

  if (pzip->m_pWrite(pzip->m_pIO_opaque, zip->entry.offset, local_dir_footer, local_dir_footer_size) !=
      local_dir_footer_size) {
    // Cannot write zip entry header
    err = ZIP_EWRTHDR;
    return -1;
  }
  memcpy(tail_buffer, local_dir_footer, local_dir_footer_size);
  zip->entry.offset += local_dir_footer_size;

  pExtra_data = extra_data;
  extra_size = mz_zip_writer_create_zip64_extra_data(
      extra_data, (zip->entry.uncomp_size >= MZ_UINT32_MAX) ? &zip->entry.uncomp_size : NULL,
      (zip->entry.comp_size >= MZ_UINT32_MAX) ? &zip->entry.comp_size : NULL,
      (zip->entry.header_offset >= MZ_UINT32_MAX) ? &zip->entry.header_offset : NULL);

  if ((entrylen) && (zip->entry.name[entrylen - 1] == '/') && !zip->entry.uncomp_size) {
    /* Set DOS Subdirectory attribute bit. */
    zip->entry.external_attr |= MZ_ZIP_DOS_DIR_ATTRIBUTE_BITFLAG;
  }
  if (!mz_zip_writer_add_to_central_dir(
          pzip, zip->entry.name, entrylen, pExtra_data, (mz_uint16)extra_size, "", 0, zip->entry.uncomp_size,
          zip->entry.comp_size, zip->entry.uncomp_crc32, zip->entry.method,
          MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8 | MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR, dos_time, dos_date,
          zip->entry.header_offset, zip->entry.external_attr, NULL, 0)) {
    // Cannot write to zip central dir
    err = ZIP_EWRTDIR;
    return -1;
  }
  pzip->m_total_files++;
  pzip->m_archive_size = zip->entry.offset;
  //--------------------------------------------------------------------------------
  mz_zip_internal_state *pState;
  mz_uint64 central_dir_ofs, central_dir_size;
  mz_uint8 hdr[256];

  if ((!pzip) || (!pzip->m_pState) || (pzip->m_zip_mode != MZ_ZIP_MODE_WRITING))
    return mz_zip_set_error(pzip, MZ_ZIP_INVALID_PARAMETER);

  pState = pzip->m_pState;

  if (pState->m_zip64) {
    if ((pzip->m_total_files > MZ_UINT32_MAX) || (pState->m_central_dir.m_size >= MZ_UINT32_MAX))
      return mz_zip_set_error(pzip, MZ_ZIP_TOO_MANY_FILES);
  } else {
    if ((pzip->m_total_files > MZ_UINT16_MAX) ||
        ((pzip->m_archive_size + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) > MZ_UINT32_MAX))
      return mz_zip_set_error(pzip, MZ_ZIP_TOO_MANY_FILES);
  }

  central_dir_ofs = 0;
  central_dir_size = 0;
  if (pzip->m_total_files) {
    /* Write central directory */
    central_dir_ofs = pzip->m_archive_size;
    central_dir_size = pState->m_central_dir.m_size;
    pzip->m_central_directory_file_ofs = central_dir_ofs;
    if (pzip->m_pWrite(pzip->m_pIO_opaque, central_dir_ofs, pState->m_central_dir.m_p, (size_t)central_dir_size) !=
        central_dir_size)
      return mz_zip_set_error(pzip, MZ_ZIP_FILE_WRITE_FAILED);

    memcpy(tail_buffer + local_dir_footer_size, pState->m_central_dir.m_p, central_dir_size);
    pzip->m_archive_size += central_dir_size;
  }

  if (pState->m_zip64) {
    /* Write zip64 end of central directory header */
    mz_uint64 rel_ofs_to_zip64_ecdr = pzip->m_archive_size;

    MZ_CLEAR_OBJ(hdr);
    MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDH_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIG);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_SIZE_OF_RECORD_OFS,
                  MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE - sizeof(mz_uint32) - sizeof(mz_uint64));
    MZ_WRITE_LE16(hdr + MZ_ZIP64_ECDH_VERSION_MADE_BY_OFS, 0x031E); /* TODO: always Unix */
    MZ_WRITE_LE16(hdr + MZ_ZIP64_ECDH_VERSION_NEEDED_OFS, 0x002D);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, pzip->m_total_files);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_TOTAL_ENTRIES_OFS, pzip->m_total_files);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_SIZE_OFS, central_dir_size);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_OFS_OFS, central_dir_ofs);
    if (pzip->m_pWrite(pzip->m_pIO_opaque, pzip->m_archive_size, hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE) !=
        MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE)
      return mz_zip_set_error(pzip, MZ_ZIP_FILE_WRITE_FAILED);

    memcpy(tail_buffer + local_dir_footer_size + central_dir_size, hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE);
    pzip->m_archive_size += MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE;

    /* Write zip64 end of central directory locator */
    MZ_CLEAR_OBJ(hdr);
    MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIG);
    MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDL_REL_OFS_TO_ZIP64_ECDR_OFS, rel_ofs_to_zip64_ecdr);
    MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_TOTAL_NUMBER_OF_DISKS_OFS, 1);
    if (pzip->m_pWrite(pzip->m_pIO_opaque, pzip->m_archive_size, hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE) !=
        MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE)
      return mz_zip_set_error(pzip, MZ_ZIP_FILE_WRITE_FAILED);

    memcpy(tail_buffer + local_dir_footer_size + central_dir_size +
               MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE,
           hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE);
    pzip->m_archive_size += MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE;
  }
  /* Write end of central directory record */
  MZ_CLEAR_OBJ(hdr);
  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_SIG_OFS, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG);
  MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, MZ_MIN(MZ_UINT16_MAX, pzip->m_total_files));
  MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS, MZ_MIN(MZ_UINT16_MAX, pzip->m_total_files));
  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_SIZE_OFS, MZ_MIN(MZ_UINT32_MAX, central_dir_size));
  MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_OFS_OFS, MZ_MIN(MZ_UINT32_MAX, central_dir_ofs));

  if (pzip->m_pWrite(pzip->m_pIO_opaque, pzip->m_archive_size, hdr, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) !=
      MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)
    return mz_zip_set_error(pzip, MZ_ZIP_FILE_WRITE_FAILED);

#ifndef MINIZ_NO_STDIO
  if ((pState->m_pFile) && (MZ_FFLUSH(pState->m_pFile) == EOF)) return mz_zip_set_error(pzip, MZ_ZIP_FILE_CLOSE_FAILED);
#endif /* #ifndef MINIZ_NO_STDIO */
  memcpy(tail_buffer + local_dir_footer_size + central_dir_size +
             MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE +
             MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE,
         hdr, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE);
  pzip->m_archive_size += MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE;
  pzip->m_zip_mode = MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED;
  *tail_length = local_dir_footer_size + central_dir_size +
                 MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE +
                 MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE +
                 MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE;
  //--------------------------------------------------------------------------------
  if ((pzip->m_pWrite == mz_zip_heap_write_func) && (pState->m_pMem)) {
    return 0;
  }
  if (pzip->m_zip_mode == MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED) {
    if (pState->m_pFile) {
      int fd = fileno(pState->m_pFile);
      ftruncate(fd, pzip->m_archive_size);
      if (flag_sync) {
        fdatasync(fd);
      }
    }
  }

  return 0;
}

[ubuntuser]

Additionally, why is it that when the attachment test.zip is uncompressed on Ubuntu system, there are no error messages? Thanks

[Igor Pavlov]

It can be difficult for me to search the bugs in your big code.
Just follow deflate RFC format description and write correct deflate stream to file.
Correct deflate stream must contain BFINAL bit set to 1.
Your file has no BFINAL bit. It means that file is not correct. So 7-Zip shows error message. And that error message helps you to know that file is not correct, and that code that was used to create that archive is not correct also.

[ubuntuser]

Thanks, but why this file corretly unzip with version 9.2 7zip.

[Igor Pavlov]

Because new 7-Zip is better.
Old 7-zip couldn't detect some errors in archive.
And new 7-Zip can detect such errors.