Skip to content

perf(Unzip): better unzip by using in memory unzip #1274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BotellaA merged 2 commits into from
May 18, 2026
Merged

perf(Unzip): better unzip by using in memory unzip #1274

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aadaa96
perf(Unzip): better unzip by using in memory unzip
BotellaA May 18, 2026
7c19a2c
fix
BotellaA May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"${workspaceFolder}/build/third_party/asyncplusplus/install/include",
"${workspaceFolder}/build/third_party/earcut/install/include",
"${workspaceFolder}/build/third_party/bitsery/install/include",
"${workspaceFolder}/build/third_party/minizip/install/include",
"${workspaceFolder}/build/third_party/minizip/install/include/minizip-ng",
"${workspaceFolder}/build/third_party/nanoflann/install/include",
"${workspaceFolder}/build/third_party/pybind11/install/include",
"${workspaceFolder}/build/third_party/spdlog/install/include",
Expand All @@ -21,7 +21,7 @@
"${workspaceFolder}/build/third_party/asyncplusplus/install/include",
"${workspaceFolder}/build/third_party/earcut/install/include",
"${workspaceFolder}/build/third_party/bitsery/install/include",
"${workspaceFolder}/build/third_party/minizip/install/include",
"${workspaceFolder}/build/third_party/minizip/install/include/minizip-ng",
"${workspaceFolder}/build/third_party/nanoflann/install/include",
"${workspaceFolder}/build/third_party/pybind11/install/include",
"${workspaceFolder}/build/third_party/spdlog/install/include",
Expand Down
119 changes: 83 additions & 36 deletions src/geode/basic/zip_file.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@
#include <fstream>
#include <string_view>

#include <mz.h>

Check failure on line 30 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:30:10 [clang-diagnostic-error] 

'mz.h' file not found
#include <mz_strm.h>
#include <mz_strm_mem.h>
#include <mz_zip.h>
#include <mz_zip_rw.h>

Expand All @@ -38,7 +39,7 @@
namespace
{
std::filesystem::path create_directory(
std::string_view file, std::string_view temp_filename )

Check warning on line 42 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:42:9 [bugprone-easily-swappable-parameters] 

2 adjacent parameters of 'create_directory' of similar type ('std::string_view') are easily swapped by mistake
{
const auto file_string = geode::to_string( file );
auto directory = std::filesystem::path{ file_string }.parent_path()
Expand All @@ -50,38 +51,36 @@

namespace geode
{
class ZipFile::Impl

Check warning on line 54 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:54:20 [cppcoreguidelines-special-member-functions] 

class 'Impl' defines a non-default destructor but does not define a copy constructor, a copy assignment operator, a move constructor or a move assignment operator
{
public:
Impl( std::string_view file, std::string_view archive_temp_filename )
{
directory_ = create_directory( file, archive_temp_filename );
writer_ = mz_zip_writer_create();

Check warning on line 60 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:60:13 [cppcoreguidelines-prefer-member-initializer] 

'writer_' should be initialized in a member initializer of the constructor
mz_zip_writer_set_compress_method(
writer_, MZ_COMPRESS_METHOD_STORE );
const auto status = mz_zip_writer_open_file(
writer_, to_string( file ).c_str(), 0, 0 );
if( status != MZ_OK )
{
std::filesystem::remove_all( directory_ );
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[ZipFile] Error opening zip for writing." );
"[ZipFile] Error opening zip for writing (", status, ")" );
}
}

~Impl()
{
std::filesystem::remove_all( directory_ );
const auto status = mz_zip_writer_close( writer_ );
if( status != MZ_OK )
if( writer_ )

Check warning on line 75 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:75:17 [readability-implicit-bool-conversion] 

implicit conversion 'void *' -> 'bool'
{
Logger::error( "[ZipFile] Error closing zip for writing" );
mz_zip_writer_close( writer_ );
mz_zip_writer_delete( &writer_ );
}
mz_zip_writer_delete( &writer_ );
std::filesystem::remove_all( directory_ );
}

void archive_files( absl::Span< const std::string_view >& files ) const
void archive_files( absl::Span< const std::string_view > files ) const
{
for( const auto& file : files )
{
Expand All @@ -96,15 +95,15 @@
writer_, file_path.string().c_str(), nullptr, 0, 1 );
if( status != MZ_OK )
{
std::filesystem::remove_all( directory_ );
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[ZipFile::archive_file] Error adding path to zip" );
"[ZipFile::archive_file] Error adding path to zip (",
status, ")" );
}
std::filesystem::remove( file_path );
}

std::string directory() const

Check warning on line 106 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:106:9 [modernize-use-nodiscard] 

function 'directory' should be marked [[nodiscard]]
{
return directory_.string();
}
Expand Down Expand Up @@ -138,18 +137,20 @@
return impl_->directory();
}

class UnzipFile::Impl

Check warning on line 140 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:140:22 [cppcoreguidelines-special-member-functions] 

class 'Impl' defines a non-default destructor but does not define a copy constructor, a copy assignment operator, a move constructor or a move assignment operator
{
public:
Impl( std::string_view file, std::string_view unarchive_temp_filename )

Check warning on line 143 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:143:9 [cppcoreguidelines-pro-type-member-init] 

constructor does not initialize these fields: zip_data_
{
directory_ = create_directory( file, unarchive_temp_filename );
reader_ = mz_zip_reader_create();
const auto status =
mz_zip_reader_open_file( reader_, to_string( file ).c_str() );
if( status != MZ_OK )
if( !load_zip_into_memory( file ) )
{
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[UnzipFile] Failed to read zip file into memory" );
}
if( !open_reader() )
{
std::filesystem::remove_all( directory_ );
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[UnzipFile] Error opening zip for reading" );
Expand All @@ -158,38 +159,56 @@

~Impl()
{
if( reader_ )

Check warning on line 162 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:162:17 [readability-implicit-bool-conversion] 

implicit conversion 'void *' -> 'bool'
{
mz_zip_reader_close( reader_ );
mz_zip_reader_delete( &reader_ );
reader_ = nullptr;
}
if( memory_stream_ )

Check warning on line 168 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:168:17 [readability-implicit-bool-conversion] 

implicit conversion 'void *' -> 'bool'
{
mz_stream_close( memory_stream_ );
mz_stream_delete( &memory_stream_ );
memory_stream_ = nullptr;
}
std::filesystem::remove_all( directory_ );
mz_zip_reader_close( reader_ );
mz_zip_reader_delete( &reader_ );
}

void extract_all() const
{
auto status = mz_zip_reader_goto_first_entry( reader_ );
constexpr size_t BUF_SIZE = 1024 * 1024; // 1 MB

Check warning on line 179 in src/geode/basic/zip_file.cpp

View workflow job for this annotation

GitHub Actions / test / tidy

src/geode/basic/zip_file.cpp:179:41 [bugprone-implicit-widening-of-multiplication-result] 

performing an implicit widening conversion to type 'const size_t' (aka 'const unsigned long') of a multiplication performed in type 'int'
std::vector< uint8_t > buffer( BUF_SIZE );
int status = mz_zip_reader_goto_first_entry( reader_ );
while( status == MZ_OK )
{
mz_zip_file* file_info{ nullptr };
status = mz_zip_reader_entry_get_info( reader_, &file_info );
if( status != MZ_OK )
mz_zip_file* info = nullptr;
mz_zip_reader_entry_get_info( reader_, &info );
if( !info )
{
std::filesystem::remove_all( directory_ );
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[UnzipFile::extract_all] Error getting entry info in "
"zip file" );
status = mz_zip_reader_goto_next_entry( reader_ );
continue;
}

auto file = directory_ / file_info->filename;
status = mz_zip_reader_entry_save_file(
reader_, file.string().c_str() );
if( status != MZ_OK )
auto out_path = directory_ / info->filename;
std::filesystem::create_directories( out_path.parent_path() );
FILE* f = fopen( out_path.string().c_str(), "wb" );
if( !f )
{
std::filesystem::remove_all( directory_ );
throw OpenGeodeBasicException( nullptr,
OpenGeodeException::TYPE::internal,
"[UnzipFile::extract_all] Error extracting entry "
"file" );
status = mz_zip_reader_goto_next_entry( reader_ );
continue;
}
status = mz_zip_reader_entry_open( reader_ );
if( status == MZ_OK )
{
int32_t bytes_read = 0;
while( ( bytes_read = mz_zip_reader_entry_read(
reader_, buffer.data(), BUF_SIZE ) )
> 0 )
{
fwrite( buffer.data(), 1, bytes_read, f );
}
mz_zip_reader_entry_close( reader_ );
}
fclose( f );
status = mz_zip_reader_goto_next_entry( reader_ );
}
}
Expand All @@ -199,9 +218,37 @@
return directory_.string();
}

private:
bool load_zip_into_memory( std::string_view file )
{
std::ifstream ifs(
to_string( file ), std::ios::binary | std::ios::ate );
if( !ifs.is_open() )
{
return false;
}
auto size = ifs.tellg();
zip_data_.resize( static_cast< size_t >( size ) );
ifs.seekg( 0 );
ifs.read( reinterpret_cast< char* >( zip_data_.data() ), size );
return ifs.good();
}

bool open_reader()
{
memory_stream_ = mz_stream_mem_create();
mz_stream_mem_set_buffer( memory_stream_, (void*) zip_data_.data(),
(int32_t) zip_data_.size() );
reader_ = mz_zip_reader_create();
return mz_zip_reader_open( reader_, memory_stream_ ) == MZ_OK;
}

private:
std::filesystem::path directory_;
std::vector< uint8_t > zip_data_;

void* reader_{ nullptr };
void* memory_stream_{ nullptr };
};

UnzipFile::UnzipFile(
Expand Down
Loading