From aadaa963496ba2a31c5949dd4ac92f74de2d0a57 Mon Sep 17 00:00:00 2001 From: Arnaud Botella Date: Mon, 18 May 2026 16:43:17 +0200 Subject: [PATCH 1/2] perf(Unzip): better unzip by using in memory unzip --- .vscode/settings.json | 4 +- src/geode/basic/zip_file.cpp | 119 ++++++++++++++++++++++++----------- 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6dc2f2d86..0b76d4650 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "${workspaceFolder}/build/third_party/asyncplusplus/install/include", "${workspaceFolder}/build/third_party/earcut/install/include", "${workspaceFolder}/build/third_party/bitsery/install/include", - "${workspaceFolder}/build/third_party/minizip/install/include", + "${workspaceFolder}/build/third_party/minizip/install/include/minizip-ng", "${workspaceFolder}/build/third_party/nanoflann/install/include", "${workspaceFolder}/build/third_party/pybind11/install/include", "${workspaceFolder}/build/third_party/spdlog/install/include", @@ -21,7 +21,7 @@ "${workspaceFolder}/build/third_party/asyncplusplus/install/include", "${workspaceFolder}/build/third_party/earcut/install/include", "${workspaceFolder}/build/third_party/bitsery/install/include", - "${workspaceFolder}/build/third_party/minizip/install/include", + "${workspaceFolder}/build/third_party/minizip/install/include/minizip-ng", "${workspaceFolder}/build/third_party/nanoflann/install/include", "${workspaceFolder}/build/third_party/pybind11/install/include", "${workspaceFolder}/build/third_party/spdlog/install/include", diff --git a/src/geode/basic/zip_file.cpp b/src/geode/basic/zip_file.cpp index 0b374212c..934512140 100644 --- a/src/geode/basic/zip_file.cpp +++ b/src/geode/basic/zip_file.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -63,25 +64,23 @@ namespace geode writer_, to_string( file ).c_str(), 0, 0 ); if( status != MZ_OK ) { - std::filesystem::remove_all( directory_ ); throw OpenGeodeBasicException( nullptr, OpenGeodeException::TYPE::internal, - "[ZipFile] Error opening zip for writing." ); + "[ZipFile] Error opening zip for writing (", status, ")" ); } } ~Impl() { - std::filesystem::remove_all( directory_ ); - const auto status = mz_zip_writer_close( writer_ ); - if( status != MZ_OK ) + if( writer_ ) { - Logger::error( "[ZipFile] Error closing zip for writing" ); + mz_zip_writer_close( writer_ ); + mz_zip_writer_delete( &writer_ ); } - mz_zip_writer_delete( &writer_ ); + std::filesystem::remove_all( directory_ ); } - void archive_files( absl::Span< const std::string_view >& files ) const + void archive_files( absl::Span< const std::string_view > files ) const { for( const auto& file : files ) { @@ -96,10 +95,10 @@ namespace geode writer_, file_path.string().c_str(), nullptr, 0, 1 ); if( status != MZ_OK ) { - std::filesystem::remove_all( directory_ ); throw OpenGeodeBasicException( nullptr, OpenGeodeException::TYPE::internal, - "[ZipFile::archive_file] Error adding path to zip" ); + "[ZipFile::archive_file] Error adding path to zip (", + status, ")" ); } std::filesystem::remove( file_path ); } @@ -144,12 +143,14 @@ namespace geode Impl( std::string_view file, std::string_view unarchive_temp_filename ) { directory_ = create_directory( file, unarchive_temp_filename ); - reader_ = mz_zip_reader_create(); - const auto status = - mz_zip_reader_open_file( reader_, to_string( file ).c_str() ); - if( status != MZ_OK ) + if( !load_zip_into_memory( file ) ) + { + throw OpenGeodeBasicException( nullptr, + OpenGeodeException::TYPE::internal, + "[UnzipFile] Failed to read zip file into memory" ); + } + if( !open_reader() ) { - std::filesystem::remove_all( directory_ ); throw OpenGeodeBasicException( nullptr, OpenGeodeException::TYPE::internal, "[UnzipFile] Error opening zip for reading" ); @@ -158,38 +159,56 @@ namespace geode ~Impl() { + if( reader_ ) + { + mz_zip_reader_close( reader_ ); + mz_zip_reader_delete( &reader_ ); + reader_ = nullptr; + } + if( memory_stream_ ) + { + mz_stream_close( memory_stream_ ); + mz_stream_delete( &memory_stream_ ); + memory_stream_ = nullptr; + } std::filesystem::remove_all( directory_ ); - mz_zip_reader_close( reader_ ); - mz_zip_reader_delete( &reader_ ); } void extract_all() const { - auto status = mz_zip_reader_goto_first_entry( reader_ ); + constexpr size_t BUF_SIZE = 1024 * 1024; // 1 MB + std::vector< uint8_t > buffer( BUF_SIZE ); + int status = mz_zip_reader_goto_first_entry( reader_ ); while( status == MZ_OK ) { - mz_zip_file* file_info{ nullptr }; - status = mz_zip_reader_entry_get_info( reader_, &file_info ); - if( status != MZ_OK ) + mz_zip_file* info = nullptr; + mz_zip_reader_entry_get_info( reader_, &info ); + if( !info ) { - std::filesystem::remove_all( directory_ ); - throw OpenGeodeBasicException( nullptr, - OpenGeodeException::TYPE::internal, - "[UnzipFile::extract_all] Error getting entry info in " - "zip file" ); + status = mz_zip_reader_goto_next_entry( reader_ ); + continue; } - - auto file = directory_ / file_info->filename; - status = mz_zip_reader_entry_save_file( - reader_, file.string().c_str() ); - if( status != MZ_OK ) + auto out_path = directory_ / info->filename; + std::filesystem::create_directories( out_path.parent_path() ); + FILE* f = fopen( out_path.string().c_str(), "wb" ); + if( !f ) { - std::filesystem::remove_all( directory_ ); - throw OpenGeodeBasicException( nullptr, - OpenGeodeException::TYPE::internal, - "[UnzipFile::extract_all] Error extracting entry " - "file" ); + status = mz_zip_reader_goto_next_entry( reader_ ); + continue; } + status = mz_zip_reader_entry_open( reader_ ); + if( status == MZ_OK ) + { + int32_t bytes_read = 0; + while( ( bytes_read = mz_zip_reader_entry_read( + reader_, buffer.data(), BUF_SIZE ) ) + > 0 ) + { + fwrite( buffer.data(), 1, bytes_read, f ); + } + mz_zip_reader_entry_close( reader_ ); + } + fclose( f ); status = mz_zip_reader_goto_next_entry( reader_ ); } } @@ -199,9 +218,37 @@ namespace geode return directory_.string(); } + private: + bool load_zip_into_memory( std::string_view file ) + { + std::ifstream ifs( + to_string( file ), std::ios::binary | std::ios::ate ); + if( !ifs.is_open() ) + { + return false; + } + auto size = ifs.tellg(); + zip_data_.resize( static_cast< size_t >( size ) ); + ifs.seekg( 0 ); + ifs.read( reinterpret_cast< char* >( zip_data_.data() ), size ); + return ifs.good(); + } + + bool open_reader() + { + memory_stream_ = mz_stream_mem_create(); + mz_stream_mem_set_buffer( memory_stream_, (void*) zip_data_.data(), + (int32_t) zip_data_.size() ); + reader_ = mz_zip_reader_create(); + return mz_zip_reader_open( reader_, memory_stream_ ) == MZ_OK; + } + private: std::filesystem::path directory_; + std::vector< uint8_t > zip_data_; + void* reader_{ nullptr }; + void* memory_stream_{ nullptr }; }; UnzipFile::UnzipFile( From 7c19a2ce0f011a1ee14ce57b457c52d08cee6f1b Mon Sep 17 00:00:00 2001 From: Arnaud Botella Date: Mon, 18 May 2026 20:52:55 +0200 Subject: [PATCH 2/2] fix --- src/geode/basic/zip_file.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/geode/basic/zip_file.cpp b/src/geode/basic/zip_file.cpp index 934512140..87f176b20 100644 --- a/src/geode/basic/zip_file.cpp +++ b/src/geode/basic/zip_file.cpp @@ -237,8 +237,9 @@ namespace geode bool open_reader() { memory_stream_ = mz_stream_mem_create(); - mz_stream_mem_set_buffer( memory_stream_, (void*) zip_data_.data(), - (int32_t) zip_data_.size() ); + mz_stream_mem_set_buffer( memory_stream_, + static_cast< void* >( zip_data_.data() ), + static_cast< int32_t >( zip_data_.size() ) ); reader_ = mz_zip_reader_create(); return mz_zip_reader_open( reader_, memory_stream_ ) == MZ_OK; }