feat(C++): remove hardcoded row group size, keep 64M default (#872)

Author: SYaoJun

cpp/src/graphar/filesystem.cc

@@ -266,8 +266,9 @@ Status FileSystem::WriteTableToFile(
   }
   case FileType::PARQUET: {
     auto schema = table->schema();
+    auto row_group_size = options->getParquetMaxRowGroupLength();
     RETURN_NOT_ARROW_OK(parquet::arrow::WriteTable(
-        *table, arrow::default_memory_pool(), output_stream, 64 * 1024 * 1024,
+        *table, arrow::default_memory_pool(), output_stream, row_group_size,
         options->getParquetWriterProperties(),
         options->getArrowWriterProperties()));
     break;
@@ -300,8 +301,9 @@ Status FileSystem::WriteLabelTableToFile(
   parquet::WriterProperties::Builder builder;
   builder.compression(arrow::Compression::type::ZSTD);  // enable compression
   builder.encoding(parquet::Encoding::RLE);
+  auto row_group_size = builder.build()->max_row_group_length();
   RETURN_NOT_ARROW_OK(parquet::arrow::WriteTable(
-      *table, arrow::default_memory_pool(), output_stream, 64 * 1024 * 1024,
+      *table, arrow::default_memory_pool(), output_stream, row_group_size,
       builder.build(), parquet::default_arrow_writer_properties()));
   return Status::OK();
 }

cpp/src/graphar/writer_util.cc

@@ -86,6 +86,13 @@ WriterOptions::getParquetWriterProperties() const {
   return builder.build();
 }
 
+int64_t WriterOptions::getParquetMaxRowGroupLength() const {
+  if (parquetOption_) {
+    return parquetOption_->max_row_group_length;
+  }
+  return parquet::WriterProperties::Builder().build()->max_row_group_length();
+}
+
 std::shared_ptr<parquet::ArrowWriterProperties>
 WriterOptions::getArrowWriterProperties() const {
   parquet::ArrowWriterProperties::Builder builder;

cpp/src/graphar/writer_util.h

@@ -94,7 +94,7 @@ class WriterOptions {
     std::vector<::parquet::SortingColumn> sorting_columns;
     int64_t dictionary_pagesize_limit = 1024 * 1024;
     int64_t write_batch_size = 1024;
-    int64_t max_row_group_length = 1024 * 1024;
+    int64_t max_row_group_length = 64 * 1024 * 1024;
     int64_t data_pagesize = 1024 * 1024;
     size_t max_statistics_size = 4096;
     int compression_level = std::numeric_limits<int>::min();
@@ -429,6 +429,7 @@ class WriterOptions {
   std::shared_ptr<parquet::WriterProperties> getParquetWriterProperties() const;
   std::shared_ptr<parquet::ArrowWriterProperties> getArrowWriterProperties()
       const;
+  int64_t getParquetMaxRowGroupLength() const;
 #ifdef ARROW_ORC
   arrow::adapters::orc::WriteOptions getOrcOption() const;
 #endif