Merge branch 'main' into adityasn/add-spec-partition-field-reuse

Author: liurenjie1024

…ation_tests/testdata/spark/entrypoint.sh .github/actions/get-msrv/action.ymlcrates/integration_tests/testdata/spark/entrypoint.sh renamed to .github/actions/get-msrv/action.yml crates/integration_tests/testdata/spark/entrypoint.sh renamed to .github/actions/get-msrv/action.yml

@@ -1,5 +1,3 @@
-#!/bin/bash
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,18 +14,22 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
-
-set -e
-
-start-master.sh -p 7077
-start-worker.sh spark://spark-iceberg:7077
 
-echo "Starting provision"
-python3 ./provision.py
+name: Get MSRV
+description: Get the Minimum Supported Rust Version from Cargo.toml
 
-echo "Finished provisioning"
-touch /tmp/ready
+outputs:
+  msrv:
+    description: The MSRV extracted from Cargo.toml
+    value: ${{ steps.get-msrv.outputs.msrv }}
 
-echo "Print logs"
-tail -f $SPARK_HOME/logs/*
+runs:
+  using: composite
+  steps:
+    - name: Get MSRV
+      id: get-msrv
+      shell: bash
+      run: |
+        msrv=$(grep '^rust-version' Cargo.toml | sed 's/.*"\(.*\)".*/\1/')
+        echo "msrv=$msrv" >> $GITHUB_OUTPUT
+        echo "MSRV is $msrv"

.github/workflows/ci.yml

@@ -40,9 +40,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
-env:
-  rust_msrv: "1.87"
-
 jobs:
   check:
     runs-on: ${{ matrix.os }}
@@ -184,10 +181,13 @@ jobs:
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get MSRV
+        id: get-msrv
+        uses: ./.github/actions/get-msrv
       - name: Setup MSRV Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: ${{ env.rust_msrv }}
+          rust-version: ${{ steps.get-msrv.outputs.msrv }}
       - name: Setup Nightly Rust toolchain
         uses: ./.github/actions/setup-builder
       - name: Check MSRV

.github/workflows/publish.yml

@@ -26,9 +26,6 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
   workflow_dispatch:
 
-env:
-  rust_msrv: "1.87"
-
 jobs:
   publish:
     runs-on: ubuntu-latest
@@ -48,10 +45,14 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
+      - name: Get MSRV
+        id: get-msrv
+        uses: ./.github/actions/get-msrv
+
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: ${{ env.rust_msrv }}
+          rust-version: ${{ steps.get-msrv.outputs.msrv }}
 
       - name: Publish ${{ matrix.package }}
         working-directory: ${{ matrix.package }}

.github/workflows/release_python.yml

@@ -24,9 +24,6 @@ on:
       - completed
   workflow_dispatch:
 
-env:
-  rust_msrv: "1.87"
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}-${{ github.event_name }}
   cancel-in-progress: true
@@ -147,10 +144,14 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: 3.12
+      - name: Get MSRV
+        id: get-msrv
+        uses: ./.github/actions/get-msrv
+
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: ${{ env.rust_msrv }}
+          rust-version: ${{ steps.get-msrv.outputs.msrv }}
       - uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}

.github/workflows/release_python_nightly.yml

@@ -22,9 +22,6 @@ on:
     - cron: "0 0 * * *" # Runs at midnight UTC every day
   workflow_dispatch: # Allows manual triggering
 
-env:
-  rust_msrv: "1.87"
-
 permissions:
   contents: read
 
@@ -88,10 +85,14 @@ jobs:
         with:
           python-version: 3.12
 
+      - name: Get MSRV
+        id: get-msrv
+        uses: ./.github/actions/get-msrv
+
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: ${{ env.rust_msrv }}
+          rust-version: ${{ steps.get-msrv.outputs.msrv }}
 
       - uses: PyO3/maturin-action@v1
         with:

Cargo.lock

@@ -272,7 +272,7 @@ pub(super) mod _serde {
 
     #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
     #[serde(rename_all = "kebab-case")]
-    /// Defines the structure of a v2 snapshot for serialization/deserialization
+    /// Defines the structure of a v3 snapshot for serialization/deserialization
     pub(crate) struct SnapshotV3 {
         pub snapshot_id: i64,
         #[serde(skip_serializing_if = "Option::is_none")]
@@ -283,8 +283,10 @@ pub(super) mod _serde {
         pub summary: Summary,
         #[serde(skip_serializing_if = "Option::is_none")]
         pub schema_id: Option<SchemaId>,
-        pub first_row_id: u64,
-        pub added_rows: u64,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        pub first_row_id: Option<u64>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        pub added_rows: Option<u64>,
         #[serde(skip_serializing_if = "Option::is_none")]
         pub key_id: Option<String>,
     }
@@ -333,10 +335,13 @@ pub(super) mod _serde {
                 summary: s.summary,
                 schema_id: s.schema_id,
                 encryption_key_id: s.key_id,
-                row_range: Some(SnapshotRowRange {
-                    first_row_id: s.first_row_id,
-                    added_rows: s.added_rows,
-                }),
+                row_range: match (s.first_row_id, s.added_rows) {
+                    (Some(first_row_id), Some(added_rows)) => Some(SnapshotRowRange {
+                        first_row_id,
+                        added_rows,
+                    }),
+                    _ => None,
+                },
             }
         }
     }
@@ -345,12 +350,10 @@ pub(super) mod _serde {
         type Error = Error;
 
         fn try_from(s: Snapshot) -> Result<Self, Self::Error> {
-            let row_range = s.row_range.ok_or_else(|| {
-                Error::new(
-                    crate::ErrorKind::DataInvalid,
-                    "v3 Snapshots must have first-row-id and rows-added fields set.".to_string(),
-                )
-            })?;
+            let (first_row_id, added_rows) = match s.row_range {
+                Some(row_range) => (Some(row_range.first_row_id), Some(row_range.added_rows)),
+                None => (None, None),
+            };
 
             Ok(SnapshotV3 {
                 snapshot_id: s.snapshot_id,
@@ -360,8 +363,8 @@ pub(super) mod _serde {
                 manifest_list: s.manifest_list,
                 summary: s.summary,
                 schema_id: s.schema_id,
-                first_row_id: row_range.first_row_id,
-                added_rows: row_range.added_rows,
+                first_row_id,
+                added_rows,
                 key_id: s.encryption_key_id,
             })
         }