From 8e43370d28ef02c771037dda4b4b35805bfae0bc Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 5 Dec 2025 17:11:57 -0800 Subject: [PATCH 1/9] c --- rust/otap-dataflow/Cargo.toml | 8 ++++---- rust/otap-dataflow/Dockerfile | 4 ++++ rust/otap-dataflow/configs/fake-parquet.yaml | 9 ++++++--- rust/otap-dataflow/crates/otap/Cargo.toml | 2 +- rust/otap-dataflow/cross-arch-build.sh | 12 ++++++++---- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/rust/otap-dataflow/Cargo.toml b/rust/otap-dataflow/Cargo.toml index 1df5338591..5d20e9b9ce 100644 --- a/rust/otap-dataflow/Cargo.toml +++ b/rust/otap-dataflow/Cargo.toml @@ -85,7 +85,7 @@ miette = { version="7.6.0", features = ["fancy"] } mimalloc-rust = "0.2.1" nix = { version = "0.30.1", features = ["process", "signal"] } num_enum = "0.7" -object_store = "0.12.3" +object_store = {version = "0.12.3", default-features = false, features = ["fs"]} once_cell = "1.20.2" opentelemetry = "0.31.0" opentelemetry-proto = { version = "0.31", default-features = false, features = ["gen-tonic-messages", "logs"]} #TODO - use it from submodule instead of crate(?) @@ -150,10 +150,10 @@ zip = "=4.2.0" byte-unit = "5.2.0" # Azure Monnitor Exporter -azure_identity = "0.30.0" +azure_core = {version = "0.30.1", default-features = false, features = ["reqwest"] } +azure_identity = {version = "0.30.0", default-features = false, features = [] } flate2 = "1.1.5" -reqwest = "0.12.24" -azure_core = "0.30.1" +reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots"] } time = "0.3.44" wiremock = "0.6.5" diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index 612eff8477..ca1eb82246 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -7,10 +7,12 @@ # with the arrow enabled go collector build files in the repo root, the image # must be built with additional --build-context flags as follows: # docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . +ARG FEATURES="" FROM --platform=$BUILDPLATFORM docker.io/rust:1.91@sha256:4a29b0db5c961cd530f39276ece3eb6e66925b59599324c8c19723b72a423615 AS dataflow-build WORKDIR /build/rust/dataflow ARG BUILDPLATFORM ARG TARGETPLATFORM +ARG FEATURES RUN apt-get update && apt-get install -y protobuf-compiler @@ -23,11 +25,13 @@ COPY --from=otel-arrow /proto/opentelemetry-proto /build/proto/opentelemetry-pro COPY . /build/rust/dataflow/. # Build dataflow engine +ENV FEATURES=${FEATURES} RUN ./cross-arch-build.sh # The runtime image FROM docker.io/alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 LABEL maintainer="The OpenTelemetry Authors" + RUN addgroup dataflow \ && adduser \ --ingroup dataflow \ diff --git a/rust/otap-dataflow/configs/fake-parquet.yaml b/rust/otap-dataflow/configs/fake-parquet.yaml index 83b93d0921..89c00f25a7 100644 --- a/rust/otap-dataflow/configs/fake-parquet.yaml +++ b/rust/otap-dataflow/configs/fake-parquet.yaml @@ -18,14 +18,17 @@ nodes: signals_per_second: 30 log_weight: 100 registry_path: https://github.com/open-telemetry/semantic-conventions.git[model] + exporter: kind: exporter plugin_urn: "urn:otel:otap:parquet:exporter" config: storage: - file: - base_uri: /tmp + azure: + base_uri: https://barquettest.blob.core.windows.net/barquet1 + auth: + type: azure_cli partitioning_strategies: - schema_metadata: ["_part_id"] writer_options: - flush_when_older_than: 10s + flush_when_older_than: 5s diff --git a/rust/otap-dataflow/crates/otap/Cargo.toml b/rust/otap-dataflow/crates/otap/Cargo.toml index c9aab12218..dd8f96ba25 100644 --- a/rust/otap-dataflow/crates/otap/Cargo.toml +++ b/rust/otap-dataflow/crates/otap/Cargo.toml @@ -70,7 +70,7 @@ geneva-uploader = { version = "0.3.0", optional = true } # Azure Monitor Exporter dependencies azure_identity = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } -reqwest = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true, default-features = false, features = ["rustls-tls-native-roots"] } azure_core = { workspace = true, optional = true } # OpenTelemetry proto with tonic feature enabled diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index b1da50a7cd..7eaa8851e8 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -1,14 +1,18 @@ set -exu +if [ -z "${FEATURES:-}" ]; then + FEATURES="" +fi + if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then RUST_TARGET=x86_64-unknown-linux-musl if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-x86-64-linux-gnu + apt-get update && apt-get install -y gcc-x86-64-linux-musl fi elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then RUST_TARGET=aarch64-unknown-linux-musl if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-aarch64-linux-gnu + apt-get update && apt-get install -y gcc-aarch64-linux-musl fi else echo "Unsupported target platform: ${TARGETPLATFORM}" @@ -16,5 +20,5 @@ else fi rustup target add "${RUST_TARGET}" -cargo build --release --target="${RUST_TARGET}" -cp "target/${RUST_TARGET}/release/df_engine" . \ No newline at end of file +cargo build --release --features "$FEATURES" --target="${RUST_TARGET}" +cp "target/${RUST_TARGET}/release/df_engine" . From 20d07ebbf03646abb2c2df94cba321093ef46278 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sat, 6 Dec 2025 11:59:08 -0800 Subject: [PATCH 2/9] Add more options --- rust/otap-dataflow/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index ca1eb82246..c8e72bd46b 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -8,6 +8,7 @@ # must be built with additional --build-context flags as follows: # docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . ARG FEATURES="" +ARG RUSTFLAGS="" FROM --platform=$BUILDPLATFORM docker.io/rust:1.91@sha256:4a29b0db5c961cd530f39276ece3eb6e66925b59599324c8c19723b72a423615 AS dataflow-build WORKDIR /build/rust/dataflow ARG BUILDPLATFORM @@ -26,12 +27,16 @@ COPY . /build/rust/dataflow/. # Build dataflow engine ENV FEATURES=${FEATURES} +ENV RUSTFLAGS=${RUSTFLAGS} RUN ./cross-arch-build.sh # The runtime image FROM docker.io/alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 LABEL maintainer="The OpenTelemetry Authors" +ARG DEBUG="0" +RUN if [ "$DEBUG" = "1" ]; then apk add --no-cache gdb; fi + RUN addgroup dataflow \ && adduser \ --ingroup dataflow \ From b57ac8dab34bbff10aa4d4cc5ae300c985888b03 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sat, 6 Dec 2025 12:46:15 -0800 Subject: [PATCH 3/9] misc fixes --- rust/otap-dataflow/Dockerfile | 10 ++++++---- rust/otap-dataflow/configs/fake-parquet.yaml | 8 +++----- rust/otap-dataflow/crates/otap/Cargo.toml | 2 +- rust/otap-dataflow/cross-arch-build.sh | 8 ++------ 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index c8e72bd46b..470624da81 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -7,13 +7,15 @@ # with the arrow enabled go collector build files in the repo root, the image # must be built with additional --build-context flags as follows: # docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . -ARG FEATURES="" -ARG RUSTFLAGS="" FROM --platform=$BUILDPLATFORM docker.io/rust:1.91@sha256:4a29b0db5c961cd530f39276ece3eb6e66925b59599324c8c19723b72a423615 AS dataflow-build WORKDIR /build/rust/dataflow ARG BUILDPLATFORM ARG TARGETPLATFORM + +ARG RUSTFLAGS +ENV RUSTFLAGS=${RUSTFLAGS} ARG FEATURES +ENV FEATURES=${FEATURES} RUN apt-get update && apt-get install -y protobuf-compiler @@ -26,8 +28,8 @@ COPY --from=otel-arrow /proto/opentelemetry-proto /build/proto/opentelemetry-pro COPY . /build/rust/dataflow/. # Build dataflow engine -ENV FEATURES=${FEATURES} -ENV RUSTFLAGS=${RUSTFLAGS} +# ENV FEATURES=${FEATURES} +# ENV RUSTFLAGS=${RUSTFLAGS} RUN ./cross-arch-build.sh # The runtime image diff --git a/rust/otap-dataflow/configs/fake-parquet.yaml b/rust/otap-dataflow/configs/fake-parquet.yaml index 89c00f25a7..b7085261a5 100644 --- a/rust/otap-dataflow/configs/fake-parquet.yaml +++ b/rust/otap-dataflow/configs/fake-parquet.yaml @@ -24,11 +24,9 @@ nodes: plugin_urn: "urn:otel:otap:parquet:exporter" config: storage: - azure: - base_uri: https://barquettest.blob.core.windows.net/barquet1 - auth: - type: azure_cli + file: + base_uri: /tmp partitioning_strategies: - schema_metadata: ["_part_id"] writer_options: - flush_when_older_than: 5s + flush_when_older_than: 10s diff --git a/rust/otap-dataflow/crates/otap/Cargo.toml b/rust/otap-dataflow/crates/otap/Cargo.toml index dd8f96ba25..65d9a0abce 100644 --- a/rust/otap-dataflow/crates/otap/Cargo.toml +++ b/rust/otap-dataflow/crates/otap/Cargo.toml @@ -70,7 +70,7 @@ geneva-uploader = { version = "0.3.0", optional = true } # Azure Monitor Exporter dependencies azure_identity = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } -reqwest = { workspace = true, optional = true, default-features = false, features = ["rustls-tls-native-roots"] } +reqwest = { workspace = true, optional = true, default-features = false, features = ["rustls-tls"] } azure_core = { workspace = true, optional = true } # OpenTelemetry proto with tonic feature enabled diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index 7eaa8851e8..74b78dd38d 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -1,18 +1,14 @@ set -exu -if [ -z "${FEATURES:-}" ]; then - FEATURES="" -fi - if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then RUST_TARGET=x86_64-unknown-linux-musl if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-x86-64-linux-musl + apt-get update && apt-get install -y gcc-x86-64-linux-gnu fi elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then RUST_TARGET=aarch64-unknown-linux-musl if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-aarch64-linux-musl + apt-get update && apt-get install -y gcc-aarch64-linux-gnu fi else echo "Unsupported target platform: ${TARGETPLATFORM}" From 217fe5470e7d8323984f6ce138d43738e3cecde5 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sat, 6 Dec 2025 12:54:43 -0800 Subject: [PATCH 4/9] update dockerfile --- rust/otap-dataflow/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index 470624da81..09fae80353 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -12,11 +12,6 @@ WORKDIR /build/rust/dataflow ARG BUILDPLATFORM ARG TARGETPLATFORM -ARG RUSTFLAGS -ENV RUSTFLAGS=${RUSTFLAGS} -ARG FEATURES -ENV FEATURES=${FEATURES} - RUN apt-get update && apt-get install -y protobuf-compiler # The build process relies on these libraries outside of the current context. @@ -27,15 +22,24 @@ COPY --from=otel-arrow /proto/opentelemetry-proto /build/proto/opentelemetry-pro COPY . /build/rust/dataflow/. +# RUSTFLAGS can be used to pass argument to rustc e.g. +# --build-arg RUSTFLAGS="-C target-feature=-gfni" +ARG RUSTFLAGS +ENV RUSTFLAGS=${RUSTFLAGS} + +# These are used to enable features and are passed to carg's --features argument +# e.g. --build-arg FEATURES="azure,quiver-persistence" +ARG FEATURES +ENV FEATURES=${FEATURES} + # Build dataflow engine -# ENV FEATURES=${FEATURES} -# ENV RUSTFLAGS=${RUSTFLAGS} RUN ./cross-arch-build.sh # The runtime image FROM docker.io/alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 LABEL maintainer="The OpenTelemetry Authors" +# Passing --build-arg BUILD=1 adds additional debugging features to the image. ARG DEBUG="0" RUN if [ "$DEBUG" = "1" ]; then apk add --no-cache gdb; fi From fc4b96767231c1e7bb7d9ee466aef52d70102972 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sat, 6 Dec 2025 12:55:45 -0800 Subject: [PATCH 5/9] revert yaml change --- rust/otap-dataflow/configs/fake-parquet.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/otap-dataflow/configs/fake-parquet.yaml b/rust/otap-dataflow/configs/fake-parquet.yaml index b7085261a5..83b93d0921 100644 --- a/rust/otap-dataflow/configs/fake-parquet.yaml +++ b/rust/otap-dataflow/configs/fake-parquet.yaml @@ -18,7 +18,6 @@ nodes: signals_per_second: 30 log_weight: 100 registry_path: https://github.com/open-telemetry/semantic-conventions.git[model] - exporter: kind: exporter plugin_urn: "urn:otel:otap:parquet:exporter" From 2875e2670e50899a6c81336a72a9dd5b56d79358 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sun, 7 Dec 2025 10:57:07 -0800 Subject: [PATCH 6/9] Fix up features --- rust/otap-dataflow/Cargo.toml | 2 +- rust/otap-dataflow/crates/otap/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/otap-dataflow/Cargo.toml b/rust/otap-dataflow/Cargo.toml index 5d20e9b9ce..194705b1cd 100644 --- a/rust/otap-dataflow/Cargo.toml +++ b/rust/otap-dataflow/Cargo.toml @@ -153,7 +153,7 @@ byte-unit = "5.2.0" azure_core = {version = "0.30.1", default-features = false, features = ["reqwest"] } azure_identity = {version = "0.30.0", default-features = false, features = [] } flate2 = "1.1.5" -reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots"] } +reqwest = { version = "0.12.24", default-features = false } time = "0.3.44" wiremock = "0.6.5" diff --git a/rust/otap-dataflow/crates/otap/Cargo.toml b/rust/otap-dataflow/crates/otap/Cargo.toml index 65d9a0abce..e9a23a876e 100644 --- a/rust/otap-dataflow/crates/otap/Cargo.toml +++ b/rust/otap-dataflow/crates/otap/Cargo.toml @@ -70,7 +70,7 @@ geneva-uploader = { version = "0.3.0", optional = true } # Azure Monitor Exporter dependencies azure_identity = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } -reqwest = { workspace = true, optional = true, default-features = false, features = ["rustls-tls"] } +reqwest = { workspace = true, optional = true, features = ["rustls-tls"] } azure_core = { workspace = true, optional = true } # OpenTelemetry proto with tonic feature enabled From bb931caf737fb6746f88daee47d0c186b302481e Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sun, 7 Dec 2025 11:06:04 -0800 Subject: [PATCH 7/9] Move fs feature from workspace to otap crate --- rust/otap-dataflow/Cargo.toml | 2 +- rust/otap-dataflow/crates/otap/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/otap-dataflow/Cargo.toml b/rust/otap-dataflow/Cargo.toml index 194705b1cd..e95db6a6b8 100644 --- a/rust/otap-dataflow/Cargo.toml +++ b/rust/otap-dataflow/Cargo.toml @@ -85,7 +85,7 @@ miette = { version="7.6.0", features = ["fancy"] } mimalloc-rust = "0.2.1" nix = { version = "0.30.1", features = ["process", "signal"] } num_enum = "0.7" -object_store = {version = "0.12.3", default-features = false, features = ["fs"]} +object_store = {version = "0.12.3", default-features = false} once_cell = "1.20.2" opentelemetry = "0.31.0" opentelemetry-proto = { version = "0.31", default-features = false, features = ["gen-tonic-messages", "logs"]} #TODO - use it from submodule instead of crate(?) diff --git a/rust/otap-dataflow/crates/otap/Cargo.toml b/rust/otap-dataflow/crates/otap/Cargo.toml index e9a23a876e..52eb77b700 100644 --- a/rust/otap-dataflow/crates/otap/Cargo.toml +++ b/rust/otap-dataflow/crates/otap/Cargo.toml @@ -22,7 +22,7 @@ futures-timer.workspace = true http.workspace = true humantime-serde.workspace = true log.workspace = true -object_store.workspace = true +object_store = {workspace = true, features = ["fs"]} parquet.workspace = true thiserror = { workspace = true } serde_with = { workspace = true } From de50e4778a60e74666df56af9824fabfa92bee5a Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sun, 7 Dec 2025 11:08:09 -0800 Subject: [PATCH 8/9] Move azure features to otap crate from workspace --- rust/otap-dataflow/Cargo.toml | 4 ++-- rust/otap-dataflow/crates/otap/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/otap-dataflow/Cargo.toml b/rust/otap-dataflow/Cargo.toml index e95db6a6b8..3085269653 100644 --- a/rust/otap-dataflow/Cargo.toml +++ b/rust/otap-dataflow/Cargo.toml @@ -150,8 +150,8 @@ zip = "=4.2.0" byte-unit = "5.2.0" # Azure Monnitor Exporter -azure_core = {version = "0.30.1", default-features = false, features = ["reqwest"] } -azure_identity = {version = "0.30.0", default-features = false, features = [] } +azure_core = {version = "0.30.1", default-features = false } +azure_identity = {version = "0.30.0", default-features = false } flate2 = "1.1.5" reqwest = { version = "0.12.24", default-features = false } time = "0.3.44" diff --git a/rust/otap-dataflow/crates/otap/Cargo.toml b/rust/otap-dataflow/crates/otap/Cargo.toml index 52eb77b700..5aed531a17 100644 --- a/rust/otap-dataflow/crates/otap/Cargo.toml +++ b/rust/otap-dataflow/crates/otap/Cargo.toml @@ -71,7 +71,7 @@ geneva-uploader = { version = "0.3.0", optional = true } azure_identity = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } reqwest = { workspace = true, optional = true, features = ["rustls-tls"] } -azure_core = { workspace = true, optional = true } +azure_core = { workspace = true, optional = true, features = ["reqwest"] } # OpenTelemetry proto with tonic feature enabled opentelemetry-proto = { workspace = true, optional = true } From c6171b62b04db2945f1e6c7e72fbd441687d2766 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 8 Dec 2025 07:42:50 -0800 Subject: [PATCH 9/9] Default RUSTFLAGS in dockerfile --- rust/otap-dataflow/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index 09fae80353..672a6e795e 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -24,7 +24,7 @@ COPY . /build/rust/dataflow/. # RUSTFLAGS can be used to pass argument to rustc e.g. # --build-arg RUSTFLAGS="-C target-feature=-gfni" -ARG RUSTFLAGS +ARG RUSTFLAGS="-C target-cpu=native" ENV RUSTFLAGS=${RUSTFLAGS} # These are used to enable features and are passed to carg's --features argument