Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
86 commits
Select commit Hold shift + click to select a range
b2bab91
chore: switch sync dependencies to irokle
St4NNi Jun 1, 2026
186ce5a
feat: add core document sync events
St4NNi Jun 1, 2026
749818e
refactor: replace core sync protocol / gossip / automerge with irokle
St4NNi Jun 1, 2026
b8818bc
refactor: remove legacy sync helpers from core
St4NNi Jun 1, 2026
bf57c11
refactor: Delete core automerge and gossip modules
St4NNi Jun 1, 2026
f5f4443
feat: replace net gossip runtime with irokle
St4NNi Jun 1, 2026
76da173
test: remove obsolete net gossip tests
St4NNi Jun 1, 2026
d6ca7c1
refactor: store core documents without automerge
St4NNi Jun 1, 2026
f1eff8c
feat: add document sync operations
St4NNi Jun 1, 2026
889da79
refactor: remove automerge runtime dispatch
St4NNi Jun 1, 2026
b48978d
refactor: remove legacy automerge operations
St4NNi Jun 1, 2026
2c564b1
feat: use irokle-backed metadata sync
St4NNi Jun 1, 2026
01dbbdd
feat: wire irokle into startup onboarding
St4NNi Jun 1, 2026
6ec37a7
refactor: publish domain documents via document sync
St4NNi Jun 1, 2026
0153b43
feat: sync metadata registry documents with irokle
St4NNi Jun 1, 2026
3c43bca
refactor: restore document sync targets on startup
St4NNi Jun 1, 2026
e48ae08
refactor: update api tests for document sync runtime
St4NNi Jun 1, 2026
8bc4270
test: remove automerge handle from operation contexts
St4NNi Jun 1, 2026
c17c355
test: update operation integration tests for irokle
St4NNi Jun 1, 2026
25c4712
chore: update doctor and docs for irokle sync
St4NNi Jun 1, 2026
7041f51
chore: Fix tests, clippy + fmt
St4NNi Jun 2, 2026
cd3ef8d
fix: Flaky tests
St4NNi Jun 2, 2026
d290436
feat: Add sync placement primitives for irokle
St4NNi Jun 2, 2026
f7efc57
feat: make realm config the source of truth for realm peers
St4NNi Jun 2, 2026
0a25564
feat: replicate via selected peers and backfill placements
St4NNi Jun 2, 2026
e34e99c
refactor: route document announcements through realm replication
St4NNi Jun 2, 2026
7042cc9
tests: install realm config directly in replication tests
St4NNi Jun 2, 2026
204c2db
feat: refresh realm peers and backfill placements on lifecycle events
St4NNi Jun 2, 2026
0aa65fc
feat: add topic/placement and node-state diagnostics
St4NNi Jun 2, 2026
bba9379
chore: Rename functions to shorter more concise names
St4NNi Jun 2, 2026
396b894
chore: Fmt
St4NNi Jun 2, 2026
bccbde3
fix: avoid blocking onboarding bootstrap on sync
St4NNi Jun 3, 2026
71123a5
feat: prevent onboarding and document sync waits
St4NNi Jun 3, 2026
48f5736
feat: retry failed document syncs
St4NNi Jun 3, 2026
8ea0851
feat: task persistence
St4NNi Jun 3, 2026
595ef74
chore: Add onboarding tests
St4NNi Jun 3, 2026
b5d55ec
feat: re-arm failed task timers
St4NNi Jun 3, 2026
60e8822
feat: Add realm-scoped user storage keys
St4NNi Jun 3, 2026
552916b
feat: add durable realm-scoped sync placements
St4NNi Jun 3, 2026
52452af
feat: add shared metadata storage entry helpers
St4NNi Jun 3, 2026
1cfafab
feat: restore document and metadata lifecycle topics at startup
St4NNi Jun 3, 2026
a4211d2
feat: Add durable graph tombstones for deletes
St4NNi Jun 3, 2026
64bf633
feat: make metadata and realm config sync best effort
St4NNi Jun 3, 2026
d43e824
fix: Include self in required replicas
St4NNi Jun 4, 2026
5aee6fb
feat: Use async persisted tasks for document sync scheduling
St4NNi Jun 4, 2026
aba71e0
feat: Added durable outbox for irokle topic
St4NNi Jun 4, 2026
6dfea32
fix: Broken tests
St4NNi Jun 4, 2026
9eb34e6
feat: Ensure that events are only applied once
St4NNi Jun 4, 2026
a08a115
fix: missing taskhandle in tests
St4NNi Jun 4, 2026
9430166
feat: Update to irokle in-mem sync
St4NNi Jun 4, 2026
d1294c5
chore: Upgrade dependencies and fix tests
St4NNi Jun 4, 2026
d2a6e75
feat: Expand info/users/group API
St4NNi Jun 5, 2026
c93c1ab
chore: Upgrade craqle
St4NNi Jun 5, 2026
82c3e59
fix: Tests by force-flushing search
St4NNi Jun 5, 2026
e3e2a19
chore: Bump version to alpha.20
St4NNi Jun 5, 2026
3e02d88
chore: Add traces to debug performance issues
St4NNi Jun 5, 2026
ae4ec0f
chore: Add IO perf tests
St4NNi Jun 5, 2026
0f55d49
chore: Update craqle + irokle
St4NNi Jun 5, 2026
ae71718
feat: Add explicit irokle / fjall persists
St4NNi Jun 5, 2026
331cee8
feat: add dht lookup statistics
St4NNi Jun 6, 2026
8d62e70
feat: extend doctor explorer diagnostics
St4NNi Jun 8, 2026
fb6325a
feat: batch document sync effects
St4NNi Jun 10, 2026
e176b81
feat: event-sourced metadata pipeline with deterministic actors
St4NNi Jun 10, 2026
2db3a5c
feat: single-flight task scheduling per key
St4NNi Jun 10, 2026
94656dd
fix: page metadata document listing
St4NNi Jun 11, 2026
2fee115
test: add throughput and convergence gates
St4NNi Jun 11, 2026
da26de3
perf: skip projection for materialized documents
St4NNi Jun 11, 2026
5b2cd81
perf: warm metadata caches at boot
St4NNi Jun 11, 2026
d5045db
perf: drain document sync outbox in fifo batches
St4NNi Jun 11, 2026
a22fa35
test: add query concurrency repro
St4NNi Jun 11, 2026
baa2425
perf: rework metadata handle visibility and dispatch
St4NNi Jun 11, 2026
a18ad9b
perf: bound materialization and trust prevalidated applies
St4NNi Jun 11, 2026
b430240
perf: coalesce inbound irokle reconciles
St4NNi Jun 11, 2026
e6018e3
feat: latency aggregation and queue lag telemetry
St4NNi Jun 11, 2026
542ee96
feat: distributed query partiality, limits and stage telemetry
St4NNi Jun 11, 2026
92ec174
perf: pipeline outbox drain publish and sync stages
St4NNi Jun 12, 2026
275be91
test: add propagation tail gate
St4NNi Jun 12, 2026
61d83f0
perf: serve group listings from visible registry cache
St4NNi Jun 12, 2026
e5ded40
feat: batched irokle document sync with stage telemetry
St4NNi Jun 12, 2026
8df8d0f
chore: pin craqle and irokle revisions
St4NNi Jun 12, 2026
688fe21
style: nightly fmt and clippy fixes
St4NNi Jun 12, 2026
5b3f043
merge: Merge branch 'feat/listobjectsv2' into feat/irokle_perf
St4NNi Jun 12, 2026
3ddb3a5
feat: add batched point reads to the storage engine
St4NNi Jun 12, 2026
98e669c
feat: support inclusive iter start bounds
St4NNi Jun 12, 2026
faf93b8
perf: batch ListObjectsV2 page hydration and seek past common-prefix …
St4NNi Jun 12, 2026
e8614ab
fix: Read after write in batch, errors on abort
St4NNi Jun 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,358 changes: 538 additions & 820 deletions Cargo.lock

Large diffs are not rendered by default.

62 changes: 34 additions & 28 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ resolver = "3"

[workspace.package]
description = "A federated data orchestration network"
version = "3.0.0-alpha.1"
version = "3.0.0-alpha.32"
edition = "2024"
license = "MIT OR Apache-2.0"
repository = "https://github.com/arunaengine/aruna"
Expand All @@ -35,23 +35,23 @@ aruna-tasks = { path = "tasks" }
# Third-party crates
ahash = "0.8.12"
async-trait = "0.1.89"
automerge = "0.8.0"
autosurgeon = "0.11.0"
aws-config = "1.8.16"
aws-sdk-s3 = "1.132.0"
aws-config = "1.8.18"
aws-sdk-s3 = "1.135.0"
axum = "0.8.9"
axum-extra = { version = "0.12.5", features = ["query"] }
axum-extra = { version = "0.12.6", features = ["query"] }
bao-tree = "0.16.0"
base64 = "0.22.1"
blake3 = { version = "1.8.5", features = ["serde"] }
byteview = "0.10.1"
bytes = "1"
chrono = { version = "0.4.44", features = ["serde"] }
bytes = "1.11.1"
chrono = { version = "0.4.45", features = ["serde"] }
clap = { version = "4.6.1", features = ["derive"] }
console-subscriber = "0.5.0"
craqle = { git = "https://github.com/arunaengine/craqle" }
crc-fast = "1.9.0"
crossfire = "3.1.12"
craqle = { git = "https://github.com/arunaengine/craqle", branch = "feat/irokle", features = [
"iroh",
] }
crc-fast = "1.10.0"
crossfire = "3.1.16"
crypto_box = "0.9.1"
data-encoding = "2.11.0"
dotenvy = "0.15.7"
Expand All @@ -61,9 +61,9 @@ futures = "0.3.32"
futures-core = "0.3.32"
futures-util = "0.3.32"
globset = "0.4.18"
hex = "0.4"
http = "1.4.0"
hyper = { version = "1.9.0", features = ["full"] }
hex = "0.4.3"
http = "1.4.1"
hyper = { version = "1.10.1", features = ["full"] }
hyper-util = { version = "0.1.20", features = [
"server",
"server-auto",
Expand All @@ -72,52 +72,58 @@ hyper-util = { version = "0.1.20", features = [
"tokio",
"service",
] }
iroh = "1.0.0-rc.0"
iroh-base = "1.0.0-rc.0"
iroh-gossip = "0.99.0"
iroh = "1.0.0-rc.1"
iroh-base = "1.0.0-rc.1"
irokle = { git = "https://github.com/arunaengine/irokle.git", branch = "main", features = [
"fjall",
"iroh",
] }
iroh-io = "0.6.2"
iroh-quinn = "0.16.1"
jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] }
jsonwebtoken = { version = "10.4.0", features = ["rust_crypto"] }
md5 = "0.8.0"
n0-future = "0.3.2"
opendal = { version = "0.56", features = [
opendal = { version = "0.57", features = [
"services-postgresql",
"services-fs",
"services-ftp",
"services-s3",
"services-http",
"services-webdav",
] }
opentelemetry = { version = "0.31.0", default-features = false, features = ["trace"] }
opentelemetry-otlp = { version = "0.31.1", default-features = false, features = [
opentelemetry = { version = "0.32.0", default-features = false, features = [
"trace",
] }
opentelemetry-otlp = { version = "0.32.0", default-features = false, features = [
"trace",
"grpc-tonic",
] }
opentelemetry_sdk = { version = "0.31.0", default-features = false, features = [
opentelemetry_sdk = { version = "0.32.1", default-features = false, features = [
"trace",
"rt-tokio",
] }
oxrdf = "0.3"
oxrdf = "0.3.3"
parking_lot = { version = "0.12.5", features = ["deadlock_detection"] }
percent-encoding = "2.3.2"
postcard = { version = "1.1.3", features = ["alloc"] }
rand = "0.10.1"
reqwest = { version = "0.13.3", default-features = false, features = [
reqwest = { version = "0.13.4", default-features = false, features = [
"json",
"rustls",
] }
s3s = { git = "https://github.com/s3s-project/s3s" }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
serde_json = "1.0.150"
sha1 = "0.11.0"
sha2 = "0.11.0"
smallvec = "1.15.1"
spargebra = "0.4.6"
tempfile = "3.27.0"
thiserror = "2.0.18"
tokio = { version = "1.52.1", features = ["full", "tracing"] }
tokio = { version = "1.52.3", features = ["full", "tracing"] }
tokio-util = "0.7.18"
tracing = "0.1.44"
tracing-opentelemetry = { version = "0.32.1", default-features = false }
tracing-opentelemetry = { version = "0.33.0", default-features = false }
tracing-subscriber = { version = "0.3.23", features = [
"env-filter",
"time",
Expand All @@ -126,7 +132,7 @@ tracing-subscriber = { version = "0.3.23", features = [
] }
ulid = { version = "1.2.1", features = ["serde"] }
url = "2.5.8"
utoipa = { version = "5.4.0", features = [
utoipa = { version = "5.5.0", features = [
"chrono",
"axum_extras",
"ulid",
Expand Down
6 changes: 3 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Build Stage
FROM rust:1-alpine3.23 AS builder
FROM rust:1-alpine3.24 AS builder
WORKDIR /build
RUN apk update
RUN apk upgrade
Expand All @@ -9,9 +9,9 @@ RUN apk add llvm cmake gcc ca-certificates libc-dev pkgconfig openssl-dev musl-d
COPY . .
RUN cargo build --release -p aruna
RUN cargo build --release -p aruna-doctor
RUN cargo install --root target iroh-doctor
RUN cargo install --locked --root target iroh-doctor

FROM alpine:3.23
FROM alpine:3.24
WORKDIR /run
RUN apk update
RUN apk upgrade
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ The system is organized around **realms**. A realm is an organizational trust bo

Each Aruna node exposes an **S3-compatible API**, so researchers can keep using the tools, scripts, workflow systems, and libraries they already have instead of learning a new storage protocol. Buckets are virtual collections that mix local data, replicated data, and references to remote resources. To a user, this looks like one coherent access point. Underneath, Aruna tracks where data actually lives, which permissions apply, and whether an object should be materialized locally or fetched on demand.

Metadata is part of the core system, not an external catalog bolted on afterwards. Descriptions are stored as **RO-Crate JSON-LD**, so datasets, files, people, instruments, workflows, software, and process runs can be described in a shared format. These descriptions live in a CRDT-based triple store, which allows concurrent edits on different nodes and merges them without a single authority arbitrating the result. Management resources such as users and groups follow the same idea through Automerge documents, which lets nodes keep working through network outages and reconcile state once they reconnect.
Metadata is part of the core system, not an external catalog bolted on afterwards. Descriptions are stored as **RO-Crate JSON-LD**, so datasets, files, people, instruments, workflows, software, and process runs can be described in a shared format. These descriptions live in a CRDT-based triple store, which allows concurrent edits on different nodes and merges them without a single authority arbitrating the result. Management resources such as users and groups are synchronized through durable Irokle document topics, which lets nodes keep working through network outages and reconcile state once they reconnect.

File contents go into a content-addressed blob layer. Objects are hashed with **BLAKE3**, making integrity checks and deduplication part of the storage model rather than a separate step. If the same file shows up under different paths or on different nodes, it is recognized by its content instead of its location. Replication uses Bao-tree verified streaming, so data can be checked incrementally as it arrives.

Expand Down Expand Up @@ -156,4 +156,4 @@ at your option. Unless you explicitly state otherwise, any contribution intentio

## Feedback & Contributions

If you have any ideas, suggestions, or issues, please don't hesitate to open an issue and/or PR. Contributions to this project are always welcome ! We appreciate your help in making this project better. Please have a look at our [Contributor Guidelines](./CONTRIBUTING.md) as well as our [Code of Conduct](./CODE_OF_CONDUCT.md) for more information.
If you have any ideas, suggestions, or issues, please don't hesitate to open an issue and/or PR. Contributions to this project are always welcome ! We appreciate your help in making this project better. Please have a look at our [Contributor Guidelines](./CONTRIBUTING.md) as well as our [Code of Conduct](./CODE_OF_CONDUCT.md) for more information.
2 changes: 2 additions & 0 deletions api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ opentelemetry = { workspace = true }
percent-encoding = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
spargebra = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
Expand All @@ -44,6 +45,7 @@ utoipa-swagger-ui = { workspace = true }
# S3
async-trait = { workspace = true }
futures-core = { workspace = true }
futures-util = { workspace = true }
http = { workspace = true }
hyper = { workspace = true }
hyper-util = { workspace = true }
Expand Down
24 changes: 12 additions & 12 deletions api/src/auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,8 @@ pub async fn auth_middleware(
// Extract and validate token, get Option<AuthContext>
// We clone headers to avoid borrowing issues with the async function
let headers = request.headers().clone();
let auth_ctx: Option<AuthContext> = extract_auth_context(&state, &headers).await;
let auth_ctx: Option<AuthContext> =
aruna_core::telemetry::time_stage("auth", extract_auth_context(&state, &headers)).await;
record_auth_context(auth_ctx.as_ref());

// Always insert (Some or None) - handlers decide if auth is required
Expand Down Expand Up @@ -471,13 +472,16 @@ pub(crate) async fn ensure_permission(
path: String,
required_permission: Permission,
) -> ServerResult<()> {
let allowed = drive(
CheckPermissionsOperation::new(CheckPermissionsConfig {
auth_context: auth.clone(),
path,
required_permission,
}),
&state.get_ctx(),
let allowed = aruna_core::telemetry::time_stage(
"permission",
drive(
CheckPermissionsOperation::new(CheckPermissionsConfig {
auth_context: auth.clone(),
path,
required_permission,
}),
&state.get_ctx(),
),
)
.await
.map_err(|err| ServerError::InternalError(err.to_string()))?;
Expand Down Expand Up @@ -563,7 +567,6 @@ mod test {
storage_handle,
net_handle: None,
blob_handle: None,
automerge_handle: None,
metadata_handle: None,
task_handle: None,
}),
Expand Down Expand Up @@ -1103,7 +1106,6 @@ mod test {
let driver_ctx = Arc::new(DriverContext {
storage_handle,
net_handle: None,
automerge_handle: None,
metadata_handle: None,
task_handle: None,
blob_handle: None,
Expand Down Expand Up @@ -1284,7 +1286,6 @@ mod test {
let driver_ctx = Arc::new(DriverContext {
storage_handle,
net_handle: Some(net_handle.clone()),
automerge_handle: None,
metadata_handle: None,
task_handle: Some(task_handle),
blob_handle: None,
Expand Down Expand Up @@ -1381,7 +1382,6 @@ mod test {
let driver_ctx = Arc::new(DriverContext {
storage_handle,
net_handle: None,
automerge_handle: None,
metadata_handle: None,
task_handle: None,
blob_handle: None,
Expand Down
13 changes: 12 additions & 1 deletion api/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ pub enum ServerError {
BadRequest,
#[error("Bad gateway")]
BadGateway,
#[error("Service unavailable")]
ServiceUnavailable,
}

#[derive(Debug, Error)]
Expand Down Expand Up @@ -145,7 +147,14 @@ impl IntoResponse for ServerError {

let body = ErrorResponse::new(&message).with_code(code);

(status, Json(body)).into_response()
let mut response = (status, Json(body)).into_response();
if matches!(self, ServerError::ServiceUnavailable) {
response.headers_mut().insert(
axum::http::header::RETRY_AFTER,
axum::http::HeaderValue::from_static("1"),
);
}
response
}
}

Expand All @@ -159,6 +168,7 @@ impl ServerError {
ServerError::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR,
ServerError::BadRequest => StatusCode::BAD_REQUEST,
ServerError::BadGateway => StatusCode::BAD_GATEWAY,
ServerError::ServiceUnavailable => StatusCode::SERVICE_UNAVAILABLE,
}
}

Expand All @@ -171,6 +181,7 @@ impl ServerError {
ServerError::InternalError(_) => "Internal error".to_string(),
ServerError::BadRequest => "Bad request".to_string(),
ServerError::BadGateway => "Bad gateway".to_string(),
ServerError::ServiceUnavailable => "Service unavailable".to_string(),
}
}

Expand Down
1 change: 0 additions & 1 deletion api/src/routes/connectors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,6 @@ mod tests {
storage_handle,
net_handle: None,
blob_handle: None,
automerge_handle: None,
metadata_handle: None,
task_handle: None,
});
Expand Down
Loading
Loading