(2+3+4) concurrency / dpdk

.github/workflows/dev.yml

@@ -144,8 +144,7 @@ jobs:
             profile: "debug"
             sanitize: ""
             instrument: "none"
-          - &release-build
-            name: "release"
+          - name: "release"
             profile: "release"
             sanitize: "" # TODO: enable cfi and safe-stack when possible
             instrument: "none"
@@ -502,45 +501,71 @@ jobs:
           recipe_args: "${{ matrix.recipe.args }}"
       - *tmate
 
-  features:
+  concurrency:
     if: >-
       ${{
         needs.check_changes.outputs.devfiles == 'true'
         || startsWith(github.event.ref, 'refs/tags/v')
         || github.event_name == 'workflow_dispatch'
       }}
-    name: "features/${{ matrix.build.name }}/${{ matrix.features }}"
+    name: "concurrency"
     runs-on: "lab"
     needs:
       - check_changes
       - check
     permissions: *check-perms
-    env: *check-env
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        include:
-          # The `loom` feature flips `concurrency::sync` to loom's
-          # primitives workspace-wide, which breaks crates that rely on
-          # `Weak`, `Arc::downgrade`, etc. (those aren't in
-          # `loom::sync`).  Scope the loom build to only the concurrency
-          # package (which hosts the quiescent tests) so workspace
-          # feature unification doesn't poison unrelated crates.
-          - build: *release-build
-            features: "loom"
-            test_package: "concurrency"
-          - build: *release-build
-            features: "shuttle"
-            test_package: ""
+    # This job doesn't use the `*check-env` anchor: that anchor's
+    # `JUST_VARS` references `matrix.build.*`, and this job has no
+    # matrix. Each step below sets `JUST_VARS` itself, inlining the
+    # docker_sock / debug_justfile / oci_repo settings that the
+    # anchor would have provided.
+    env:
+      USER: "runner"
     steps:
       - *checkout
       - *nix-setup
-      - name: "test/${{ matrix.features }}"
+      - name: "shuttle"
+        env:
+          JUST_VARS: >-
+            docker_sock=/run/docker/docker.sock
+            debug_justfile=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_justfile || false }}
+            profile=release
+            features=shuttle
+            oci_repo=ghcr.io
+        uses: *just
+        with:
+          recipe: "test"
+      - name: "shuttle_pct"
+        env:
+          JUST_VARS: >-
+            docker_sock=/run/docker/docker.sock
+            debug_justfile=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_justfile || false }}
+            profile=release
+            features=shuttle_pct
+            oci_repo=ghcr.io
+        uses: *just
+        with:
+          recipe: "test"
+      # The `loom` feature flips `concurrency::sync` to loom's
+      # primitives workspace-wide, which breaks crates that rely on
+      # `Weak`, `Arc::downgrade`, etc. (those aren't in
+      # `loom::sync`).  Scope the loom build to only the concurrency
+      # package (which hosts the core concurrency tests) so workspace
+      # feature unification doesn't poison unrelated crates.
+      #
+      # TODO: gate tests which can't be used with loom
+      - name: "loom"
+        env:
+          JUST_VARS: >-
+            docker_sock=/run/docker/docker.sock
+            debug_justfile=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_justfile || false }}
+            profile=release
+            features=loom
+            oci_repo=ghcr.io
         uses: *just
         with:
           recipe: "test"
-          recipe_args: "${{ matrix.test_package }}"
+          recipe_args: "concurrency"
       - *tmate
 
   vlab:
@@ -633,7 +658,7 @@ jobs:
     needs:
       - check
       - sanitize
-      - features
+      - concurrency
       - build
       - vlab
       - test_each
@@ -653,10 +678,10 @@ jobs:
         run: |
           echo '::error:: Some check job(s) failed'
           exit 1
-      - name: "Flag any features matrix failures"
-        if: ${{ needs.features.result != 'success' && needs.features.result != 'skipped' }}
+      - name: "Flag any concurrency job failures"
+        if: ${{ needs.concurrency.result != 'success' && needs.concurrency.result != 'skipped' }}
         run: |
-          echo '::error:: Some features job(s) failed'
+          echo '::error:: concurrency job failed'
           exit 1
       - name: "Flag any test_each matrix failures"
         if: ${{ needs.test_each.result != 'success' && needs.test_each.result != 'skipped' }}

Cargo.toml

@@ -53,6 +53,9 @@ repository = "https://github.com/githedgehog/dataplane/"
 #
 # 1. correctly documenting what each package actually depends on,
 # 2. allowing builds under different environments (e.g. cross-compilation, wasm, miri, and so on).
+#
+# Exceptions: see `tokio = { ..., features = ["parking_lot"] }` below.  Each exception has a comment
+# justifying why it is workspace-wide.
 
 # Internal
 args = { path = "./args", package = "dataplane-args", features = [] }
@@ -110,6 +113,7 @@ chrono = { version = "0.4.44", default-features = false, features = [] }
 clap = { version = "4.6.1", default-features = true, features = [] }
 color-eyre = { version = "0.6.5", default-features = false, features = [] }
 colored = { version = "3.1.1", default-features = false, features = [] }
+crossbeam-utils = { version = "0.8.21", default-features = false, features = [] }
 ctrlc = { version = "3.5.2", default-features = false, features = [] }
 dashmap = { version = "6.2.1", default-features = false, features = [] }
 derive_builder = { version = "0.20.2", default-features = false, features = [] }
@@ -158,6 +162,7 @@ pci-ids = { version = "0.2.6", default-features = false, features = [] }
 prefix-trie = { version = "0.9.2", default-features = false, features = [] }
 pretty_assertions = { version = "1.4.1", default-features = false, features = [] }
 priority-queue = { version = "2.7.0", default-features = false, features = [] }
+proc-macro-crate = { version = "3.5.0", default-features = false, features = [] }
 proc-macro2 = { version = "1.0.106", default-features = false, features = [] }
 procfs = { version = "0.18.0", default-features = false, features = [] }
 pyroscope = { version = "2.0.3", default-features = false, features = [] }
@@ -183,7 +188,13 @@ strum_macros = { version = "0.28.0", default-features = false, features = [] }
 syn = { version = "2.0.117", default-features = false, features = [] }
 thiserror = { version = "2.0.18", default-features = false, features = [] }
 thread_local = { version = "1.1.9", default-features = false, features = [] }
-tokio = { version = "1.52.3", default-features = false, features = [] }
+# Exception to the "no workspace-wide features" rule above.  `tokio/parking_lot` is
+# not an additive feature in the usual sense -- it picks the lock implementation
+# tokio uses internally for its runtime.  Production builds already pull
+# parking_lot in via `concurrency::sync`, so the only thing scoping this
+# per-crate would buy is divergent runtime behaviour between test binaries
+# and the real dataplane.  Keep it global.
+tokio = { version = "1.52.3", default-features = false, features = ["parking_lot"] }
 tokio-util = { version = "0.7.18", default-features = false, features = [] }
 tonic = { version = "0.14.6", default-features = false, features = [] }
 tracing = { version = "0.1.44", default-features = false, features = [] }
@@ -247,6 +258,12 @@ package = "dataplane-cli"
 miri = true
 wasm = false # split
 
+[workspace.metadata.package.concurrency]
+package = "dataplane-concurrency"
+miri = true # must ALWAYS work
+# NOTE: concurrency should likely be dependency gated as `cfg(unix)` or `cfg(not(target_arch = "wasm32"))`
+wasm = false # hopeless + pointless
+
 [workspace.metadata.package.dataplane]
 package = "dataplane"
 miri = false # hopeless + pointless

common/Cargo.toml

@@ -7,5 +7,6 @@ publish.workspace = true
 repository.workspace = true
 
 [dependencies]
-left-right = { workspace = true }
 arc-swap = { workspace = true }
+concurrency = { workspace = true }
+left-right = { workspace = true }