bench-incremental: add nix-dynamic tools (experimental builder)

docs/src/benchmarking.md

@@ -28,7 +28,7 @@ verifies that the binary links.
 | `-runs N` | `3` | Runs per scenario; the median is reported |
 | `-scenario S` | `all` | One of `no_change`, `leaf`, `mid`, `deep`, `all` |
 | `-touch-mode M` | `private` | `private` edits an unexported symbol; `exported` edits an exported one |
-| `-tools L` | `nix-nocgo,nix-ca-nocgo` | Comma-separated tool variants: `nix`, `nix-ca`, `nix-nocgo`, `nix-ca-nocgo` |
+| `-tools L` | `nix-nocgo,nix-ca-nocgo` | Comma-separated tool variants: `nix`, `nix-ca`, `nix-nocgo`, `nix-ca-nocgo`, `nix-dynamic`, `nix-dynamic-nocgo`, `bazel` |
 | `-fixture F` | `light` | `light` or `torture` (see below) |
 | `-json PATH` | — | Write raw results as JSON |
 | `-assert-cascade N` | — | Fail (non-zero exit) if any tool builds more than `N` derivations on a touch scenario |
@@ -38,6 +38,15 @@ variants set `CGO_ENABLED = 0`. Comparing `nix-nocgo` against
 `nix-ca-nocgo` with `-touch-mode private` shows the `iface` early-cutoff in
 action.
 
+The `nix-dynamic*` variants use the
+[experimental builder](modes/experimental-mode.md)
+(`buildGoApplicationExperimental` — recursive-nix + dynamic-derivations +
+ca-derivations). They need a Nix that provides those experimental features
+*and* the `recursive-nix` system feature in the build sandbox; the harness
+probes once at startup and drops the tool with a `SKIP` notice if the probe
+fails, so the rest of the run continues. Remote builders that don't advertise
+`recursive-nix` will not work — this is a local-only comparison for now.
+
 ## Fixtures and scenarios
 
 Two synthetic projects under `tests/fixtures/`:

flake.nix

@@ -259,6 +259,7 @@
               pkgs.lib.makeBinPath [
                 pkgs.nixVersions.nix_2_34
                 pkgs.bazel_7
+                pkgs.stdenv.cc
               ]
             }:$PATH"
             exec ${bench}/bin/bench-incremental "$@"

go/bench-incremental/main.go

@@ -86,6 +86,31 @@ goEnv.buildGoApplication {
   doCheck = false;
   %s
 }
+`
+	// Experimental builder (recursive-nix + dynamic-derivations + ca-derivations).
+	// buildGoApplicationExperimental returns a CA wrapper drv whose .target
+	// is the actual binary; wrap in runCommand so the existing instantiate
+	// → realise flow in nixTool.Build works unchanged.
+	dynExprTemplate = `{ srcPath ? %s }:
+let
+  pkgs = import <nixpkgs> { system = "%s"; };
+  go2nixLib = import %s/lib.nix {};
+  goEnv = go2nixLib.mkGoEnv {
+    go = pkgs.go_1_26;
+    go2nix = import %s/packages/go2nix { inherit pkgs; };
+    nixPackage = pkgs.nixVersions.nix_2_34;
+    inherit (pkgs) callPackage;
+  };
+  app = goEnv.buildGoApplicationExperimental {
+    src = srcPath;
+    modRoot = "%s";
+    goLock = "${srcPath}/%s/go2nix.toml";
+    pname = "bench";
+    subPackages = [ "%s" ];
+    %s
+  };
+in
+pkgs.runCommand "bench-dynamic" { } "ln -s ${app.target} $out"
 `
 )
 
@@ -220,6 +245,10 @@ type buildTool interface {
 	Name() string
 	Build(srcPath string) (buildResult, error)
 	Cleanup() error
+	// SkipOnFail reports whether a probe-build failure should drop this
+	// tool with a notice (rather than aborting the whole run). Used for
+	// nix-dynamic*, which needs recursive-nix in the build sandbox.
+	SkipOnFail() bool
 }
 
 type nixTool struct {
@@ -231,10 +260,12 @@ type nixTool struct {
 	extraOpts   []string
 	storeRoot   string // local store root (NIX_REMOTE=local?root=...)
 	stderrTail  int    // bytes of stderr to keep in error messages
+	skipOnFail  bool
 }
 
-func (t *nixTool) Name() string   { return t.name }
-func (t *nixTool) Cleanup() error { return nil }
+func (t *nixTool) Name() string     { return t.name }
+func (t *nixTool) Cleanup() error   { return nil }
+func (t *nixTool) SkipOnFail() bool { return t.skipOnFail }
 
 type buildResult struct {
 	total     time.Duration
@@ -311,7 +342,8 @@ type bazelTool struct {
 	stderrTail int
 }
 
-func (t *bazelTool) Name() string { return t.name }
+func (t *bazelTool) Name() string     { return t.name }
+func (t *bazelTool) SkipOnFail() bool { return false }
 
 // bazelProcessRe parses Bazel's action summary line, e.g.:
 //
@@ -321,8 +353,14 @@ var bazelProcessRe = regexp.MustCompile(`INFO: (\d+) process`)
 func (t *bazelTool) Build(_ string) (buildResult, error) {
 	baseArgs := []string{"--output_base=" + t.outputBase}
 
+	// Redirect convenience symlinks (bazel-bin, bazel-out, …) to the
+	// output base so they don't land in the shared fixture copy.  Stray
+	// symlinks in the workspace change the nix store path when
+	// nix-dynamic re-evaluates, forcing a full wrapper rebuild that fails.
+	symlinkPrefix := t.outputBase + "/convenience-"
+
 	// Phase 1: loading + analysis only (analogous to nix-instantiate).
-	analysisArgs := append(append([]string{}, baseArgs...), "build", "--nobuild", t.target)
+	analysisArgs := append(append([]string{}, baseArgs...), "build", "--nobuild", "--symlink_prefix="+symlinkPrefix, t.target)
 	analysisElapsed, _, analysisStderr, err := runBazelCommand(t.workspace, analysisArgs)
 	if err != nil {
 		tail := analysisStderr
@@ -333,7 +371,7 @@ func (t *bazelTool) Build(_ string) (buildResult, error) {
 	}
 
 	// Phase 2: full build (analogous to nix-store --realise).
-	buildArgs := append(append([]string{}, baseArgs...), "build", t.target)
+	buildArgs := append(append([]string{}, baseArgs...), "build", "--symlink_prefix="+symlinkPrefix, t.target)
 	buildElapsed, _, buildStderr, err := runBazelCommand(t.workspace, buildArgs)
 	if err != nil {
 		tail := buildStderr
@@ -424,8 +462,8 @@ func resolvePaths(repoRoot string) (nixpkgsPath, pluginPath, gomodcache string,
 	return nixpkgsPath, pluginPath, gomodcache, nil
 }
 
-func writeNixExpr(tmpdir, name, fixturePath, go2nixSrc, system, mr, subPkg, extraAttrs string) (string, error) {
-	content := fmt.Sprintf(exprTemplate, fixturePath, system, go2nixSrc, go2nixSrc, mr, mr, subPkg, extraAttrs)
+func writeNixExpr(tmpdir, name, tmpl, fixturePath, go2nixSrc, system, mr, subPkg, extraAttrs string) (string, error) {
+	content := fmt.Sprintf(tmpl, fixturePath, system, go2nixSrc, go2nixSrc, mr, mr, subPkg, extraAttrs)
 	path := filepath.Join(tmpdir, "bench-"+name+".nix")
 	return path, os.WriteFile(path, []byte(content), 0o644)
 }
@@ -718,7 +756,7 @@ func main() {
 	touchMode := flag.String("touch-mode", "private",
 		"edit type: private=internal symbol, exported=API change")
 	toolsCSV := flag.String("tools", "nix-nocgo,nix-ca-nocgo",
-		"comma-separated tools (nix,nix-ca,nix-nocgo,nix-ca-nocgo,bazel)")
+		"comma-separated tools (nix,nix-ca,nix-nocgo,nix-ca-nocgo,nix-dynamic,nix-dynamic-nocgo,bazel)")
 	fixtureName := flag.String("fixture", "light",
 		"fixture to use (torture|light)")
 	jsonOut := flag.String("json", "", "export results as JSON to this path")
@@ -803,31 +841,47 @@ func main() {
 
 		mr := fc.modRoot
 		sp := fc.subPackage
-		exprNix, err := writeNixExpr(tmpdir, "nix", fixtureSrc, repoRoot, system, mr, sp, "")
+		exprNix, err := writeNixExpr(tmpdir, "nix", exprTemplate, fixtureSrc, repoRoot, system, mr, sp, "")
+		if err != nil {
+			_, _ = fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		exprCA, err := writeNixExpr(tmpdir, "nix-ca", exprTemplate, fixtureSrc, repoRoot, system, mr, sp, "contentAddressed = true;")
 		if err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 
-		exprCA, err := writeNixExpr(tmpdir, "nix-ca", fixtureSrc, repoRoot, system, mr, sp, "contentAddressed = true;")
+		exprNoCgo, err := writeNixExpr(tmpdir, "nix-nocgo", exprTemplate, fixtureSrc, repoRoot, system, mr, sp, "CGO_ENABLED = \"0\";")
 		if err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 
-		exprNoCgo, err := writeNixExpr(tmpdir, "nix-nocgo", fixtureSrc, repoRoot, system, mr, sp, "CGO_ENABLED = \"0\";")
+		exprCANoCgo, err := writeNixExpr(tmpdir, "nix-ca-nocgo", exprTemplate, fixtureSrc, repoRoot, system, mr, sp, "contentAddressed = true; CGO_ENABLED = \"0\";")
 		if err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 
-		exprCANoCgo, err := writeNixExpr(tmpdir, "nix-ca-nocgo", fixtureSrc, repoRoot, system, mr, sp, "contentAddressed = true; CGO_ENABLED = \"0\";")
+		exprDyn, err := writeNixExpr(tmpdir, "nix-dynamic", dynExprTemplate, fixtureSrc, repoRoot, system, mr, sp, "")
+		if err != nil {
+			_, _ = fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		exprDynNoCgo, err := writeNixExpr(tmpdir, "nix-dynamic-nocgo", dynExprTemplate, fixtureSrc, repoRoot, system, mr, sp, `CGO_ENABLED = "0";`)
 		if err != nil {
 			_, _ = fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 
 		caOpts := []string{"--option", "extra-experimental-features", "ca-derivations"}
+		dynOpts := []string{
+			"--option", "extra-experimental-features", "dynamic-derivations ca-derivations recursive-nix",
+			"--option", "extra-system-features", "recursive-nix",
+		}
 
 		available["nix"] = &nixTool{
 			name: "nix", nixpkgsPath: nixpkgsPath, pluginPath: pluginPath,
@@ -849,6 +903,23 @@ func main() {
 			gomodcache: gomodcache, exprPath: exprCANoCgo, storeRoot: storeRoot,
 			extraOpts: caOpts, stderrTail: *stderrTail,
 		}
+		// Dynamic tools use the host store (storeRoot=""). The recursive-nix
+		// inner daemon serves whichever store the outer build runs against,
+		// and a rooted store doesn't have the FOD inputDrvs the inner
+		// `go2nix resolve` registers — AddToStore fails with "path is not
+		// valid". The host store does. Less isolated than the dag tools'
+		// rooted store, but the wrapper drv is unique per fixture path so
+		// runs don't interfere.
+		available["nix-dynamic"] = &nixTool{
+			name: "nix-dynamic", nixpkgsPath: nixpkgsPath, pluginPath: pluginPath,
+			gomodcache: gomodcache, exprPath: exprDyn,
+			extraOpts: dynOpts, stderrTail: *stderrTail, skipOnFail: true,
+		}
+		available["nix-dynamic-nocgo"] = &nixTool{
+			name: "nix-dynamic-nocgo", nixpkgsPath: nixpkgsPath, pluginPath: pluginPath,
+			gomodcache: gomodcache, exprPath: exprDynNoCgo,
+			extraOpts: dynOpts, stderrTail: *stderrTail, skipOnFail: true,
+		}
 	}
 
 	// Bazel tool: only available for fixtures that have BUILD files.
@@ -894,6 +965,31 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Probe each tool once. nix-dynamic* needs recursive-nix in the build
+	// sandbox; on stores or remotes that don't provide it the wrapper build
+	// fails — drop those tools with a notice rather than aborting the whole
+	// run, mirroring benchmark-build's "SKIPPED (ca-derivations not enabled)"
+	// behaviour. Other tools still fail fast.
+	fmt.Println("Probing tools...")
+	probed := tools[:0]
+	for _, t := range tools {
+		if _, err := t.Build(fixtureCopy); err != nil {
+			if t.SkipOnFail() {
+				fmt.Printf("  SKIP %s: %v\n", t.Name(), err)
+				continue
+			}
+			fmt.Fprintf(os.Stderr, "  FATAL: probe failed for %s: %v\n", t.Name(), err)
+			os.Exit(1)
+		}
+		fmt.Printf("  OK   %s\n", t.Name())
+		probed = append(probed, t)
+	}
+	tools = probed
+	if len(tools) == 0 {
+		fmt.Fprintln(os.Stderr, "no runnable tools after probe")
+		os.Exit(1)
+	}
+
 	fmt.Printf("\n%s\nINCREMENTAL BUILD BENCHMARK\n%s\n",
 		strings.Repeat("=", 70), strings.Repeat("=", 70))
 	fmt.Printf("Fixture:    %s/%s\n", fc.dir, fc.modRoot)

nix/dynamic/default.nix

@@ -162,6 +162,7 @@ let
         ${lib.optionalString (netrcFile != null) "--netrc-file ${netrcFile}"} \
         ${lib.optionalString (pgoProfile != null) "--pgo-profile ${pgoProfile}"} \
         --daemon-socket "''${NIX_REMOTE#unix://}" \
+        --nix-jobs "''${NIX_BUILD_CORES:-0}" \
         --output $out
     '';