From 402b5a10ad4b37edd97a939ac0d352205567aa26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 17:48:57 -0400 Subject: [PATCH 01/56] feat: add value-free resolution report via check --json/--explain Surface the resolution waterfall the resolver already computes as a stable, versioned, machine-readable contract that never carries secret values. This is phase 1 of polyglot language support: the per-secret provenance type every later phase (FFI, codegen, SDKs) depends on, shipped standalone as the check --explain/--json quick win. - New public ResolutionReport / SecretResolution / ResolutionStatus types (schema_version 1), serde-serializable, with to_explain_string() and all_required_present() helpers. - validate_audited now records per-secret provenance (status, the serving provider's credential-free URI, generated, default_applied, as_path) instead of discarding it; entries sorted by name for deterministic output. - ValidatedSecrets and ValidationErrors carry the resolution and expose report(), so the report is available on both success and missing-required. - check --json (versioned JSON) and check --explain (human trace) skip the prompt-for-missing flow and exit non-zero when a required secret is missing, so CI can gate on them. No secret values are ever printed. - Canonical JSON Schema committed at schema/resolution-report.schema.json. - Golden wire-format test plus an end-to-end provenance test through a real dotenv backend; CLI reference and CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 17 ++ docs/src/content/docs/reference/cli.md | 42 ++++ schema/resolution-report.schema.json | 67 +++++ secretspec/src/cli/mod.rs | 40 +++ secretspec/src/lib.rs | 4 + secretspec/src/report.rs | 231 ++++++++++++++++++ secretspec/src/secrets.rs | 62 ++++- secretspec/src/tests.rs | 92 +++++++ secretspec/src/validation.rs | 29 +++ .../fixtures/resolution_report.golden.json | 48 ++++ 10 files changed, 624 insertions(+), 8 deletions(-) create mode 100644 schema/resolution-report.schema.json create mode 100644 secretspec/src/report.rs create mode 100644 secretspec/tests/fixtures/resolution_report.golden.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ad41a..3b10d64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- `secretspec check --json` and `secretspec check --explain` surface a + value-free resolution report describing how every declared secret resolved + for the active profile: its status (`resolved`, `missing_required`, + `missing_optional`), whether the value came from a provider (with the serving + provider's credential-free URI), a generator, or a committed default, and + whether it is exposed `as_path`. Secret values are never included. Both flags + skip the interactive prompt-for-missing flow and exit non-zero when a required + secret is missing, so CI can gate on them. `--json` emits a versioned + (`schema_version`) machine-readable object; its canonical JSON Schema is + committed at `schema/resolution-report.schema.json`. The same report is + available to the Rust SDK via `ValidatedSecrets::report()` / + `ValidationErrors::report()`, returning the new public `ResolutionReport`, + `SecretResolution`, and `ResolutionStatus` types. + ## [0.12.0] - 2026-06-08 ### Added diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 869805b..db9b5c1 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -130,6 +130,9 @@ secretspec check [OPTIONS] **Options:** - `-p, --provider ` - Provider backend to use - `-P, --profile ` - Profile to use +- `-n, --no-prompt` - Don't prompt for missing secrets (exit with error if any are missing) +- `--json` - Print a value-free resolution report as JSON instead of prompting +- `--explain` - Print a value-free, human-readable resolution trace instead of prompting **Example:** ```bash @@ -140,6 +143,45 @@ Enter value for API_KEY (profile: production): **** ✓ Secret 'API_KEY' saved to keyring (profile: production) ``` +#### Resolution report (`--json` / `--explain`) + +`--json` and `--explain` report how every declared secret resolved for the +active profile without prompting and without ever printing a secret value. Both +exit non-zero when a required secret is missing, so they work as a CI gate. + +`--explain` prints a human-readable trace: + +```bash +$ secretspec check --profile production --explain +profile: production +provider: keyring:// + DATABASE_URL ok source keyring:// + JWT_SECRET ok generated + LOG_LEVEL ok default value + SENTRY_DSN missing optional + STRIPE_KEY MISSING required +``` + +`--json` emits a versioned, machine-readable object for tooling and CI. Each +entry reports the `status` (`resolved`, `missing_required`, `missing_optional`), +whether the value came from a provider (`source_provider`, credential-free), a +generator (`generated`), or a committed default (`default_applied`), and whether +it is exposed `as_path`. No secret values appear. The canonical JSON Schema is +committed at `schema/resolution-report.schema.json`. + +```bash +$ secretspec check --profile production --json +{ + "schema_version": 1, + "provider": "keyring://", + "profile": "production", + "secrets": [ + { "name": "DATABASE_URL", "status": "resolved", "required": true, "source_provider": "keyring://", "default_applied": false, "generated": false, "as_path": false }, + { "name": "STRIPE_KEY", "status": "missing_required", "required": true, "default_applied": false, "generated": false, "as_path": false } + ] +} +``` + ### get Get a secret value. diff --git a/schema/resolution-report.schema.json b/schema/resolution-report.schema.json new file mode 100644 index 0000000..214683f --- /dev/null +++ b/schema/resolution-report.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://secretspec.dev/schema/resolution-report.schema.json", + "title": "SecretSpec resolution report", + "description": "Value-free, versioned description of how every declared secret resolved for one profile. Emitted by `secretspec check --json`. Never contains secret values.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "provider", "profile", "secrets"], + "properties": { + "schema_version": { + "description": "Wire-format version. Consumers should refuse versions they do not understand.", + "type": "integer", + "const": 1 + }, + "provider": { + "description": "Credential-free URI of the provider the resolution is reported against.", + "type": "string" + }, + "profile": { + "description": "The profile that was resolved.", + "type": "string" + }, + "secrets": { + "description": "One entry per declared secret, sorted by name.", + "type": "array", + "items": { "$ref": "#/$defs/secretResolution" } + } + }, + "$defs": { + "secretResolution": { + "type": "object", + "additionalProperties": false, + "required": ["name", "status", "required", "default_applied", "generated", "as_path"], + "properties": { + "name": { + "description": "Declared secret name (the UPPER_SNAKE manifest key).", + "type": "string" + }, + "status": { + "description": "Whether the secret resolved, and if not whether that is an error.", + "type": "string", + "enum": ["resolved", "missing_required", "missing_optional"] + }, + "required": { + "description": "Whether the active profile marks this secret as required.", + "type": "boolean" + }, + "source_provider": { + "description": "Credential-free URI of the provider that answered. Present only when the value came from a provider (absent when generated, defaulted, or missing).", + "type": "string" + }, + "default_applied": { + "description": "Whether the value came from the manifest's committed default.", + "type": "boolean" + }, + "generated": { + "description": "Whether the value was freshly minted by the secret's generate config.", + "type": "boolean" + }, + "as_path": { + "description": "Whether the value is materialized to a temp file and exposed as a path.", + "type": "boolean" + } + } + } + } +} diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index 8d3f03a..89148bd 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -94,6 +94,14 @@ enum Commands { /// Don't prompt for missing secrets (exit with error if any are missing) #[arg(short = 'n', long)] no_prompt: bool, + /// Print the value-free resolution report as JSON (no secret values). + /// Never prompts; exits non-zero if a required secret is missing. + #[arg(long, conflicts_with = "explain")] + json: bool, + /// Print a value-free, human-readable resolution trace (no secret + /// values). Never prompts; exits non-zero if a required secret is missing. + #[arg(long)] + explain: bool, }, /// Init or show ~/.config/secretspec/config.toml Config { @@ -617,6 +625,8 @@ pub fn main() -> Result<()> { provider, profile, no_prompt, + json, + explain, } => { let mut app = load_secrets(&cli.file, &cli.reason)?; if let Some(p) = provider { @@ -625,6 +635,36 @@ pub fn main() -> Result<()> { if let Some(p) = profile { app.set_profile(p); } + + // `--json`/`--explain` surface the value-free resolution report + // instead of the interactive prompt-for-missing flow. They report + // on every declared secret (including missing required ones) and + // exit non-zero when a required secret is missing, so CI can gate. + if json || explain { + let report = match app + .validate() + .into_diagnostic() + .wrap_err("Failed to resolve secrets")? + { + Ok(validated) => validated.report(), + Err(errors) => errors.report(), + }; + + if json { + let rendered = serde_json::to_string_pretty(&report) + .into_diagnostic() + .wrap_err("Failed to serialize resolution report")?; + println!("{}", rendered); + } else { + print!("{}", report.to_explain_string()); + } + + if !report.all_required_present() { + std::process::exit(1); + } + return Ok(()); + } + let mut validated = app .check(no_prompt) .into_diagnostic() diff --git a/secretspec/src/lib.rs b/secretspec/src/lib.rs index f38a54f..2e81f4f 100644 --- a/secretspec/src/lib.rs +++ b/secretspec/src/lib.rs @@ -45,6 +45,7 @@ mod audit; mod config; mod error; pub(crate) mod generator; +mod report; mod secrets; mod validation; @@ -70,6 +71,9 @@ pub use config::{GenerateConfig, GenerateOptions, Secret}; // Public API exports pub use error::{Result, SecretSpecError}; pub use provider::Provider; +pub use report::{ + RESOLUTION_REPORT_SCHEMA_VERSION, ResolutionReport, ResolutionStatus, SecretResolution, +}; pub use secrets::Secrets; pub use validation::ValidatedSecrets; diff --git a/secretspec/src/report.rs b/secretspec/src/report.rs new file mode 100644 index 0000000..1ee9ef9 --- /dev/null +++ b/secretspec/src/report.rs @@ -0,0 +1,231 @@ +//! The resolution report: a value-free, versioned description of how every +//! declared secret resolved. +//! +//! This is the stable, machine-readable contract that surfaces the resolution +//! waterfall the resolver already computes (which provider answered, whether a +//! value was generated, whether a default was applied, whether a required +//! secret is missing) without ever exposing a secret value. It is emitted by +//! `secretspec check --json` and rendered by `secretspec check --explain`. +//! +//! The shape is versioned via [`RESOLUTION_REPORT_SCHEMA_VERSION`] so that +//! out-of-process consumers (other-language SDKs, CI tooling) can refuse a +//! mismatched version rather than silently misparse. The canonical JSON Schema +//! lives at `schema/resolution-report.schema.json` in the repository root. + +use serde::{Deserialize, Serialize}; + +/// Version of the [`ResolutionReport`] wire format. +/// +/// Bump this whenever the serialized shape changes in a way that is not purely +/// additive-and-optional, and update `schema/resolution-report.schema.json`. +pub const RESOLUTION_REPORT_SCHEMA_VERSION: u32 = 1; + +/// How a single declared secret resolved. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolutionStatus { + /// A value was produced (from a provider, a generator, or a default). + Resolved, + /// Required by the active profile but not found anywhere. + MissingRequired, + /// Optional and not found; resolution still succeeds overall. + MissingOptional, +} + +/// The resolution outcome for one declared secret. Never carries the value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SecretResolution { + /// The declared secret name (the `UPPER_SNAKE` key from the manifest). + pub name: String, + /// Whether the secret resolved, and if not, whether that is an error. + pub status: ResolutionStatus, + /// Whether the active profile marks this secret as required. + pub required: bool, + /// Credential-free URI of the provider that actually answered, when the + /// value came from a provider. `None` when generated, defaulted, or missing. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_provider: Option, + /// Whether the value came from the manifest's committed `default`. + pub default_applied: bool, + /// Whether the value was freshly minted by the secret's `generate` config. + pub generated: bool, + /// Whether the value is materialized to a temp file and exposed as a path. + pub as_path: bool, +} + +/// A complete, value-free snapshot of one resolution pass over a profile. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolutionReport { + /// Wire-format version; see [`RESOLUTION_REPORT_SCHEMA_VERSION`]. + pub schema_version: u32, + /// Credential-free URI of the provider resolution reported against. + pub provider: String, + /// The profile that was resolved. + pub profile: String, + /// One entry per declared secret, sorted by name for deterministic output. + pub secrets: Vec, +} + +impl ResolutionReport { + /// Build a report from its parts, stamping the current schema version and + /// sorting entries by name so the output is deterministic (important for + /// golden conformance vectors). + pub fn new(provider: String, profile: String, mut secrets: Vec) -> Self { + secrets.sort_by(|a, b| a.name.cmp(&b.name)); + Self { + schema_version: RESOLUTION_REPORT_SCHEMA_VERSION, + provider, + profile, + secrets, + } + } + + /// True when no required secret is missing (i.e. resolution would succeed). + pub fn all_required_present(&self) -> bool { + !self + .secrets + .iter() + .any(|s| s.status == ResolutionStatus::MissingRequired) + } + + /// Render a human-readable resolution trace. Value-free, word-based status + /// (no reliance on color) for accessibility. + pub fn to_explain_string(&self) -> String { + let mut out = String::new(); + out.push_str(&format!("profile: {}\n", self.profile)); + out.push_str(&format!("provider: {}\n", self.provider)); + + let width = self.secrets.iter().map(|s| s.name.len()).max().unwrap_or(0); + + for s in &self.secrets { + let detail = match s.status { + ResolutionStatus::Resolved => { + if s.generated { + "ok generated".to_string() + } else if s.default_applied { + "ok default value".to_string() + } else if let Some(uri) = &s.source_provider { + format!("ok source {}", uri) + } else { + "ok".to_string() + } + } + ResolutionStatus::MissingRequired => "MISSING required".to_string(), + ResolutionStatus::MissingOptional => "missing optional".to_string(), + }; + let path = if s.as_path { " (as path)" } else { "" }; + out.push_str(&format!( + " {:width$} {}{}\n", + s.name, + detail, + path, + width = width + )); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> ResolutionReport { + // Deliberately unsorted input to exercise the sort in `new`. + ResolutionReport::new( + "keyring://".to_string(), + "production".to_string(), + vec![ + SecretResolution { + name: "STRIPE_KEY".to_string(), + status: ResolutionStatus::MissingRequired, + required: true, + source_provider: None, + default_applied: false, + generated: false, + as_path: false, + }, + SecretResolution { + name: "DATABASE_URL".to_string(), + status: ResolutionStatus::Resolved, + required: true, + source_provider: Some("keyring://".to_string()), + default_applied: false, + generated: false, + as_path: false, + }, + SecretResolution { + name: "JWT_SECRET".to_string(), + status: ResolutionStatus::Resolved, + required: true, + source_provider: None, + default_applied: false, + generated: true, + as_path: false, + }, + SecretResolution { + name: "LOG_LEVEL".to_string(), + status: ResolutionStatus::Resolved, + required: false, + source_provider: None, + default_applied: true, + generated: false, + as_path: false, + }, + SecretResolution { + name: "SENTRY_DSN".to_string(), + status: ResolutionStatus::MissingOptional, + required: false, + source_provider: None, + default_applied: false, + generated: false, + as_path: false, + }, + ], + ) + } + + #[test] + fn entries_are_sorted_by_name() { + let report = sample(); + let names: Vec<&str> = report.secrets.iter().map(|s| s.name.as_str()).collect(); + assert_eq!( + names, + vec![ + "DATABASE_URL", + "JWT_SECRET", + "LOG_LEVEL", + "SENTRY_DSN", + "STRIPE_KEY" + ] + ); + } + + #[test] + fn all_required_present_tracks_missing_required() { + assert!(!sample().all_required_present()); + let mut report = sample(); + report + .secrets + .retain(|s| s.status != ResolutionStatus::MissingRequired); + assert!(report.all_required_present()); + } + + /// Locks the wire format. The golden file is the contract other-language + /// SDKs and CI consumers parse; any change here is a deliberate contract + /// change that must bump `RESOLUTION_REPORT_SCHEMA_VERSION` and the schema. + #[test] + fn serializes_to_golden_wire_format() { + let golden = include_str!("../tests/fixtures/resolution_report.golden.json"); + let actual = serde_json::to_string_pretty(&sample()).unwrap(); + assert_eq!(actual.trim(), golden.trim()); + } + + #[test] + fn round_trips_through_json() { + let report = sample(); + let json = serde_json::to_string(&report).unwrap(); + let back: ResolutionReport = serde_json::from_str(&json).unwrap(); + assert_eq!(report, back); + } +} diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index 685cd43..f8ff9b2 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -4,6 +4,7 @@ use crate::audit::{AuditAction, AuditContext, AuditLogger, AuditOutcome}; use crate::config::{Config, GlobalConfig, Profile, RequireReason, Resolved}; use crate::error::{Result, SecretSpecError}; use crate::provider::Provider as ProviderTrait; +use crate::report::{ResolutionStatus, SecretResolution}; use crate::validation::{ValidatedSecrets, ValidationErrors}; use colored::Colorize; use secrecy::{ExposeSecret, SecretString}; @@ -2020,6 +2021,12 @@ impl Secrets { let mut missing_optional = Vec::new(); let mut with_defaults = Vec::new(); let mut temp_files = Vec::new(); + // Per-secret provenance for the value-free resolution report. + let mut resolution: Vec = Vec::new(); + // Credential-free `uri()` of each successfully built primary + // provider group, keyed by the configured group URI, so a + // primary hit can be attributed to the provider that answered. + let mut group_uris: HashMap, String> = HashMap::new(); let profile = self.resolve_profile(Some(&profile_name))?; let all_secrets: Vec<(String, crate::config::Secret)> = @@ -2085,6 +2092,12 @@ impl Secrets { } }; + // Attribute primary hits to the provider's own credential-free + // `uri()`, never the raw configured alias (which may embed a + // token). Recorded before the fetch so attribution survives a + // partial batch. + group_uris.insert(provider_uri.clone(), provider.uri()); + let keys: Vec<&str> = secret_names.iter().map(|s| s.as_str()).collect(); match provider.get_batch(&self.config.project.name, &keys, &profile_name) { Ok(batch_results) => fetched_values.extend(batch_results), @@ -2096,7 +2109,10 @@ impl Secrets { } } - // Process results - apply defaults, handle as_path, track missing + // Process results - apply defaults, handle as_path, track missing. + // Each secret also records a value-free provenance entry for the + // resolution report (status, which provider answered, generated, + // defaulted). for (name, _) in all_secrets { let secret_config = self .resolve_secret_config(&name, Some(&profile_name)) @@ -2105,8 +2121,17 @@ impl Secrets { let default = secret_config.default.clone(); let as_path = secret_config.as_path.unwrap_or(false); + // `name` is consumed by whichever arm resolves/records it; + // keep a copy for the provenance entry pushed at the end. + let report_name = name.clone(); + let status; + let mut source_provider = None; + let mut default_applied = false; + let mut generated = false; + match fetched_values.remove(&name) { Some(value) => { + source_provider = group_uris.get(&secret_primary_uris[&name]).cloned(); self.insert_resolved( &mut secrets, &mut temp_files, @@ -2114,13 +2139,14 @@ impl Secrets { value, as_path, )?; + status = ResolutionStatus::Resolved; } None => { let primary_uri = &secret_primary_uris[&name]; let primary_failed = failed_primary_uris.contains_key(primary_uri); // An explicit override collapses the chain to one provider, no fallback. - let fallback_value = + let (fallback_value, fallback_uri) = match (override_uri.as_ref(), secret_config.providers.as_deref()) { (None, Some(providers)) if providers.len() > 1 => { let fallback_uris = @@ -2132,7 +2158,6 @@ impl Secrets { fallback_uris.as_deref(), None, )? - .0 } // No alternative chain to try and the primary failed: surface the // original error rather than reporting the secret as merely @@ -2144,10 +2169,11 @@ impl Secrets { .expect("primary_failed implies entry present"); return Err(err); } - _ => None, + _ => (None, None), }; if let Some(value) = fallback_value { + source_provider = fallback_uri; self.insert_resolved( &mut secrets, &mut temp_files, @@ -2155,17 +2181,21 @@ impl Secrets { value, as_path, )?; - } else if let Some(generated) = + status = ResolutionStatus::Resolved; + } else if let Some(generated_value) = self.try_generate_secret(&name, &secret_config, &profile_name)? { + generated = true; self.insert_resolved( &mut secrets, &mut temp_files, name, - generated, + generated_value, as_path, )?; + status = ResolutionStatus::Resolved; } else if let Some(default_value) = default { + default_applied = true; self.insert_resolved( &mut secrets, &mut temp_files, @@ -2174,13 +2204,26 @@ impl Secrets { as_path, )?; with_defaults.push((name, default_value)); + status = ResolutionStatus::Resolved; } else if required { missing_required.push(name); + status = ResolutionStatus::MissingRequired; } else { missing_optional.push(name); + status = ResolutionStatus::MissingOptional; } } } + + resolution.push(SecretResolution { + name: report_name, + status, + required, + source_provider, + default_applied, + generated, + as_path, + }); } let report_provider_uri = self.validation_report_provider_uri( @@ -2189,13 +2232,15 @@ impl Secrets { )?; if !missing_required.is_empty() { - Ok(Err(ValidationErrors::new( + let mut errors = ValidationErrors::new( missing_required, missing_optional, with_defaults, report_provider_uri, profile_name.to_string(), - ))) + ); + errors.resolution = resolution; + Ok(Err(errors)) } else { Ok(Ok(ValidatedSecrets { resolved: Resolved::new( @@ -2205,6 +2250,7 @@ impl Secrets { ), missing_optional, with_defaults, + resolution, temp_files, })) } diff --git a/secretspec/src/tests.rs b/secretspec/src/tests.rs index 2205ff4..7e17988 100644 --- a/secretspec/src/tests.rs +++ b/secretspec/src/tests.rs @@ -316,6 +316,7 @@ fn test_validation_result_structure() { resolved: Resolved::new(HashMap::new(), "keyring".to_string(), "default".to_string()), missing_optional: vec!["optional_secret".to_string()], with_defaults: Vec::new(), + resolution: Vec::new(), temp_files: Vec::new(), }; assert_eq!(valid_result.missing_optional.len(), 1); @@ -333,6 +334,97 @@ fn test_validation_result_structure() { assert_eq!(validation_errors.missing_required.len(), 1); } +#[test] +fn test_resolution_report_provenance() { + use crate::report::ResolutionStatus; + + let temp_dir = TempDir::new().unwrap(); + // Only DATABASE_URL is present in the backend; everything else exercises a + // different resolution arm (default, missing-optional, missing-required). + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "DATABASE_URL=postgres://localhost/db\n").unwrap(); + + let secret = |required: bool, default: Option<&str>| Secret { + description: Some("test".to_string()), + required: Some(required), + default: default.map(String::from), + ..Default::default() + }; + + let mut secrets = HashMap::new(); + secrets.insert("DATABASE_URL".to_string(), secret(true, None)); + secrets.insert("LOG_LEVEL".to_string(), secret(false, Some("info"))); + secrets.insert("SENTRY_DSN".to_string(), secret(false, None)); + secrets.insert("STRIPE_KEY".to_string(), secret(true, None)); + + let mut profiles = HashMap::new(); + profiles.insert( + "default".to_string(), + Profile { + defaults: None, + secrets, + }, + ); + + let config = Config { + project: Project { + name: "report-test".to_string(), + ..Default::default() + }, + profiles, + providers: None, + }; + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(config, None, Some(provider), None); + + // A required secret (STRIPE_KEY) is missing, so validation reports an error, + // but the report still describes every declared secret. + let report = match spec.validate().unwrap() { + Ok(validated) => validated.report(), + Err(errors) => errors.report(), + }; + + assert_eq!(report.schema_version, 1); + assert_eq!(report.profile, "default"); + assert!(!report.all_required_present()); + + // Entries are sorted by name for deterministic output. + let names: Vec<&str> = report.secrets.iter().map(|s| s.name.as_str()).collect(); + assert_eq!( + names, + vec!["DATABASE_URL", "LOG_LEVEL", "SENTRY_DSN", "STRIPE_KEY"] + ); + + let by_name = |name: &str| { + report + .secrets + .iter() + .find(|s| s.name == name) + .unwrap_or_else(|| panic!("missing entry {name}")) + }; + + let db = by_name("DATABASE_URL"); + assert_eq!(db.status, ResolutionStatus::Resolved); + assert!(db.required); + assert!(db.source_provider.is_some(), "provider hit is attributed"); + assert!(!db.default_applied); + assert!(!db.generated); + + let log = by_name("LOG_LEVEL"); + assert_eq!(log.status, ResolutionStatus::Resolved); + assert!(log.default_applied); + assert!(log.source_provider.is_none(), "a default has no provider"); + + let sentry = by_name("SENTRY_DSN"); + assert_eq!(sentry.status, ResolutionStatus::MissingOptional); + assert!(!sentry.required); + + let stripe = by_name("STRIPE_KEY"); + assert_eq!(stripe.status, ResolutionStatus::MissingRequired); + assert!(stripe.required); +} + #[test] fn test_secretspec_new() { let config = Config { diff --git a/secretspec/src/validation.rs b/secretspec/src/validation.rs index a23f1c7..338bbc1 100644 --- a/secretspec/src/validation.rs +++ b/secretspec/src/validation.rs @@ -1,6 +1,7 @@ //! Validation results for secret checking use crate::config::Resolved; +use crate::report::{ResolutionReport, SecretResolution}; use secrecy::SecretString; use std::collections::HashMap; use std::fmt; @@ -17,6 +18,9 @@ pub struct ValidatedSecrets { pub missing_optional: Vec, /// List of secrets using their default values (name, default_value) pub with_defaults: Vec<(String, String)>, + /// Value-free per-secret resolution provenance (which provider answered, + /// generated, defaulted, as_path). Drives `check --json`/`--explain`. + pub resolution: Vec, /// Temporary files for secrets with as_path=true. /// These are kept alive for the lifetime of ValidatedSecrets and automatically /// cleaned up when dropped. @@ -25,6 +29,15 @@ pub struct ValidatedSecrets { } impl ValidatedSecrets { + /// Build the value-free [`ResolutionReport`] for this successful resolution. + pub fn report(&self) -> ResolutionReport { + ResolutionReport::new( + self.resolved.provider.clone(), + self.resolved.profile.clone(), + self.resolution.clone(), + ) + } + /// Persist all temporary files, preventing automatic cleanup. /// /// This method consumes the temporary file handles and persists them, @@ -71,6 +84,10 @@ pub struct ValidationErrors { pub provider: String, /// The profile that was used pub profile: String, + /// Value-free per-secret resolution provenance, including the missing + /// required secrets that caused this error. Empty unless populated by the + /// resolver; see [`ValidationErrors::report`]. + pub resolution: Vec, } impl ValidationErrors { @@ -88,6 +105,7 @@ impl ValidationErrors { with_defaults, provider, profile, + resolution: Vec::new(), } } @@ -95,6 +113,17 @@ impl ValidationErrors { pub fn has_errors(&self) -> bool { !self.missing_required.is_empty() } + + /// Build the value-free [`ResolutionReport`] for this failed resolution. + /// The report still describes every declared secret, including the ones + /// that resolved, so consumers see the full picture, not just the gaps. + pub fn report(&self) -> ResolutionReport { + ResolutionReport::new( + self.provider.clone(), + self.profile.clone(), + self.resolution.clone(), + ) + } } impl fmt::Display for ValidationErrors { diff --git a/secretspec/tests/fixtures/resolution_report.golden.json b/secretspec/tests/fixtures/resolution_report.golden.json new file mode 100644 index 0000000..3f8958d --- /dev/null +++ b/secretspec/tests/fixtures/resolution_report.golden.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "provider": "keyring://", + "profile": "production", + "secrets": [ + { + "name": "DATABASE_URL", + "status": "resolved", + "required": true, + "source_provider": "keyring://", + "default_applied": false, + "generated": false, + "as_path": false + }, + { + "name": "JWT_SECRET", + "status": "resolved", + "required": true, + "default_applied": false, + "generated": true, + "as_path": false + }, + { + "name": "LOG_LEVEL", + "status": "resolved", + "required": false, + "default_applied": true, + "generated": false, + "as_path": false + }, + { + "name": "SENTRY_DSN", + "status": "missing_optional", + "required": false, + "default_applied": false, + "generated": false, + "as_path": false + }, + { + "name": "STRIPE_KEY", + "status": "missing_required", + "required": true, + "default_applied": false, + "generated": false, + "as_path": false + } + ] +} From 91a85791ed7486674c031a28d1830a73c1811fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 17:57:01 -0400 Subject: [PATCH 02/56] feat: add value-carrying resolve payload and `secretspec resolve --json` Phase 2, slice 1: the authoritative value-carrying resolution output the C ABI and other-language SDKs consume. The FFI crate (next slice) is a thin wrapper over this, keeping resolution logic in one place. - New Secrets::resolve() -> ResolveResponse, building on phase 1 provenance: per-secret value (or persisted temp-file path for as_path), source (provider/generated/default), and the serving provider's credential-free URI. On a missing required secret it returns an empty secrets map plus missing_required, mirroring the derive crate's load(). - New public ResolveResponse / ResolvedSecret / ResolvedSource types (schema_version 1, BTreeMap for deterministic key order) with is_ok() and without_values(). - secretspec resolve --json prints the payload (values to stdout, meant to be piped); --no-values emits the same structure value-free. Exits non-zero when a required secret is missing. - Canonical JSON Schema at schema/resolve-response.schema.json; CLI reference and CHANGELOG updated; tests cover values, provenance, missing-required, and as_path path persistence. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 13 +++ docs/src/content/docs/reference/cli.md | 41 ++++++++ schema/resolve-response.schema.json | 69 +++++++++++++ secretspec/src/cli/mod.rs | 45 +++++++++ secretspec/src/lib.rs | 2 + secretspec/src/resolve.rs | 92 +++++++++++++++++ secretspec/src/secrets.rs | 87 +++++++++++++++- secretspec/src/tests.rs | 131 +++++++++++++++++++++++++ 8 files changed, 479 insertions(+), 1 deletion(-) create mode 100644 schema/resolve-response.schema.json create mode 100644 secretspec/src/resolve.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b10d64..a581b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `secretspec resolve --json` resolves every declared secret and prints a + versioned, value-carrying JSON object: the SDK boundary other-language + clients consume. Each entry reports the value (or, for `as_path` secrets, the + path to a persisted temp file), its `source` (`provider`, `generated`, + `default`), and the serving provider's credential-free URI. When a required + secret is missing the command exits non-zero with an empty `secrets` object + and a populated `missing_required` list, mirroring the derive crate's + `load()`. `--no-values` emits the same structure without secret values. Unlike + `check`, this command prints secret values to stdout and is meant to be piped, + not displayed. The same payload is available to the Rust SDK via + `Secrets::resolve()`, returning the new public `ResolveResponse`, + `ResolvedSecret`, and `ResolvedSource` types; its JSON Schema is committed at + `schema/resolve-response.schema.json`. - `secretspec check --json` and `secretspec check --explain` surface a value-free resolution report describing how every declared secret resolved for the active profile: its status (`resolved`, `missing_required`, diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index db9b5c1..6961f5e 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -199,6 +199,47 @@ $ secretspec get DATABASE_URL --profile production postgresql://prod.example.com/mydb ``` +### resolve +Resolve every declared secret and print it as JSON. This is the SDK boundary: +other-language clients consume this payload (over a subprocess or the C ABI) +rather than reimplementing resolution. + +```bash +secretspec resolve [OPTIONS] +``` + +Unlike `check`, `resolve` prints secret **values** to stdout. Pipe it into a +program; do not display it. Use `--no-values` for a value-free structural view. +When a required secret is missing, the command exits non-zero with an empty +`secrets` object and a populated `missing_required` list (mirroring the SDK's +`load()`). + +**Options:** +- `-p, --provider ` - Provider backend to use +- `-P, --profile ` - Profile to use +- `--no-values` - Omit secret values, emitting only structure and provenance + +**Example:** +```bash +$ secretspec resolve --profile production +{ + "schema_version": 1, + "provider": "keyring://", + "profile": "production", + "secrets": { + "DATABASE_URL": { "value": "postgresql://prod.example.com/mydb", "as_path": false, "source": "provider", "source_provider": "keyring://" }, + "TLS_CERT": { "path": "/tmp/.tmpAbc123", "as_path": true, "source": "provider", "source_provider": "keyring://" } + }, + "missing_required": [], + "missing_optional": [] +} +``` + +Each entry reports the value (or, for `as_path` secrets, the `path` to a +persisted temp file), its `source` (`provider`, `generated`, or `default`), and +the serving provider's credential-free URI. The canonical JSON Schema is +committed at `schema/resolve-response.schema.json`. + ### set Set a secret value. diff --git a/schema/resolve-response.schema.json b/schema/resolve-response.schema.json new file mode 100644 index 0000000..3d7fa99 --- /dev/null +++ b/schema/resolve-response.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://secretspec.dev/schema/resolve-response.schema.json", + "title": "SecretSpec resolve response", + "description": "Value-carrying resolution result for one profile, emitted by `secretspec resolve --json` and the C ABI. CARRIES SECRET VALUES; treat as sensitive. When a required secret is missing, `secrets` is empty and `missing_required` is populated.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "provider", "profile", "secrets", "missing_required", "missing_optional"], + "properties": { + "schema_version": { + "description": "Wire-format version. Consumers should refuse versions they do not understand.", + "type": "integer", + "const": 1 + }, + "provider": { + "description": "Credential-free URI of the provider the resolution is reported against.", + "type": "string" + }, + "profile": { + "description": "The profile that was resolved.", + "type": "string" + }, + "secrets": { + "description": "Resolved secrets by name. Empty when a required secret is missing.", + "type": "object", + "additionalProperties": { "$ref": "#/$defs/resolvedSecret" } + }, + "missing_required": { + "description": "Required secrets not found anywhere. Non-empty means resolution failed.", + "type": "array", + "items": { "type": "string" } + }, + "missing_optional": { + "description": "Optional secrets not found.", + "type": "array", + "items": { "type": "string" } + } + }, + "$defs": { + "resolvedSecret": { + "type": "object", + "additionalProperties": false, + "required": ["as_path", "source"], + "properties": { + "value": { + "description": "The secret value, present when exposed inline (mutually exclusive with `path`).", + "type": "string" + }, + "path": { + "description": "Path to the temp file holding the value, present when `as_path` is set (mutually exclusive with `value`).", + "type": "string" + }, + "as_path": { + "description": "Whether this secret is exposed as a file path rather than inline.", + "type": "boolean" + }, + "source": { + "description": "Where the value came from.", + "type": "string", + "enum": ["provider", "generated", "default"] + }, + "source_provider": { + "description": "Credential-free URI of the provider that answered, present when `source` is `provider`.", + "type": "string" + } + } + } + } +} diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index 89148bd..4a2c270 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -103,6 +103,22 @@ enum Commands { #[arg(long)] explain: bool, }, + /// Resolve all secrets and print them as JSON (the SDK boundary). + /// + /// Unlike `check`, this prints secret VALUES (to stdout). It is intended for + /// programmatic consumption by other-language SDKs and tooling; pipe it, do + /// not display it. Use `--no-values` for a value-free structural view. + Resolve { + /// Provider backend to use + #[arg(short, long, env = "SECRETSPEC_PROVIDER")] + provider: Option, + /// Profile to use + #[arg(short = 'P', long, env = "SECRETSPEC_PROFILE")] + profile: Option, + /// Omit secret values, emitting only structure and provenance + #[arg(long)] + no_values: bool, + }, /// Init or show ~/.config/secretspec/config.toml Config { #[command(subcommand)] @@ -676,6 +692,35 @@ pub fn main() -> Result<()> { .wrap_err("Failed to persist temporary files")?; Ok(()) } + // Resolve all secrets to JSON (the SDK boundary; prints values) + Commands::Resolve { + provider, + profile, + no_values, + } => { + let mut app = load_secrets(&cli.file, &cli.reason)?; + if let Some(p) = provider { + app.set_provider(p); + } + if let Some(p) = profile { + app.set_profile(p); + } + let mut response = app + .resolve() + .into_diagnostic() + .wrap_err("Failed to resolve secrets")?; + if no_values { + response = response.without_values(); + } + let rendered = serde_json::to_string_pretty(&response) + .into_diagnostic() + .wrap_err("Failed to serialize resolve response")?; + println!("{}", rendered); + if !response.is_ok() { + std::process::exit(1); + } + Ok(()) + } // Import secrets from one provider to another Commands::Import { from_provider } => { let app = load_secrets(&cli.file, &cli.reason)?; diff --git a/secretspec/src/lib.rs b/secretspec/src/lib.rs index 2e81f4f..36289a0 100644 --- a/secretspec/src/lib.rs +++ b/secretspec/src/lib.rs @@ -46,6 +46,7 @@ mod config; mod error; pub(crate) mod generator; mod report; +mod resolve; mod secrets; mod validation; @@ -74,6 +75,7 @@ pub use provider::Provider; pub use report::{ RESOLUTION_REPORT_SCHEMA_VERSION, ResolutionReport, ResolutionStatus, SecretResolution, }; +pub use resolve::{RESOLVE_SCHEMA_VERSION, ResolveResponse, ResolvedSecret, ResolvedSource}; pub use secrets::Secrets; pub use validation::ValidatedSecrets; diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs new file mode 100644 index 0000000..b29ed89 --- /dev/null +++ b/secretspec/src/resolve.rs @@ -0,0 +1,92 @@ +//! The value-carrying resolve payload: the FFI/SDK boundary. +//! +//! Unlike the value-free [`crate::report::ResolutionReport`] (which powers +//! `check --json` and must never expose a value), this payload **does** carry +//! the resolved secret values. It is the single authoritative output that any +//! other-language SDK consumes, either over the C ABI (in-process) or via +//! `secretspec resolve --json` (subprocess). Producing it deliberately exposes +//! secrets, so it is only built at an explicit resolve boundary and its bytes +//! must be treated as sensitive by the caller. +//! +//! On a successful resolution `secrets` holds one entry per declared secret +//! that produced a value; `missing_optional` lists optional secrets with no +//! value. When a required secret is missing, resolution is an error: `secrets` +//! is empty and `missing_required` is populated, mirroring the derive crate's +//! `load()` which fails rather than returning partial secrets. +//! +//! The shape is versioned via [`RESOLVE_SCHEMA_VERSION`]. The canonical JSON +//! Schema lives at `schema/resolve-response.schema.json`. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Version of the [`ResolveResponse`] wire format. +pub const RESOLVE_SCHEMA_VERSION: u32 = 1; + +/// Where a resolved value came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolvedSource { + /// Returned by a storage provider. + Provider, + /// Freshly minted by the secret's `generate` config. + Generated, + /// The manifest's committed `default` value. + Default, +} + +/// One resolved secret. Exactly one of `value` or `path` is set: `path` when +/// the secret is materialized to a temp file (`as_path`), `value` otherwise. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedSecret { + /// The secret value, when exposed inline. + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, + /// Path to the temp file holding the value, when `as_path` is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + /// Whether this secret is exposed as a file path rather than inline. + pub as_path: bool, + /// Whether the value came from a provider, a generator, or a default. + pub source: ResolvedSource, + /// Credential-free URI of the provider that answered, when `source` is + /// `provider`. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_provider: Option, +} + +/// A complete value-carrying resolution result for one profile. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolveResponse { + /// Wire-format version; see [`RESOLVE_SCHEMA_VERSION`]. + pub schema_version: u32, + /// Credential-free URI of the provider the resolution reported against. + pub provider: String, + /// The profile that was resolved. + pub profile: String, + /// Resolved secrets by name. Empty when a required secret is missing. + /// `BTreeMap` keeps the JSON object key order deterministic. + pub secrets: BTreeMap, + /// Required secrets that were not found anywhere. Non-empty means the + /// resolution failed; `secrets` is then empty. + pub missing_required: Vec, + /// Optional secrets that were not found. + pub missing_optional: Vec, +} + +impl ResolveResponse { + /// True when no required secret is missing (the resolution succeeded). + pub fn is_ok(&self) -> bool { + self.missing_required.is_empty() + } + + /// Drop every inline value, keeping structure and provenance. Useful for an + /// inventory/policy consumer that wants the resolve shape without secrets. + pub fn without_values(mut self) -> Self { + for secret in self.secrets.values_mut() { + secret.value = None; + secret.path = None; + } + self + } +} diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index f8ff9b2..39eec37 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -5,10 +5,11 @@ use crate::config::{Config, GlobalConfig, Profile, RequireReason, Resolved}; use crate::error::{Result, SecretSpecError}; use crate::provider::Provider as ProviderTrait; use crate::report::{ResolutionStatus, SecretResolution}; +use crate::resolve::{RESOLVE_SCHEMA_VERSION, ResolveResponse, ResolvedSecret, ResolvedSource}; use crate::validation::{ValidatedSecrets, ValidationErrors}; use colored::Colorize; use secrecy::{ExposeSecret, SecretString}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryFrom; use std::env; use std::io::{self, IsTerminal, Read}; @@ -1981,6 +1982,90 @@ impl Secrets { self.validate_audited(true) } + /// Resolve every declared secret into a value-carrying [`ResolveResponse`], + /// the authoritative output other-language SDKs consume (over the C ABI or + /// `secretspec resolve --json`). + /// + /// Unlike [`Self::validate`], the returned payload **carries secret + /// values** (or, for `as_path` secrets, the path to a persisted temp file). + /// Treat its bytes as sensitive. When a required secret is missing the + /// resolution failed: `secrets` is empty and `missing_required` is + /// populated, mirroring the derive crate's `load()`. + /// + /// `as_path` temp files are persisted so the returned paths stay valid for + /// the caller; this is a one-shot boundary and the caller owns their + /// lifetime thereafter. + pub fn resolve(&self) -> Result { + match self.validate()? { + Ok(mut validated) => { + // Persist as_path temp files so returned paths outlive this call. + validated.keep_temp_files()?; + + let mut secrets = BTreeMap::new(); + for entry in &validated.resolution { + if entry.status != ResolutionStatus::Resolved { + continue; + } + let raw = validated + .resolved + .secrets + .get(&entry.name) + .expect("a Resolved entry always has a value") + .expose_secret() + .to_string(); + let source = if entry.generated { + ResolvedSource::Generated + } else if entry.default_applied { + ResolvedSource::Default + } else { + ResolvedSource::Provider + }; + let (value, path) = if entry.as_path { + (None, Some(raw)) + } else { + (Some(raw), None) + }; + secrets.insert( + entry.name.clone(), + ResolvedSecret { + value, + path, + as_path: entry.as_path, + source, + source_provider: entry.source_provider.clone(), + }, + ); + } + + let mut missing_optional = validated.missing_optional.clone(); + missing_optional.sort(); + + Ok(ResolveResponse { + schema_version: RESOLVE_SCHEMA_VERSION, + provider: validated.resolved.provider.clone(), + profile: validated.resolved.profile.clone(), + secrets, + missing_required: Vec::new(), + missing_optional, + }) + } + Err(errors) => { + let mut missing_required = errors.missing_required.clone(); + missing_required.sort(); + let mut missing_optional = errors.missing_optional.clone(); + missing_optional.sort(); + Ok(ResolveResponse { + schema_version: RESOLVE_SCHEMA_VERSION, + provider: errors.provider.clone(), + profile: errors.profile.clone(), + secrets: BTreeMap::new(), + missing_required, + missing_optional, + }) + } + } + } + /// Resolves all secrets. `emit_check` controls whether this pass records a /// `Check` audit event. /// diff --git a/secretspec/src/tests.rs b/secretspec/src/tests.rs index 7e17988..9625f08 100644 --- a/secretspec/src/tests.rs +++ b/secretspec/src/tests.rs @@ -425,6 +425,137 @@ fn test_resolution_report_provenance() { assert!(stripe.required); } +fn resolve_test_config(secrets: HashMap) -> Config { + let mut profiles = HashMap::new(); + profiles.insert( + "default".to_string(), + Profile { + defaults: None, + secrets, + }, + ); + Config { + project: Project { + name: "resolve-test".to_string(), + ..Default::default() + }, + profiles, + providers: None, + } +} + +#[test] +fn test_resolve_carries_values_and_provenance() { + use crate::resolve::ResolvedSource; + + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "DATABASE_URL=postgres://localhost/db\n").unwrap(); + + let secret = |required: bool, default: Option<&str>| Secret { + description: Some("test".to_string()), + required: Some(required), + default: default.map(String::from), + ..Default::default() + }; + + let mut secrets = HashMap::new(); + secrets.insert("DATABASE_URL".to_string(), secret(true, None)); + secrets.insert("LOG_LEVEL".to_string(), secret(false, Some("info"))); + secrets.insert("SENTRY_DSN".to_string(), secret(false, None)); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + let response = spec.resolve().unwrap(); + assert_eq!(response.schema_version, 1); + assert_eq!(response.profile, "default"); + assert!(response.is_ok()); + assert_eq!(response.missing_optional, vec!["SENTRY_DSN".to_string()]); + + // Provider value is exposed with provenance. + let db = &response.secrets["DATABASE_URL"]; + assert_eq!(db.value.as_deref(), Some("postgres://localhost/db")); + assert!(db.path.is_none()); + assert!(!db.as_path); + assert_eq!(db.source, ResolvedSource::Provider); + assert!(db.source_provider.is_some()); + + // Default value is exposed and attributed to the default source. + let log = &response.secrets["LOG_LEVEL"]; + assert_eq!(log.value.as_deref(), Some("info")); + assert_eq!(log.source, ResolvedSource::Default); + assert!(log.source_provider.is_none()); + + // Optional-missing does not appear in secrets. + assert!(!response.secrets.contains_key("SENTRY_DSN")); + + // without_values strips values but keeps structure. + let stripped = response.without_values(); + assert!(stripped.secrets["DATABASE_URL"].value.is_none()); + assert_eq!( + stripped.secrets["DATABASE_URL"].source, + ResolvedSource::Provider + ); +} + +#[test] +fn test_resolve_missing_required_is_empty_with_error_list() { + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "").unwrap(); + + let mut secrets = HashMap::new(); + secrets.insert( + "STRIPE_KEY".to_string(), + Secret { + description: Some("stripe".to_string()), + required: Some(true), + ..Default::default() + }, + ); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + let response = spec.resolve().unwrap(); + assert!(!response.is_ok()); + assert_eq!(response.missing_required, vec!["STRIPE_KEY".to_string()]); + // A failed resolution returns no values, mirroring the derive crate's load(). + assert!(response.secrets.is_empty()); +} + +#[test] +fn test_resolve_as_path_returns_persisted_path() { + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "TLS_CERT=----cert-bytes----\n").unwrap(); + + let mut secrets = HashMap::new(); + secrets.insert( + "TLS_CERT".to_string(), + Secret { + description: Some("cert".to_string()), + required: Some(true), + as_path: Some(true), + ..Default::default() + }, + ); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + let response = spec.resolve().unwrap(); + let cert = &response.secrets["TLS_CERT"]; + assert!(cert.as_path); + assert!(cert.value.is_none()); + let path = cert.path.as_deref().expect("as_path yields a path"); + // The temp file is persisted, so the path is readable after resolve returns. + let contents = fs::read_to_string(path).unwrap(); + assert_eq!(contents, "----cert-bytes----"); + fs::remove_file(path).ok(); +} + #[test] fn test_secretspec_new() { let config = Config { From f927037ce655c23c9f11ab994cfe9ac5c5c866fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 18:03:41 -0400 Subject: [PATCH 03/56] feat: add secretspec-ffi crate exposing the narrow C ABI Phase 2, slice 2: the in-process boundary other-language SDKs bind to. A deliberately narrow, JSON-in/JSON-out C ABI keeps every binding thin and keeps resolution logic in the secretspec crate alone. - Three-function surface: secretspec_resolve(request_json) -> response_json, secretspec_free(ptr), secretspec_abi_version(). Request and response are the versioned JSON contract; the response envelope separates transport failure ({"ok":false,"error":{kind,message}}) from a successful resolution ({"ok":true,"response":ResolveResponse}) that still reports missing_required. - Panics are caught at the boundary (never unwind across FFI); returned strings are caller-owned and freed via secretspec_free; null and bad input are handled. - Hand-written C header at secretspec-ffi/include/secretspec.h; crate builds as cdylib + staticlib + rlib. - SecretSpecError::kind() promoted to pub for typed SDK error handling. - Tests drive the real extern \"C\" entry points (values, no_values, missing-required, invalid input, missing manifest); a committed smoke.c plus a drafted per-platform ffi-build workflow build and smoke-test the cdylib on linux/macos/windows (native per-runner; portable packaging is follow-up). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ffi-build.yml | 80 +++++++++++ CHANGELOG.md | 12 ++ Cargo.lock | 10 ++ Cargo.toml | 2 +- secretspec-ffi/Cargo.toml | 23 ++++ secretspec-ffi/include/secretspec.h | 51 +++++++ secretspec-ffi/src/lib.rs | 204 ++++++++++++++++++++++++++++ secretspec-ffi/tests/c_abi.rs | 143 +++++++++++++++++++ secretspec-ffi/tests/smoke.c | 35 +++++ secretspec/src/error.rs | 5 +- 10 files changed, 562 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ffi-build.yml create mode 100644 secretspec-ffi/Cargo.toml create mode 100644 secretspec-ffi/include/secretspec.h create mode 100644 secretspec-ffi/src/lib.rs create mode 100644 secretspec-ffi/tests/c_abi.rs create mode 100644 secretspec-ffi/tests/smoke.c diff --git a/.github/workflows/ffi-build.yml b/.github/workflows/ffi-build.yml new file mode 100644 index 0000000..4463b16 --- /dev/null +++ b/.github/workflows/ffi-build.yml @@ -0,0 +1,80 @@ +name: "FFI cdylib" + +# Builds the secretspec-ffi C ABI library for each platform the language SDKs +# bundle. Builds NATIVELY on a per-platform runner rather than cross-compiling, +# because the crate links system libraries (e.g. dbus/keyring on Linux) that +# make cross toolchains painful. +# +# NOTE: This produces development-grade artifacts. Production packaging +# (manylinux compliance for Python wheels, macOS code signing/notarization, +# Windows runtime considerations) is follow-up work tracked with the SDK +# distribution effort. + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-ffi/**" + - "secretspec/**" + - ".github/workflows/ffi-build.yml" + +jobs: + build: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-latest + artifact: libsecretspec_ffi.so + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + artifact: libsecretspec_ffi.so + - target: x86_64-apple-darwin + runner: macos-13 + artifact: libsecretspec_ffi.dylib + - target: aarch64-apple-darwin + runner: macos-14 + artifact: libsecretspec_ffi.dylib + - target: x86_64-pc-windows-msvc + runner: windows-latest + artifact: secretspec_ffi.dll + + steps: + - uses: actions/checkout@v5 + + - name: Install Linux system dependencies + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Build cdylib + run: cargo build -p secretspec-ffi --release --target ${{ matrix.target }} + + - name: Smoke test the C ABI (Unix) + if: runner.os != 'Windows' + run: | + cc secretspec-ffi/tests/smoke.c \ + -I secretspec-ffi/include \ + -L target/${{ matrix.target }}/release \ + -lsecretspec_ffi -o smoke + LD_LIBRARY_PATH=target/${{ matrix.target }}/release \ + DYLD_LIBRARY_PATH=target/${{ matrix.target }}/release \ + ./smoke + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: secretspec-ffi-${{ matrix.target }} + path: | + target/${{ matrix.target }}/release/${{ matrix.artifact }} + secretspec-ffi/include/secretspec.h diff --git a/CHANGELOG.md b/CHANGELOG.md index a581b7d..a2e25f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec-ffi` crate exposing a deliberately narrow C ABI for resolving + secrets from any language. The entire native surface is three functions + (`secretspec_resolve`, `secretspec_free`, `secretspec_abi_version`); all + richness lives in the versioned JSON contract, so language bindings stay thin. + `secretspec_resolve` takes a JSON request (`path`, `provider`, `profile`, + `reason`, `no_values`, all optional) and returns a JSON envelope that + separates transport failure (`{"ok": false, "error": {...}}`) from a + successful resolution (`{"ok": true, "response": {...}}`), which still reports + domain results like `missing_required` in its own fields. Panics are caught at + the boundary; returned strings are owned by the caller and freed with + `secretspec_free`. A C header ships at `secretspec-ffi/include/secretspec.h`. + `SecretSpecError::kind()` is now public so SDKs can do typed error handling. - `secretspec resolve --json` resolves every declared secret and prints a versioned, value-carrying JSON object: the SDK boundary other-language clients consume. Each entry reports the value (or, for `as_path` secrets, the diff --git a/Cargo.lock b/Cargo.lock index 6627e4b..db62096 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3901,6 +3901,16 @@ dependencies = [ "serde", ] +[[package]] +name = "secretspec-ffi" +version = "0.12.0" +dependencies = [ + "secretspec", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "security-framework" version = "2.11.1" diff --git a/Cargo.toml b/Cargo.toml index 5792527..749ef5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["secretspec", "secretspec-derive", "examples/derive"] +members = ["secretspec", "secretspec-derive", "secretspec-ffi", "examples/derive"] resolver = "2" [workspace.package] diff --git a/secretspec-ffi/Cargo.toml b/secretspec-ffi/Cargo.toml new file mode 100644 index 0000000..f57b1c7 --- /dev/null +++ b/secretspec-ffi/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "secretspec-ffi" +version.workspace = true +edition.workspace = true +repository = "https://github.com/cachix/secretspec" +description = "C ABI for SecretSpec: resolve secrets from any language" +license = "Apache-2.0" + +[lib] +name = "secretspec_ffi" +# cdylib for dynamic loading (the SDK distribution target); staticlib for +# embedding; lib (rlib) so Rust integration tests can call the entry points. +# All expose the same narrow C ABI. +crate-type = ["cdylib", "staticlib", "lib"] + +[dependencies] +secretspec = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } +tempfile = { workspace = true } diff --git a/secretspec-ffi/include/secretspec.h b/secretspec-ffi/include/secretspec.h new file mode 100644 index 0000000..4b36858 --- /dev/null +++ b/secretspec-ffi/include/secretspec.h @@ -0,0 +1,51 @@ +/* + * SecretSpec C ABI. + * + * A deliberately narrow, JSON-in / JSON-out boundary. The entire native surface + * is the three functions below; all richness lives in the versioned JSON + * contract so language bindings stay thin. + * + * Request JSON (all fields optional): + * { "path": ".../secretspec.toml", "provider": "keyring://", + * "profile": "production", "reason": "boot", "no_values": false } + * + * Response envelope: + * { "ok": true, "response": { ...resolve response... } } + * { "ok": false, "error": { "kind": "io", "message": "..." } } + * + * The response (when ok) carries secret values unless "no_values" was set. + * Treat returned strings as sensitive and free them promptly. + */ +#ifndef SECRETSPEC_H +#define SECRETSPEC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Resolve secrets described by `request_json` (a NUL-terminated UTF-8 JSON + * string). Returns a newly allocated, NUL-terminated JSON response envelope + * that the caller OWNS and must release with secretspec_free(). + * + * Returns NULL only on catastrophic allocation failure. + */ +char *secretspec_resolve(const char *request_json); + +/* + * Free a string previously returned by secretspec_resolve(). NULL is ignored. + * Must not be called twice on the same pointer. + */ +void secretspec_free(char *ptr); + +/* + * Return the ABI version as a static NUL-terminated string. Do NOT free; the + * pointer is valid for the lifetime of the loaded library. + */ +const char *secretspec_abi_version(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SECRETSPEC_H */ diff --git a/secretspec-ffi/src/lib.rs b/secretspec-ffi/src/lib.rs new file mode 100644 index 0000000..0922cd3 --- /dev/null +++ b/secretspec-ffi/src/lib.rs @@ -0,0 +1,204 @@ +//! The SecretSpec C ABI: a deliberately narrow, JSON-in/JSON-out boundary. +//! +//! The entire native surface is three functions. Richness lives in the +//! versioned JSON contract, not in a wide C API, so that every language binding +//! (Python via cffi, Go via purego, Ruby via ffi, Node via napi-rs) stays a +//! thin shell: marshal a request string in, get a response string out, free it. +//! Resolution logic lives only in the `secretspec` crate; this is a wrapper. +//! +//! # Contract +//! +//! - [`secretspec_resolve`] takes a UTF-8, NUL-terminated JSON **request** and +//! returns a heap-allocated, NUL-terminated JSON **response envelope**. The +//! caller owns the returned pointer and must free it with [`secretspec_free`]. +//! - [`secretspec_abi_version`] returns a static version string (do not free). +//! +//! ## Request JSON +//! +//! ```json +//! { "path": "…/secretspec.toml", "provider": "keyring://", +//! "profile": "production", "reason": "boot", "no_values": false } +//! ``` +//! +//! All fields are optional. `path` omitted means "walk up from the working +//! directory" like the CLI. `no_values` strips secret values from the response. +//! +//! ## Response envelope +//! +//! ```json +//! { "ok": true, "response": { …ResolveResponse… } } +//! { "ok": false, "error": { "kind": "io", "message": "…" } } +//! ``` +//! +//! `ok: true` carries the value-carrying [`secretspec::ResolveResponse`] (which +//! still reports domain failures like `missing_required` in its own fields). +//! `ok: false` means the call itself failed (bad manifest, provider error, +//! reason policy). This separates transport failure from "a required secret is +//! missing", which the SDK surfaces differently. +//! +//! # Safety +//! +//! Returned response strings carry secret values (unless `no_values`). Treat +//! them as sensitive and free them promptly. The host language's heap cannot be +//! zeroized; for file-shaped secrets prefer `as_path`, whose value never crosses +//! the boundary (only the temp-file path does). + +use std::ffi::{CStr, CString, c_char}; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::Path; + +use secretspec::{ResolveResponse, Secrets}; +use serde::{Deserialize, Serialize}; + +/// ABI version, NUL-terminated for direct return as a C string. +const ABI_VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), "\0"); + +#[derive(Debug, Default, Deserialize)] +struct ResolveRequest { + #[serde(default)] + path: Option, + #[serde(default)] + provider: Option, + #[serde(default)] + profile: Option, + #[serde(default)] + reason: Option, + #[serde(default)] + no_values: bool, +} + +#[derive(Debug, Serialize)] +struct FfiError { + kind: String, + message: String, +} + +#[derive(Debug, Serialize)] +struct Envelope { + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + response: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +impl Envelope { + fn ok(response: ResolveResponse) -> Self { + Self { + ok: true, + response: Some(response), + error: None, + } + } + + fn err(kind: &str, message: impl Into) -> Self { + Self { + ok: false, + response: None, + error: Some(FfiError { + kind: kind.to_string(), + message: message.into(), + }), + } + } +} + +/// Returns the ABI version as a static NUL-terminated string. Do not free. +/// +/// # Safety +/// The returned pointer is valid for the lifetime of the loaded library. +#[unsafe(no_mangle)] +pub extern "C" fn secretspec_abi_version() -> *const c_char { + ABI_VERSION.as_ptr().cast() +} + +/// Frees a string previously returned by [`secretspec_resolve`]. +/// +/// # Safety +/// `ptr` must be either null or a pointer returned by [`secretspec_resolve`] +/// that has not already been freed. Passing anything else is undefined +/// behavior. Null is accepted and ignored. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn secretspec_free(ptr: *mut c_char) { + if ptr.is_null() { + return; + } + // Retake ownership and drop. + unsafe { + drop(CString::from_raw(ptr)); + } +} + +/// Resolves secrets described by a JSON request, returning a JSON response +/// envelope. See the module docs for the request/response shapes. +/// +/// # Safety +/// `request_json` must be null or a valid pointer to a NUL-terminated C string. +/// The returned pointer is owned by the caller and must be freed with +/// [`secretspec_free`]. Returns null only on catastrophic allocation failure. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn secretspec_resolve(request_json: *const c_char) -> *mut c_char { + // Never let a panic unwind across the FFI boundary (that is UB). + let envelope = match catch_unwind(AssertUnwindSafe(|| resolve_inner(request_json))) { + Ok(env) => env, + Err(_) => Envelope::err("panic", "internal panic during resolve"), + }; + + let json = serde_json::to_string(&envelope).unwrap_or_else(|_| { + // Should be unreachable; fall back to a hand-built valid envelope. + "{\"ok\":false,\"error\":{\"kind\":\"serialize\",\"message\":\"failed to serialize response\"}}" + .to_string() + }); + + match CString::new(json) { + Ok(c) => c.into_raw(), + Err(_) => std::ptr::null_mut(), + } +} + +fn resolve_inner(request_json: *const c_char) -> Envelope { + if request_json.is_null() { + return Envelope::err("invalid_input", "request_json was null"); + } + + // Safety: caller contract guarantees a NUL-terminated string when non-null. + let raw = unsafe { CStr::from_ptr(request_json) }; + let text = match raw.to_str() { + Ok(s) => s, + Err(_) => return Envelope::err("invalid_input", "request_json was not valid UTF-8"), + }; + + let request: ResolveRequest = match serde_json::from_str(text) { + Ok(req) => req, + Err(e) => return Envelope::err("invalid_request", format!("invalid request JSON: {e}")), + }; + + let loaded = match &request.path { + Some(path) => Secrets::load_from(Path::new(path)), + None => Secrets::load(), + }; + let mut app = match loaded { + Ok(app) => app, + Err(e) => return Envelope::err(e.kind(), e.to_string()), + }; + + if let Some(provider) = request.provider { + app.set_provider(provider); + } + if let Some(profile) = request.profile { + app.set_profile(profile); + } + if let Some(reason) = request.reason { + app = app.with_reason(reason); + } + + match app.resolve() { + Ok(mut response) => { + if request.no_values { + response = response.without_values(); + } + Envelope::ok(response) + } + Err(e) => Envelope::err(e.kind(), e.to_string()), + } +} diff --git a/secretspec-ffi/tests/c_abi.rs b/secretspec-ffi/tests/c_abi.rs new file mode 100644 index 0000000..ba2703e --- /dev/null +++ b/secretspec-ffi/tests/c_abi.rs @@ -0,0 +1,143 @@ +//! Exercises the C ABI through the real extern "C" entry points, as a native +//! caller would: build a request JSON, call `secretspec_resolve`, parse the +//! returned envelope, then `secretspec_free`. + +use std::ffi::{CStr, CString, c_char}; +use std::fs; + +use secretspec_ffi::{secretspec_abi_version, secretspec_free, secretspec_resolve}; +use serde_json::Value; +use tempfile::TempDir; + +/// Call the C ABI with a Rust string request and return the parsed JSON +/// envelope, freeing the native allocation. +fn resolve(request: &str) -> Value { + let c_request = CString::new(request).unwrap(); + let ptr: *mut c_char = unsafe { secretspec_resolve(c_request.as_ptr()) }; + assert!(!ptr.is_null(), "resolve returned null"); + let json = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap().to_string(); + unsafe { secretspec_free(ptr) }; + serde_json::from_str(&json).unwrap() +} + +fn write_project(dir: &TempDir, manifest: &str, dotenv: &str) -> (String, String) { + let manifest_path = dir.path().join("secretspec.toml"); + let env_path = dir.path().join(".env"); + fs::write(&manifest_path, manifest).unwrap(); + fs::write(&env_path, dotenv).unwrap(); + ( + manifest_path.display().to_string(), + format!("dotenv://{}", env_path.display()), + ) +} + +const MANIFEST: &str = r#" +[project] +name = "ffi-test" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } +"#; + +#[test] +fn abi_version_is_nonempty() { + let ptr = secretspec_abi_version(); + assert!(!ptr.is_null()); + let version = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap(); + assert!(!version.is_empty()); + // Static string: no free. +} + +#[test] +fn resolve_returns_values_and_provenance() { + let dir = TempDir::new().unwrap(); + let (manifest_path, provider) = write_project(&dir, MANIFEST, "DATABASE_URL=postgres://db\n"); + + let request = serde_json::json!({ + "path": manifest_path, + "provider": provider, + "reason": "ffi test", + }) + .to_string(); + + let env = resolve(&request); + assert_eq!(env["ok"], true, "envelope: {env}"); + let response = &env["response"]; + assert_eq!(response["schema_version"], 1); + assert_eq!(response["profile"], "default"); + assert_eq!( + response["secrets"]["DATABASE_URL"]["value"], + "postgres://db" + ); + assert_eq!(response["secrets"]["DATABASE_URL"]["source"], "provider"); + assert_eq!(response["secrets"]["LOG_LEVEL"]["value"], "info"); + assert_eq!(response["secrets"]["LOG_LEVEL"]["source"], "default"); + assert_eq!(response["missing_optional"][0], "SENTRY_DSN"); + assert!(response["missing_required"].as_array().unwrap().is_empty()); +} + +#[test] +fn resolve_no_values_strips_secrets() { + let dir = TempDir::new().unwrap(); + let (manifest_path, provider) = write_project(&dir, MANIFEST, "DATABASE_URL=postgres://db\n"); + + let request = serde_json::json!({ + "path": manifest_path, + "provider": provider, + "reason": "ffi test", + "no_values": true, + }) + .to_string(); + + let env = resolve(&request); + assert_eq!(env["ok"], true); + // Structure and provenance remain, but no value is present. + let db = &env["response"]["secrets"]["DATABASE_URL"]; + assert_eq!(db["source"], "provider"); + assert!(db.get("value").is_none(), "value should be stripped: {db}"); +} + +#[test] +fn resolve_missing_required_is_ok_envelope_with_error_list() { + let dir = TempDir::new().unwrap(); + // DATABASE_URL is required but absent from the backend. + let (manifest_path, provider) = write_project(&dir, MANIFEST, ""); + + let request = serde_json::json!({ + "path": manifest_path, + "provider": provider, + "reason": "ffi test", + }) + .to_string(); + + let env = resolve(&request); + // A missing required secret is a domain result, not a transport error: + // the envelope is ok, but the response reports it. + assert_eq!(env["ok"], true, "envelope: {env}"); + assert_eq!(env["response"]["missing_required"][0], "DATABASE_URL"); + assert!(env["response"]["secrets"].as_object().unwrap().is_empty()); +} + +#[test] +fn invalid_request_json_yields_error_envelope() { + let env = resolve("not json at all"); + assert_eq!(env["ok"], false); + assert_eq!(env["error"]["kind"], "invalid_request"); +} + +#[test] +fn missing_manifest_yields_error_envelope() { + let request = serde_json::json!({ + "path": "/definitely/does/not/exist/secretspec.toml", + "reason": "ffi test", + }) + .to_string(); + + let env = resolve(&request); + assert_eq!(env["ok"], false, "envelope: {env}"); + assert!(env["error"]["kind"].is_string()); + assert!(env["error"]["message"].is_string()); +} diff --git a/secretspec-ffi/tests/smoke.c b/secretspec-ffi/tests/smoke.c new file mode 100644 index 0000000..42c3437 --- /dev/null +++ b/secretspec-ffi/tests/smoke.c @@ -0,0 +1,35 @@ +/* + * Minimal C smoke test for the SecretSpec C ABI. Proves the cdylib links, the + * three entry points are callable from C, and the malloc/free roundtrip works. + * Run by the ffi-build workflow against the freshly built library. + */ +#include +#include +#include "secretspec.h" + +int main(void) { + const char *version = secretspec_abi_version(); + if (version == NULL || version[0] == '\0') { + printf("FAIL: abi_version empty\n"); + return 1; + } + printf("abi_version: %s\n", version); + + /* A deliberately invalid request must yield a well-formed error envelope. */ + char *out = secretspec_resolve("not json"); + if (out == NULL) { + printf("FAIL: resolve returned NULL\n"); + return 1; + } + printf("resolve(bad): %s\n", out); + if (strstr(out, "\"ok\":false") == NULL) { + printf("FAIL: expected an error envelope\n"); + secretspec_free(out); + return 1; + } + secretspec_free(out); + secretspec_free(NULL); /* must be a no-op */ + + printf("OK\n"); + return 0; +} diff --git a/secretspec/src/error.rs b/secretspec/src/error.rs index 1fe8285..7d47b71 100644 --- a/secretspec/src/error.rs +++ b/secretspec/src/error.rs @@ -67,11 +67,12 @@ pub enum SecretSpecError { } impl SecretSpecError { - /// A stable, non-sensitive token identifying the error variant, for audit logs. + /// A stable, non-sensitive token identifying the error variant, for audit + /// logs and typed handling by other-language SDKs over the FFI boundary. /// /// Returns only the variant name, never the error message: messages can embed /// secret names, provider URIs, or backend detail that must not reach the log. - pub(crate) fn kind(&self) -> &'static str { + pub fn kind(&self) -> &'static str { match self { SecretSpecError::Io(_) => "io", SecretSpecError::Toml(_) => "toml", From 5f80f5388a6e7dd94ce761af4a76c4b9f053c488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 18:11:28 -0400 Subject: [PATCH 04/56] feat: add shared codegen IR (secretspec::codegen) Phase 3, slice 1: the single brain for typed-accessor generation. Every generator (the Rust derive macro and the future TS/Python/Go/Ruby emitters) needs the same manifest decisions; computing them in one place stops drift. - build_ir(&Config) -> CodegenIr reduces a manifest to a language-neutral IR: project name, sorted profile list, the union field set (optional if optional-in or missing-from any profile, a path if as_path in any profile), and the per-profile raw (non-merged) field sets. - Faithfully reproduces derive macro semantics, including the long-standing quirk that an unspecified `required` is treated as optional (differs from the runtime resolver) so generated output stays stable. - IR types are serde-serializable for emitters and tooling. Unit tests cover union optionality, missing-in-profile, as_path-in-any, per-profile exactness, descriptions, and the empty-profiles default case. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 + secretspec/src/codegen.rs | 324 ++++++++++++++++++++++++++++++++++++++ secretspec/src/lib.rs | 1 + 3 files changed, 331 insertions(+) create mode 100644 secretspec/src/codegen.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a2e25f1..61dd9ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec::codegen` module: a shared, language-neutral intermediate + representation (IR) that reduces a manifest to the typed-accessor decisions + every generator needs (union vs per-profile field sets, optionality, `as_path`, + profile list). It is the single source of truth those decisions are computed + in, so the Rust derive macro and the eventual TypeScript/Python/Go/Ruby + emitters cannot drift. `build_ir(&Config) -> CodegenIr`. - New `secretspec-ffi` crate exposing a deliberately narrow C ABI for resolving secrets from any language. The entire native surface is three functions (`secretspec_resolve`, `secretspec_free`, `secretspec_abi_version`); all diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs new file mode 100644 index 0000000..2cf51bb --- /dev/null +++ b/secretspec/src/codegen.rs @@ -0,0 +1,324 @@ +//! The shared codegen intermediate representation (IR). +//! +//! Every typed-accessor generator computes the *same* decisions from a manifest: +//! which secrets exist, whether a field is optional, whether it is a file path, +//! how profiles map to types. If each generator (the Rust derive macro plus the +//! eventual TypeScript/Python/Go/Ruby emitters) recomputed those decisions, they +//! would drift. This module is the single brain: a manifest is reduced to a +//! language-neutral [`CodegenIr`] once, and every emitter is a thin template +//! over it. +//! +//! The IR deliberately mirrors the two shapes the derive crate exposes: +//! - a **union** field set (`SecretSpec`) safe to use without knowing the +//! profile: a field is optional if it is optional in, or missing from, *any* +//! profile, and a path if it is a path in *any* profile; +//! - **per-profile** field sets (`SecretSpecProfile`) with exact, raw types: a +//! field is optional iff that profile does not mark it `required = true`, and a +//! path iff that profile sets `as_path = true`. Per-profile sets are NOT +//! inheritance-merged with the `default` profile, matching the derive macro. +//! +//! Note: the union/per-profile "optional iff not `required = true`" rule means a +//! secret with `required` unspecified is treated as optional here, which is the +//! derive crate's long-standing behavior (and differs from the runtime +//! resolver, where unspecified means required). The IR reproduces the derive +//! behavior so generated code stays stable. + +use crate::config::{Config, Secret}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; + +/// One field in a generated type. `name` is the canonical `UPPER_SNAKE` env key +/// and the source of truth; each emitter applies its own casing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IrField { + /// The declared secret name (the `UPPER_SNAKE` manifest key). + pub name: String, + /// Whether the generated field is optional (nullable) rather than required. + pub optional: bool, + /// Whether the value is exposed as a file path rather than inline. + pub as_path: bool, + /// The secret's description, when one is declared. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, +} + +/// A profile and its exact (raw, non-merged) field set. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IrProfile { + /// The profile name as written in the manifest (e.g. `production`). + pub name: String, + /// The profile's fields, sorted by name for deterministic output. + pub fields: Vec, +} + +/// The complete, language-neutral codegen description of a manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CodegenIr { + /// The project name. + pub project: String, + /// Profile names in sorted order; `["default"]` when the manifest declares + /// none. + pub profiles: Vec, + /// Union fields safe across every profile, sorted by name. + pub union: Vec, + /// Per-profile exact field sets, in the same order as [`Self::profiles`]. + pub profile_fields: Vec, +} + +/// A secret is optional unless the profile explicitly marks it `required = true`. +/// Matches the derive crate (and differs from the runtime resolver default). +fn is_secret_optional(secret: &Secret) -> bool { + secret.required != Some(true) +} + +/// For the union type a field is optional if it is optional in, or absent from, +/// any profile, so the union can safely represent secrets from any profile. +fn is_field_optional_across_profiles(name: &str, config: &Config) -> bool { + for profile in config.profiles.values() { + match profile.secrets.get(name) { + Some(secret) if !is_secret_optional(secret) => {} + // Optional in this profile, or missing from it. + _ => return true, + } + } + false +} + +/// For the union type a field is a path if any profile declares it `as_path`. +fn is_field_as_path_across_profiles(name: &str, config: &Config) -> bool { + config + .profiles + .values() + .any(|profile| profile.secrets.get(name).and_then(|s| s.as_path) == Some(true)) +} + +/// Pick a description for a union field: the first profile (by sorted name) that +/// declares one. +fn union_description(name: &str, config: &Config) -> Option { + let mut profile_names: Vec<&String> = config.profiles.keys().collect(); + profile_names.sort(); + profile_names.into_iter().find_map(|profile_name| { + config.profiles[profile_name] + .secrets + .get(name) + .and_then(|s| s.description.clone()) + }) +} + +/// Build the union field set: every unique secret across all profiles, sorted. +fn build_union(config: &Config) -> Vec { + let names: BTreeSet<&String> = config + .profiles + .values() + .flat_map(|profile| profile.secrets.keys()) + .collect(); + + names + .into_iter() + .map(|name| IrField { + name: name.clone(), + optional: is_field_optional_across_profiles(name, config), + as_path: is_field_as_path_across_profiles(name, config), + description: union_description(name, config), + }) + .collect() +} + +/// Build the exact field set for one profile's raw secrets, sorted by name. +fn build_profile_fields(secrets: &HashMap) -> Vec { + let mut fields: Vec = secrets + .iter() + .map(|(name, secret)| IrField { + name: name.clone(), + optional: is_secret_optional(secret), + as_path: secret.as_path.unwrap_or(false), + description: secret.description.clone(), + }) + .collect(); + fields.sort_by(|a, b| a.name.cmp(&b.name)); + fields +} + +/// Reduce a manifest to the language-neutral [`CodegenIr`] every emitter +/// consumes. This is the only place manifest typing decisions are made. +pub fn build_ir(config: &Config) -> CodegenIr { + let union = build_union(config); + + let profile_fields = if config.profiles.is_empty() { + // No declared profiles: a single `default` profile carrying every field, + // matching the derive macro's empty-profile case. + vec![IrProfile { + name: "default".to_string(), + fields: union.clone(), + }] + } else { + let mut names: Vec<&String> = config.profiles.keys().collect(); + names.sort(); + names + .into_iter() + .map(|name| IrProfile { + name: name.clone(), + fields: build_profile_fields(&config.profiles[name].secrets), + }) + .collect() + }; + + let profiles = profile_fields.iter().map(|p| p.name.clone()).collect(); + + CodegenIr { + project: config.project.name.clone(), + profiles, + union, + profile_fields, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{Profile, Project}; + + fn secret(required: Option, as_path: Option, desc: Option<&str>) -> Secret { + Secret { + description: desc.map(String::from), + required, + as_path, + ..Default::default() + } + } + + fn config_with(profiles: Vec<(&str, Vec<(&str, Secret)>)>) -> Config { + let mut map = HashMap::new(); + for (name, secrets) in profiles { + let mut secret_map = HashMap::new(); + for (sname, s) in secrets { + secret_map.insert(sname.to_string(), s); + } + map.insert( + name.to_string(), + Profile { + defaults: None, + secrets: secret_map, + }, + ); + } + Config { + project: Project { + name: "ir-test".to_string(), + ..Default::default() + }, + profiles: map, + providers: None, + } + } + + fn union_field<'a>(ir: &'a CodegenIr, name: &str) -> &'a IrField { + ir.union.iter().find(|f| f.name == name).unwrap() + } + + #[test] + fn union_optional_if_optional_or_missing_in_any_profile() { + let ir = build_ir(&config_with(vec![ + ( + "development", + vec![ + ("DATABASE_URL", secret(Some(true), None, None)), + ("API_KEY", secret(Some(false), None, None)), + ], + ), + ( + "production", + vec![ + ("DATABASE_URL", secret(Some(true), None, None)), + ("API_KEY", secret(Some(true), None, None)), + ("REDIS_URL", secret(Some(true), None, None)), + ], + ), + ])); + + // Required in every profile it appears in -> required in the union. + assert!(!union_field(&ir, "DATABASE_URL").optional); + // Optional in development -> optional in the union. + assert!(union_field(&ir, "API_KEY").optional); + // Missing from development -> optional in the union. + assert!(union_field(&ir, "REDIS_URL").optional); + + // Union is sorted and complete. + let names: Vec<&str> = ir.union.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["API_KEY", "DATABASE_URL", "REDIS_URL"]); + } + + #[test] + fn union_as_path_if_any_profile_marks_it() { + let ir = build_ir(&config_with(vec![ + ( + "development", + vec![("CERT", secret(Some(true), None, None))], + ), + ( + "production", + vec![("CERT", secret(Some(true), Some(true), None))], + ), + ])); + assert!(union_field(&ir, "CERT").as_path); + } + + #[test] + fn per_profile_fields_are_raw_and_exact() { + let ir = build_ir(&config_with(vec![ + ( + "development", + vec![ + ("DATABASE_URL", secret(Some(true), None, Some("dev db"))), + ("API_KEY", secret(Some(false), None, None)), + ], + ), + ( + "production", + vec![("DATABASE_URL", secret(Some(true), Some(true), None))], + ), + ])); + + assert_eq!(ir.profiles, vec!["development", "production"]); + + let dev = ir + .profile_fields + .iter() + .find(|p| p.name == "development") + .unwrap(); + let dev_names: Vec<&str> = dev.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(dev_names, vec!["API_KEY", "DATABASE_URL"]); + // Description flows through per profile. + assert_eq!(dev.fields[1].description.as_deref(), Some("dev db")); + + // production has only DATABASE_URL, here as a path. + let prod = ir + .profile_fields + .iter() + .find(|p| p.name == "production") + .unwrap(); + assert_eq!(prod.fields.len(), 1); + assert!(prod.fields[0].as_path); + assert!(!prod.fields[0].optional); + } + + #[test] + fn unspecified_required_is_optional_matching_derive() { + let ir = build_ir(&config_with(vec![( + "default", + vec![("TOKEN", secret(None, None, None))], + )])); + assert!(union_field(&ir, "TOKEN").optional); + } + + #[test] + fn empty_profiles_yield_single_default_with_union_fields() { + let mut config = config_with(vec![]); + config.profiles.clear(); + let ir = build_ir(&config); + assert_eq!(ir.profiles, vec!["default"]); + assert_eq!(ir.profile_fields.len(), 1); + assert_eq!(ir.profile_fields[0].name, "default"); + assert!(ir.union.is_empty()); + } +} diff --git a/secretspec/src/lib.rs b/secretspec/src/lib.rs index 36289a0..f980c20 100644 --- a/secretspec/src/lib.rs +++ b/secretspec/src/lib.rs @@ -42,6 +42,7 @@ // Internal modules mod audit; +pub mod codegen; mod config; mod error; pub(crate) mod generator; From d8ed866de795be295743c5b80ae7190688a8a372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 18:52:44 -0400 Subject: [PATCH 05/56] refactor: drive the derive macro through the shared codegen IR Phase 3, slice 2: validate the IR against the known-good consumer and remove the duplicated typing logic, so the derive macro and the future TS/Python/Go/Ruby emitters share one brain. - declare_secrets now calls secretspec::codegen::build_ir(&config) once and sources every typing decision from it: the union struct fields, per-profile enum variants, the Profile list, and the load_profile arms. The empty-profiles special case disappears because the IR already models it. - Removed the derive's own is_secret_optional / is_field_optional_across_profiles / is_field_as_path / analyze_field_types / get_profile_variants; the token emitters now read optional/as_path straight off the IR (only the Rust type mapping stays local). Dropped the unit tests that covered those moved helpers (now tested in secretspec::codegen). - Generated API is unchanged: 15 derive unit + 3 trybuild UI + 12 integration tests pass, and the example crate resolves through the generated builder. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 7 + secretspec-derive/src/lib.rs | 333 +++++++++----------------------- secretspec-derive/src/tests.rs | 343 --------------------------------- 3 files changed, 98 insertions(+), 585 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61dd9ff..93866d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 profile list). It is the single source of truth those decisions are computed in, so the Rust derive macro and the eventual TypeScript/Python/Go/Ruby emitters cannot drift. `build_ir(&Config) -> CodegenIr`. + +### Changed +- The `secretspec-derive` macro now computes all of its typing decisions through + the shared `secretspec::codegen` IR instead of its own duplicated logic. The + generated `SecretSpec`/`SecretSpecProfile`/`Profile` API and builder are + unchanged (verified by the existing integration and trybuild tests); this + guarantees the macro and the future other-language emitters stay consistent. - New `secretspec-ffi` crate exposing a deliberately narrow C ABI for resolving secrets from any language. The entire native surface is three functions (`secretspec_resolve`, `secretspec_free`, `secretspec_abi_version`); all diff --git a/secretspec-derive/src/lib.rs b/secretspec-derive/src/lib.rs index 880edf6..093f088 100644 --- a/secretspec-derive/src/lib.rs +++ b/secretspec-derive/src/lib.rs @@ -21,7 +21,8 @@ use proc_macro::TokenStream; use quote::{format_ident, quote}; -use secretspec::{Config, Secret}; +use secretspec::Config; +use secretspec::codegen::{CodegenIr, IrField, build_ir}; use std::collections::{BTreeMap, HashSet}; use syn::{LitStr, parse_macro_input}; @@ -69,6 +70,17 @@ impl FieldInfo { } } + /// Build a `FieldInfo` from a shared-IR field. The IR is the single source + /// of the optionality/as_path decisions; this only maps them to a Rust type. + fn from_ir(field: &IrField) -> Self { + Self::new( + field.name.clone(), + ir_field_type(field), + field.optional, + field.as_path, + ) + } + /// Get the field name as a Rust identifier. /// /// Converts the secret name to a valid Rust field name by: @@ -478,80 +490,17 @@ fn field_name_ident(name: &str) -> proc_macro2::Ident { format_ident!("{}", name.to_lowercase()) } -/// Helper function to check if a secret is optional. -/// -/// A secret is considered optional only if: -/// - It has `required = false` in the config -/// -/// Having a default value does not make a secret optional. -/// -/// # Arguments -/// -/// * `secret_config` - The secret's configuration -/// -/// # Returns -/// -/// `true` if the secret is optional, `false` if required -fn is_secret_optional(secret_config: &Secret) -> bool { - secret_config.required != Some(true) -} - -/// Determines if a field should be optional across all profiles. -/// -/// For the union struct (SecretSpec), a field is optional if it's optional -/// in ANY profile or missing from ANY profile. This ensures the union type -/// can safely represent secrets from any profile. -/// -/// # Arguments -/// -/// * `secret_name` - The name of the secret to check -/// * `config` - The project configuration -/// -/// # Returns -/// -/// `true` if the field should be Option in the union struct -/// -/// # Logic -/// -/// - If the secret is missing from any profile → optional -/// - If the secret is optional in any profile → optional -/// - Only if required in ALL profiles → not optional -fn is_field_optional_across_profiles(secret_name: &str, config: &Config) -> bool { - // Check each profile - for profile_config in config.profiles.values() { - if let Some(secret_config) = profile_config.secrets.get(secret_name) { - if is_secret_optional(secret_config) { - return true; - } - } else { - // Secret doesn't exist in this profile, so it's optional - return true; - } - } - false -} - -/// Check if a field should be represented as a path across all profiles. +/// Map a shared-IR field's optionality and path-ness to its Rust type. /// -/// A field is considered `as_path` if any profile defines it with `as_path = true`. -/// -/// # Arguments -/// -/// * `secret_name` - The name of the secret to check -/// * `config` - The configuration to analyze -/// -/// # Returns -/// -/// `true` if any profile has this secret with `as_path = true`, `false` otherwise -fn is_field_as_path(secret_name: &str, config: &Config) -> bool { - for profile_config in config.profiles.values() { - if let Some(secret_config) = profile_config.secrets.get(secret_name) - && secret_config.as_path == Some(true) - { - return true; - } +/// This is the only typing decision the derive macro still makes locally; the +/// underlying optional/as_path facts come from [`secretspec::codegen`]. +fn ir_field_type(field: &IrField) -> proc_macro2::TokenStream { + match (field.optional, field.as_path) { + (true, true) => quote! { Option }, + (true, false) => quote! { Option }, + (false, true) => quote! { std::path::PathBuf }, + (false, false) => quote! { String }, } - false } /// Generate a unified secret assignment from a HashMap. @@ -630,78 +579,27 @@ fn generate_secret_assignment( } } -/// Analyzes all profiles to determine field types for the union struct. +/// Build the union struct's fields from the shared IR. /// -/// This function examines all secrets across all profiles to determine: -/// - Which secrets exist across profiles -/// - Whether each secret should be optional in the union type -/// - The appropriate Rust type for each field -/// -/// # Arguments -/// -/// * `config` - The project configuration -/// -/// # Returns -/// -/// A BTreeMap (for consistent ordering) mapping secret names to their FieldInfo -/// -/// # Algorithm -/// -/// 1. Collect all unique secret names from all profiles -/// 2. For each secret, determine if it's optional across profiles -/// 3. Generate appropriate type (String or Option) -/// 4. Create FieldInfo with all metadata needed for code generation -fn analyze_field_types(config: &Config) -> BTreeMap { - let mut field_info = BTreeMap::new(); - - // Collect all unique secrets across all profiles - for profile_config in config.profiles.values() { - for secret_name in profile_config.secrets.keys() { - field_info.entry(secret_name.clone()).or_insert_with(|| { - let is_optional = is_field_optional_across_profiles(secret_name, config); - let as_path = is_field_as_path(secret_name, config); - let field_type = match (is_optional, as_path) { - (true, true) => quote! { Option }, - (true, false) => quote! { Option }, - (false, true) => quote! { std::path::PathBuf }, - (false, false) => quote! { String }, - }; - FieldInfo::new(secret_name.clone(), field_type, is_optional, as_path) - }); - } - } - - field_info +/// The IR already determined the union field set and each field's +/// optionality/as_path; this just maps them to `FieldInfo`, keyed and ordered +/// by name (the IR union is pre-sorted). +fn union_field_info(ir: &CodegenIr) -> BTreeMap { + ir.union + .iter() + .map(|field| (field.name.clone(), FieldInfo::from_ir(field))) + .collect() } -/// Get normalized profile variants for enum generation. -/// -/// Converts profile names into ProfileVariant structs, handling the special -/// case of empty profiles (generates a "Default" variant). -/// -/// # Arguments -/// -/// * `profiles` - Set of profile names from the configuration -/// -/// # Returns -/// -/// A sorted vector of ProfileVariant structs +/// Profile variants for enum generation, taken from the shared IR. /// -/// # Special Cases -/// -/// - Empty profiles → returns vec![ProfileVariant("default", "Default")] -/// - Otherwise → sorted list of profile variants -fn get_profile_variants(profiles: &HashSet) -> Vec { - if profiles.is_empty() { - vec![ProfileVariant::new("default".to_string())] - } else { - let mut variants: Vec<_> = profiles - .iter() - .map(|name| ProfileVariant::new(name.clone())) - .collect(); - variants.sort_by(|a, b| a.name.cmp(&b.name)); - variants - } +/// The IR's profile list is already sorted and already substitutes a single +/// `default` profile when the manifest declares none, so this is a direct map. +fn profile_variants_from_ir(ir: &CodegenIr) -> Vec { + ir.profiles + .iter() + .map(|name| ProfileVariant::new(name.clone())) + .collect() } // ===== Profile Generation Module ===== @@ -953,51 +851,27 @@ mod secret_spec_generation { /// /// - Empty profiles → generates a Default variant with all fields /// - Each profile → generates variant with profile-specific fields - pub fn generate_profile_enum_variants( - config: &Config, - field_info: &BTreeMap, - variants: &[ProfileVariant], - ) -> Vec { - if config.profiles.is_empty() { - // If no profiles, create a Default variant with all fields - let fields = field_info.values().map(|info| info.generate_struct_field()); - vec![quote! { - Default { - #(#fields,)* + pub fn generate_profile_enum_variants(ir: &CodegenIr) -> Vec { + // The IR's per-profile field sets already handle the empty-profiles case + // (a single `default` profile carrying the union), so there is no special + // branch here: one variant per IR profile, with that profile's exact + // (raw, non-merged) fields. + ir.profile_fields + .iter() + .map(|profile| { + let variant_ident = ProfileVariant::new(profile.name.clone()).as_ident(); + let fields = profile.fields.iter().map(|field| { + let field_name = field_name_ident(&field.name); + let field_type = ir_field_type(field); + quote! { #field_name: #field_type } + }); + quote! { + #variant_ident { + #(#fields,)* + } } - }] - } else { - variants - .iter() - .filter_map(|variant| { - config.profiles.get(&variant.name).map(|profile_config| { - let variant_ident = variant.as_ident(); - let fields = - profile_config - .secrets - .iter() - .map(|(secret_name, secret_config)| { - let field_name = field_name_ident(secret_name); - let is_optional = is_secret_optional(secret_config); - let as_path = secret_config.as_path.unwrap_or(false); - let field_type = match (is_optional, as_path) { - (true, true) => quote! { Option }, - (true, false) => quote! { Option }, - (false, true) => quote! { std::path::PathBuf }, - (false, false) => quote! { String }, - }; - quote! { #field_name: #field_type } - }); - - quote! { - #variant_ident { - #(#fields,)* - } - } - }) - }) - .collect() - } + }) + .collect() } /// Generate load_profile match arms. @@ -1025,52 +899,29 @@ mod secret_spec_generation { /// api_key: secrets.get("API_KEY").cloned(), /// }) /// ``` - pub fn generate_load_profile_arms( - config: &Config, - field_info: &BTreeMap, - variants: &[ProfileVariant], - ) -> Vec { - if config.profiles.is_empty() { - // Handle Default profile - let assignments = field_info - .values() - .map(|info| info.generate_assignment(quote! { secrets })); - - vec![quote! { - Profile::Default => Ok(SecretSpecProfile::Default { - #(#assignments,)* - }) - }] - } else { - variants - .iter() - .filter_map(|variant| { - config.profiles.get(&variant.name).map(|profile_config| { - let variant_ident = variant.as_ident(); - let assignments = - profile_config - .secrets - .iter() - .map(|(secret_name, secret_config)| { - let field_name = field_name_ident(secret_name); - generate_secret_assignment( - &field_name, - secret_name, - quote! { secrets }, - is_secret_optional(secret_config), - secret_config.as_path.unwrap_or(false), - ) - }); - - quote! { - Profile::#variant_ident => Ok(SecretSpecProfile::#variant_ident { - #(#assignments,)* - }) - } + pub fn generate_load_profile_arms(ir: &CodegenIr) -> Vec { + // One arm per IR profile, assigning that profile's exact fields. The + // empty-profiles case is already a single `default` profile in the IR. + ir.profile_fields + .iter() + .map(|profile| { + let variant_ident = ProfileVariant::new(profile.name.clone()).as_ident(); + let assignments = profile.fields.iter().map(|field| { + generate_secret_assignment( + &field_name_ident(&field.name), + &field.name, + quote! { secrets }, + field.optional, + field.as_path, + ) + }); + quote! { + Profile::#variant_ident => Ok(SecretSpecProfile::#variant_ident { + #(#assignments,)* }) - }) - .collect() - } + } + }) + .collect() } /// Generate the shared load_internal implementation. @@ -1507,12 +1358,15 @@ mod builder_generation { /// 5. Generate builder pattern implementation /// 6. Combine all components with necessary imports fn generate_secret_spec_code(config: Config) -> proc_macro2::TokenStream { - // Collect all profiles - let all_profiles: HashSet = config.profiles.keys().cloned().collect(); - let profile_variants = get_profile_variants(&all_profiles); + // Reduce the manifest to the shared codegen IR once. Every typing decision + // (union vs per-profile fields, optionality, as_path, profile list) comes + // from here, so this macro and the other-language emitters cannot drift. + let ir = build_ir(&config); - // Analyze field types - let field_info = analyze_field_types(&config); + let profile_variants = profile_variants_from_ir(&ir); + + // Union struct fields. + let field_info = union_field_info(&ir); // Generate field assignments for load() let load_assignments: Vec<_> = field_info @@ -1531,15 +1385,10 @@ fn generate_secret_spec_code(config: Config) -> proc_macro2::TokenStream { // Generate SecretSpec components let secret_spec_struct = secret_spec_generation::generate_struct(&field_info); - let profile_enum_variants = secret_spec_generation::generate_profile_enum_variants( - &config, - &field_info, - &profile_variants, - ); + let profile_enum_variants = secret_spec_generation::generate_profile_enum_variants(&ir); let secret_spec_profile_enum = secret_spec_generation::generate_profile_enum(&profile_enum_variants); - let load_profile_arms = - secret_spec_generation::generate_load_profile_arms(&config, &field_info, &profile_variants); + let load_profile_arms = secret_spec_generation::generate_load_profile_arms(&ir); let load_internal = secret_spec_generation::generate_load_internal(); let secret_spec_impl = secret_spec_generation::generate_impl(&load_assignments, env_setters, &field_info); diff --git a/secretspec-derive/src/tests.rs b/secretspec-derive/src/tests.rs index f54c376..acc0629 100644 --- a/secretspec-derive/src/tests.rs +++ b/secretspec-derive/src/tests.rs @@ -548,314 +548,6 @@ HAS_DEFAULT = { description = "Secret with default", required = true, default = assert_eq!(field_name_ident("Mixed_Case").to_string(), "mixed_case"); } - #[test] - fn test_is_secret_optional() { - use crate::is_secret_optional; - use secretspec::Secret; - - // Required without default - let required_no_default = Secret { - description: Some("Required".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }; - assert!(!is_secret_optional(&required_no_default)); - - // Required with default (should NOT be optional) - let required_with_default = Secret { - description: Some("Required with default".to_string()), - required: Some(true), - default: Some("default_value".to_string()), - providers: None, - ..Default::default() - }; - assert!(!is_secret_optional(&required_with_default)); - - // Not required - let not_required = Secret { - description: Some("Not required".to_string()), - required: Some(false), - default: None, - providers: None, - ..Default::default() - }; - assert!(is_secret_optional(¬_required)); - - // Not required with default - let not_required_with_default = Secret { - description: Some("Not required with default".to_string()), - required: Some(false), - default: Some("default_value".to_string()), - providers: None, - ..Default::default() - }; - assert!(is_secret_optional(¬_required_with_default)); - } - - #[test] - fn test_is_field_optional_across_profiles() { - use crate::is_field_optional_across_profiles; - use secretspec::{Profile, Project, Secret}; - use std::collections::HashMap; - - // Setup config with multiple profiles - let mut profiles = HashMap::new(); - - // Default profile: API_KEY required, DATABASE_URL optional - let mut default_secrets = HashMap::new(); - default_secrets.insert( - "API_KEY".to_string(), - Secret { - description: Some("API Key".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - default_secrets.insert( - "DATABASE_URL".to_string(), - Secret { - description: Some("Database URL".to_string()), - required: Some(false), - default: None, - providers: None, - ..Default::default() - }, - ); - profiles.insert( - "default".to_string(), - Profile { - defaults: None, - secrets: default_secrets, - }, - ); - - // Development profile: API_KEY with default (optional), DATABASE_URL required - let mut dev_secrets = HashMap::new(); - dev_secrets.insert( - "API_KEY".to_string(), - Secret { - description: Some("API Key".to_string()), - required: Some(true), - default: Some("dev-key".to_string()), - providers: None, - ..Default::default() - }, - ); - dev_secrets.insert( - "DATABASE_URL".to_string(), - Secret { - description: Some("Database URL".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - // Note: CACHE_URL only exists in development - dev_secrets.insert( - "CACHE_URL".to_string(), - Secret { - description: Some("Cache URL".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - profiles.insert( - "development".to_string(), - Profile { - defaults: None, - secrets: dev_secrets, - }, - ); - - let config = Config { - project: Project { - name: "test".to_string(), - ..Default::default() - }, - profiles, - providers: None, - }; - - // API_KEY is NOT optional (required in all profiles, default doesn't make it optional) - assert!(!is_field_optional_across_profiles("API_KEY", &config)); - - // DATABASE_URL is optional because it's not required in default - assert!(is_field_optional_across_profiles("DATABASE_URL", &config)); - - // CACHE_URL is optional because it doesn't exist in default profile - assert!(is_field_optional_across_profiles("CACHE_URL", &config)); - - // Test a secret that's always required - let mut strict_profiles = HashMap::new(); - let mut strict_default = HashMap::new(); - strict_default.insert( - "ALWAYS_REQUIRED".to_string(), - Secret { - description: Some("Always required".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - let mut strict_dev = HashMap::new(); - strict_dev.insert( - "ALWAYS_REQUIRED".to_string(), - Secret { - description: Some("Always required".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - strict_profiles.insert( - "default".to_string(), - Profile { - defaults: None, - secrets: strict_default, - }, - ); - strict_profiles.insert( - "development".to_string(), - Profile { - defaults: None, - secrets: strict_dev, - }, - ); - - let strict_config = Config { - project: Project { - name: "test".to_string(), - ..Default::default() - }, - profiles: strict_profiles, - providers: None, - }; - - // ALWAYS_REQUIRED should not be optional - assert!(!is_field_optional_across_profiles( - "ALWAYS_REQUIRED", - &strict_config - )); - } - - #[test] - fn test_analyze_field_types() { - use crate::analyze_field_types; - use secretspec::{Profile, Project, Secret}; - use std::collections::HashMap; - - let mut profiles = HashMap::new(); - - // Default profile - let mut default_secrets = HashMap::new(); - default_secrets.insert( - "REQUIRED_SECRET".to_string(), - Secret { - description: Some("Always required".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - default_secrets.insert( - "OPTIONAL_SECRET".to_string(), - Secret { - description: Some("Optional".to_string()), - required: Some(false), - default: None, - providers: None, - ..Default::default() - }, - ); - default_secrets.insert( - "DEFAULT_SECRET".to_string(), - Secret { - description: Some("Has default".to_string()), - required: Some(true), - default: Some("default_value".to_string()), - providers: None, - ..Default::default() - }, - ); - profiles.insert( - "default".to_string(), - Profile { - defaults: None, - secrets: default_secrets, - }, - ); - - // Development profile with additional secret - let mut dev_secrets = HashMap::new(); - dev_secrets.insert( - "REQUIRED_SECRET".to_string(), - Secret { - description: Some("Always required".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - dev_secrets.insert( - "DEV_ONLY_SECRET".to_string(), - Secret { - description: Some("Development only".to_string()), - required: Some(true), - default: None, - providers: None, - ..Default::default() - }, - ); - profiles.insert( - "development".to_string(), - Profile { - defaults: None, - secrets: dev_secrets, - }, - ); - - let config = Config { - project: Project { - name: "test".to_string(), - ..Default::default() - }, - profiles, - providers: None, - }; - - let field_info = analyze_field_types(&config); - - // Should have 4 unique secrets across all profiles - assert_eq!(field_info.len(), 4); - - // REQUIRED_SECRET exists in both profiles and is always required -> String - let required_field = field_info.get("REQUIRED_SECRET").unwrap(); - assert!(!required_field.is_optional); - - // OPTIONAL_SECRET only exists in default and is optional -> Option - let optional_field = field_info.get("OPTIONAL_SECRET").unwrap(); - assert!(optional_field.is_optional); - - // DEFAULT_SECRET has default value -> Option - let default_field = field_info.get("DEFAULT_SECRET").unwrap(); - assert!(default_field.is_optional); - - // DEV_ONLY_SECRET only exists in development -> Option - let dev_only_field = field_info.get("DEV_ONLY_SECRET").unwrap(); - assert!(dev_only_field.is_optional); - } - #[test] fn test_field_info_methods() { use crate::FieldInfo; @@ -910,41 +602,6 @@ HAS_DEFAULT = { description = "Secret with default", required = true, default = assert_eq!(prod_variant.as_ident().to_string(), "Production"); } - #[test] - fn test_get_profile_variants() { - use crate::get_profile_variants; - use std::collections::HashSet; - - // Test empty profiles - let empty_profiles = HashSet::new(); - let variants = get_profile_variants(&empty_profiles); - assert_eq!(variants.len(), 1); - assert_eq!(variants[0].name, "default"); - - // Test with multiple profiles - let mut profiles = HashSet::new(); - profiles.insert("production".to_string()); - profiles.insert("development".to_string()); - profiles.insert("staging".to_string()); - profiles.insert("default".to_string()); - - let variants = get_profile_variants(&profiles); - assert_eq!(variants.len(), 4); - - // Should be sorted alphabetically - let names: Vec<&String> = variants.iter().map(|v| &v.name).collect(); - assert_eq!( - names, - vec!["default", "development", "production", "staging"] - ); - - // Check capitalization - assert_eq!(variants[0].capitalized, "Default"); - assert_eq!(variants[1].capitalized, "Development"); - assert_eq!(variants[2].capitalized, "Production"); - assert_eq!(variants[3].capitalized, "Staging"); - } - #[test] fn test_validate_config_for_codegen() { use crate::validate_config_for_codegen; From 07c9f0347990496a22b245fd58b251f6b6dcfcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:18:49 -0400 Subject: [PATCH 06/56] feat: add secretspec-py, the Python reference SDK over the C ABI Phase 4: the first non-Rust consumer, proving the whole stack from a generic dlopen caller (the same C ABI path Go/purego and Ruby/ffi will reuse). - secretspec-py binds secretspec-ffi via cffi: marshals a JSON request to secretspec_resolve, parses the envelope, frees the buffer. No resolution logic in Python; every provider is inherited from the Rust core. - Mirrors the derive crate's vocabulary: SecretSpec.builder().with_provider() .with_profile().with_reason().load() -> Resolved(.secrets/.provider/.profile), plus set_as_env(). MissingRequiredError vs SecretSpecError(.kind) separate a missing required secret from a transport failure. as_path yields a file path. - Library discovery via SECRETSPEC_FFI_LIB, a wheel-bundled copy, or a Cargo target dir. pyproject packages it; README documents it. - pytest suite (6 tests) drives the real cdylib end to end: values, default source, missing-optional, set_as_env, missing-required, as_path, invalid input. conftest builds the crate and locates the library automatically. - devenv.nix now provides Python + cffi + pytest + maturin. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 ++ devenv.nix | 14 ++ secretspec-py/.gitignore | 7 + secretspec-py/README.md | 33 ++++ secretspec-py/pyproject.toml | 24 +++ secretspec-py/secretspec/__init__.py | 247 +++++++++++++++++++++++++++ secretspec-py/tests/conftest.py | 44 +++++ secretspec-py/tests/test_resolve.py | 128 ++++++++++++++ 8 files changed, 507 insertions(+) create mode 100644 secretspec-py/.gitignore create mode 100644 secretspec-py/README.md create mode 100644 secretspec-py/pyproject.toml create mode 100644 secretspec-py/secretspec/__init__.py create mode 100644 secretspec-py/tests/conftest.py create mode 100644 secretspec-py/tests/test_resolve.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 93866d2..3d3c774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec-py` Python SDK: a thin client over the `secretspec-ffi` C ABI + (loaded via cffi), so Python apps inherit every provider with no Python-side + resolution logic. Mirrors the derive crate's vocabulary + (`SecretSpec.builder().with_provider(...).with_profile(...).with_reason(...).load()` + returning a `Resolved` with `.secrets`/`.provider`/`.profile`, plus + `set_as_env()`). A missing required secret raises `MissingRequiredError`; other + failures raise `SecretSpecError` with a stable `.kind`. `as_path` secrets are + returned as a readable file path. The native library is found via + `SECRETSPEC_FFI_LIB`, a wheel-bundled copy, or a Cargo target directory. + `devenv.nix` now provides Python, cffi, pytest, and maturin. - New `secretspec::codegen` module: a shared, language-neutral intermediate representation (IR) that reduces a manifest to the typed-accessor decisions every generator needs (union vs per-profile field sets, optionality, `as_path`, diff --git a/devenv.nix b/devenv.nix index ae55b1d..1be0131 100644 --- a/devenv.nix +++ b/devenv.nix @@ -12,6 +12,18 @@ install.enable = true; }; }; + # Python is used by the reference SDK (secretspec-py), which binds the + # secretspec-ffi C ABI via cffi (dlopen) over the prebuilt cdylib. + languages.python = { + enable = true; + venv = { + enable = true; + requirements = '' + cffi + pytest + ''; + }; + }; packages = [ # keyring @@ -20,6 +32,8 @@ pkgs.cargo-tarpaulin # installers pkgs.cargo-dist + # packaging the Python SDK wheel that bundles the cdylib + pkgs.maturin ]; git-hooks.hooks = { diff --git a/secretspec-py/.gitignore b/secretspec-py/.gitignore new file mode 100644 index 0000000..619d235 --- /dev/null +++ b/secretspec-py/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +build/ +dist/ +*.egg-info/ +secretspec/_lib/ diff --git a/secretspec-py/README.md b/secretspec-py/README.md new file mode 100644 index 0000000..60c8814 --- /dev/null +++ b/secretspec-py/README.md @@ -0,0 +1,33 @@ +# secretspec (Python SDK) + +Python bindings for [SecretSpec](https://secretspec.dev/), a declarative secrets +manager. This package is a thin client over the `secretspec-ffi` C ABI: +resolution (providers, chains, profiles, generation, `as_path`) happens in the +Rust core, so the SDK inherits every provider with no Python-side logic. + +```python +from secretspec import SecretSpec + +resolved = ( + SecretSpec.builder() + .with_provider("keyring://") + .with_profile("production") + .with_reason("boot web app") + .load() +) + +print(resolved.provider, resolved.profile) +db = resolved.secrets["DATABASE_URL"] +print(db.get) # the value, or the file path for as_path secrets +resolved.set_as_env() # export everything into os.environ +``` + +A missing required secret raises `MissingRequiredError`; any other failure +raises `SecretSpecError` (with a stable `.kind`). + +## Library discovery + +The SDK loads the native library from, in order: the `SECRETSPEC_FFI_LIB` +environment variable, a copy bundled in the installed wheel, or a Cargo `target` +directory found by searching up from the working directory (useful in a source +checkout). diff --git a/secretspec-py/pyproject.toml b/secretspec-py/pyproject.toml new file mode 100644 index 0000000..3761aeb --- /dev/null +++ b/secretspec-py/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "secretspec" +version = "0.12.0" +description = "Declarative secrets, every environment, any provider (Python SDK)" +readme = "README.md" +requires-python = ">=3.9" +license = { text = "Apache-2.0" } +dependencies = ["cffi>=1.15"] + +[project.urls] +Homepage = "https://secretspec.dev/" +Repository = "https://github.com/cachix/secretspec" + +[tool.setuptools] +packages = ["secretspec"] + +[tool.setuptools.package-data] +# The cdylib is bundled here by the packaging step; absent in a source checkout, +# where the SDK falls back to SECRETSPEC_FFI_LIB or a Cargo target directory. +secretspec = ["_lib/*"] diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py new file mode 100644 index 0000000..58dd5a5 --- /dev/null +++ b/secretspec-py/secretspec/__init__.py @@ -0,0 +1,247 @@ +"""SecretSpec Python SDK. + +A thin client over the ``secretspec-ffi`` C ABI. Resolution (providers, chains, +profiles, generation, ``as_path``) happens entirely in the Rust core; this +package marshals a JSON request to ``secretspec_resolve``, parses the response +envelope, and exposes it with the same vocabulary as the Rust derive crate +(a builder with ``with_provider``/``with_profile``/``with_reason`` and ``load``, +returning a ``Resolved`` with ``.secrets``/``.provider``/``.profile``). + +The library is loaded via cffi (dlopen) from, in order: the ``SECRETSPEC_FFI_LIB`` +environment variable, a copy bundled in the wheel, or a Cargo target directory. +""" + +from __future__ import annotations + +import json +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from cffi import FFI + +__all__ = [ + "SecretSpec", + "Resolved", + "ResolvedSecret", + "SecretSpecError", + "MissingRequiredError", + "resolve", + "abi_version", +] + +# The narrow C ABI. Mirrors secretspec-ffi/include/secretspec.h. +_ffi = FFI() +_ffi.cdef( + """ + char *secretspec_resolve(const char *request_json); + void secretspec_free(char *ptr); + const char *secretspec_abi_version(void); + """ +) + + +def _candidate_lib_names() -> list[str]: + if sys.platform == "darwin": + return ["libsecretspec_ffi.dylib"] + if sys.platform == "win32": + return ["secretspec_ffi.dll"] + return ["libsecretspec_ffi.so"] + + +def _find_library() -> str: + # 1. Explicit override (used in development and tests). + override = os.environ.get("SECRETSPEC_FFI_LIB") + if override: + return override + + names = _candidate_lib_names() + + # 2. Bundled next to this package (the wheel distribution layout). + here = Path(__file__).resolve().parent + for name in names: + bundled = here / "_lib" / name + if bundled.exists(): + return str(bundled) + + # 3. A Cargo target directory, searching up from the current directory. + for base in [Path.cwd(), *Path.cwd().parents]: + target = base / "target" + if target.is_dir(): + for profile in ("release", "debug"): + for name in names: + candidate = target / profile / name + if candidate.exists(): + return str(candidate) + + raise SecretSpecError( + "load", + "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB " + "to its path", + ) + + +_lib = None + + +def _load() -> object: + global _lib + if _lib is None: + _lib = _ffi.dlopen(_find_library()) + return _lib + + +class SecretSpecError(Exception): + """A resolution call failed (bad manifest, provider error, reason policy).""" + + def __init__(self, kind: str, message: str): + super().__init__(f"{message} (kind: {kind})") + self.kind = kind + self.message = message + + +class MissingRequiredError(SecretSpecError): + """One or more required secrets were not found anywhere.""" + + def __init__(self, missing: list[str]): + super().__init__( + "missing_required", + "missing required secret(s): " + ", ".join(missing), + ) + self.missing = missing + + +@dataclass(frozen=True) +class ResolvedSecret: + """One resolved secret. Exactly one of ``value`` / ``path`` is set.""" + + value: Optional[str] + path: Optional[str] + as_path: bool + source: str + source_provider: Optional[str] + + @property + def get(self) -> Optional[str]: + """The usable string: the file path for ``as_path`` secrets, else the value.""" + return self.path if self.as_path else self.value + + +@dataclass(frozen=True) +class Resolved: + """A successful resolution, mirroring the Rust ``Resolved`` wrapper.""" + + provider: str + profile: str + secrets: dict[str, ResolvedSecret] + missing_optional: list[str] = field(default_factory=list) + + def set_as_env(self) -> None: + """Export each resolved secret into ``os.environ`` by its declared name.""" + for name, secret in self.secrets.items(): + usable = secret.get + if usable is not None: + os.environ[name] = usable + + +def abi_version() -> str: + """The ABI version reported by the loaded library.""" + ptr = _load().secretspec_abi_version() + return _ffi.string(ptr).decode() + + +def _resolve_envelope(request: dict) -> dict: + lib = _load() + payload = json.dumps(request).encode("utf-8") + ptr = lib.secretspec_resolve(payload) + if ptr == _ffi.NULL: + raise SecretSpecError("ffi", "secretspec_resolve returned null") + try: + raw = _ffi.string(ptr).decode("utf-8") + finally: + lib.secretspec_free(ptr) + return json.loads(raw) + + +def _resolve_response(request: dict) -> dict: + envelope = _resolve_envelope(request) + if not envelope.get("ok", False): + err = envelope.get("error", {}) + raise SecretSpecError(err.get("kind", "unknown"), err.get("message", "")) + return envelope["response"] + + +def resolve( + *, + path: Optional[str] = None, + provider: Optional[str] = None, + profile: Optional[str] = None, + reason: Optional[str] = None, +) -> Resolved: + """Resolve secrets and return a :class:`Resolved`. + + Raises :class:`MissingRequiredError` if a required secret is missing, and + :class:`SecretSpecError` for any other failure. + """ + return SecretSpec.builder().with_path(path).with_provider(provider).with_profile( + profile + ).with_reason(reason).load() + + +class SecretSpec: + """Entry point mirroring the derive crate's ``SecretSpec::builder()``.""" + + @staticmethod + def builder() -> "_Builder": + return _Builder() + + +class _Builder: + def __init__(self) -> None: + self._request: dict = {} + + def with_path(self, path: Optional[str]) -> "_Builder": + if path is not None: + self._request["path"] = path + return self + + def with_provider(self, provider: Optional[str]) -> "_Builder": + if provider is not None: + self._request["provider"] = provider + return self + + def with_profile(self, profile: Optional[str]) -> "_Builder": + if profile is not None: + self._request["profile"] = profile + return self + + def with_reason(self, reason: Optional[str]) -> "_Builder": + if reason is not None: + self._request["reason"] = reason + return self + + def load(self) -> Resolved: + response = _resolve_response(self._request) + + missing_required = response.get("missing_required", []) + if missing_required: + raise MissingRequiredError(missing_required) + + secrets = { + name: ResolvedSecret( + value=entry.get("value"), + path=entry.get("path"), + as_path=entry.get("as_path", False), + source=entry.get("source", "provider"), + source_provider=entry.get("source_provider"), + ) + for name, entry in response.get("secrets", {}).items() + } + return Resolved( + provider=response["provider"], + profile=response["profile"], + secrets=secrets, + missing_optional=response.get("missing_optional", []), + ) diff --git a/secretspec-py/tests/conftest.py b/secretspec-py/tests/conftest.py new file mode 100644 index 0000000..e92fabc --- /dev/null +++ b/secretspec-py/tests/conftest.py @@ -0,0 +1,44 @@ +"""Ensure the secretspec-ffi cdylib is built and discoverable before tests run. + +If SECRETSPEC_FFI_LIB is not already set, build the crate and point the SDK at +the freshly built library in the Cargo target directory. +""" + +import json +import os +import pathlib +import subprocess +import sys + +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def _lib_name() -> str: + if sys.platform == "darwin": + return "libsecretspec_ffi.dylib" + if sys.platform == "win32": + return "secretspec_ffi.dll" + return "libsecretspec_ffi.so" + + +def _ensure_lib() -> None: + if os.environ.get("SECRETSPEC_FFI_LIB"): + return + + subprocess.run( + ["cargo", "build", "-p", "secretspec-ffi"], + cwd=_REPO_ROOT, + check=True, + ) + meta = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + target_dir = pathlib.Path(json.loads(meta.stdout)["target_directory"]) + os.environ["SECRETSPEC_FFI_LIB"] = str(target_dir / "debug" / _lib_name()) + + +_ensure_lib() diff --git a/secretspec-py/tests/test_resolve.py b/secretspec-py/tests/test_resolve.py new file mode 100644 index 0000000..8cc7acb --- /dev/null +++ b/secretspec-py/tests/test_resolve.py @@ -0,0 +1,128 @@ +"""Exercise the Python SDK end to end against the real C ABI.""" + +import pathlib + +import pytest + +from secretspec import ( + MissingRequiredError, + SecretSpec, + SecretSpecError, + abi_version, +) + +MANIFEST = """ +[project] +name = "py-test" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } +""" + + +def _project(tmp_path: pathlib.Path, dotenv: str) -> tuple[str, str]: + manifest_path = tmp_path / "secretspec.toml" + env_path = tmp_path / ".env" + manifest_path.write_text(MANIFEST) + env_path.write_text(dotenv) + return str(manifest_path), f"dotenv://{env_path}" + + +def test_abi_version_nonempty(): + assert abi_version() + + +def test_load_returns_values_and_provenance(tmp_path): + manifest, provider = _project(tmp_path, "DATABASE_URL=postgres://db\n") + + resolved = ( + SecretSpec.builder() + .with_path(manifest) + .with_provider(provider) + .with_reason("py test") + .load() + ) + + assert resolved.profile == "default" + db = resolved.secrets["DATABASE_URL"] + assert db.get == "postgres://db" + assert db.source == "provider" + assert db.source_provider is not None + + log = resolved.secrets["LOG_LEVEL"] + assert log.get == "info" + assert log.source == "default" + + assert resolved.missing_optional == ["SENTRY_DSN"] + assert "SENTRY_DSN" not in resolved.secrets + + +def test_set_as_env(tmp_path, monkeypatch): + manifest, provider = _project(tmp_path, "DATABASE_URL=postgres://db\n") + monkeypatch.delenv("DATABASE_URL", raising=False) + + resolved = ( + SecretSpec.builder() + .with_path(manifest) + .with_provider(provider) + .with_reason("py test") + .load() + ) + resolved.set_as_env() + + import os + + assert os.environ["DATABASE_URL"] == "postgres://db" + + +def test_missing_required_raises(tmp_path): + manifest, provider = _project(tmp_path, "") # DATABASE_URL absent + + with pytest.raises(MissingRequiredError) as exc: + SecretSpec.builder().with_path(manifest).with_provider(provider).with_reason( + "py test" + ).load() + + assert "DATABASE_URL" in exc.value.missing + + +def test_as_path_returns_readable_file(tmp_path): + manifest_path = tmp_path / "secretspec.toml" + env_path = tmp_path / ".env" + manifest_path.write_text( + """ +[project] +name = "py-test" +revision = "1.0" + +[profiles.default] +TLS_CERT = { description = "cert", required = true, as_path = true } +""" + ) + env_path.write_text("TLS_CERT=----cert-bytes----\n") + + resolved = ( + SecretSpec.builder() + .with_path(str(manifest_path)) + .with_provider(f"dotenv://{env_path}") + .with_reason("py test") + .load() + ) + + cert = resolved.secrets["TLS_CERT"] + assert cert.as_path + assert cert.value is None + assert pathlib.Path(cert.get).read_text() == "----cert-bytes----" + + +def test_invalid_manifest_raises_secretspec_error(tmp_path): + with pytest.raises(SecretSpecError) as exc: + SecretSpec.builder().with_path( + "/definitely/does/not/exist/secretspec.toml" + ).with_reason("py test").load() + + assert not isinstance(exc.value, MissingRequiredError) + assert exc.value.kind From 7dffc9826b006ca5b9940644432f9ea366fd04df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:28:18 -0400 Subject: [PATCH 07/56] feat: add `secretspec codegen --lang python` (IR-driven typed accessors) Completes phase 4: the codegen half of the Python reference SDK, so typed accessors and the runtime SDK ship together. The emitter is a thin template over the shared codegen IR, so it cannot drift from the derive macro or future language emitters. - codegen::python::emit(&CodegenIr) -> String generates a module that mirrors the derive crate's shape over the runtime SDK: a SecretSpec union dataclass plus one Secrets dataclass per profile, each with a builder-style load(). Idiomatic Python: snake_case attributes typed str / Optional[str] / Path, required pulled directly, optional guarded, as_path wrapped in Path. - New `secretspec codegen --lang python [-o FILE]` CLI command (value-free; reads only the manifest via the now-non-test-gated Secrets::config()). - Rust test asserts the emitted types/assignments; two Python e2e tests generate a module via the CLI, import it, and resolve through the generated accessors (union + profile-pinned, including as_path). conftest builds the CLI too. - Generated code avoids `from __future__ import annotations` so it is robust when imported/exec'd in any context. CLI reference and CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 + docs/src/content/docs/reference/cli.md | 27 ++++ secretspec-py/tests/conftest.py | 16 ++- secretspec-py/tests/test_codegen.py | 84 ++++++++++++ secretspec/src/cli/mod.rs | 37 ++++- secretspec/src/codegen.rs | 181 +++++++++++++++++++++++++ secretspec/src/secrets.rs | 4 +- 7 files changed, 348 insertions(+), 7 deletions(-) create mode 100644 secretspec-py/tests/test_codegen.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3c774..64220bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 profile list). It is the single source of truth those decisions are computed in, so the Rust derive macro and the eventual TypeScript/Python/Go/Ruby emitters cannot drift. `build_ir(&Config) -> CodegenIr`. +- New `secretspec codegen --lang python` command: emits typed Python accessors + over the `secretspec` Python SDK from the manifest, driven by the shared IR. + It mirrors the derive crate's shape (a `SecretSpec` union dataclass plus one + `Secrets` dataclass per profile, each with a builder-style `load`) + with idiomatic snake_case attributes typed as `str`/`Optional[str]`/`Path`. + Value-free: reads only the manifest. `-o` writes to a file instead of stdout. ### Changed - The `secretspec-derive` macro now computes all of its typing decisions through diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 6961f5e..58ebd8f 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -240,6 +240,33 @@ persisted temp file), its `source` (`provider`, `generated`, or `default`), and the serving provider's credential-free URI. The canonical JSON Schema is committed at `schema/resolve-response.schema.json`. +### codegen +Generate typed accessors for another language from the manifest, driven by the +shared codegen IR (so every language stays consistent). Value-free: reads only +the manifest, never a provider. + +```bash +secretspec codegen --lang [-o FILE] +``` + +**Options:** +- `--lang ` - Target language (`python`) +- `-o, --output ` - Write to this file instead of stdout + +For Python it emits a `SecretSpec` union dataclass plus one `Secrets` +dataclass per profile, each with a builder-style `load`, over the `secretspec` +Python SDK: + +```bash +$ secretspec codegen --lang python -o secrets_gen.py +``` +```python +from secrets_gen import SecretSpec + +s = SecretSpec.load(profile="production", reason="boot") +print(s.database_url) # typed str +``` + ### set Set a secret value. diff --git a/secretspec-py/tests/conftest.py b/secretspec-py/tests/conftest.py index e92fabc..9c8e12f 100644 --- a/secretspec-py/tests/conftest.py +++ b/secretspec-py/tests/conftest.py @@ -21,12 +21,19 @@ def _lib_name() -> str: return "libsecretspec_ffi.so" +def _bin_name() -> str: + return "secretspec.exe" if sys.platform == "win32" else "secretspec" + + def _ensure_lib() -> None: - if os.environ.get("SECRETSPEC_FFI_LIB"): + have_lib = bool(os.environ.get("SECRETSPEC_FFI_LIB")) + have_bin = bool(os.environ.get("SECRETSPEC_BIN")) + if have_lib and have_bin: return + # Build the cdylib (for the runtime SDK) and the CLI (for codegen tests). subprocess.run( - ["cargo", "build", "-p", "secretspec-ffi"], + ["cargo", "build", "-p", "secretspec-ffi", "-p", "secretspec"], cwd=_REPO_ROOT, check=True, ) @@ -37,8 +44,9 @@ def _ensure_lib() -> None: capture_output=True, text=True, ) - target_dir = pathlib.Path(json.loads(meta.stdout)["target_directory"]) - os.environ["SECRETSPEC_FFI_LIB"] = str(target_dir / "debug" / _lib_name()) + debug_dir = pathlib.Path(json.loads(meta.stdout)["target_directory"]) / "debug" + os.environ.setdefault("SECRETSPEC_FFI_LIB", str(debug_dir / _lib_name())) + os.environ.setdefault("SECRETSPEC_BIN", str(debug_dir / _bin_name())) _ensure_lib() diff --git a/secretspec-py/tests/test_codegen.py b/secretspec-py/tests/test_codegen.py new file mode 100644 index 0000000..e77a695 --- /dev/null +++ b/secretspec-py/tests/test_codegen.py @@ -0,0 +1,84 @@ +"""End-to-end test of `secretspec codegen --lang python`. + +Generates a typed module from a manifest, imports it, and resolves through the +generated accessors, proving the IR-driven codegen produces working code on top +of the runtime SDK. +""" + +import importlib.util +import os +import pathlib +import subprocess +import sys + +MANIFEST = """ +[project] +name = "codegen-test" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } + +[profiles.production] +DATABASE_URL = { description = "DB", required = true } +TLS_CERT = { description = "cert", required = true, as_path = true } +""" + + +def _generate(tmp_path: pathlib.Path, name: str = "generated"): + manifest = tmp_path / "secretspec.toml" + manifest.write_text(MANIFEST) + out = tmp_path / f"{name}.py" + subprocess.run( + [ + os.environ["SECRETSPEC_BIN"], + "-f", + str(manifest), + "codegen", + "--lang", + "python", + "-o", + str(out), + ], + check=True, + ) + spec = importlib.util.spec_from_file_location(name, out) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # so the module resolves its own references + spec.loader.exec_module(module) # also validates the generated syntax + return module, manifest + + +def test_generated_union_resolves_with_typed_attributes(tmp_path): + env = tmp_path / ".env" + env.write_text("DATABASE_URL=postgres://db\n") + module, manifest = _generate(tmp_path, "gen_union") + + secrets = module.SecretSpec.load( + provider=f"dotenv://{env}", + reason="codegen test", + path=str(manifest), + ) + + # Typed snake_case attributes mirror the derive crate, idiomatic for Python. + assert secrets.database_url == "postgres://db" + assert secrets.log_level == "info" # from default + assert secrets.sentry_dsn is None # optional, missing + + +def test_generated_profile_class_pins_profile_and_paths(tmp_path): + env = tmp_path / ".env" + env.write_text("DATABASE_URL=postgres://prod\nTLS_CERT=----cert----\n") + module, manifest = _generate(tmp_path, "gen_profile") + + prod = module.ProductionSecrets.load( + provider=f"dotenv://{env}", + reason="codegen test", + path=str(manifest), + ) + + assert prod.database_url == "postgres://prod" + # as_path field is exposed as a readable file path. + assert pathlib.Path(prod.tls_cert).read_text() == "----cert----" diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index 4a2c270..f999069 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -1,6 +1,6 @@ use crate::provider::{Provider, providers}; use crate::{Config, GlobalConfig, GlobalDefaults, Profile, Project, Secrets}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use miette::{IntoDiagnostic, Result, WrapErr, miette}; use std::collections::HashMap; use std::fs; @@ -103,6 +103,19 @@ enum Commands { #[arg(long)] explain: bool, }, + /// Generate typed accessors for another language from the manifest. + /// + /// Emits code that wraps that language's runtime SDK, driven by the shared + /// codegen IR so every language stays consistent. Value-free: reads only the + /// manifest, never a provider. + Codegen { + /// Target language to generate + #[arg(long, value_enum)] + lang: CodegenLang, + /// Write to this file instead of stdout + #[arg(short, long)] + output: Option, + }, /// Resolve all secrets and print them as JSON (the SDK boundary). /// /// Unlike `check`, this prints secret VALUES (to stdout). It is intended for @@ -146,6 +159,13 @@ enum Commands { }, } +/// Target languages for `secretspec codegen`. +#[derive(Clone, Debug, ValueEnum)] +enum CodegenLang { + /// Python typed accessors over the `secretspec` Python SDK + Python, +} + /// Configuration-related subcommands. /// /// These actions handle the user's global configuration settings, @@ -692,6 +712,21 @@ pub fn main() -> Result<()> { .wrap_err("Failed to persist temporary files")?; Ok(()) } + // Generate typed accessors for another language (value-free) + Commands::Codegen { lang, output } => { + let app = load_secrets(&cli.file, &cli.reason)?; + let ir = crate::codegen::build_ir(app.config()); + let code = match lang { + CodegenLang::Python => crate::codegen::python::emit(&ir), + }; + match output { + Some(path) => fs::write(&path, code) + .into_diagnostic() + .wrap_err_with(|| format!("Failed to write {}", path.display()))?, + None => print!("{}", code), + } + Ok(()) + } // Resolve all secrets to JSON (the SDK boundary; prints values) Commands::Resolve { provider, diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs index 2cf51bb..79f4625 100644 --- a/secretspec/src/codegen.rs +++ b/secretspec/src/codegen.rs @@ -173,6 +173,147 @@ pub fn build_ir(config: &Config) -> CodegenIr { } } +/// Language emitters: thin templates that turn the [`CodegenIr`] into typed +/// accessors. Each mirrors the derive crate's shape (a union type plus +/// per-profile types, a builder-style `load`) using the target language's +/// idioms. +pub mod python { + use super::{CodegenIr, IrField}; + use std::fmt::Write; + + /// The Python attribute name (snake_case) for an `UPPER_SNAKE` secret name. + fn attr(name: &str) -> String { + name.to_lowercase() + } + + /// The Python type annotation for a field. + fn type_ann(field: &IrField) -> &'static str { + match (field.optional, field.as_path) { + (false, false) => "str", + (true, false) => "Optional[str]", + (false, true) => "Path", + (true, true) => "Optional[Path]", + } + } + + /// The constructor keyword-argument line that pulls one field out of the + /// runtime `Resolved`. + fn assignment(field: &IrField) -> String { + let env = &field.name; + let name = attr(env); + match (field.optional, field.as_path) { + (false, false) => format!(" {name}=r.secrets[\"{env}\"].get,"), + (false, true) => format!(" {name}=Path(r.secrets[\"{env}\"].get),"), + (true, false) => format!( + " {name}=(r.secrets[\"{env}\"].get if \"{env}\" in r.secrets else None)," + ), + (true, true) => format!( + " {name}=(Path(r.secrets[\"{env}\"].get) if \"{env}\" in r.secrets else None)," + ), + } + } + + fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + } + } + + /// Emit one frozen dataclass plus its `load` classmethod. When + /// `pinned_profile` is set the class always loads that profile (no `profile` + /// argument); otherwise `load` takes a `profile` argument. + fn emit_class( + out: &mut String, + class: &str, + doc: &str, + fields: &[IrField], + pinned_profile: Option<&str>, + ) { + writeln!(out, "@dataclass(frozen=True)").unwrap(); + writeln!(out, "class {class}:").unwrap(); + writeln!(out, " \"\"\"{doc}\"\"\"").unwrap(); + writeln!(out).unwrap(); + // No field has a default, so any order is valid for the dataclass. + for field in fields { + writeln!(out, " {}: {}", attr(&field.name), type_ann(field)).unwrap(); + } + writeln!(out).unwrap(); + + writeln!(out, " @classmethod").unwrap(); + if pinned_profile.is_some() { + writeln!( + out, + " def load(cls, *, provider: Optional[str] = None, reason: Optional[str] = None, path: Optional[str] = None) -> \"{class}\":" + ) + .unwrap(); + } else { + writeln!( + out, + " def load(cls, *, provider: Optional[str] = None, profile: Optional[str] = None, reason: Optional[str] = None, path: Optional[str] = None) -> \"{class}\":" + ) + .unwrap(); + } + writeln!(out, " r = (").unwrap(); + writeln!(out, " _secretspec.SecretSpec.builder()").unwrap(); + writeln!(out, " .with_path(path)").unwrap(); + writeln!(out, " .with_provider(provider)").unwrap(); + match pinned_profile { + Some(profile) => writeln!(out, " .with_profile(\"{profile}\")").unwrap(), + None => writeln!(out, " .with_profile(profile)").unwrap(), + } + writeln!(out, " .with_reason(reason)").unwrap(); + writeln!(out, " .load()").unwrap(); + writeln!(out, " )").unwrap(); + if fields.is_empty() { + writeln!(out, " return cls()").unwrap(); + } else { + writeln!(out, " return cls(").unwrap(); + for field in fields { + writeln!(out, "{}", assignment(field)).unwrap(); + } + writeln!(out, " )").unwrap(); + } + writeln!(out).unwrap(); + writeln!(out).unwrap(); + } + + /// Emit a Python module of typed accessors over the runtime `secretspec` + /// package: a `SecretSpec` union dataclass plus one `Secrets` + /// dataclass per profile. + pub fn emit(ir: &CodegenIr) -> String { + let mut out = String::new(); + writeln!( + out, + "# Code generated by `secretspec codegen --lang python`. Do not edit." + ) + .unwrap(); + writeln!(out, "# Project: {}", ir.project).unwrap(); + writeln!(out, "from dataclasses import dataclass").unwrap(); + writeln!(out, "from pathlib import Path").unwrap(); + writeln!(out, "from typing import Optional").unwrap(); + writeln!(out).unwrap(); + writeln!(out, "import secretspec as _secretspec").unwrap(); + writeln!(out).unwrap(); + writeln!(out).unwrap(); + + emit_class( + &mut out, + "SecretSpec", + "Union of all profiles; safe to use without knowing the active profile.", + &ir.union, + None, + ); + for profile in &ir.profile_fields { + let class = format!("{}Secrets", capitalize(&profile.name)); + let doc = format!("Secrets for the '{}' profile.", profile.name); + emit_class(&mut out, &class, &doc, &profile.fields, Some(&profile.name)); + } + out + } +} + #[cfg(test)] mod tests { use super::*; @@ -311,6 +452,46 @@ mod tests { assert!(union_field(&ir, "TOKEN").optional); } + #[test] + fn python_emitter_types_fields_and_assigns_from_runtime() { + let ir = build_ir(&config_with(vec![ + ( + "development", + vec![ + ("DATABASE_URL", secret(Some(true), None, None)), + ("API_KEY", secret(Some(false), None, None)), + ], + ), + ( + "production", + vec![ + ("DATABASE_URL", secret(Some(true), None, None)), + ("TLS_CERT", secret(Some(true), Some(true), None)), + ], + ), + ])); + let code = python::emit(&ir); + + // Union types: required -> str, optional/missing -> Optional[...], path. + assert!(code.contains("class SecretSpec:")); + assert!(code.contains(" database_url: str")); + assert!(code.contains(" api_key: Optional[str]")); + assert!(code.contains(" tls_cert: Optional[Path]")); + + // Required field is pulled directly; optional is guarded; path is wrapped. + assert!(code.contains("database_url=r.secrets[\"DATABASE_URL\"].get,")); + assert!(code.contains( + "api_key=(r.secrets[\"API_KEY\"].get if \"API_KEY\" in r.secrets else None)," + )); + + // Per-profile classes mirror derive's profile-specific shape. + assert!(code.contains("class DevelopmentSecrets:")); + assert!(code.contains("class ProductionSecrets:")); + // production pins its profile and types TLS_CERT as a required Path. + assert!(code.contains(".with_profile(\"production\")")); + assert!(code.contains(" tls_cert: Path")); + } + #[test] fn empty_profiles_yield_single_default_with_union_fields() { let mut config = config_with(vec![]); diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index 39eec37..eb6004d 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -510,8 +510,8 @@ impl Secrets { Ok(()) } - /// Get a reference to the project configuration (for testing) - #[cfg(test)] + /// Get a reference to the project configuration. Used by `secretspec + /// codegen` (which needs the manifest, not a provider) and by tests. pub(crate) fn config(&self) -> &Config { &self.config } From 2a3cbd1ab25f75b5ec5e23dff6582be07084ed5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:41:56 -0400 Subject: [PATCH 08/56] feat: add secretspec-go, the Go SDK via purego Phase 5: the Go binding over the same C ABI, reusing the generic dlopen path the Python SDK validated (here in a static language, for the devops/k8s audience). - secretspec-go binds secretspec-ffi via purego (dlopen, no cgo): marshals a JSON request to secretspec_resolve, reads the C string, frees it. No resolution logic in Go; every provider comes from the Rust core. - Mirrors the derive vocabulary with idiomatic Go (PascalCase): New() .WithProvider().WithProfile().WithReason().Load() -> *Resolved with Provider/Profile/Secrets and SetAsEnv(). *MissingRequiredError vs *Error{Kind} separate a missing required secret from a transport failure. as_path yields a readable file path. - Library discovery via SECRETSPEC_FFI_LIB or a Cargo target dir. Tests (gofmt clean) drive the real cdylib end to end via a TestMain that builds and locates it: abi version, values+provenance, missing-required, as_path, invalid input. - devenv.nix now provides Go. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 9 + devenv.nix | 2 + secretspec-go/README.md | 42 +++++ secretspec-go/go.mod | 5 + secretspec-go/go.sum | 2 + secretspec-go/secretspec.go | 273 +++++++++++++++++++++++++++++++ secretspec-go/secretspec_test.go | 191 +++++++++++++++++++++ 7 files changed, 524 insertions(+) create mode 100644 secretspec-go/README.md create mode 100644 secretspec-go/go.mod create mode 100644 secretspec-go/go.sum create mode 100644 secretspec-go/secretspec.go create mode 100644 secretspec-go/secretspec_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 64220bb..1f85d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec-go` Go SDK: a thin client over the `secretspec-ffi` C ABI, + loaded at runtime via purego (dlopen, no cgo), so Go apps inherit every + provider with no Go-side resolution logic. Mirrors the derive crate's + vocabulary (`secretspec.New().WithProvider(...).WithProfile(...).WithReason(...).Load()` + returning a `*Resolved` with `Provider`/`Profile`/`Secrets`, plus `SetAsEnv()`). + A missing required secret returns `*MissingRequiredError`; other failures + return `*Error` with a stable `.Kind`. `as_path` secrets are returned as a + readable file path. The library is found via `SECRETSPEC_FFI_LIB` or a Cargo + target directory. `devenv.nix` now provides Go. - New `secretspec-py` Python SDK: a thin client over the `secretspec-ffi` C ABI (loaded via cffi), so Python apps inherit every provider with no Python-side resolution logic. Mirrors the derive crate's vocabulary diff --git a/devenv.nix b/devenv.nix index 1be0131..3cb97cb 100644 --- a/devenv.nix +++ b/devenv.nix @@ -24,6 +24,8 @@ ''; }; }; + # Go SDK (secretspec-go) binds the C ABI via purego (dlopen, no cgo). + languages.go.enable = true; packages = [ # keyring diff --git a/secretspec-go/README.md b/secretspec-go/README.md new file mode 100644 index 0000000..1d7a3dc --- /dev/null +++ b/secretspec-go/README.md @@ -0,0 +1,42 @@ +# secretspec (Go SDK) + +Go bindings for [SecretSpec](https://secretspec.dev/), a declarative secrets +manager. A thin client over the `secretspec-ffi` C ABI, loaded at runtime via +[purego](https://github.com/ebitengine/purego) (dlopen, no cgo). Resolution +happens in the Rust core, so the SDK inherits every provider with no Go-side +logic. + +```go +package main + +import ( + "fmt" + "log" + + secretspec "github.com/cachix/secretspec/secretspec-go" +) + +func main() { + resolved, err := secretspec.New(). + WithProvider("keyring://"). + WithProfile("production"). + WithReason("boot web app"). + Load() + if err != nil { + log.Fatal(err) + } + + fmt.Println(resolved.Provider, resolved.Profile) + db := resolved.Secrets["DATABASE_URL"] + fmt.Println(db.Get()) // the value, or the file path for as_path secrets + resolved.SetAsEnv() // export everything into the process environment +} +``` + +A missing required secret returns `*MissingRequiredError`; any other failure +returns `*Error` (with a stable `.Kind`). + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/secretspec-go/go.mod b/secretspec-go/go.mod new file mode 100644 index 0000000..6b0df66 --- /dev/null +++ b/secretspec-go/go.mod @@ -0,0 +1,5 @@ +module github.com/cachix/secretspec/secretspec-go + +go 1.23 + +require github.com/ebitengine/purego v0.8.2 diff --git a/secretspec-go/go.sum b/secretspec-go/go.sum new file mode 100644 index 0000000..38eca3d --- /dev/null +++ b/secretspec-go/go.sum @@ -0,0 +1,2 @@ +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go new file mode 100644 index 0000000..46df4ea --- /dev/null +++ b/secretspec-go/secretspec.go @@ -0,0 +1,273 @@ +// Package secretspec is a Go SDK for SecretSpec, a declarative secrets manager. +// +// It is a thin client over the secretspec-ffi C ABI, loaded at runtime via +// purego (dlopen, no cgo). Resolution (providers, chains, profiles, generation, +// as_path) happens entirely in the Rust core; this package marshals a JSON +// request to secretspec_resolve, parses the response envelope, and exposes it +// with the same vocabulary as the Rust derive crate. +// +// The native library is located via, in order: the SECRETSPEC_FFI_LIB +// environment variable, or a Cargo target directory found by searching up from +// the working directory. +package secretspec + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "unsafe" + + "github.com/ebitengine/purego" +) + +var ( + loadOnce sync.Once + loadErr error + cResolve func(string) uintptr + cFree func(uintptr) + cABI func() uintptr +) + +func libNames() []string { + switch runtime.GOOS { + case "darwin": + return []string{"libsecretspec_ffi.dylib"} + case "windows": + return []string{"secretspec_ffi.dll"} + default: + return []string{"libsecretspec_ffi.so"} + } +} + +func findLibrary() (string, error) { + if p := os.Getenv("SECRETSPEC_FFI_LIB"); p != "" { + return p, nil + } + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + for _, profile := range []string{"release", "debug"} { + for _, name := range libNames() { + candidate := filepath.Join(dir, "target", profile, name) + if _, err := os.Stat(candidate); err == nil { + return candidate, nil + } + } + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", &Error{ + Kind: "load", + Message: "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB", + } +} + +func ensureLoaded() error { + loadOnce.Do(func() { + path, err := findLibrary() + if err != nil { + loadErr = err + return + } + handle, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + loadErr = err + return + } + purego.RegisterLibFunc(&cResolve, handle, "secretspec_resolve") + purego.RegisterLibFunc(&cFree, handle, "secretspec_free") + purego.RegisterLibFunc(&cABI, handle, "secretspec_abi_version") + }) + return loadErr +} + +// goString copies a NUL-terminated C string at ptr into a Go string. The +// pointer comes from the C ABI (a Rust allocation), not Go's heap, so this is a +// legitimate FFI read; `go vet`'s unsafeptr check flags it as a false positive +// (it is not part of the `go test` vet subset). +func goString(ptr uintptr) string { + if ptr == 0 { + return "" + } + base := unsafe.Pointer(ptr) + length := 0 + for *(*byte)(unsafe.Add(base, length)) != 0 { + length++ + } + return string(unsafe.Slice((*byte)(base), length)) +} + +// Error is a resolution failure (bad manifest, provider error, reason policy). +type Error struct { + Kind string + Message string +} + +func (e *Error) Error() string { + return fmt.Sprintf("%s (kind: %s)", e.Message, e.Kind) +} + +// MissingRequiredError reports required secrets that were not found anywhere. +type MissingRequiredError struct { + Missing []string +} + +func (e *MissingRequiredError) Error() string { + return "missing required secret(s): " + strings.Join(e.Missing, ", ") +} + +// ResolvedSecret is one resolved secret. Exactly one of Value / Path is set. +type ResolvedSecret struct { + Value *string + Path *string + AsPath bool + Source string + SourceProvider *string +} + +// Get returns the usable string: the file path for as_path secrets, else the value. +func (s ResolvedSecret) Get() string { + if s.AsPath { + if s.Path != nil { + return *s.Path + } + return "" + } + if s.Value != nil { + return *s.Value + } + return "" +} + +// Resolved is a successful resolution, mirroring the Rust Resolved wrapper. +type Resolved struct { + Provider string + Profile string + Secrets map[string]ResolvedSecret + MissingOptional []string +} + +// SetAsEnv exports each resolved secret into the process environment by name. +func (r *Resolved) SetAsEnv() error { + for name, secret := range r.Secrets { + if err := os.Setenv(name, secret.Get()); err != nil { + return err + } + } + return nil +} + +// ABIVersion returns the version reported by the loaded library. +func ABIVersion() (string, error) { + if err := ensureLoaded(); err != nil { + return "", err + } + return goString(cABI()), nil +} + +// Builder configures a resolution, mirroring the derive crate's SecretSpec::builder(). +type Builder struct { + req map[string]any +} + +// New starts a resolution builder. +func New() *Builder { + return &Builder{req: map[string]any{}} +} + +func (b *Builder) WithPath(path string) *Builder { b.req["path"] = path; return b } +func (b *Builder) WithProvider(p string) *Builder { b.req["provider"] = p; return b } +func (b *Builder) WithProfile(p string) *Builder { b.req["profile"] = p; return b } +func (b *Builder) WithReason(reason string) *Builder { b.req["reason"] = reason; return b } +func (b *Builder) WithNoValues(v bool) *Builder { b.req["no_values"] = v; return b } + +type envelopeJSON struct { + OK bool `json:"ok"` + Response *responseJSON `json:"response"` + Error *errorJSON `json:"error"` +} + +type errorJSON struct { + Kind string `json:"kind"` + Message string `json:"message"` +} + +type secretJSON struct { + Value *string `json:"value"` + Path *string `json:"path"` + AsPath bool `json:"as_path"` + Source string `json:"source"` + SourceProvider *string `json:"source_provider"` +} + +type responseJSON struct { + SchemaVersion int `json:"schema_version"` + Provider string `json:"provider"` + Profile string `json:"profile"` + Secrets map[string]secretJSON `json:"secrets"` + MissingRequired []string `json:"missing_required"` + MissingOptional []string `json:"missing_optional"` +} + +// Load resolves the secrets. It returns *MissingRequiredError if a required +// secret is missing, and *Error for any other failure. +func (b *Builder) Load() (*Resolved, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + payload, err := json.Marshal(b.req) + if err != nil { + return nil, err + } + + ptr := cResolve(string(payload)) + if ptr == 0 { + return nil, &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + } + raw := goString(ptr) + cFree(ptr) + + var env envelopeJSON + if err := json.Unmarshal([]byte(raw), &env); err != nil { + return nil, err + } + if !env.OK { + kind, message := "unknown", "" + if env.Error != nil { + kind, message = env.Error.Kind, env.Error.Message + } + return nil, &Error{Kind: kind, Message: message} + } + + resp := env.Response + if len(resp.MissingRequired) > 0 { + return nil, &MissingRequiredError{Missing: resp.MissingRequired} + } + + secrets := make(map[string]ResolvedSecret, len(resp.Secrets)) + for name, entry := range resp.Secrets { + secrets[name] = ResolvedSecret{ + Value: entry.Value, + Path: entry.Path, + AsPath: entry.AsPath, + Source: entry.Source, + SourceProvider: entry.SourceProvider, + } + } + return &Resolved{ + Provider: resp.Provider, + Profile: resp.Profile, + Secrets: secrets, + MissingOptional: resp.MissingOptional, + }, nil +} diff --git a/secretspec-go/secretspec_test.go b/secretspec-go/secretspec_test.go new file mode 100644 index 0000000..ea10294 --- /dev/null +++ b/secretspec-go/secretspec_test.go @@ -0,0 +1,191 @@ +package secretspec + +import ( + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +const manifest = ` +[project] +name = "go-test" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } +` + +// TestMain builds the secretspec-ffi cdylib and points the SDK at it, unless +// SECRETSPEC_FFI_LIB is already set. +func TestMain(m *testing.M) { + if err := ensureLib(); err != nil { + panic(err) + } + os.Exit(m.Run()) +} + +func ensureLib() error { + if os.Getenv("SECRETSPEC_FFI_LIB") != "" { + return nil + } + wd, err := os.Getwd() + if err != nil { + return err + } + repo := filepath.Dir(wd) // secretspec-go lives directly under the repo root + + build := exec.Command("cargo", "build", "-p", "secretspec-ffi") + build.Dir = repo + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + return err + } + + meta := exec.Command("cargo", "metadata", "--no-deps", "--format-version", "1") + meta.Dir = repo + out, err := meta.Output() + if err != nil { + return err + } + var parsed struct { + TargetDirectory string `json:"target_directory"` + } + if err := json.Unmarshal(out, &parsed); err != nil { + return err + } + name := "libsecretspec_ffi.so" + if runtime.GOOS == "darwin" { + name = "libsecretspec_ffi.dylib" + } + return os.Setenv("SECRETSPEC_FFI_LIB", filepath.Join(parsed.TargetDirectory, "debug", name)) +} + +func writeProject(t *testing.T, dotenv string) (string, string) { + t.Helper() + dir := t.TempDir() + manifestPath := filepath.Join(dir, "secretspec.toml") + envPath := filepath.Join(dir, ".env") + if err := os.WriteFile(manifestPath, []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(envPath, []byte(dotenv), 0o600); err != nil { + t.Fatal(err) + } + return manifestPath, "dotenv://" + envPath +} + +func TestABIVersion(t *testing.T) { + version, err := ABIVersion() + if err != nil { + t.Fatal(err) + } + if version == "" { + t.Fatal("empty ABI version") + } +} + +func TestLoadValuesAndProvenance(t *testing.T) { + manifestPath, provider := writeProject(t, "DATABASE_URL=postgres://db\n") + + resolved, err := New(). + WithPath(manifestPath). + WithProvider(provider). + WithReason("go test"). + Load() + if err != nil { + t.Fatal(err) + } + + if resolved.Profile != "default" { + t.Fatalf("profile = %q", resolved.Profile) + } + db := resolved.Secrets["DATABASE_URL"] + if db.Get() != "postgres://db" { + t.Fatalf("DATABASE_URL = %q", db.Get()) + } + if db.Source != "provider" || db.SourceProvider == nil { + t.Fatalf("DATABASE_URL provenance: source=%q provider=%v", db.Source, db.SourceProvider) + } + + log := resolved.Secrets["LOG_LEVEL"] + if log.Get() != "info" || log.Source != "default" { + t.Fatalf("LOG_LEVEL = %q source=%q", log.Get(), log.Source) + } + + if len(resolved.MissingOptional) != 1 || resolved.MissingOptional[0] != "SENTRY_DSN" { + t.Fatalf("missing_optional = %v", resolved.MissingOptional) + } + if _, ok := resolved.Secrets["SENTRY_DSN"]; ok { + t.Fatal("missing optional should not appear in secrets") + } +} + +func TestMissingRequired(t *testing.T) { + manifestPath, provider := writeProject(t, "") // DATABASE_URL absent + + _, err := New().WithPath(manifestPath).WithProvider(provider).WithReason("go test").Load() + var missing *MissingRequiredError + if !errors.As(err, &missing) { + t.Fatalf("expected MissingRequiredError, got %v", err) + } + if len(missing.Missing) != 1 || missing.Missing[0] != "DATABASE_URL" { + t.Fatalf("missing = %v", missing.Missing) + } +} + +func TestAsPath(t *testing.T) { + dir := t.TempDir() + manifestPath := filepath.Join(dir, "secretspec.toml") + envPath := filepath.Join(dir, ".env") + os.WriteFile(manifestPath, []byte(` +[project] +name = "go-test" +revision = "1.0" + +[profiles.default] +TLS_CERT = { description = "cert", required = true, as_path = true } +`), 0o600) + os.WriteFile(envPath, []byte("TLS_CERT=----cert----\n"), 0o600) + + resolved, err := New(). + WithPath(manifestPath). + WithProvider("dotenv://" + envPath). + WithReason("go test"). + Load() + if err != nil { + t.Fatal(err) + } + + cert := resolved.Secrets["TLS_CERT"] + if !cert.AsPath || cert.Value != nil { + t.Fatalf("expected as_path with nil value, got %+v", cert) + } + contents, err := os.ReadFile(cert.Get()) + if err != nil { + t.Fatal(err) + } + if string(contents) != "----cert----" { + t.Fatalf("cert contents = %q", contents) + } +} + +func TestInvalidManifest(t *testing.T) { + _, err := New(). + WithPath("/definitely/does/not/exist/secretspec.toml"). + WithReason("go test"). + Load() + var sErr *Error + if !errors.As(err, &sErr) { + t.Fatalf("expected *Error, got %v", err) + } + var missing *MissingRequiredError + if errors.As(err, &missing) { + t.Fatal("should not be a MissingRequiredError") + } +} From 73c2eeb56a27db21a9563993796021c685aedd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:44:58 -0400 Subject: [PATCH 09/56] feat: add secretspec-rb, the Ruby SDK via stdlib Fiddle Phase 5: the Ruby binding over the same C ABI. Uses stdlib Fiddle (dlopen) rather than the ffi gem, so there is no native gem build; same generic C ABI path as Python/Go. - secretspec-rb binds secretspec-ffi via Fiddle: marshals a JSON request to secretspec_resolve, reads the C string, frees it. No resolution logic in Ruby. - Mirrors the derive vocabulary idiomatically: Secretspec::SecretSpec.builder.with_provider.with_profile.with_reason.load -> Resolved(#provider/#profile/#secrets) plus set_as_env!. MissingRequiredError vs Error(#kind) separate a missing required secret from a transport failure. as_path yields a readable file path. - Library discovery via SECRETSPEC_FFI_LIB or a Cargo target dir. minitest suite (6 tests) drives the real cdylib end to end, building/locating it first. - gemspec + README; devenv.nix now provides Ruby. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 ++ devenv.nix | 2 + secretspec-rb/README.md | 30 +++++ secretspec-rb/lib/secretspec.rb | 188 +++++++++++++++++++++++++++++ secretspec-rb/secretspec.gemspec | 15 +++ secretspec-rb/test/test_resolve.rb | 138 +++++++++++++++++++++ 6 files changed, 383 insertions(+) create mode 100644 secretspec-rb/README.md create mode 100644 secretspec-rb/lib/secretspec.rb create mode 100644 secretspec-rb/secretspec.gemspec create mode 100644 secretspec-rb/test/test_resolve.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f85d91..8b865ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec-rb` Ruby SDK: a thin client over the `secretspec-ffi` C ABI, + loaded at runtime via the stdlib Fiddle (dlopen, no native gem), so Ruby apps + inherit every provider with no Ruby-side resolution logic. Mirrors the derive + crate's vocabulary (`Secretspec::SecretSpec.builder.with_provider(...).with_profile(...).with_reason(...).load` + returning a `Resolved` with `#provider`/`#profile`/`#secrets`, plus + `set_as_env!`). A missing required secret raises + `Secretspec::MissingRequiredError`; other failures raise `Secretspec::Error` + with a stable `#kind`. `as_path` secrets are returned as a readable file path. + The library is found via `SECRETSPEC_FFI_LIB` or a Cargo target directory. + `devenv.nix` now provides Ruby. - New `secretspec-go` Go SDK: a thin client over the `secretspec-ffi` C ABI, loaded at runtime via purego (dlopen, no cgo), so Go apps inherit every provider with no Go-side resolution logic. Mirrors the derive crate's diff --git a/devenv.nix b/devenv.nix index 3cb97cb..d8c2bf0 100644 --- a/devenv.nix +++ b/devenv.nix @@ -26,6 +26,8 @@ }; # Go SDK (secretspec-go) binds the C ABI via purego (dlopen, no cgo). languages.go.enable = true; + # Ruby SDK (secretspec-rb) binds the C ABI via stdlib Fiddle (dlopen). + languages.ruby.enable = true; packages = [ # keyring diff --git a/secretspec-rb/README.md b/secretspec-rb/README.md new file mode 100644 index 0000000..d3c26ec --- /dev/null +++ b/secretspec-rb/README.md @@ -0,0 +1,30 @@ +# secretspec (Ruby SDK) + +Ruby bindings for [SecretSpec](https://secretspec.dev/), a declarative secrets +manager. A thin client over the `secretspec-ffi` C ABI, loaded at runtime via +the stdlib [Fiddle](https://docs.ruby-lang.org/en/master/Fiddle.html) (dlopen, +no native gem). Resolution happens in the Rust core, so the SDK inherits every +provider with no Ruby-side logic. + +```ruby +require "secretspec" + +resolved = Secretspec::SecretSpec.builder + .with_provider("keyring://") + .with_profile("production") + .with_reason("boot web app") + .load + +puts resolved.provider, resolved.profile +db = resolved.secrets["DATABASE_URL"] +puts db.get # the value, or the file path for as_path secrets +resolved.set_as_env! # export everything into ENV +``` + +A missing required secret raises `Secretspec::MissingRequiredError`; any other +failure raises `Secretspec::Error` (with a stable `#kind`). + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb new file mode 100644 index 0000000..b891ee0 --- /dev/null +++ b/secretspec-rb/lib/secretspec.rb @@ -0,0 +1,188 @@ +# frozen_string_literal: true + +# Ruby SDK for SecretSpec, a declarative secrets manager. +# +# A thin client over the secretspec-ffi C ABI, loaded at runtime via the stdlib +# Fiddle (dlopen, no native gem). Resolution happens entirely in the Rust core, +# so the SDK inherits every provider with no Ruby-side logic. Mirrors the Rust +# derive crate's vocabulary. +# +# The native library is located via SECRETSPEC_FFI_LIB, or a Cargo target +# directory found by searching up from the working directory. + +require "fiddle" +require "json" +require "rbconfig" + +module Secretspec + # A resolution failure (bad manifest, provider error, reason policy). + class Error < StandardError + attr_reader :kind + + def initialize(kind, message) + @kind = kind + super("#{message} (kind: #{kind})") + end + end + + # One or more required secrets were not found anywhere. + class MissingRequiredError < Error + attr_reader :missing + + def initialize(missing) + @missing = missing + super("missing_required", "missing required secret(s): #{missing.join(', ')}") + end + end + + # One resolved secret. Exactly one of +value+ / +path+ is set. + ResolvedSecret = Struct.new(:value, :path, :as_path, :source, :source_provider) do + # The usable string: the file path for as_path secrets, else the value. + def get + as_path ? path : value + end + end + + # A successful resolution, mirroring the Rust Resolved wrapper. + Resolved = Struct.new(:provider, :profile, :secrets, :missing_optional) do + # Export each resolved secret into ENV by its declared name. + def set_as_env! + secrets.each { |name, secret| ENV[name] = secret.get } + end + end + + # The narrow C ABI, loaded lazily via Fiddle. + module Native + class << self + def resolve(request_json) + ensure_loaded + ptr = @resolve.call(request_json) + raise Error.new("ffi", "secretspec_resolve returned null") if ptr.null? + + begin + ptr.to_s + ensure + @free.call(ptr) + end + end + + def abi_version + ensure_loaded + @abi.call.to_s + end + + private + + def ensure_loaded + return if @loaded + + handle = Fiddle.dlopen(find_library) + @resolve = Fiddle::Function.new( + handle["secretspec_resolve"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOIDP + ) + @free = Fiddle::Function.new( + handle["secretspec_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID + ) + @abi = Fiddle::Function.new( + handle["secretspec_abi_version"], [], Fiddle::TYPE_VOIDP + ) + @loaded = true + end + + def lib_names + case RbConfig::CONFIG["host_os"] + when /darwin/ then ["libsecretspec_ffi.dylib"] + when /mswin|mingw/ then ["secretspec_ffi.dll"] + else ["libsecretspec_ffi.so"] + end + end + + def find_library + env = ENV["SECRETSPEC_FFI_LIB"] + return env if env && !env.empty? + + dir = Dir.pwd + loop do + %w[release debug].each do |profile| + lib_names.each do |name| + candidate = File.join(dir, "target", profile, name) + return candidate if File.exist?(candidate) + end + end + parent = File.dirname(dir) + break if parent == dir + + dir = parent + end + raise Error.new("load", + "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB") + end + end + end + + # Entry point mirroring the derive crate's SecretSpec::builder(). + class SecretSpec + def self.builder + Builder.new + end + end + + # Fluent builder for a resolution. + class Builder + def initialize + @request = {} + end + + def with_path(path) + @request["path"] = path if path + self + end + + def with_provider(provider) + @request["provider"] = provider if provider + self + end + + def with_profile(profile) + @request["profile"] = profile if profile + self + end + + def with_reason(reason) + @request["reason"] = reason if reason + self + end + + # Resolve the secrets. Raises MissingRequiredError if a required secret is + # missing, and Error for any other failure. + def load + envelope = JSON.parse(Native.resolve(JSON.generate(@request))) + + unless envelope["ok"] + err = envelope["error"] || {} + raise Error.new(err["kind"] || "unknown", err["message"] || "") + end + + response = envelope["response"] + missing = response["missing_required"] || [] + raise MissingRequiredError.new(missing) unless missing.empty? + + secrets = {} + (response["secrets"] || {}).each do |name, entry| + secrets[name] = ResolvedSecret.new( + entry["value"], entry["path"], entry["as_path"] || false, + entry["source"], entry["source_provider"] + ) + end + + Resolved.new( + response["provider"], response["profile"], secrets, + response["missing_optional"] || [] + ) + end + end + + def self.abi_version + Native.abi_version + end +end diff --git a/secretspec-rb/secretspec.gemspec b/secretspec-rb/secretspec.gemspec new file mode 100644 index 0000000..0a62c38 --- /dev/null +++ b/secretspec-rb/secretspec.gemspec @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +Gem::Specification.new do |spec| + spec.name = "secretspec" + spec.version = "0.12.0" + spec.summary = "Declarative secrets, every environment, any provider (Ruby SDK)" + spec.description = "Ruby bindings for SecretSpec, a thin client over the " \ + "secretspec-ffi C ABI (loaded via stdlib Fiddle)." + spec.authors = ["Cachix"] + spec.license = "Apache-2.0" + spec.homepage = "https://secretspec.dev/" + spec.files = Dir["lib/**/*.rb"] + ["README.md"] + spec.require_paths = ["lib"] + spec.required_ruby_version = ">= 3.0" +end diff --git a/secretspec-rb/test/test_resolve.rb b/secretspec-rb/test/test_resolve.rb new file mode 100644 index 0000000..bfa872a --- /dev/null +++ b/secretspec-rb/test/test_resolve.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require "json" +require "rbconfig" +require "tmpdir" +require "minitest/autorun" + +# Build the secretspec-ffi cdylib and point the SDK at it, unless +# SECRETSPEC_FFI_LIB is already set. +def ensure_lib + return if ENV["SECRETSPEC_FFI_LIB"] && !ENV["SECRETSPEC_FFI_LIB"].empty? + + repo = File.expand_path("../..", __dir__) + system("cargo", "build", "-p", "secretspec-ffi", chdir: repo) || raise("cargo build failed") + meta = JSON.parse(`cd #{repo} && cargo metadata --no-deps --format-version 1`) + name = RbConfig::CONFIG["host_os"] =~ /darwin/ ? "libsecretspec_ffi.dylib" : "libsecretspec_ffi.so" + ENV["SECRETSPEC_FFI_LIB"] = File.join(meta["target_directory"], "debug", name) +end + +ensure_lib +require_relative "../lib/secretspec" + +MANIFEST = <<~TOML + [project] + name = "rb-test" + revision = "1.0" + + [profiles.default] + DATABASE_URL = { description = "DB", required = true } + LOG_LEVEL = { description = "log", required = false, default = "info" } + SENTRY_DSN = { description = "sentry", required = false } +TOML + +def project(dir, dotenv, manifest: MANIFEST) + manifest_path = File.join(dir, "secretspec.toml") + env_path = File.join(dir, ".env") + File.write(manifest_path, manifest) + File.write(env_path, dotenv) + [manifest_path, "dotenv://#{env_path}"] +end + +class ResolveTest < Minitest::Test + def test_abi_version_nonempty + refute_empty Secretspec.abi_version + end + + def test_load_values_and_provenance + Dir.mktmpdir do |dir| + manifest, provider = project(dir, "DATABASE_URL=postgres://db\n") + + resolved = Secretspec::SecretSpec.builder + .with_path(manifest) + .with_provider(provider) + .with_reason("rb test") + .load + + assert_equal "default", resolved.profile + db = resolved.secrets["DATABASE_URL"] + assert_equal "postgres://db", db.get + assert_equal "provider", db.source + refute_nil db.source_provider + + log = resolved.secrets["LOG_LEVEL"] + assert_equal "info", log.get + assert_equal "default", log.source + + assert_equal ["SENTRY_DSN"], resolved.missing_optional + refute resolved.secrets.key?("SENTRY_DSN") + end + end + + def test_set_as_env + Dir.mktmpdir do |dir| + manifest, provider = project(dir, "DATABASE_URL=postgres://db\n") + ENV.delete("DATABASE_URL") + + Secretspec::SecretSpec.builder + .with_path(manifest) + .with_provider(provider) + .with_reason("rb test") + .load + .set_as_env! + + assert_equal "postgres://db", ENV.fetch("DATABASE_URL") + end + end + + def test_missing_required_raises + Dir.mktmpdir do |dir| + manifest, provider = project(dir, "") + + error = assert_raises(Secretspec::MissingRequiredError) do + Secretspec::SecretSpec.builder + .with_path(manifest) + .with_provider(provider) + .with_reason("rb test") + .load + end + assert_includes error.missing, "DATABASE_URL" + end + end + + def test_as_path_returns_readable_file + Dir.mktmpdir do |dir| + manifest = <<~TOML + [project] + name = "rb-test" + revision = "1.0" + + [profiles.default] + TLS_CERT = { description = "cert", required = true, as_path = true } + TOML + manifest_path, provider = project(dir, "TLS_CERT=----cert----\n", manifest: manifest) + + resolved = Secretspec::SecretSpec.builder + .with_path(manifest_path) + .with_provider(provider) + .with_reason("rb test") + .load + + cert = resolved.secrets["TLS_CERT"] + assert cert.as_path + assert_nil cert.value + assert_equal "----cert----", File.read(cert.get) + end + end + + def test_invalid_manifest_raises_error + error = assert_raises(Secretspec::Error) do + Secretspec::SecretSpec.builder + .with_path("/definitely/does/not/exist/secretspec.toml") + .with_reason("rb test") + .load + end + refute_instance_of Secretspec::MissingRequiredError, error + refute_empty error.kind + end +end From 0e8ad5f60752fefe29f873dfa33500b5f6a4d0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:47:45 -0400 Subject: [PATCH 10/56] feat: add secretspec Node.js/TypeScript SDK via koffi Phase 5: the Node binding over the same C ABI, via koffi (dlopen), keeping Node on the identical generic C ABI path as Python/Go/Ruby. (napi-rs remains the future production-distribution option; koffi keeps the reference uniform.) - secretspec-node binds secretspec-ffi via koffi: marshals a JSON request to secretspec_resolve, decodes the C string, frees it. No resolution logic in JS. - Mirrors the derive vocabulary idiomatically (camelCase): SecretSpec.builder().withProvider().withProfile().withReason().load() -> Resolved(provider/profile/secrets) plus setAsEnv(). MissingRequiredError vs SecretSpecError(.kind) separate a missing required secret from a transport failure. as_path yields a readable file path. TypeScript types in index.d.ts. - Library discovery via SECRETSPEC_FFI_LIB or a Cargo target dir. node:test suite (6 tests) drives the real cdylib end to end, building/locating it first. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 ++ secretspec-node/.gitignore | 2 + secretspec-node/README.md | 30 +++++ secretspec-node/index.d.ts | 46 ++++++++ secretspec-node/index.js | 167 +++++++++++++++++++++++++++ secretspec-node/package.json | 16 +++ secretspec-node/test/resolve.test.js | 142 +++++++++++++++++++++++ 7 files changed, 413 insertions(+) create mode 100644 secretspec-node/.gitignore create mode 100644 secretspec-node/README.md create mode 100644 secretspec-node/index.d.ts create mode 100644 secretspec-node/index.js create mode 100644 secretspec-node/package.json create mode 100644 secretspec-node/test/resolve.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b865ce..e15cf9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec` Node.js / TypeScript SDK (`secretspec-node`): a thin client + over the `secretspec-ffi` C ABI, loaded at runtime via koffi (dlopen), so + Node apps inherit every provider with no JS-side resolution logic. Mirrors the + derive crate's vocabulary + (`SecretSpec.builder().withProvider(...).withProfile(...).withReason(...).load()` + returning a `Resolved` with `provider`/`profile`/`secrets`, plus `setAsEnv()`). + A missing required secret throws `MissingRequiredError`; other failures throw + `SecretSpecError` with a stable `.kind`. `as_path` secrets are returned as a + readable file path. TypeScript declarations ship in `index.d.ts`. The library + is found via `SECRETSPEC_FFI_LIB` or a Cargo target directory. - New `secretspec-rb` Ruby SDK: a thin client over the `secretspec-ffi` C ABI, loaded at runtime via the stdlib Fiddle (dlopen, no native gem), so Ruby apps inherit every provider with no Ruby-side resolution logic. Mirrors the derive diff --git a/secretspec-node/.gitignore b/secretspec-node/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/secretspec-node/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/secretspec-node/README.md b/secretspec-node/README.md new file mode 100644 index 0000000..4ad49b7 --- /dev/null +++ b/secretspec-node/README.md @@ -0,0 +1,30 @@ +# secretspec (Node.js SDK) + +Node.js / TypeScript bindings for [SecretSpec](https://secretspec.dev/), a +declarative secrets manager. A thin client over the `secretspec-ffi` C ABI, +loaded at runtime via [koffi](https://koffi.dev/) (dlopen). Resolution happens +in the Rust core, so the SDK inherits every provider with no JS-side logic. + +```js +const { SecretSpec } = require('secretspec'); + +const resolved = SecretSpec.builder() + .withProvider('keyring://') + .withProfile('production') + .withReason('boot web app') + .load(); + +console.log(resolved.provider, resolved.profile); +const db = resolved.secrets.DATABASE_URL; +console.log(db.get()); // the value, or the file path for as_path secrets +resolved.setAsEnv(); // export everything into process.env +``` + +A missing required secret throws `MissingRequiredError`; any other failure +throws `SecretSpecError` (with a stable `.kind`). TypeScript declarations ship +in `index.d.ts`. + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/secretspec-node/index.d.ts b/secretspec-node/index.d.ts new file mode 100644 index 0000000..c336f31 --- /dev/null +++ b/secretspec-node/index.d.ts @@ -0,0 +1,46 @@ +// Type definitions for the SecretSpec Node.js SDK. + +export class SecretSpecError extends Error { + kind: string; +} + +export class MissingRequiredError extends SecretSpecError { + missing: string[]; +} + +export class ResolvedSecret { + value: string | null; + path: string | null; + asPath: boolean; + source: string; + sourceProvider: string | null; + /** The usable string: the file path for as_path secrets, else the value. */ + get(): string | null; +} + +export class Resolved { + provider: string; + profile: string; + secrets: Record; + missingOptional: string[]; + /** Export each resolved secret into process.env by its declared name. */ + setAsEnv(): void; +} + +export class Builder { + withPath(path: string): this; + withProvider(provider: string): this; + withProfile(profile: string): this; + withReason(reason: string): this; + /** + * Resolve the secrets. Throws MissingRequiredError if a required secret is + * missing, and SecretSpecError for any other failure. + */ + load(): Resolved; +} + +export const SecretSpec: { + builder(): Builder; +}; + +export function abiVersion(): string; diff --git a/secretspec-node/index.js b/secretspec-node/index.js new file mode 100644 index 0000000..9a59468 --- /dev/null +++ b/secretspec-node/index.js @@ -0,0 +1,167 @@ +'use strict'; + +// Node.js SDK for SecretSpec, a declarative secrets manager. +// +// A thin client over the secretspec-ffi C ABI, loaded at runtime via koffi +// (dlopen). Resolution happens entirely in the Rust core, so the SDK inherits +// every provider with no JS-side logic. Mirrors the Rust derive crate's +// vocabulary. + +const fs = require('fs'); +const path = require('path'); +const koffi = require('koffi'); + +class SecretSpecError extends Error { + constructor(kind, message) { + super(`${message} (kind: ${kind})`); + this.name = 'SecretSpecError'; + this.kind = kind; + } +} + +class MissingRequiredError extends SecretSpecError { + constructor(missing) { + super('missing_required', `missing required secret(s): ${missing.join(', ')}`); + this.name = 'MissingRequiredError'; + this.missing = missing; + } +} + +function libNames() { + if (process.platform === 'darwin') return ['libsecretspec_ffi.dylib']; + if (process.platform === 'win32') return ['secretspec_ffi.dll']; + return ['libsecretspec_ffi.so']; +} + +function findLibrary() { + const override = process.env.SECRETSPEC_FFI_LIB; + if (override) return override; + + let dir = process.cwd(); + for (;;) { + for (const profile of ['release', 'debug']) { + for (const name of libNames()) { + const candidate = path.join(dir, 'target', profile, name); + if (fs.existsSync(candidate)) return candidate; + } + } + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new SecretSpecError( + 'load', + 'could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB', + ); +} + +let _lib = null; + +function lib() { + if (_lib) return _lib; + const handle = koffi.load(findLibrary()); + _lib = { + // void* return so we own the pointer and can free it after decoding. + resolve: handle.func('void *secretspec_resolve(const char *)'), + free: handle.func('void secretspec_free(void *)'), + // const char* return is a static string; koffi decodes it directly. + abi: handle.func('const char *secretspec_abi_version()'), + }; + return _lib; +} + +function resolveRaw(request) { + const l = lib(); + const ptr = l.resolve(JSON.stringify(request)); + if (!ptr) { + throw new SecretSpecError('ffi', 'secretspec_resolve returned null'); + } + try { + return koffi.decode(ptr, 'char', -1); // NUL-terminated C string + } finally { + l.free(ptr); + } +} + +class ResolvedSecret { + constructor(entry) { + this.value = entry.value ?? null; + this.path = entry.path ?? null; + this.asPath = entry.as_path ?? false; + this.source = entry.source; + this.sourceProvider = entry.source_provider ?? null; + } + + /** The usable string: the file path for as_path secrets, else the value. */ + get() { + return this.asPath ? this.path : this.value; + } +} + +class Resolved { + constructor(response) { + this.provider = response.provider; + this.profile = response.profile; + this.secrets = {}; + for (const [name, entry] of Object.entries(response.secrets || {})) { + this.secrets[name] = new ResolvedSecret(entry); + } + this.missingOptional = response.missing_optional || []; + } + + /** Export each resolved secret into process.env by its declared name. */ + setAsEnv() { + for (const [name, secret] of Object.entries(this.secrets)) { + process.env[name] = secret.get(); + } + } +} + +class Builder { + constructor() { + this._request = {}; + } + + withPath(p) { if (p != null) this._request.path = p; return this; } + withProvider(p) { if (p != null) this._request.provider = p; return this; } + withProfile(p) { if (p != null) this._request.profile = p; return this; } + withReason(r) { if (r != null) this._request.reason = r; return this; } + + /** + * Resolve the secrets. Throws MissingRequiredError if a required secret is + * missing, and SecretSpecError for any other failure. + */ + load() { + const envelope = JSON.parse(resolveRaw(this._request)); + if (!envelope.ok) { + const err = envelope.error || {}; + throw new SecretSpecError(err.kind || 'unknown', err.message || ''); + } + const response = envelope.response; + const missing = response.missing_required || []; + if (missing.length) { + throw new MissingRequiredError(missing); + } + return new Resolved(response); + } +} + +const SecretSpec = { + builder() { + return new Builder(); + }, +}; + +function abiVersion() { + return lib().abi(); +} + +module.exports = { + SecretSpec, + Builder, + Resolved, + ResolvedSecret, + SecretSpecError, + MissingRequiredError, + abiVersion, +}; diff --git a/secretspec-node/package.json b/secretspec-node/package.json new file mode 100644 index 0000000..9aa96cb --- /dev/null +++ b/secretspec-node/package.json @@ -0,0 +1,16 @@ +{ + "name": "secretspec", + "version": "0.12.0", + "description": "Declarative secrets, every environment, any provider (Node.js SDK)", + "license": "Apache-2.0", + "homepage": "https://secretspec.dev/", + "main": "index.js", + "types": "index.d.ts", + "files": ["index.js", "index.d.ts"], + "scripts": { + "test": "node --test" + }, + "dependencies": { + "koffi": "^2.9.0" + } +} diff --git a/secretspec-node/test/resolve.test.js b/secretspec-node/test/resolve.test.js new file mode 100644 index 0000000..29933ee --- /dev/null +++ b/secretspec-node/test/resolve.test.js @@ -0,0 +1,142 @@ +'use strict'; + +const assert = require('node:assert'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +// Build the secretspec-ffi cdylib and point the SDK at it, unless +// SECRETSPEC_FFI_LIB is already set. +function ensureLib() { + if (process.env.SECRETSPEC_FFI_LIB) return; + const repo = path.resolve(__dirname, '..', '..'); + execFileSync('cargo', ['build', '-p', 'secretspec-ffi'], { cwd: repo, stdio: 'inherit' }); + const meta = JSON.parse( + execFileSync('cargo', ['metadata', '--no-deps', '--format-version', '1'], { cwd: repo }), + ); + const name = + process.platform === 'darwin' ? 'libsecretspec_ffi.dylib' : 'libsecretspec_ffi.so'; + process.env.SECRETSPEC_FFI_LIB = path.join(meta.target_directory, 'debug', name); +} + +ensureLib(); +const { + SecretSpec, + MissingRequiredError, + SecretSpecError, + abiVersion, +} = require('../index.js'); + +const MANIFEST = ` +[project] +name = "node-test" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } +`; + +function project(dotenv, manifest = MANIFEST) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ss-node-')); + const manifestPath = path.join(dir, 'secretspec.toml'); + const envPath = path.join(dir, '.env'); + fs.writeFileSync(manifestPath, manifest); + fs.writeFileSync(envPath, dotenv); + return { manifestPath, provider: `dotenv://${envPath}` }; +} + +test('abiVersion is non-empty', () => { + assert.ok(abiVersion().length > 0); +}); + +test('load returns values and provenance', () => { + const { manifestPath, provider } = project('DATABASE_URL=postgres://db\n'); + + const resolved = SecretSpec.builder() + .withPath(manifestPath) + .withProvider(provider) + .withReason('node test') + .load(); + + assert.equal(resolved.profile, 'default'); + const db = resolved.secrets.DATABASE_URL; + assert.equal(db.get(), 'postgres://db'); + assert.equal(db.source, 'provider'); + assert.ok(db.sourceProvider); + + const log = resolved.secrets.LOG_LEVEL; + assert.equal(log.get(), 'info'); + assert.equal(log.source, 'default'); + + assert.deepEqual(resolved.missingOptional, ['SENTRY_DSN']); + assert.ok(!('SENTRY_DSN' in resolved.secrets)); +}); + +test('setAsEnv exports secrets', () => { + const { manifestPath, provider } = project('DATABASE_URL=postgres://db\n'); + delete process.env.DATABASE_URL; + + SecretSpec.builder() + .withPath(manifestPath) + .withProvider(provider) + .withReason('node test') + .load() + .setAsEnv(); + + assert.equal(process.env.DATABASE_URL, 'postgres://db'); +}); + +test('missing required throws MissingRequiredError', () => { + const { manifestPath, provider } = project(''); + + assert.throws( + () => + SecretSpec.builder() + .withPath(manifestPath) + .withProvider(provider) + .withReason('node test') + .load(), + (err) => err instanceof MissingRequiredError && err.missing.includes('DATABASE_URL'), + ); +}); + +test('as_path returns a readable file path', () => { + const manifest = ` +[project] +name = "node-test" +revision = "1.0" + +[profiles.default] +TLS_CERT = { description = "cert", required = true, as_path = true } +`; + const { manifestPath, provider } = project('TLS_CERT=----cert----\n', manifest); + + const resolved = SecretSpec.builder() + .withPath(manifestPath) + .withProvider(provider) + .withReason('node test') + .load(); + + const cert = resolved.secrets.TLS_CERT; + assert.equal(cert.asPath, true); + assert.equal(cert.value, null); + assert.equal(fs.readFileSync(cert.get(), 'utf8'), '----cert----'); +}); + +test('invalid manifest throws SecretSpecError (not MissingRequired)', () => { + assert.throws( + () => + SecretSpec.builder() + .withPath('/definitely/does/not/exist/secretspec.toml') + .withReason('node test') + .load(), + (err) => + err instanceof SecretSpecError && + !(err instanceof MissingRequiredError) && + typeof err.kind === 'string', + ); +}); From 5cc327d2723256dc301701f3314b35a594e13e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:51:44 -0400 Subject: [PATCH 11/56] test: add cross-language conformance suite The capstone of phase 5: prove the Python, Go, Ruby, and Node SDKs agree. They are all thin clients over one C ABI, so the risk is in each SDK's parsing and exposure, not the resolver. This suite locks that down. - conformance/fixtures/* are self-contained cases (manifest + .env + expected.json). Each SDK resolves them and projects its result to a canonical shape (profile, per-secret {value, source, as_path}, missing lists), then asserts equality with expected.json. For as_path secrets the canonical value is the materialized file's contents, so it is deterministic across languages. - Each SDK runs the fixtures inside its own native runner (pytest, go test, minitest, node:test), reading conformance/ relative to the repo root. All four pass the same two fixtures (basic: provider + default + optional-missing; as_path). - Fixtures cover successful resolutions; error behavior stays in per-SDK suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 ++ conformance/README.md | 48 +++++++++ conformance/fixtures/as_path/.env | 1 + conformance/fixtures/as_path/expected.json | 8 ++ conformance/fixtures/as_path/secretspec.toml | 6 ++ conformance/fixtures/basic/.env | 1 + conformance/fixtures/basic/expected.json | 9 ++ conformance/fixtures/basic/secretspec.toml | 8 ++ secretspec-go/conformance_test.go | 100 +++++++++++++++++++ secretspec-node/test/conformance.test.js | 57 +++++++++++ secretspec-py/tests/test_conformance.py | 50 ++++++++++ secretspec-rb/test/test_resolve.rb | 32 ++++++ 12 files changed, 326 insertions(+) create mode 100644 conformance/README.md create mode 100644 conformance/fixtures/as_path/.env create mode 100644 conformance/fixtures/as_path/expected.json create mode 100644 conformance/fixtures/as_path/secretspec.toml create mode 100644 conformance/fixtures/basic/.env create mode 100644 conformance/fixtures/basic/expected.json create mode 100644 conformance/fixtures/basic/secretspec.toml create mode 100644 secretspec-go/conformance_test.go create mode 100644 secretspec-node/test/conformance.test.js create mode 100644 secretspec-py/tests/test_conformance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e15cf9e..f651004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New cross-language conformance suite (`conformance/`): shared fixtures + (manifest + `.env` + a canonical `expected.json`) that every SDK resolves and + must reduce to the identical canonical result, guaranteeing the Python, Go, + Ruby, and Node SDKs agree. Each SDK runs the fixtures inside its own native + test runner. For `as_path` secrets the canonical value is the materialized + file's contents, so the comparison is deterministic across languages. - New `secretspec` Node.js / TypeScript SDK (`secretspec-node`): a thin client over the `secretspec-ffi` C ABI, loaded at runtime via koffi (dlopen), so Node apps inherit every provider with no JS-side resolution logic. Mirrors the diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..a1798b0 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,48 @@ +# Cross-language conformance suite + +Every SecretSpec SDK (Python, Go, Ruby, Node) is a thin client over the same +`secretspec-ffi` C ABI. This suite proves they agree: each SDK resolves the same +fixtures and must produce the identical **canonical** result. + +## Fixtures + +Each directory under `fixtures/` is one case: + +- `secretspec.toml` — the manifest +- `.env` — backing values (resolved via the `dotenv` provider) +- `expected.json` — the canonical result every SDK must produce + +Fixtures only cover successful resolutions; per-SDK test suites cover error +behavior (missing-required, invalid input). + +## Canonical form + +Environmental details (the absolute `dotenv://` path, `as_path` temp-file paths) +are not comparable across runs, so each SDK projects its resolved result to a +canonical shape before comparing: + +```json +{ + "profile": "", + "secrets": { + "": { "value": "", + "source": "provider|generated|default", + "as_path": false } + }, + "missing_required": [], + "missing_optional": [""] +} +``` + +For `as_path` secrets, `value` is the **contents** of the materialized file, so +the comparison is deterministic and meaningful across languages. + +## Running + +Each SDK runs the fixtures as part of its own test suite (so it uses that +language's native runner), reading this directory relative to the repo root: + +- Python: `cd secretspec-py && pytest` +- Go: `cd secretspec-go && go test ./...` +- Ruby: `cd secretspec-rb && ruby test/test_resolve.rb` +- Node: `cd secretspec-node && node --test` diff --git a/conformance/fixtures/as_path/.env b/conformance/fixtures/as_path/.env new file mode 100644 index 0000000..2a5fff9 --- /dev/null +++ b/conformance/fixtures/as_path/.env @@ -0,0 +1 @@ +TLS_CERT=cert-bytes-here diff --git a/conformance/fixtures/as_path/expected.json b/conformance/fixtures/as_path/expected.json new file mode 100644 index 0000000..064fe99 --- /dev/null +++ b/conformance/fixtures/as_path/expected.json @@ -0,0 +1,8 @@ +{ + "profile": "default", + "secrets": { + "TLS_CERT": { "value": "cert-bytes-here", "source": "provider", "as_path": true } + }, + "missing_required": [], + "missing_optional": [] +} diff --git a/conformance/fixtures/as_path/secretspec.toml b/conformance/fixtures/as_path/secretspec.toml new file mode 100644 index 0000000..13aad0b --- /dev/null +++ b/conformance/fixtures/as_path/secretspec.toml @@ -0,0 +1,6 @@ +[project] +name = "conformance-as-path" +revision = "1.0" + +[profiles.default] +TLS_CERT = { description = "cert", required = true, as_path = true } diff --git a/conformance/fixtures/basic/.env b/conformance/fixtures/basic/.env new file mode 100644 index 0000000..f310f57 --- /dev/null +++ b/conformance/fixtures/basic/.env @@ -0,0 +1 @@ +DATABASE_URL=postgres://db diff --git a/conformance/fixtures/basic/expected.json b/conformance/fixtures/basic/expected.json new file mode 100644 index 0000000..35f626d --- /dev/null +++ b/conformance/fixtures/basic/expected.json @@ -0,0 +1,9 @@ +{ + "profile": "default", + "secrets": { + "DATABASE_URL": { "value": "postgres://db", "source": "provider", "as_path": false }, + "LOG_LEVEL": { "value": "info", "source": "default", "as_path": false } + }, + "missing_required": [], + "missing_optional": ["SENTRY_DSN"] +} diff --git a/conformance/fixtures/basic/secretspec.toml b/conformance/fixtures/basic/secretspec.toml new file mode 100644 index 0000000..76321a0 --- /dev/null +++ b/conformance/fixtures/basic/secretspec.toml @@ -0,0 +1,8 @@ +[project] +name = "conformance-basic" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { description = "DB", required = true } +LOG_LEVEL = { description = "log", required = false, default = "info" } +SENTRY_DSN = { description = "sentry", required = false } diff --git a/secretspec-go/conformance_test.go b/secretspec-go/conformance_test.go new file mode 100644 index 0000000..9102066 --- /dev/null +++ b/secretspec-go/conformance_test.go @@ -0,0 +1,100 @@ +package secretspec + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" +) + +// TestConformance resolves the shared cross-language fixtures and asserts this +// SDK produces the canonical result every other SDK must also produce. +func TestConformance(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + fixtures := filepath.Join(filepath.Dir(wd), "conformance", "fixtures") + + entries, err := os.ReadDir(fixtures) + if err != nil { + t.Fatal(err) + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + t.Run(entry.Name(), func(t *testing.T) { + dir := filepath.Join(fixtures, entry.Name()) + + resolved, err := New(). + WithPath(filepath.Join(dir, "secretspec.toml")). + WithProvider("dotenv://" + filepath.Join(dir, ".env")). + WithReason("conformance"). + Load() + if err != nil { + t.Fatal(err) + } + + actual := canonical(t, resolved) + + expectedBytes, err := os.ReadFile(filepath.Join(dir, "expected.json")) + if err != nil { + t.Fatal(err) + } + var expected any + if err := json.Unmarshal(expectedBytes, &expected); err != nil { + t.Fatal(err) + } + + // Round-trip actual through JSON so both sides are the same generic + // shape (map[string]any, []any) for DeepEqual. + actualBytes, err := json.Marshal(actual) + if err != nil { + t.Fatal(err) + } + var actualGeneric any + if err := json.Unmarshal(actualBytes, &actualGeneric); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(actualGeneric, expected) { + t.Fatalf("canonical mismatch\n got: %s\nwant: %s", actualBytes, expectedBytes) + } + }) + } +} + +func canonical(t *testing.T, resolved *Resolved) map[string]any { + t.Helper() + secrets := map[string]any{} + for name, secret := range resolved.Secrets { + var value string + if secret.AsPath { + contents, err := os.ReadFile(secret.Get()) + if err != nil { + t.Fatal(err) + } + value = string(contents) + } else if secret.Value != nil { + value = *secret.Value + } + secrets[name] = map[string]any{ + "value": value, + "source": secret.Source, + "as_path": secret.AsPath, + } + } + missingOptional := resolved.MissingOptional + if missingOptional == nil { + missingOptional = []string{} + } + return map[string]any{ + "profile": resolved.Profile, + "secrets": secrets, + "missing_required": []string{}, + "missing_optional": missingOptional, + } +} diff --git a/secretspec-node/test/conformance.test.js b/secretspec-node/test/conformance.test.js new file mode 100644 index 0000000..85b356c --- /dev/null +++ b/secretspec-node/test/conformance.test.js @@ -0,0 +1,57 @@ +'use strict'; + +// Cross-language conformance: resolve the shared fixtures and assert this SDK +// produces the canonical result every other SDK must also produce. + +const assert = require('node:assert'); +const test = require('node:test'); +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +function ensureLib() { + if (process.env.SECRETSPEC_FFI_LIB) return; + const repo = path.resolve(__dirname, '..', '..'); + execFileSync('cargo', ['build', '-p', 'secretspec-ffi'], { cwd: repo, stdio: 'inherit' }); + const meta = JSON.parse( + execFileSync('cargo', ['metadata', '--no-deps', '--format-version', '1'], { cwd: repo }), + ); + const name = + process.platform === 'darwin' ? 'libsecretspec_ffi.dylib' : 'libsecretspec_ffi.so'; + process.env.SECRETSPEC_FFI_LIB = path.join(meta.target_directory, 'debug', name); +} + +ensureLib(); +const { SecretSpec } = require('../index.js'); + +const FIXTURES = path.resolve(__dirname, '..', '..', 'conformance', 'fixtures'); + +function canonical(resolved) { + const secrets = {}; + for (const [name, secret] of Object.entries(resolved.secrets)) { + const value = secret.asPath ? fs.readFileSync(secret.get(), 'utf8') : secret.value; + secrets[name] = { value, source: secret.source, as_path: secret.asPath }; + } + return { + profile: resolved.profile, + secrets, + missing_required: [], + missing_optional: [...resolved.missingOptional].sort(), + }; +} + +for (const fixture of fs.readdirSync(FIXTURES).sort()) { + const dir = path.join(FIXTURES, fixture); + if (!fs.statSync(dir).isDirectory()) continue; + + test(`conformance: ${fixture}`, () => { + const expected = JSON.parse(fs.readFileSync(path.join(dir, 'expected.json'), 'utf8')); + const resolved = SecretSpec.builder() + .withPath(path.join(dir, 'secretspec.toml')) + .withProvider(`dotenv://${path.join(dir, '.env')}`) + .withReason('conformance') + .load(); + + assert.deepStrictEqual(canonical(resolved), expected); + }); +} diff --git a/secretspec-py/tests/test_conformance.py b/secretspec-py/tests/test_conformance.py new file mode 100644 index 0000000..b542a5f --- /dev/null +++ b/secretspec-py/tests/test_conformance.py @@ -0,0 +1,50 @@ +"""Cross-language conformance: resolve the shared fixtures and assert the SDK +produces the canonical result every other SDK must also produce.""" + +import json +import pathlib + +import pytest + +from secretspec import SecretSpec + +FIXTURES = pathlib.Path(__file__).resolve().parents[2] / "conformance" / "fixtures" + + +def _canonical(resolved) -> dict: + secrets = {} + for name, secret in resolved.secrets.items(): + value = ( + pathlib.Path(secret.path).read_text() if secret.as_path else secret.value + ) + secrets[name] = { + "value": value, + "source": secret.source, + "as_path": secret.as_path, + } + return { + "profile": resolved.profile, + "secrets": secrets, + "missing_required": [], + "missing_optional": sorted(resolved.missing_optional), + } + + +def _fixtures(): + return sorted(p.name for p in FIXTURES.iterdir() if p.is_dir()) + + +@pytest.mark.parametrize("fixture", _fixtures()) +def test_conformance(fixture): + directory = FIXTURES / fixture + expected = json.loads((directory / "expected.json").read_text()) + + resolved = ( + SecretSpec.builder() + .with_path(str(directory / "secretspec.toml")) + .with_provider(f"dotenv://{directory / '.env'}") + .with_reason("conformance") + .load() + ) + + assert _canonical(resolved) == expected diff --git a/secretspec-rb/test/test_resolve.rb b/secretspec-rb/test/test_resolve.rb index bfa872a..7909b4b 100644 --- a/secretspec-rb/test/test_resolve.rb +++ b/secretspec-rb/test/test_resolve.rb @@ -136,3 +136,35 @@ def test_invalid_manifest_raises_error refute_empty error.kind end end + +# Cross-language conformance: resolve the shared fixtures and assert this SDK +# produces the canonical result every other SDK must also produce. +class ConformanceTest < Minitest::Test + FIXTURES = File.expand_path("../../conformance/fixtures", __dir__) + + def canonical(resolved) + secrets = {} + resolved.secrets.each do |name, secret| + value = secret.as_path ? File.read(secret.get) : secret.value + secrets[name] = { "value" => value, "source" => secret.source, "as_path" => secret.as_path } + end + { + "profile" => resolved.profile, + "secrets" => secrets, + "missing_required" => [], + "missing_optional" => resolved.missing_optional.sort + } + end + + Dir.glob(File.join(FIXTURES, "*")).select { |p| File.directory?(p) }.sort.each do |dir| + define_method("test_conformance_#{File.basename(dir)}") do + expected = JSON.parse(File.read(File.join(dir, "expected.json"))) + resolved = Secretspec::SecretSpec.builder + .with_path(File.join(dir, "secretspec.toml")) + .with_provider("dotenv://#{File.join(dir, '.env')}") + .with_reason("conformance") + .load + assert_equal expected, canonical(resolved) + end + end +end From e046a062f072c2cdd6c977793a001ca64ec4e7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 20:57:57 -0400 Subject: [PATCH 12/56] test: add aggregate cross-language conformance runner conformance/run.sh builds the secretspec-ffi cdylib once, points every SDK at it via SECRETSPEC_FFI_LIB (so they don't each rebuild), runs all four conformance suites in their native runners, and prints a combined PASS/FAIL/SKIP summary. Exits non-zero if any language fails; a missing toolchain is SKIP, not FAIL. README documents it as the one-command entry point. Co-Authored-By: Claude Opus 4.8 (1M context) --- conformance/README.md | 13 ++++++-- conformance/run.sh | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100755 conformance/run.sh diff --git a/conformance/README.md b/conformance/README.md index a1798b0..551264d 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -39,8 +39,17 @@ the comparison is deterministic and meaningful across languages. ## Running -Each SDK runs the fixtures as part of its own test suite (so it uses that -language's native runner), reading this directory relative to the repo root: +Run everything with the aggregate runner (inside the project devenv shell). It +builds the `secretspec-ffi` cdylib once, points every SDK at it via +`SECRETSPEC_FFI_LIB`, runs each language's conformance suite, and prints a +combined PASS/FAIL/SKIP summary (exiting non-zero if any language fails): + +```sh +devenv shell -- bash conformance/run.sh +``` + +Or run a single language in its own native runner, reading this directory +relative to the repo root: - Python: `cd secretspec-py && pytest` - Go: `cd secretspec-go && go test ./...` diff --git a/conformance/run.sh b/conformance/run.sh new file mode 100755 index 0000000..820d294 --- /dev/null +++ b/conformance/run.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Aggregate cross-language conformance runner. +# +# Builds the secretspec-ffi cdylib once, then runs every SDK's conformance suite +# against the shared fixtures and reports a combined result. Run inside the +# project devenv shell (which provides cargo, python, go, ruby, node): +# +# devenv shell -- bash conformance/run.sh +# +# Exits non-zero if any language's conformance suite fails. A language whose +# toolchain is missing is reported as SKIP and does not fail the run. +set -uo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +echo "==> Building secretspec-ffi cdylib" +cargo build -p secretspec-ffi || exit 1 + +target_dir="$(cargo metadata --no-deps --format-version 1 \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +case "$(uname -s)" in + Darwin) lib_name="libsecretspec_ffi.dylib" ;; + *) lib_name="libsecretspec_ffi.so" ;; +esac +export SECRETSPEC_FFI_LIB="$target_dir/debug/$lib_name" +echo "==> SECRETSPEC_FFI_LIB=$SECRETSPEC_FFI_LIB" + +names=() +statuses=() + +run() { + local name="$1" tool="$2" fn="$3" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "==> SKIP $name ($tool not found)" + names+=("$name"); statuses+=("SKIP") + return + fi + echo "==> $name conformance" + if "$fn"; then + names+=("$name"); statuses+=("PASS") + else + names+=("$name"); statuses+=("FAIL") + fi +} + +run_python() { ( cd secretspec-py && python -m pytest tests/test_conformance.py -q ); } +run_go() { ( cd secretspec-go && go test -run TestConformance ./... ); } +run_ruby() { ( cd secretspec-rb && ruby test/test_resolve.rb -n "/conformance/" ); } +run_node() { ( + cd secretspec-node + [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null + node --test test/conformance.test.js +); } + +run "Python" python run_python +run "Go" go run_go +run "Ruby" ruby run_ruby +run "Node" node run_node + +echo +echo "==> Conformance summary" +overall=0 +for i in "${!names[@]}"; do + printf " %-8s %s\n" "${names[$i]}" "${statuses[$i]}" + [ "${statuses[$i]}" = "FAIL" ] && overall=1 +done +exit "$overall" From 10b46a7ef3885fa3bbce3fc0656f1dad186d836e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 21:42:14 -0400 Subject: [PATCH 13/56] refactor: generate typed shapes via JSON Schema + quicktype, not per-language emitters Replaces the hand-written per-language codegen emitter approach (the `codegen --lang python` command and the WIP Go/Ruby/TypeScript emitters) with a single JSON Schema emitter, so we no longer maintain a typed-accessor generator per language. quicktype turns the schema into idiomatic types AND deserializers for any language; we maintain only a tiny generic `fields()` helper per SDK. - `secretspec codegen --lang ...` becomes `secretspec schema`: emits a JSON Schema (draft-06) with a `SecretSpec` union type plus one `Secrets` per profile, property names = secret names, optionals nullable. Driven by the same shared IR (so it can't drift from the derive macro). - Each runtime SDK gains a generic `fields()` returning a flat `{SECRET_NAME: value}` map (the file path for as_path): Python/Ruby return the map, Go/Node also expose a JSON variant (FieldsJSON / fieldsJson) for quicktype's bytes/string deserializers. - The loader the user writes is one line, e.g. `SecretSpec.from_dict(resolved.fields())`. quicktype owns naming, optionality, and converters for all current and future languages. - Deleted codegen::{python,go,ruby,typescript} and their emitter tests; added a schema emitter test. Python e2e now drives the real pipeline: `secretspec schema | quicktype --lang python` then `SecretSpec.from_dict(resolved.fields())`. CLI reference + CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 23 ++- docs/src/content/docs/reference/cli.md | 30 +-- secretspec-go/secretspec.go | 15 ++ secretspec-node/index.d.ts | 4 + secretspec-node/index.js | 17 ++ secretspec-node/package.json | 8 +- secretspec-py/secretspec/__init__.py | 9 + secretspec-py/tests/test_codegen.py | 92 +++++----- secretspec-rb/lib/secretspec.rb | 7 + secretspec/src/cli/mod.rs | 35 ++-- secretspec/src/codegen.rs | 241 ++++++++++--------------- 11 files changed, 250 insertions(+), 231 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f651004..a5efe73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,16 +55,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `devenv.nix` now provides Python, cffi, pytest, and maturin. - New `secretspec::codegen` module: a shared, language-neutral intermediate representation (IR) that reduces a manifest to the typed-accessor decisions - every generator needs (union vs per-profile field sets, optionality, `as_path`, - profile list). It is the single source of truth those decisions are computed - in, so the Rust derive macro and the eventual TypeScript/Python/Go/Ruby - emitters cannot drift. `build_ir(&Config) -> CodegenIr`. -- New `secretspec codegen --lang python` command: emits typed Python accessors - over the `secretspec` Python SDK from the manifest, driven by the shared IR. - It mirrors the derive crate's shape (a `SecretSpec` union dataclass plus one - `Secrets` dataclass per profile, each with a builder-style `load`) - with idiomatic snake_case attributes typed as `str`/`Optional[str]`/`Path`. - Value-free: reads only the manifest. `-o` writes to a file instead of stdout. + (union vs per-profile field sets, optionality, `as_path`, profile list). It is + the single source of truth those decisions are computed in, so the Rust derive + macro and the JSON Schema emitter cannot drift. `build_ir(&Config) -> CodegenIr`. +- New `secretspec schema` command: emits a JSON Schema for the manifest's typed + shapes (a `SecretSpec` union type plus one `Secrets` per profile). + Rather than hand-write a typed-accessor generator per language, feed this to + [quicktype](https://quicktype.io) to generate idiomatic types and + deserializers for any language, then hand the deserializer the flat + `{SECRET_NAME: value}` map from each SDK's new `fields()` helper (e.g. in + Python, `SecretSpec.from_dict(resolved.fields())`). This keeps the per-language + maintenance to the small `fields()` method. `fields()` (and a JSON variant for + Go/Node) is available on the resolved result in every SDK. Value-free: `schema` + reads only the manifest; `-o` writes to a file instead of stdout. ### Changed - The `secretspec-derive` macro now computes all of its typing decisions through diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 58ebd8f..60fecbd 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -240,33 +240,39 @@ persisted temp file), its `source` (`provider`, `generated`, or `default`), and the serving provider's credential-free URI. The canonical JSON Schema is committed at `schema/resolve-response.schema.json`. -### codegen -Generate typed accessors for another language from the manifest, driven by the -shared codegen IR (so every language stays consistent). Value-free: reads only -the manifest, never a provider. +### schema +Emit a JSON Schema for the manifest's typed shapes: a `SecretSpec` type (the +union, safe for any profile) plus one `Secrets` type per profile. +Value-free: reads only the manifest, never a provider. ```bash -secretspec codegen --lang [-o FILE] +secretspec schema [-o FILE] ``` **Options:** -- `--lang ` - Target language (`python`) - `-o, --output ` - Write to this file instead of stdout -For Python it emits a `SecretSpec` union dataclass plus one `Secrets` -dataclass per profile, each with a builder-style `load`, over the `secretspec` -Python SDK: +Rather than ship a typed-accessor generator per language, feed this schema to +[quicktype](https://quicktype.io), which generates idiomatic types **and** +deserializers for any language. At runtime, hand the generated deserializer the +flat `{SECRET_NAME: value}` map from the SDK's `fields()` helper: ```bash -$ secretspec codegen --lang python -o secrets_gen.py +$ secretspec schema | quicktype -s schema --lang python -o secrets_gen.py ``` ```python -from secrets_gen import SecretSpec +from secretspec import SecretSpec +from secrets_gen import SecretSpec as Secrets # quicktype-generated, typed -s = SecretSpec.load(profile="production", reason="boot") +resolved = SecretSpec.builder().with_profile("production").with_reason("boot").load() +s = Secrets.from_dict(resolved.fields()) print(s.database_url) # typed str ``` +The same pattern works in every SDK: Go `UnmarshalSecretSpec(resolved.FieldsJSON())`, +TypeScript `Convert.toSecretSpec(resolved.fieldsJson())`, Ruby +`SecretSpec.from_dynamic!(resolved.fields)`. + ### set Set a secret value. diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 46df4ea..37c257a 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -167,6 +167,21 @@ func (r *Resolved) SetAsEnv() error { return nil } +// Fields returns a flat map of SECRET_NAME -> value (the file path for as_path). +func (r *Resolved) Fields() map[string]string { + out := make(map[string]string, len(r.Secrets)) + for name, secret := range r.Secrets { + out[name] = secret.Get() + } + return out +} + +// FieldsJSON marshals Fields() to JSON, the input for a quicktype-generated +// deserializer (e.g. UnmarshalSecretSpec). See `secretspec schema`. +func (r *Resolved) FieldsJSON() ([]byte, error) { + return json.Marshal(r.Fields()) +} + // ABIVersion returns the version reported by the loaded library. func ABIVersion() (string, error) { if err := ensureLoaded(); err != nil { diff --git a/secretspec-node/index.d.ts b/secretspec-node/index.d.ts index c336f31..a6a0590 100644 --- a/secretspec-node/index.d.ts +++ b/secretspec-node/index.d.ts @@ -25,6 +25,10 @@ export class Resolved { missingOptional: string[]; /** Export each resolved secret into process.env by its declared name. */ setAsEnv(): void; + /** Flat { SECRET_NAME: value } object (the file path for as_path secrets). */ + fields(): Record; + /** fields() as a JSON string, the input for a quicktype-generated deserializer. */ + fieldsJson(): string; } export class Builder { diff --git a/secretspec-node/index.js b/secretspec-node/index.js index 9a59468..6fe1b7d 100644 --- a/secretspec-node/index.js +++ b/secretspec-node/index.js @@ -115,6 +115,23 @@ class Resolved { process.env[name] = secret.get(); } } + + /** Flat { SECRET_NAME: value } object (the file path for as_path secrets). */ + fields() { + const out = {}; + for (const [name, secret] of Object.entries(this.secrets)) { + out[name] = secret.get(); + } + return out; + } + + /** + * fields() as a JSON string, the input for a quicktype-generated deserializer + * (e.g. Convert.toSecretSpec). See `secretspec schema`. + */ + fieldsJson() { + return JSON.stringify(this.fields()); + } } class Builder { diff --git a/secretspec-node/package.json b/secretspec-node/package.json index 9aa96cb..0ed56c8 100644 --- a/secretspec-node/package.json +++ b/secretspec-node/package.json @@ -6,11 +6,17 @@ "homepage": "https://secretspec.dev/", "main": "index.js", "types": "index.d.ts", - "files": ["index.js", "index.d.ts"], + "files": [ + "index.js", + "index.d.ts" + ], "scripts": { "test": "node --test" }, "dependencies": { "koffi": "^2.9.0" + }, + "devDependencies": { + "typescript": "^6.0.3" } } diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py index 58dd5a5..c5aeba9 100644 --- a/secretspec-py/secretspec/__init__.py +++ b/secretspec-py/secretspec/__init__.py @@ -145,6 +145,15 @@ def set_as_env(self) -> None: if usable is not None: os.environ[name] = usable + def fields(self) -> dict: + """Flat ``{SECRET_NAME: value}`` map (the file path for ``as_path``). + + This is the input for a quicktype-generated deserializer: feed it to the + generated type's ``from_dict`` to get a typed object. See + ``secretspec schema``. + """ + return {name: secret.get for name, secret in self.secrets.items()} + def abi_version() -> str: """The ABI version reported by the loaded library.""" diff --git a/secretspec-py/tests/test_codegen.py b/secretspec-py/tests/test_codegen.py index e77a695..ec00743 100644 --- a/secretspec-py/tests/test_codegen.py +++ b/secretspec-py/tests/test_codegen.py @@ -1,16 +1,22 @@ -"""End-to-end test of `secretspec codegen --lang python`. +"""End-to-end test of the typed-codegen pipeline: -Generates a typed module from a manifest, imports it, and resolves through the -generated accessors, proving the IR-driven codegen produces working code on top -of the runtime SDK. + secretspec schema -> quicktype -> Type.from_dict(resolved.fields()) + +This proves the schema we emit drives quicktype to a typed class whose +deserializer consumes the runtime SDK's flat fields() map. """ import importlib.util import os import pathlib +import shutil import subprocess import sys +import pytest + +from secretspec import SecretSpec + MANIFEST = """ [project] name = "codegen-test" @@ -20,65 +26,65 @@ DATABASE_URL = { description = "DB", required = true } LOG_LEVEL = { description = "log", required = false, default = "info" } SENTRY_DSN = { description = "sentry", required = false } - -[profiles.production] -DATABASE_URL = { description = "DB", required = true } -TLS_CERT = { description = "cert", required = true, as_path = true } """ +pytestmark = pytest.mark.skipif( + shutil.which("npx") is None, reason="npx (for quicktype) not available" +) -def _generate(tmp_path: pathlib.Path, name: str = "generated"): + +def _generate_types(tmp_path: pathlib.Path, name: str): manifest = tmp_path / "secretspec.toml" manifest.write_text(MANIFEST) - out = tmp_path / f"{name}.py" + + schema = tmp_path / "schema.json" + subprocess.run( + [os.environ["SECRETSPEC_BIN"], "-f", str(manifest), "schema", "-o", str(schema)], + check=True, + ) + + generated = tmp_path / f"{name}.py" subprocess.run( [ - os.environ["SECRETSPEC_BIN"], - "-f", - str(manifest), - "codegen", + "npx", + "--yes", + "quicktype", + "-s", + "schema", + str(schema), "--lang", "python", + "--python-version", + "3.7", "-o", - str(out), + str(generated), ], check=True, ) - spec = importlib.util.spec_from_file_location(name, out) + + spec = importlib.util.spec_from_file_location(name, generated) module = importlib.util.module_from_spec(spec) - sys.modules[name] = module # so the module resolves its own references + sys.modules[name] = module spec.loader.exec_module(module) # also validates the generated syntax return module, manifest -def test_generated_union_resolves_with_typed_attributes(tmp_path): +def test_quicktype_types_consume_runtime_fields(tmp_path): env = tmp_path / ".env" env.write_text("DATABASE_URL=postgres://db\n") - module, manifest = _generate(tmp_path, "gen_union") - - secrets = module.SecretSpec.load( - provider=f"dotenv://{env}", - reason="codegen test", - path=str(manifest), - ) - - # Typed snake_case attributes mirror the derive crate, idiomatic for Python. - assert secrets.database_url == "postgres://db" - assert secrets.log_level == "info" # from default - assert secrets.sentry_dsn is None # optional, missing - - -def test_generated_profile_class_pins_profile_and_paths(tmp_path): - env = tmp_path / ".env" - env.write_text("DATABASE_URL=postgres://prod\nTLS_CERT=----cert----\n") - module, manifest = _generate(tmp_path, "gen_profile") + module, manifest = _generate_types(tmp_path, "gen_python") - prod = module.ProductionSecrets.load( - provider=f"dotenv://{env}", - reason="codegen test", - path=str(manifest), + resolved = ( + SecretSpec.builder() + .with_path(str(manifest)) + .with_provider(f"dotenv://{env}") + .with_reason("codegen test") + .load() ) - assert prod.database_url == "postgres://prod" - # as_path field is exposed as a readable file path. - assert pathlib.Path(prod.tls_cert).read_text() == "----cert----" + # The quicktype-generated, typed class is constructed from the runtime SDK's + # flat fields() map. Idiomatic snake_case attributes, typed. + typed = module.SecretSpec.from_dict(resolved.fields()) + assert typed.database_url == "postgres://db" + assert typed.log_level == "info" # from default + assert typed.sentry_dsn is None # optional, missing diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index b891ee0..fac4944 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -49,6 +49,13 @@ def get def set_as_env! secrets.each { |name, secret| ENV[name] = secret.get } end + + # Flat { "SECRET_NAME" => value } hash (the file path for as_path). Feed this + # to a quicktype-generated deserializer (e.g. from_dynamic!). See + # `secretspec schema`. + def fields + secrets.transform_values(&:get) + end end # The narrow C ABI, loaded lazily via Fiddle. diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index f999069..43cf7b6 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -1,6 +1,6 @@ use crate::provider::{Provider, providers}; use crate::{Config, GlobalConfig, GlobalDefaults, Profile, Project, Secrets}; -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; use miette::{IntoDiagnostic, Result, WrapErr, miette}; use std::collections::HashMap; use std::fs; @@ -103,15 +103,15 @@ enum Commands { #[arg(long)] explain: bool, }, - /// Generate typed accessors for another language from the manifest. + /// Emit a JSON Schema for the manifest's typed shapes. /// - /// Emits code that wraps that language's runtime SDK, driven by the shared - /// codegen IR so every language stays consistent. Value-free: reads only the - /// manifest, never a provider. - Codegen { - /// Target language to generate - #[arg(long, value_enum)] - lang: CodegenLang, + /// Feed this to [quicktype](https://quicktype.io) to generate idiomatic + /// typed accessors (plus deserializers) for any language, then hand the + /// deserializer the flat map from each SDK's `fields()` helper. Value-free: + /// reads only the manifest, never a provider. + /// + /// Example: `secretspec schema | quicktype -s schema --lang typescript` + Schema { /// Write to this file instead of stdout #[arg(short, long)] output: Option, @@ -159,13 +159,6 @@ enum Commands { }, } -/// Target languages for `secretspec codegen`. -#[derive(Clone, Debug, ValueEnum)] -enum CodegenLang { - /// Python typed accessors over the `secretspec` Python SDK - Python, -} - /// Configuration-related subcommands. /// /// These actions handle the user's global configuration settings, @@ -713,17 +706,15 @@ pub fn main() -> Result<()> { Ok(()) } // Generate typed accessors for another language (value-free) - Commands::Codegen { lang, output } => { + Commands::Schema { output } => { let app = load_secrets(&cli.file, &cli.reason)?; let ir = crate::codegen::build_ir(app.config()); - let code = match lang { - CodegenLang::Python => crate::codegen::python::emit(&ir), - }; + let schema = crate::codegen::schema::emit(&ir); match output { - Some(path) => fs::write(&path, code) + Some(path) => fs::write(&path, schema) .into_diagnostic() .wrap_err_with(|| format!("Failed to write {}", path.display()))?, - None => print!("{}", code), + None => print!("{}", schema), } Ok(()) } diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs index 79f4625..b0abcd4 100644 --- a/secretspec/src/codegen.rs +++ b/secretspec/src/codegen.rs @@ -173,44 +173,46 @@ pub fn build_ir(config: &Config) -> CodegenIr { } } -/// Language emitters: thin templates that turn the [`CodegenIr`] into typed -/// accessors. Each mirrors the derive crate's shape (a union type plus -/// per-profile types, a builder-style `load`) using the target language's -/// idioms. -pub mod python { +/// JSON Schema emitter. +/// +/// Rather than hand-write typed accessors per language, we emit a JSON Schema +/// describing the manifest's types and let [quicktype](https://quicktype.io) +/// generate the idiomatic types and deserializers for any target language. We +/// then maintain only the small generic `fields()` helper in each runtime SDK, +/// which hands quicktype's deserializer a flat `{SECRET_NAME: value}` map. +/// +/// The schema defines one type per shape the derive crate exposes: `SecretSpec` +/// (the union, safe for any profile) and one `Secrets` per profile. A +/// `Manifest` wrapper references them all so quicktype emits every type. +pub mod schema { use super::{CodegenIr, IrField}; - use std::fmt::Write; + use serde_json::{Map, Value, json}; - /// The Python attribute name (snake_case) for an `UPPER_SNAKE` secret name. - fn attr(name: &str) -> String { - name.to_lowercase() - } - - /// The Python type annotation for a field. - fn type_ann(field: &IrField) -> &'static str { - match (field.optional, field.as_path) { - (false, false) => "str", - (true, false) => "Optional[str]", - (false, true) => "Path", - (true, true) => "Optional[Path]", + fn property_type(field: &IrField) -> Value { + // Every secret is a string; optional secrets are nullable. `as_path` + // secrets are also strings (the file path), so they need no special type. + if field.optional { + json!({ "type": ["string", "null"] }) + } else { + json!({ "type": "string" }) } } - /// The constructor keyword-argument line that pulls one field out of the - /// runtime `Resolved`. - fn assignment(field: &IrField) -> String { - let env = &field.name; - let name = attr(env); - match (field.optional, field.as_path) { - (false, false) => format!(" {name}=r.secrets[\"{env}\"].get,"), - (false, true) => format!(" {name}=Path(r.secrets[\"{env}\"].get),"), - (true, false) => format!( - " {name}=(r.secrets[\"{env}\"].get if \"{env}\" in r.secrets else None)," - ), - (true, true) => format!( - " {name}=(Path(r.secrets[\"{env}\"].get) if \"{env}\" in r.secrets else None)," - ), + fn object_schema(fields: &[IrField]) -> Value { + let mut properties = Map::new(); + let mut required = Vec::new(); + for field in fields { + properties.insert(field.name.clone(), property_type(field)); + if !field.optional { + required.push(Value::String(field.name.clone())); + } } + json!({ + "type": "object", + "additionalProperties": false, + "properties": Value::Object(properties), + "required": required, + }) } fn capitalize(s: &str) -> String { @@ -221,96 +223,45 @@ pub mod python { } } - /// Emit one frozen dataclass plus its `load` classmethod. When - /// `pinned_profile` is set the class always loads that profile (no `profile` - /// argument); otherwise `load` takes a `profile` argument. - fn emit_class( - out: &mut String, - class: &str, - doc: &str, - fields: &[IrField], - pinned_profile: Option<&str>, - ) { - writeln!(out, "@dataclass(frozen=True)").unwrap(); - writeln!(out, "class {class}:").unwrap(); - writeln!(out, " \"\"\"{doc}\"\"\"").unwrap(); - writeln!(out).unwrap(); - // No field has a default, so any order is valid for the dataclass. - for field in fields { - writeln!(out, " {}: {}", attr(&field.name), type_ann(field)).unwrap(); - } - writeln!(out).unwrap(); - - writeln!(out, " @classmethod").unwrap(); - if pinned_profile.is_some() { - writeln!( - out, - " def load(cls, *, provider: Optional[str] = None, reason: Optional[str] = None, path: Optional[str] = None) -> \"{class}\":" - ) - .unwrap(); - } else { - writeln!( - out, - " def load(cls, *, provider: Optional[str] = None, profile: Optional[str] = None, reason: Optional[str] = None, path: Optional[str] = None) -> \"{class}\":" - ) - .unwrap(); - } - writeln!(out, " r = (").unwrap(); - writeln!(out, " _secretspec.SecretSpec.builder()").unwrap(); - writeln!(out, " .with_path(path)").unwrap(); - writeln!(out, " .with_provider(provider)").unwrap(); - match pinned_profile { - Some(profile) => writeln!(out, " .with_profile(\"{profile}\")").unwrap(), - None => writeln!(out, " .with_profile(profile)").unwrap(), - } - writeln!(out, " .with_reason(reason)").unwrap(); - writeln!(out, " .load()").unwrap(); - writeln!(out, " )").unwrap(); - if fields.is_empty() { - writeln!(out, " return cls()").unwrap(); - } else { - writeln!(out, " return cls(").unwrap(); - for field in fields { - writeln!(out, "{}", assignment(field)).unwrap(); - } - writeln!(out, " )").unwrap(); - } - writeln!(out).unwrap(); - writeln!(out).unwrap(); - } - - /// Emit a Python module of typed accessors over the runtime `secretspec` - /// package: a `SecretSpec` union dataclass plus one `Secrets` - /// dataclass per profile. + /// Emit the JSON Schema (draft-06, the dialect quicktype consumes) for the + /// manifest's types. pub fn emit(ir: &CodegenIr) -> String { - let mut out = String::new(); - writeln!( - out, - "# Code generated by `secretspec codegen --lang python`. Do not edit." - ) - .unwrap(); - writeln!(out, "# Project: {}", ir.project).unwrap(); - writeln!(out, "from dataclasses import dataclass").unwrap(); - writeln!(out, "from pathlib import Path").unwrap(); - writeln!(out, "from typing import Optional").unwrap(); - writeln!(out).unwrap(); - writeln!(out, "import secretspec as _secretspec").unwrap(); - writeln!(out).unwrap(); - writeln!(out).unwrap(); - - emit_class( - &mut out, - "SecretSpec", - "Union of all profiles; safe to use without knowing the active profile.", - &ir.union, - None, + let mut definitions = Map::new(); + let mut wrapper_props = Map::new(); + + definitions.insert("SecretSpec".to_string(), object_schema(&ir.union)); + wrapper_props.insert( + "SecretSpec".to_string(), + json!({ "$ref": "#/definitions/SecretSpec" }), ); + for profile in &ir.profile_fields { - let class = format!("{}Secrets", capitalize(&profile.name)); - let doc = format!("Secrets for the '{}' profile.", profile.name); - emit_class(&mut out, &class, &doc, &profile.fields, Some(&profile.name)); + let name = format!("{}Secrets", capitalize(&profile.name)); + definitions.insert(name.clone(), object_schema(&profile.fields)); + wrapper_props.insert( + name.clone(), + json!({ "$ref": format!("#/definitions/{name}") }), + ); } - out + + // A wrapper that references every type so quicktype emits all of them. + definitions.insert( + "Manifest".to_string(), + json!({ + "type": "object", + "additionalProperties": false, + "properties": Value::Object(wrapper_props), + }), + ); + + let schema = json!({ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/Manifest", + "title": ir.project, + "definitions": Value::Object(definitions), + }); + + format!("{}\n", serde_json::to_string_pretty(&schema).unwrap()) } } @@ -453,7 +404,7 @@ mod tests { } #[test] - fn python_emitter_types_fields_and_assigns_from_runtime() { + fn schema_emits_types_and_nullability_for_quicktype() { let ir = build_ir(&config_with(vec![ ( "development", @@ -464,32 +415,36 @@ mod tests { ), ( "production", - vec![ - ("DATABASE_URL", secret(Some(true), None, None)), - ("TLS_CERT", secret(Some(true), Some(true), None)), - ], + vec![("DATABASE_URL", secret(Some(true), None, None))], ), ])); - let code = python::emit(&ir); - - // Union types: required -> str, optional/missing -> Optional[...], path. - assert!(code.contains("class SecretSpec:")); - assert!(code.contains(" database_url: str")); - assert!(code.contains(" api_key: Optional[str]")); - assert!(code.contains(" tls_cert: Optional[Path]")); - - // Required field is pulled directly; optional is guarded; path is wrapped. - assert!(code.contains("database_url=r.secrets[\"DATABASE_URL\"].get,")); - assert!(code.contains( - "api_key=(r.secrets[\"API_KEY\"].get if \"API_KEY\" in r.secrets else None)," - )); - - // Per-profile classes mirror derive's profile-specific shape. - assert!(code.contains("class DevelopmentSecrets:")); - assert!(code.contains("class ProductionSecrets:")); - // production pins its profile and types TLS_CERT as a required Path. - assert!(code.contains(".with_profile(\"production\")")); - assert!(code.contains(" tls_cert: Path")); + + let rendered = schema::emit(&ir); + let value: serde_json::Value = serde_json::from_str(&rendered).unwrap(); + let defs = &value["definitions"]; + + // The union type and one type per profile, plus the Manifest wrapper. + assert!(defs["SecretSpec"].is_object()); + assert!(defs["DevelopmentSecrets"].is_object()); + assert!(defs["ProductionSecrets"].is_object()); + assert_eq!(value["$ref"], "#/definitions/Manifest"); + + // Required vs nullable: DATABASE_URL required everywhere; API_KEY optional + // in development, so optional in the union and nullable in the schema. + let union = &defs["SecretSpec"]; + assert_eq!(union["properties"]["DATABASE_URL"]["type"], "string"); + assert_eq!( + union["properties"]["API_KEY"]["type"], + serde_json::json!(["string", "null"]) + ); + let required: Vec<&str> = union["required"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!(required.contains(&"DATABASE_URL")); + assert!(!required.contains(&"API_KEY")); } #[test] From 8e1b31d50b172dd1497b6817e9aa54c6edaba1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 21:43:42 -0400 Subject: [PATCH 14/56] ci: run all language SDK suites in CI Adds a SDKs workflow that builds the cdylib + CLI once and runs every SDK's full test suite (unit + cross-language conformance + the schema/quicktype codegen pipeline) via scripts/ci-sdks.sh, so the Python/Go/Ruby/Node bindings cannot silently rot. The prior CI (`devenv test` -> cargo test --all) covered only the Rust crates, including secretspec-ffi, but none of the language SDKs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdks.yml | 24 ++++++++++++++++++++++++ scripts/ci-sdks.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .github/workflows/sdks.yml create mode 100755 scripts/ci-sdks.sh diff --git a/.github/workflows/sdks.yml b/.github/workflows/sdks.yml new file mode 100644 index 0000000..119745e --- /dev/null +++ b/.github/workflows/sdks.yml @@ -0,0 +1,24 @@ +name: "SDKs" + +# Runs every language SDK's full test suite (unit + cross-language conformance + +# the schema/quicktype codegen pipeline) against a freshly built cdylib, so the +# Python/Go/Ruby/Node bindings cannot silently rot. + +on: + pull_request: + push: + branches: [main] + +jobs: + sdks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: cachix/install-nix-action@v31 + - uses: cachix/cachix-action@v16 + with: + name: devenv + - name: Install devenv.sh + run: nix profile install nixpkgs#devenv + - name: Build cdylib and run all SDK suites + run: devenv shell -- bash scripts/ci-sdks.sh diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh new file mode 100755 index 0000000..e371fcd --- /dev/null +++ b/scripts/ci-sdks.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Run every language SDK's full test suite (unit + conformance + the +# schema/quicktype pipeline) against one freshly built cdylib. Run inside the +# project devenv shell: +# +# devenv shell -- bash scripts/ci-sdks.sh +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +echo "==> Building cdylib + CLI" +cargo build -p secretspec-ffi -p secretspec + +target_dir="$(cargo metadata --no-deps --format-version 1 \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +case "$(uname -s)" in + Darwin) lib_name="libsecretspec_ffi.dylib" ;; + *) lib_name="libsecretspec_ffi.so" ;; +esac +export SECRETSPEC_FFI_LIB="$target_dir/debug/$lib_name" +export SECRETSPEC_BIN="$target_dir/debug/secretspec" +echo "==> SECRETSPEC_FFI_LIB=$SECRETSPEC_FFI_LIB" + +echo "==> Python" +( cd secretspec-py && python -m pytest -q ) + +echo "==> Go" +( cd secretspec-go && go test ./... ) + +echo "==> Ruby" +( cd secretspec-rb && ruby test/test_resolve.rb ) + +echo "==> Node" +( cd secretspec-node && { [ -d node_modules ] || npm install --no-audit --no-fund; } && node --test ) + +echo "==> All SDK suites passed" From 4cdbd65c56b5a797807c20ff844be762a06d530e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 8 Jun 2026 21:46:07 -0400 Subject: [PATCH 15/56] docs: add Python/Go/Ruby/Node.js SDK pages and README section The polyglot SDKs were undocumented (the docs site had only a Rust SDK page and the README never mentioned them). Adds a docs page per SDK (quick start, error model, the schema/quicktype typed-access pattern, library discovery), wires them into the sidebar, and adds a "Language SDKs" section to the README. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/astro.config.mjs | 8 ++++- docs/src/content/docs/sdk/go.md | 52 ++++++++++++++++++++++++++++ docs/src/content/docs/sdk/nodejs.md | 50 +++++++++++++++++++++++++++ docs/src/content/docs/sdk/python.md | 53 +++++++++++++++++++++++++++++ docs/src/content/docs/sdk/ruby.md | 48 ++++++++++++++++++++++++++ secretspec/README.md | 22 ++++++++++++ 6 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 docs/src/content/docs/sdk/go.md create mode 100644 docs/src/content/docs/sdk/nodejs.md create mode 100644 docs/src/content/docs/sdk/python.md create mode 100644 docs/src/content/docs/sdk/ruby.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 8a29293..983dcbb 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -189,7 +189,13 @@ Secrets can be stored in: keyring (default), dotenv files, environment variables }, { label: "SDK", - items: [{ label: "Rust SDK", slug: "sdk/rust" }], + items: [ + { label: "Rust SDK", slug: "sdk/rust" }, + { label: "Python SDK", slug: "sdk/python" }, + { label: "Go SDK", slug: "sdk/go" }, + { label: "Ruby SDK", slug: "sdk/ruby" }, + { label: "Node.js SDK", slug: "sdk/nodejs" }, + ], }, { label: "Reference", diff --git a/docs/src/content/docs/sdk/go.md b/docs/src/content/docs/sdk/go.md new file mode 100644 index 0000000..dbb0e6a --- /dev/null +++ b/docs/src/content/docs/sdk/go.md @@ -0,0 +1,52 @@ +--- +title: Go SDK +description: Resolve SecretSpec secrets from Go +--- + +The Go SDK (`secretspec-go`) is a thin client over the `secretspec-ffi` C ABI, +loaded via [purego](https://github.com/ebitengine/purego) (dlopen, no cgo). +Resolution happens in the Rust core, so the SDK inherits every provider with no +Go-side logic. + +## Quick start + +```go +import secretspec "github.com/cachix/secretspec/secretspec-go" + +resolved, err := secretspec.New(). + WithProvider("keyring://"). + WithProfile("production"). + WithReason("boot web app"). + Load() +if err != nil { + log.Fatal(err) +} + +fmt.Println(resolved.Provider, resolved.Profile) +db := resolved.Secrets["DATABASE_URL"] +fmt.Println(db.Get()) // the value, or the file path for as_path secrets +resolved.SetAsEnv() // export everything into the process environment +``` + +A missing required secret returns `*MissingRequiredError`; any other failure +returns `*Error` (with a stable `.Kind`). + +## Typed access (codegen) + +Generate typed structs with `secretspec schema` plus +[quicktype](https://quicktype.io), then unmarshal `resolved.FieldsJSON()`: + +```bash +secretspec schema | quicktype -s schema --lang go -o secrets_gen.go +``` + +```go +data, _ := resolved.FieldsJSON() +typed, _ := UnmarshalSecretSpec(data) // typed, generated +fmt.Println(typed.DatabaseURL) +``` + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/docs/src/content/docs/sdk/nodejs.md b/docs/src/content/docs/sdk/nodejs.md new file mode 100644 index 0000000..94261c0 --- /dev/null +++ b/docs/src/content/docs/sdk/nodejs.md @@ -0,0 +1,50 @@ +--- +title: Node.js SDK +description: Resolve SecretSpec secrets from Node.js and TypeScript +--- + +The Node.js / TypeScript SDK (`secretspec`) is a thin client over the +`secretspec-ffi` C ABI, loaded via [koffi](https://koffi.dev/) (dlopen). +Resolution happens in the Rust core, so the SDK inherits every provider with no +JS-side logic. TypeScript declarations ship in `index.d.ts`. + +## Quick start + +```js +const { SecretSpec } = require('secretspec'); + +const resolved = SecretSpec.builder() + .withProvider('keyring://') + .withProfile('production') + .withReason('boot web app') + .load(); + +console.log(resolved.provider, resolved.profile); +const db = resolved.secrets.DATABASE_URL; +console.log(db.get()); // the value, or the file path for as_path secrets +resolved.setAsEnv(); // export everything into process.env +``` + +A missing required secret throws `MissingRequiredError`; any other failure +throws `SecretSpecError` (with a stable `.kind`). + +## Typed access (codegen) + +Generate typed interfaces with `secretspec schema` plus +[quicktype](https://quicktype.io), then convert `resolved.fieldsJson()`: + +```bash +secretspec schema | quicktype -s schema --lang typescript -o secrets_gen.ts +``` + +```ts +import { Convert } from './secrets_gen'; // typed, generated + +const typed = Convert.toSecretSpec(resolved.fieldsJson()); +console.log(typed.DATABASE_URL); +``` + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/docs/src/content/docs/sdk/python.md b/docs/src/content/docs/sdk/python.md new file mode 100644 index 0000000..959ccf9 --- /dev/null +++ b/docs/src/content/docs/sdk/python.md @@ -0,0 +1,53 @@ +--- +title: Python SDK +description: Resolve SecretSpec secrets from Python +--- + +The Python SDK (`secretspec`) is a thin client over the `secretspec-ffi` C ABI, +loaded via cffi. Resolution (providers, chains, profiles, generation, `as_path`) +happens in the Rust core, so the SDK inherits every provider with no Python-side +logic. + +## Quick start + +```python +from secretspec import SecretSpec + +resolved = ( + SecretSpec.builder() + .with_provider("keyring://") + .with_profile("production") + .with_reason("boot web app") + .load() +) + +print(resolved.provider, resolved.profile) +db = resolved.secrets["DATABASE_URL"] +print(db.get) # the value, or the file path for as_path secrets +resolved.set_as_env() # export everything into os.environ +``` + +A missing required secret raises `MissingRequiredError`; any other failure +raises `SecretSpecError` (with a stable `.kind`). + +## Typed access (codegen) + +Generate typed classes with `secretspec schema` plus +[quicktype](https://quicktype.io), then build them from `resolved.fields()`: + +```bash +secretspec schema | quicktype -s schema --lang python -o secrets_gen.py +``` + +```python +from secrets_gen import SecretSpec as Secrets # typed + +typed = Secrets.from_dict(resolved.fields()) +print(typed.database_url) # typed str +``` + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, a +copy bundled in the installed wheel, or a Cargo `target` directory found by +searching up from the working directory. diff --git a/docs/src/content/docs/sdk/ruby.md b/docs/src/content/docs/sdk/ruby.md new file mode 100644 index 0000000..2033ca5 --- /dev/null +++ b/docs/src/content/docs/sdk/ruby.md @@ -0,0 +1,48 @@ +--- +title: Ruby SDK +description: Resolve SecretSpec secrets from Ruby +--- + +The Ruby SDK (`secretspec`) is a thin client over the `secretspec-ffi` C ABI, +loaded via the stdlib [Fiddle](https://docs.ruby-lang.org/en/master/Fiddle.html) +(dlopen, no native gem). Resolution happens in the Rust core, so the SDK +inherits every provider with no Ruby-side logic. + +## Quick start + +```ruby +require "secretspec" + +resolved = Secretspec::SecretSpec.builder + .with_provider("keyring://") + .with_profile("production") + .with_reason("boot web app") + .load + +puts resolved.provider, resolved.profile +db = resolved.secrets["DATABASE_URL"] +puts db.get # the value, or the file path for as_path secrets +resolved.set_as_env! # export everything into ENV +``` + +A missing required secret raises `Secretspec::MissingRequiredError`; any other +failure raises `Secretspec::Error` (with a stable `#kind`). + +## Typed access (codegen) + +Generate typed classes with `secretspec schema` plus +[quicktype](https://quicktype.io), then build them from `resolved.fields`: + +```bash +secretspec schema | quicktype -s schema --lang ruby -o secrets_gen.rb +``` + +```ruby +typed = SecretSpec.from_dynamic!(resolved.fields) # typed, generated +puts typed.database_url +``` + +## Library discovery + +The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, +or a Cargo `target` directory found by searching up from the working directory. diff --git a/secretspec/README.md b/secretspec/README.md index a74d3eb..a93f535 100644 --- a/secretspec/README.md +++ b/secretspec/README.md @@ -179,6 +179,28 @@ fn main() -> Result<(), Box> { See the [Rust SDK documentation](https://secretspec.dev/sdk/rust) for advanced usage including profile-specific types. +## Language SDKs + +Beyond Rust, SecretSpec ships SDKs for other languages. Each is a thin client +over the same native core (the `secretspec-ffi` C ABI), so every provider, chain, +profile, and generator works identically with no per-language resolution logic: + +- [Python](https://secretspec.dev/sdk/python) (via cffi) +- [Go](https://secretspec.dev/sdk/go) (via purego, no cgo) +- [Ruby](https://secretspec.dev/sdk/ruby) (via stdlib Fiddle) +- [Node.js / TypeScript](https://secretspec.dev/sdk/nodejs) (via koffi) + +```python +from secretspec import SecretSpec + +resolved = SecretSpec.builder().with_provider("keyring://").with_reason("boot").load() +print(resolved.secrets["DATABASE_URL"].get) +``` + +For typed access, `secretspec schema` emits a JSON Schema you feed to +[quicktype](https://quicktype.io) to generate typed classes for any language, +built from the SDK's `fields()` map. + ## CLI Reference Common commands: From e3788c42377e4671e35d318b7d28396486a51a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 03:46:05 -0400 Subject: [PATCH 16/56] build(python): package a platform wheel bundling the cdylib Phase 6 / pillar A, Python: make the Python SDK installable without a separate native build by bundling the secretspec-ffi cdylib into the wheel. The SDK already prefers a bundled secretspec/_lib/ over SECRETSPEC_FFI_LIB / a Cargo target dir, so this wires up the build. - scripts/stage-cdylib.sh builds the cdylib (release) and stages it into secretspec/_lib/ (gitignored) per OS, including the Windows .dll case. - setup.py forces a platform (non-pure) wheel tagged py3-none- so pip installs the right native library per OS/arch; metadata stays in pyproject. - Verified locally: the produced wheel is py3-none-linux_x86_64, contains secretspec/_lib/libsecretspec_ffi.so, and a clean install (no env var, outside the repo) loads the bundled lib and resolves a secret. - Drafted python-wheels.yml: a per-platform matrix (linux x86_64/aarch64, macOS x86_64/aarch64, windows) that stages the cdylib, builds the wheel, and smoke tests it. NOTE: Linux wheels are tagged linux_*, not manylinux_*; PyPI publishing still needs an auditwheel-repair step to vendor the cdylib's system deps (libdbus from keyring) and make glibc portable. That is the remaining follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/python-wheels.yml | 71 +++++++++++++++++++++++++++ secretspec-py/scripts/stage-cdylib.sh | 22 +++++++++ secretspec-py/setup.py | 39 +++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 .github/workflows/python-wheels.yml create mode 100755 secretspec-py/scripts/stage-cdylib.sh create mode 100644 secretspec-py/setup.py diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml new file mode 100644 index 0000000..f6758aa --- /dev/null +++ b/.github/workflows/python-wheels.yml @@ -0,0 +1,71 @@ +name: "Python wheels" + +# Builds platform wheels for the Python SDK that bundle the secretspec-ffi +# cdylib (into secretspec/_lib/), so `pip install secretspec` works without a +# separate native build. +# +# NOTE: the Linux wheels produced here are tagged `linux_*`, not `manylinux_*`. +# Publishing to PyPI requires repairing them against a manylinux toolchain +# (`auditwheel repair`) so the bundled cdylib's system dependencies (notably +# libdbus, pulled in by the keyring provider) are vendored and the glibc +# requirement is portable. That repair step is the remaining follow-up before a +# PyPI release; the build + bundling mechanism below is what is validated. + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-py/**" + - "secretspec-ffi/**" + - "secretspec/**" + - ".github/workflows/python-wheels.yml" + +jobs: + wheels: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { target: linux-x86_64, runner: ubuntu-latest } + - { target: linux-aarch64, runner: ubuntu-24.04-arm } + - { target: macos-x86_64, runner: macos-13 } + - { target: macos-aarch64, runner: macos-14 } + - { target: windows-x86_64, runner: windows-latest } + + steps: + - uses: actions/checkout@v5 + + - name: Install Linux system dependencies (dbus for keyring) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Stage the cdylib and build the wheel + shell: bash + run: | + python -m pip install --upgrade build + bash secretspec-py/scripts/stage-cdylib.sh + ( cd secretspec-py && python -m build --wheel --outdir "$GITHUB_WORKSPACE/wheelhouse" ) + + - name: Smoke test the wheel + shell: bash + run: | + python -m pip install --find-links wheelhouse secretspec + # Run outside the repo with no SECRETSPEC_FFI_LIB so only the bundled + # library can satisfy the import. + cd "$RUNNER_TEMP" + python -c "import secretspec; print('abi', secretspec.abi_version())" + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.target }} + path: wheelhouse/*.whl diff --git a/secretspec-py/scripts/stage-cdylib.sh b/secretspec-py/scripts/stage-cdylib.sh new file mode 100755 index 0000000..0e9a4aa --- /dev/null +++ b/secretspec-py/scripts/stage-cdylib.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Build the secretspec-ffi cdylib (release) and stage it into +# secretspec/_lib/ so a wheel build bundles it. Run before building the wheel. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" + +target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +case "$(uname -s)" in + Darwin) lib_name="libsecretspec_ffi.dylib" ;; + MINGW* | MSYS* | CYGWIN*) lib_name="secretspec_ffi.dll" ;; + *) lib_name="libsecretspec_ffi.so" ;; +esac + +mkdir -p "$pkg_dir/secretspec/_lib" +cp "$target_dir/release/$lib_name" "$pkg_dir/secretspec/_lib/$lib_name" +echo "staged $lib_name into secretspec/_lib/" diff --git a/secretspec-py/setup.py b/secretspec-py/setup.py new file mode 100644 index 0000000..18d0923 --- /dev/null +++ b/secretspec-py/setup.py @@ -0,0 +1,39 @@ +"""Build a platform-specific wheel that bundles the secretspec-ffi cdylib. + +Project metadata lives in ``pyproject.toml``; this file only (a) forces a +platform (non-pure) wheel so pip installs the right native library per OS/arch, +and (b) tags it ``py3-none-`` because the package is pure Python apart +from the bundled library, which works on any Python 3. + +The native library must be staged into ``secretspec/_lib/`` before building it +(see ``scripts/stage-cdylib.sh``); it is declared as package data in +``pyproject.toml``. +""" + +from setuptools import setup +from setuptools.dist import Distribution + +try: # setuptools >= 70 vendors bdist_wheel + from setuptools.command.bdist_wheel import bdist_wheel +except ImportError: # pragma: no cover - older setuptools + from wheel.bdist_wheel import bdist_wheel + + +class BinaryDistribution(Distribution): + """Marks the distribution as containing a native library, so the wheel is + platform-specific rather than ``any``.""" + + def has_ext_modules(self) -> bool: + return True + + +class PlatformWheel(bdist_wheel): + def get_tag(self): + _, _, platform = super().get_tag() + return "py3", "none", platform + + +setup( + distclass=BinaryDistribution, + cmdclass={"bdist_wheel": PlatformWheel}, +) From 67b0471699850bd1c33c40ca548a2118c38b93bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 03:48:49 -0400 Subject: [PATCH 17/56] build(ruby): package a platform gem bundling the cdylib Phase 6 / pillar A, Ruby: make the Ruby SDK installable without a separate native build by bundling the secretspec-ffi cdylib into a platform gem. - The SDK now prefers a vendored vendor/ (staged into a platform gem) over SECRETSPEC_FFI_LIB / a Cargo target dir. - scripts/stage-cdylib.sh builds the cdylib (release) and stages it into vendor/ (gitignored) per OS, incl. the Windows .dll case. - The gemspec includes vendor/* and sets Gem::Platform::CURRENT when the lib is staged, so `gem build` produces a platform gem (else a pure-Ruby gem). - Verified locally: built secretspec-0.12.0-x86_64-linux.gem, and a clean install (no env var, outside the repo) loaded the bundled lib and resolved a secret. Existing Ruby suite still green. - Drafted ruby-gems.yml: per-platform matrix that stages, builds, and smoke tests the gem. NOTE: like the wheels, a portable Linux gem needs a baseline build (e.g. rake-compiler-dock) to vendor the cdylib's system deps (libdbus) and glibc; that is the remaining follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ruby-gems.yml | 71 +++++++++++++++++++++++++++ secretspec-rb/.gitignore | 2 + secretspec-rb/lib/secretspec.rb | 6 +++ secretspec-rb/scripts/stage-cdylib.sh | 22 +++++++++ secretspec-rb/secretspec.gemspec | 9 +++- 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ruby-gems.yml create mode 100644 secretspec-rb/.gitignore create mode 100755 secretspec-rb/scripts/stage-cdylib.sh diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml new file mode 100644 index 0000000..2922921 --- /dev/null +++ b/.github/workflows/ruby-gems.yml @@ -0,0 +1,71 @@ +name: "Ruby gems" + +# Builds platform-specific gems for the Ruby SDK that bundle the secretspec-ffi +# cdylib (into vendor/), so `gem install secretspec` works without a separate +# native build. +# +# NOTE: as with the Python wheels, the Linux gem here links the runner's glibc +# and system libraries (notably libdbus, via the keyring provider). A portable +# release build should use a baseline toolchain (e.g. rake-compiler-dock) and +# vendor those deps. That is the remaining follow-up; the build + bundling +# mechanism below is what is validated. + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-rb/**" + - "secretspec-ffi/**" + - "secretspec/**" + - ".github/workflows/ruby-gems.yml" + +jobs: + gems: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { target: x86_64-linux, runner: ubuntu-latest } + - { target: aarch64-linux, runner: ubuntu-24.04-arm } + - { target: x86_64-darwin, runner: macos-13 } + - { target: arm64-darwin, runner: macos-14 } + - { target: x64-mingw, runner: windows-latest } + + steps: + - uses: actions/checkout@v5 + + - name: Install Linux system dependencies (dbus for keyring) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + + - uses: dtolnay/rust-toolchain@stable + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + + - name: Stage the cdylib and build the platform gem + shell: bash + run: | + bash secretspec-rb/scripts/stage-cdylib.sh + ( cd secretspec-rb && gem build secretspec.gemspec ) + + - name: Smoke test the gem + shell: bash + run: | + cd secretspec-rb + gem install --no-document --install-dir "$RUNNER_TEMP/gemhome" secretspec-*.gem + # Run outside the repo with no SECRETSPEC_FFI_LIB so only the bundled + # library can satisfy the require. + cd "$RUNNER_TEMP" + GEM_HOME="$RUNNER_TEMP/gemhome" GEM_PATH="$RUNNER_TEMP/gemhome" \ + ruby -e 'require "secretspec"; puts "abi " + Secretspec.abi_version' + + - uses: actions/upload-artifact@v4 + with: + name: gem-${{ matrix.target }} + path: secretspec-rb/*.gem diff --git a/secretspec-rb/.gitignore b/secretspec-rb/.gitignore new file mode 100644 index 0000000..f78b421 --- /dev/null +++ b/secretspec-rb/.gitignore @@ -0,0 +1,2 @@ +vendor/ +*.gem diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index fac4944..ee7efb1 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -108,6 +108,12 @@ def find_library env = ENV["SECRETSPEC_FFI_LIB"] return env if env && !env.empty? + # A copy bundled in a platform gem (staged into vendor/ at build time). + lib_names.each do |name| + bundled = File.expand_path("../../vendor/#{name}", __FILE__) + return bundled if File.exist?(bundled) + end + dir = Dir.pwd loop do %w[release debug].each do |profile| diff --git a/secretspec-rb/scripts/stage-cdylib.sh b/secretspec-rb/scripts/stage-cdylib.sh new file mode 100755 index 0000000..5beead0 --- /dev/null +++ b/secretspec-rb/scripts/stage-cdylib.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Build the secretspec-ffi cdylib (release) and stage it into vendor/ so a +# platform gem build bundles it. Run before `gem build`. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" + +target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +case "$(uname -s)" in + Darwin) lib_name="libsecretspec_ffi.dylib" ;; + MINGW* | MSYS* | CYGWIN*) lib_name="secretspec_ffi.dll" ;; + *) lib_name="libsecretspec_ffi.so" ;; +esac + +mkdir -p "$pkg_dir/vendor" +cp "$target_dir/release/$lib_name" "$pkg_dir/vendor/$lib_name" +echo "staged $lib_name into vendor/" diff --git a/secretspec-rb/secretspec.gemspec b/secretspec-rb/secretspec.gemspec index 0a62c38..3e43f68 100644 --- a/secretspec-rb/secretspec.gemspec +++ b/secretspec-rb/secretspec.gemspec @@ -9,7 +9,14 @@ Gem::Specification.new do |spec| spec.authors = ["Cachix"] spec.license = "Apache-2.0" spec.homepage = "https://secretspec.dev/" - spec.files = Dir["lib/**/*.rb"] + ["README.md"] + spec.files = Dir["lib/**/*.rb"] + ["README.md"] + Dir["vendor/*"] spec.require_paths = ["lib"] spec.required_ruby_version = ">= 3.0" + + # When the cdylib has been staged into vendor/ (see scripts/stage-cdylib.sh), + # build a platform-specific gem that bundles it, so `gem install` needs no + # native build. Without it, a pure-Ruby gem is built (the SDK then falls back + # to SECRETSPEC_FFI_LIB or a Cargo target directory). + staged = Dir["vendor/libsecretspec_ffi.*"] + Dir["vendor/secretspec_ffi.dll"] + spec.platform = Gem::Platform::CURRENT unless staged.empty? end From cf9d34e8db557d620c94a3bb01ee3c019de72a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 03:57:53 -0400 Subject: [PATCH 18/56] build(go): embed the cdylib via go:embed (behind embed_lib tag) Phase 6 / pillar A, Go: let `go get` work with no native build by embedding the secretspec-ffi cdylib per platform (go:embed) and extracting it to a temp file at first use for purego to dlopen. - Per-platform embedded__.go files (linux/darwin/windows x amd64/arm64) embed lib/secretspec_ffi__.; embedded.go extracts it to a content-addressed temp path. findLibrary prefers SECRETSPEC_FFI_LIB, then the embedded lib, then a Cargo target dir. - Gated behind the `embed_lib` build tag: the default build (CI, `go test`, a plain checkout) compiles a nil-stub and needs no staged binary, so nothing breaks; only release/distribution builds pass `-tags embed_lib` with the libraries staged. - scripts/stage-cdylib.sh builds + stages the lib under the build-tagged name. Verified locally: default `go test` green, and a tagged build outside the repo with no SECRETSPEC_FFI_LIB embeds the lib and resolves a secret. - Drafted go-embed.yml (per-platform build + embedded smoke test). NOTE: the embedded libs are ~34 MB each; a release commits them via git-LFS (they are gitignored here) and flips embedding on by default. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/go-embed.yml | 81 +++++++++++++++++++++++++ secretspec-go/.gitignore | 3 + secretspec-go/embedded.go | 51 ++++++++++++++++ secretspec-go/embedded_darwin_amd64.go | 10 +++ secretspec-go/embedded_darwin_arm64.go | 10 +++ secretspec-go/embedded_default.go | 10 +++ secretspec-go/embedded_linux_amd64.go | 10 +++ secretspec-go/embedded_linux_arm64.go | 10 +++ secretspec-go/embedded_unsupported.go | 7 +++ secretspec-go/embedded_windows_amd64.go | 10 +++ secretspec-go/scripts/stage-cdylib.sh | 30 +++++++++ secretspec-go/secretspec.go | 5 ++ 12 files changed, 237 insertions(+) create mode 100644 .github/workflows/go-embed.yml create mode 100644 secretspec-go/.gitignore create mode 100644 secretspec-go/embedded.go create mode 100644 secretspec-go/embedded_darwin_amd64.go create mode 100644 secretspec-go/embedded_darwin_arm64.go create mode 100644 secretspec-go/embedded_default.go create mode 100644 secretspec-go/embedded_linux_amd64.go create mode 100644 secretspec-go/embedded_linux_arm64.go create mode 100644 secretspec-go/embedded_unsupported.go create mode 100644 secretspec-go/embedded_windows_amd64.go create mode 100755 secretspec-go/scripts/stage-cdylib.sh diff --git a/.github/workflows/go-embed.yml b/.github/workflows/go-embed.yml new file mode 100644 index 0000000..c499798 --- /dev/null +++ b/.github/workflows/go-embed.yml @@ -0,0 +1,81 @@ +name: "Go embedded lib" + +# Builds the per-platform cdylib the Go SDK embeds (go:embed, behind the +# `embed_lib` build tag) and verifies the embedded build works with no +# SECRETSPEC_FFI_LIB set. The libraries are uploaded as artifacts; a release +# commits them into the module via git-LFS (they are tens of MB each). + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-go/**" + - "secretspec-ffi/**" + - "secretspec/**" + - ".github/workflows/go-embed.yml" + +jobs: + embed: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { target: linux_amd64, runner: ubuntu-latest } + - { target: linux_arm64, runner: ubuntu-24.04-arm } + - { target: darwin_amd64, runner: macos-13 } + - { target: darwin_arm64, runner: macos-14 } + - { target: windows_amd64, runner: windows-latest } + + steps: + - uses: actions/checkout@v5 + + - name: Install Linux system dependencies (dbus for keyring) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + + - name: Stage the embedded cdylib + shell: bash + run: bash secretspec-go/scripts/stage-cdylib.sh + + - name: Build and smoke test the embedded SDK (no SECRETSPEC_FFI_LIB) + shell: bash + run: | + smoke="$RUNNER_TEMP/embedsmoke" + mkdir -p "$smoke" + cat > "$smoke/main.go" <<'EOF' + package main + import ( + "fmt" + secretspec "github.com/cachix/secretspec/secretspec-go" + ) + func main() { + v, err := secretspec.ABIVersion() + if err != nil { panic(err) } + fmt.Println("abi", v) + } + EOF + cat > "$smoke/go.mod" < $GITHUB_WORKSPACE/secretspec-go + EOF + cd "$smoke" + go mod tidy + unset SECRETSPEC_FFI_LIB + go run -tags embed_lib . + + - uses: actions/upload-artifact@v4 + with: + name: go-embed-${{ matrix.target }} + path: secretspec-go/lib/* diff --git a/secretspec-go/.gitignore b/secretspec-go/.gitignore new file mode 100644 index 0000000..2d6ce01 --- /dev/null +++ b/secretspec-go/.gitignore @@ -0,0 +1,3 @@ +# Embedded cdylibs are large (tens of MB); staged at build time, committed for a +# release via git-LFS rather than plain git. +lib/ diff --git a/secretspec-go/embedded.go b/secretspec-go/embedded.go new file mode 100644 index 0000000..80c5717 --- /dev/null +++ b/secretspec-go/embedded.go @@ -0,0 +1,51 @@ +package secretspec + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" +) + +// extractEmbedded writes the build-time-embedded cdylib to a content-addressed +// temp file (reused across runs and processes) and returns its path, so purego +// can dlopen it. The per-platform `embeddedLib` and `embeddedLibName` are +// defined in the build-tagged embedded__.go files (or zeroed by +// embedded_unsupported.go). +func extractEmbedded() (string, error) { + sum := sha256.Sum256(embeddedLib) + dir := filepath.Join(os.TempDir(), "secretspec-ffi-"+hex.EncodeToString(sum[:8])) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + + path := filepath.Join(dir, embeddedLibName) + if info, err := os.Stat(path); err == nil && info.Size() == int64(len(embeddedLib)) { + return path, nil + } + + tmp, err := os.CreateTemp(dir, "lib-*") + if err != nil { + return "", err + } + tmpName := tmp.Name() + if _, err := tmp.Write(embeddedLib); err != nil { + tmp.Close() + os.Remove(tmpName) + return "", err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return "", err + } + + if err := os.Rename(tmpName, path); err != nil { + os.Remove(tmpName) + // A concurrent process may have published it first. + if _, statErr := os.Stat(path); statErr == nil { + return path, nil + } + return "", err + } + return path, nil +} diff --git a/secretspec-go/embedded_darwin_amd64.go b/secretspec-go/embedded_darwin_amd64.go new file mode 100644 index 0000000..b194b80 --- /dev/null +++ b/secretspec-go/embedded_darwin_amd64.go @@ -0,0 +1,10 @@ +//go:build embed_lib && darwin && amd64 + +package secretspec + +import _ "embed" + +//go:embed lib/secretspec_ffi_darwin_amd64.dylib +var embeddedLib []byte + +const embeddedLibName = "libsecretspec_ffi.dylib" diff --git a/secretspec-go/embedded_darwin_arm64.go b/secretspec-go/embedded_darwin_arm64.go new file mode 100644 index 0000000..f9a9340 --- /dev/null +++ b/secretspec-go/embedded_darwin_arm64.go @@ -0,0 +1,10 @@ +//go:build embed_lib && darwin && arm64 + +package secretspec + +import _ "embed" + +//go:embed lib/secretspec_ffi_darwin_arm64.dylib +var embeddedLib []byte + +const embeddedLibName = "libsecretspec_ffi.dylib" diff --git a/secretspec-go/embedded_default.go b/secretspec-go/embedded_default.go new file mode 100644 index 0000000..bd88e62 --- /dev/null +++ b/secretspec-go/embedded_default.go @@ -0,0 +1,10 @@ +//go:build !embed_lib + +package secretspec + +// Built without the `embed_lib` tag: no embedded library, so the SDK uses +// SECRETSPEC_FFI_LIB or a Cargo target directory. Release/distribution builds +// pass `-tags embed_lib` (with the per-platform libraries staged into lib/). +var embeddedLib []byte + +const embeddedLibName = "" diff --git a/secretspec-go/embedded_linux_amd64.go b/secretspec-go/embedded_linux_amd64.go new file mode 100644 index 0000000..ed8fea1 --- /dev/null +++ b/secretspec-go/embedded_linux_amd64.go @@ -0,0 +1,10 @@ +//go:build embed_lib && linux && amd64 + +package secretspec + +import _ "embed" + +//go:embed lib/secretspec_ffi_linux_amd64.so +var embeddedLib []byte + +const embeddedLibName = "libsecretspec_ffi.so" diff --git a/secretspec-go/embedded_linux_arm64.go b/secretspec-go/embedded_linux_arm64.go new file mode 100644 index 0000000..9af474e --- /dev/null +++ b/secretspec-go/embedded_linux_arm64.go @@ -0,0 +1,10 @@ +//go:build embed_lib && linux && arm64 + +package secretspec + +import _ "embed" + +//go:embed lib/secretspec_ffi_linux_arm64.so +var embeddedLib []byte + +const embeddedLibName = "libsecretspec_ffi.so" diff --git a/secretspec-go/embedded_unsupported.go b/secretspec-go/embedded_unsupported.go new file mode 100644 index 0000000..7602607 --- /dev/null +++ b/secretspec-go/embedded_unsupported.go @@ -0,0 +1,7 @@ +//go:build embed_lib && !(linux && amd64) && !(linux && arm64) && !(darwin && amd64) && !(darwin && arm64) && !(windows && amd64) + +package secretspec + +var embeddedLib []byte + +const embeddedLibName = "" diff --git a/secretspec-go/embedded_windows_amd64.go b/secretspec-go/embedded_windows_amd64.go new file mode 100644 index 0000000..c11912d --- /dev/null +++ b/secretspec-go/embedded_windows_amd64.go @@ -0,0 +1,10 @@ +//go:build embed_lib && windows && amd64 + +package secretspec + +import _ "embed" + +//go:embed lib/secretspec_ffi_windows_amd64.dll +var embeddedLib []byte + +const embeddedLibName = "secretspec_ffi.dll" diff --git a/secretspec-go/scripts/stage-cdylib.sh b/secretspec-go/scripts/stage-cdylib.sh new file mode 100755 index 0000000..dc94123 --- /dev/null +++ b/secretspec-go/scripts/stage-cdylib.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Build the secretspec-ffi cdylib (release) and stage it into lib/ under the +# build-tagged name the embedded__.go files reference, so `go build` +# embeds it. Run before building/releasing the Go module. +# +# NOTE: the embedded libraries are large (tens of MB each); a release should +# commit them via git-LFS (or stage them into a release tag) rather than plain +# git. They are gitignored here. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" + +target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" + +goos="$(go env GOOS)" +goarch="$(go env GOARCH)" +case "$goos" in + darwin) src="libsecretspec_ffi.dylib"; ext="dylib" ;; + windows) src="secretspec_ffi.dll"; ext="dll" ;; + *) src="libsecretspec_ffi.so"; ext="so" ;; +esac + +mkdir -p "$pkg_dir/lib" +cp "$target_dir/release/$src" "$pkg_dir/lib/secretspec_ffi_${goos}_${goarch}.${ext}" +echo "staged secretspec_ffi_${goos}_${goarch}.${ext} into lib/" diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 37c257a..4515876 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -47,6 +47,11 @@ func findLibrary() (string, error) { if p := os.Getenv("SECRETSPEC_FFI_LIB"); p != "" { return p, nil } + // A library embedded at build time (go:embed, per platform) is extracted to + // a temp file and used, so `go get` works with no native build. + if len(embeddedLib) > 0 { + return extractEmbedded() + } dir, err := os.Getwd() if err != nil { return "", err From eb8bbeb950ac44960433c048688ad65d37eb2147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:11:28 -0400 Subject: [PATCH 19/56] feat(node): switch the Node SDK to a napi-rs addon Phase 6 / pillar A, Node: replace the koffi (dlopen) binding with a napi-rs native addon that embeds the resolver, so `npm install` needs no cdylib or SECRETSPEC_FFI_LIB. This is the standard way to ship a Rust-backed npm package. - New `secretspec::resolve_json(&str) -> String` in the core: the shared JSON-in/JSON-out boundary (request -> response envelope). secretspec-ffi is refactored into a thin wrapper over it (and drops its serde deps), so the C ABI and the napi addon define the envelope contract in exactly one place. - New secretspec-node-native crate (napi-rs) exposing resolve()/abiVersion() over resolve_json; a napi cdylib is a valid Node addon, so scripts/build-addon.sh is just `cargo build` + rename to secretspec.node. - index.js now requires ./secretspec.node instead of koffi; the JS API (builder, Resolved, fields/fieldsJson, errors) is unchanged. Dropped the koffi and unused typescript deps; the package has no runtime npm dependencies. - Test harness builds the addon instead of the cdylib; all 8 Node tests (incl. conformance) pass, and the full cross-language suite stays green. - Drafted node-addon.yml (per-platform addon build + smoke test). NOTE: full npm distribution publishes per-platform addon packages (the pattern @napi-rs/cli automates); that publish wiring is the remaining follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/node-addon.yml | 57 ++++++++++++ CHANGELOG.md | 12 ++- Cargo.lock | 99 +++++++++++++++++++- Cargo.toml | 8 +- docs/src/content/docs/sdk/nodejs.md | 14 +-- scripts/ci-sdks.sh | 4 +- secretspec-ffi/Cargo.toml | 4 +- secretspec-ffi/src/lib.rs | 112 +++-------------------- secretspec-node/.gitignore | 1 + secretspec-node/Cargo.toml | 18 ++++ secretspec-node/build.rs | 3 + secretspec-node/index.js | 85 ++++------------- secretspec-node/package.json | 12 +-- secretspec-node/scripts/build-addon.sh | 22 +++++ secretspec-node/src/lib.rs | 21 +++++ secretspec-node/test/conformance.test.js | 18 ++-- secretspec-node/test/resolve.test.js | 21 ++--- secretspec/README.md | 2 +- secretspec/src/lib.rs | 4 +- secretspec/src/resolve.rs | 96 +++++++++++++++++++ 20 files changed, 392 insertions(+), 221 deletions(-) create mode 100644 .github/workflows/node-addon.yml create mode 100644 secretspec-node/Cargo.toml create mode 100644 secretspec-node/build.rs create mode 100644 secretspec-node/scripts/build-addon.sh create mode 100644 secretspec-node/src/lib.rs diff --git a/.github/workflows/node-addon.yml b/.github/workflows/node-addon.yml new file mode 100644 index 0000000..bf3176d --- /dev/null +++ b/.github/workflows/node-addon.yml @@ -0,0 +1,57 @@ +name: "Node addon" + +# Builds the napi-rs Node addon (secretspec.node) per platform and smoke tests +# it. The addon embeds the resolver, so the published npm package needs no +# native build at install time. +# +# NOTE: this uploads a per-platform addon artifact. Full npm distribution +# publishes these as per-platform packages (the pattern @napi-rs/cli automates); +# that publish wiring is the remaining follow-up. + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-node/**" + - "secretspec/**" + - ".github/workflows/node-addon.yml" + +jobs: + addon: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { target: linux-x64, runner: ubuntu-latest } + - { target: linux-arm64, runner: ubuntu-24.04-arm } + - { target: darwin-x64, runner: macos-13 } + - { target: darwin-arm64, runner: macos-14 } + - { target: win32-x64, runner: windows-latest } + + steps: + - uses: actions/checkout@v5 + + - name: Install Linux system dependencies (dbus for keyring) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Build the addon and run the SDK tests + shell: bash + run: | + bash secretspec-node/scripts/build-addon.sh + ( cd secretspec-node && node --test ) + + - uses: actions/upload-artifact@v4 + with: + name: node-addon-${{ matrix.target }} + path: secretspec-node/secretspec.node diff --git a/CHANGELOG.md b/CHANGELOG.md index a5efe73..89e2e00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Ruby, and Node SDKs agree. Each SDK runs the fixtures inside its own native test runner. For `as_path` secrets the canonical value is the materialized file's contents, so the comparison is deterministic across languages. -- New `secretspec` Node.js / TypeScript SDK (`secretspec-node`): a thin client - over the `secretspec-ffi` C ABI, loaded at runtime via koffi (dlopen), so - Node apps inherit every provider with no JS-side resolution logic. Mirrors the - derive crate's vocabulary +- New `secretspec::resolve_json(&str) -> String`: the shared JSON-in/JSON-out + resolution boundary (request to response envelope), so every native binding + (the `secretspec-ffi` C ABI and the napi-rs Node addon) defines the envelope + contract in exactly one place. `secretspec-ffi` is now a thin wrapper over it. +- New `secretspec` Node.js / TypeScript SDK (`secretspec-node`): a thin wrapper + over a napi-rs native addon that embeds the resolver, so Node apps inherit + every provider with no JS-side resolution logic and `npm install` needs no + native build. Mirrors the derive crate's vocabulary (`SecretSpec.builder().withProvider(...).withProfile(...).withReason(...).load()` returning a `Resolved` with `provider`/`profile`/`secrets`, plus `setAsEnv()`). A missing required secret throws `MissingRequiredError`; other failures throw diff --git a/Cargo.lock b/Cargo.lock index db62096..2f6e4da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1134,6 +1134,15 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -1275,6 +1284,16 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ctutils" version = "0.4.0" @@ -1494,7 +1513,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ - "convert_case", + "convert_case 0.10.0", "proc-macro2", "quote", "rustc_version", @@ -2723,6 +2742,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2881,6 +2910,63 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "napi" +version = "2.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3" +dependencies = [ + "bitflags", + "ctor", + "napi-derive", + "napi-sys", + "once_cell", +] + +[[package]] +name = "napi-build" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1" + +[[package]] +name = "napi-derive" +version = "2.16.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" +dependencies = [ + "cfg-if", + "convert_case 0.6.0", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" +dependencies = [ + "convert_case 0.6.0", + "once_cell", + "proc-macro2", + "quote", + "regex", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" +dependencies = [ + "libloading", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -3906,11 +3992,20 @@ name = "secretspec-ffi" version = "0.12.0" dependencies = [ "secretspec", - "serde", "serde_json", "tempfile", ] +[[package]] +name = "secretspec-node-native" +version = "0.12.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "secretspec", +] + [[package]] name = "security-framework" version = "2.11.1" diff --git a/Cargo.toml b/Cargo.toml index 749ef5c..ff952b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,11 @@ [workspace] -members = ["secretspec", "secretspec-derive", "secretspec-ffi", "examples/derive"] +members = [ + "secretspec", + "secretspec-derive", + "secretspec-ffi", + "secretspec-node", + "examples/derive", +] resolver = "2" [workspace.package] diff --git a/docs/src/content/docs/sdk/nodejs.md b/docs/src/content/docs/sdk/nodejs.md index 94261c0..86f2458 100644 --- a/docs/src/content/docs/sdk/nodejs.md +++ b/docs/src/content/docs/sdk/nodejs.md @@ -3,10 +3,11 @@ title: Node.js SDK description: Resolve SecretSpec secrets from Node.js and TypeScript --- -The Node.js / TypeScript SDK (`secretspec`) is a thin client over the -`secretspec-ffi` C ABI, loaded via [koffi](https://koffi.dev/) (dlopen). -Resolution happens in the Rust core, so the SDK inherits every provider with no -JS-side logic. TypeScript declarations ship in `index.d.ts`. +The Node.js / TypeScript SDK (`secretspec`) is a thin wrapper over a +[napi-rs](https://napi.rs/) native addon that embeds the resolver. Resolution +happens in the Rust core, so the SDK inherits every provider with no JS-side +logic, and `npm install` needs no native build. TypeScript declarations ship in +`index.d.ts`. ## Quick start @@ -43,8 +44,3 @@ import { Convert } from './secrets_gen'; // typed, generated const typed = Convert.toSecretSpec(resolved.fieldsJson()); console.log(typed.DATABASE_URL); ``` - -## Library discovery - -The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, -or a Cargo `target` directory found by searching up from the working directory. diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh index e371fcd..4602cf8 100755 --- a/scripts/ci-sdks.sh +++ b/scripts/ci-sdks.sh @@ -33,6 +33,8 @@ echo "==> Ruby" ( cd secretspec-rb && ruby test/test_resolve.rb ) echo "==> Node" -( cd secretspec-node && { [ -d node_modules ] || npm install --no-audit --no-fund; } && node --test ) +# The Node SDK uses a napi-rs addon (built by its test harness), not the cdylib, +# and has no npm dependencies. +( cd secretspec-node && node --test ) echo "==> All SDK suites passed" diff --git a/secretspec-ffi/Cargo.toml b/secretspec-ffi/Cargo.toml index f57b1c7..bf188b7 100644 --- a/secretspec-ffi/Cargo.toml +++ b/secretspec-ffi/Cargo.toml @@ -14,9 +14,9 @@ name = "secretspec_ffi" crate-type = ["cdylib", "staticlib", "lib"] [dependencies] +# The envelope/resolve logic lives in the secretspec crate (resolve_json); this +# crate is only the C ABI wrapper. secretspec = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } [dev-dependencies] serde_json = { workspace = true } diff --git a/secretspec-ffi/src/lib.rs b/secretspec-ffi/src/lib.rs index 0922cd3..ae90848 100644 --- a/secretspec-ffi/src/lib.rs +++ b/secretspec-ffi/src/lib.rs @@ -45,62 +45,14 @@ use std::ffi::{CStr, CString, c_char}; use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::path::Path; - -use secretspec::{ResolveResponse, Secrets}; -use serde::{Deserialize, Serialize}; /// ABI version, NUL-terminated for direct return as a C string. const ABI_VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), "\0"); -#[derive(Debug, Default, Deserialize)] -struct ResolveRequest { - #[serde(default)] - path: Option, - #[serde(default)] - provider: Option, - #[serde(default)] - profile: Option, - #[serde(default)] - reason: Option, - #[serde(default)] - no_values: bool, -} - -#[derive(Debug, Serialize)] -struct FfiError { - kind: String, - message: String, -} - -#[derive(Debug, Serialize)] -struct Envelope { - ok: bool, - #[serde(skip_serializing_if = "Option::is_none")] - response: Option, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, -} - -impl Envelope { - fn ok(response: ResolveResponse) -> Self { - Self { - ok: true, - response: Some(response), - error: None, - } - } - - fn err(kind: &str, message: impl Into) -> Self { - Self { - ok: false, - response: None, - error: Some(FfiError { - kind: kind.to_string(), - message: message.into(), - }), - } - } +/// A hand-built error envelope for failures that occur before the request +/// reaches the shared resolver (null pointer, non-UTF-8 input). +fn input_error(message: &str) -> String { + format!("{{\"ok\":false,\"error\":{{\"kind\":\"invalid_input\",\"message\":{message:?}}}}}") } /// Returns the ABI version as a static NUL-terminated string. Do not free. @@ -139,66 +91,26 @@ pub unsafe extern "C" fn secretspec_free(ptr: *mut c_char) { #[unsafe(no_mangle)] pub unsafe extern "C" fn secretspec_resolve(request_json: *const c_char) -> *mut c_char { // Never let a panic unwind across the FFI boundary (that is UB). - let envelope = match catch_unwind(AssertUnwindSafe(|| resolve_inner(request_json))) { - Ok(env) => env, - Err(_) => Envelope::err("panic", "internal panic during resolve"), + let json = match catch_unwind(AssertUnwindSafe(|| resolve_inner(request_json))) { + Ok(json) => json, + Err(_) => input_error("internal panic during resolve"), }; - let json = serde_json::to_string(&envelope).unwrap_or_else(|_| { - // Should be unreachable; fall back to a hand-built valid envelope. - "{\"ok\":false,\"error\":{\"kind\":\"serialize\",\"message\":\"failed to serialize response\"}}" - .to_string() - }); - match CString::new(json) { Ok(c) => c.into_raw(), Err(_) => std::ptr::null_mut(), } } -fn resolve_inner(request_json: *const c_char) -> Envelope { +fn resolve_inner(request_json: *const c_char) -> String { if request_json.is_null() { - return Envelope::err("invalid_input", "request_json was null"); + return input_error("request_json was null"); } // Safety: caller contract guarantees a NUL-terminated string when non-null. let raw = unsafe { CStr::from_ptr(request_json) }; - let text = match raw.to_str() { - Ok(s) => s, - Err(_) => return Envelope::err("invalid_input", "request_json was not valid UTF-8"), - }; - - let request: ResolveRequest = match serde_json::from_str(text) { - Ok(req) => req, - Err(e) => return Envelope::err("invalid_request", format!("invalid request JSON: {e}")), - }; - - let loaded = match &request.path { - Some(path) => Secrets::load_from(Path::new(path)), - None => Secrets::load(), - }; - let mut app = match loaded { - Ok(app) => app, - Err(e) => return Envelope::err(e.kind(), e.to_string()), - }; - - if let Some(provider) = request.provider { - app.set_provider(provider); - } - if let Some(profile) = request.profile { - app.set_profile(profile); - } - if let Some(reason) = request.reason { - app = app.with_reason(reason); - } - - match app.resolve() { - Ok(mut response) => { - if request.no_values { - response = response.without_values(); - } - Envelope::ok(response) - } - Err(e) => Envelope::err(e.kind(), e.to_string()), + match raw.to_str() { + Ok(text) => secretspec::resolve_json(text), + Err(_) => input_error("request_json was not valid UTF-8"), } } diff --git a/secretspec-node/.gitignore b/secretspec-node/.gitignore index 504afef..fb4a295 100644 --- a/secretspec-node/.gitignore +++ b/secretspec-node/.gitignore @@ -1,2 +1,3 @@ node_modules/ package-lock.json +*.node diff --git a/secretspec-node/Cargo.toml b/secretspec-node/Cargo.toml new file mode 100644 index 0000000..5bc2250 --- /dev/null +++ b/secretspec-node/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "secretspec-node-native" +version.workspace = true +edition.workspace = true +repository = "https://github.com/cachix/secretspec" +description = "napi-rs Node addon for SecretSpec" +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +napi = { version = "2.16", default-features = false, features = ["napi8"] } +napi-derive = "2.16" +secretspec = { workspace = true } + +[build-dependencies] +napi-build = "2.1" diff --git a/secretspec-node/build.rs b/secretspec-node/build.rs new file mode 100644 index 0000000..0f1b010 --- /dev/null +++ b/secretspec-node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/secretspec-node/index.js b/secretspec-node/index.js index 6fe1b7d..14e47a4 100644 --- a/secretspec-node/index.js +++ b/secretspec-node/index.js @@ -2,14 +2,23 @@ // Node.js SDK for SecretSpec, a declarative secrets manager. // -// A thin client over the secretspec-ffi C ABI, loaded at runtime via koffi -// (dlopen). Resolution happens entirely in the Rust core, so the SDK inherits -// every provider with no JS-side logic. Mirrors the Rust derive crate's -// vocabulary. - -const fs = require('fs'); -const path = require('path'); -const koffi = require('koffi'); +// A thin wrapper over the native napi-rs addon (secretspec.node), which embeds +// the resolver and shares the same JSON envelope contract as every other +// language binding. Resolution (providers, chains, profiles, generation, +// as_path) happens entirely in the Rust core; this layer marshals JSON and +// exposes the builder API. TypeScript declarations ship in index.d.ts. + +let native; +try { + // The prebuilt addon for this platform. napi-rs publishes it per platform; + // in a source checkout, build it with scripts/build-addon.sh. + native = require('./secretspec.node'); +} catch (err) { + throw new Error( + 'failed to load the secretspec native addon (secretspec.node); build it ' + + `with scripts/build-addon.sh. Underlying error: ${err.message}`, + ); +} class SecretSpecError extends Error { constructor(kind, message) { @@ -27,62 +36,6 @@ class MissingRequiredError extends SecretSpecError { } } -function libNames() { - if (process.platform === 'darwin') return ['libsecretspec_ffi.dylib']; - if (process.platform === 'win32') return ['secretspec_ffi.dll']; - return ['libsecretspec_ffi.so']; -} - -function findLibrary() { - const override = process.env.SECRETSPEC_FFI_LIB; - if (override) return override; - - let dir = process.cwd(); - for (;;) { - for (const profile of ['release', 'debug']) { - for (const name of libNames()) { - const candidate = path.join(dir, 'target', profile, name); - if (fs.existsSync(candidate)) return candidate; - } - } - const parent = path.dirname(dir); - if (parent === dir) break; - dir = parent; - } - throw new SecretSpecError( - 'load', - 'could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB', - ); -} - -let _lib = null; - -function lib() { - if (_lib) return _lib; - const handle = koffi.load(findLibrary()); - _lib = { - // void* return so we own the pointer and can free it after decoding. - resolve: handle.func('void *secretspec_resolve(const char *)'), - free: handle.func('void secretspec_free(void *)'), - // const char* return is a static string; koffi decodes it directly. - abi: handle.func('const char *secretspec_abi_version()'), - }; - return _lib; -} - -function resolveRaw(request) { - const l = lib(); - const ptr = l.resolve(JSON.stringify(request)); - if (!ptr) { - throw new SecretSpecError('ffi', 'secretspec_resolve returned null'); - } - try { - return koffi.decode(ptr, 'char', -1); // NUL-terminated C string - } finally { - l.free(ptr); - } -} - class ResolvedSecret { constructor(entry) { this.value = entry.value ?? null; @@ -149,7 +102,7 @@ class Builder { * missing, and SecretSpecError for any other failure. */ load() { - const envelope = JSON.parse(resolveRaw(this._request)); + const envelope = JSON.parse(native.resolve(JSON.stringify(this._request))); if (!envelope.ok) { const err = envelope.error || {}; throw new SecretSpecError(err.kind || 'unknown', err.message || ''); @@ -170,7 +123,7 @@ const SecretSpec = { }; function abiVersion() { - return lib().abi(); + return native.abiVersion(); } module.exports = { diff --git a/secretspec-node/package.json b/secretspec-node/package.json index 0ed56c8..6799322 100644 --- a/secretspec-node/package.json +++ b/secretspec-node/package.json @@ -6,17 +6,9 @@ "homepage": "https://secretspec.dev/", "main": "index.js", "types": "index.d.ts", - "files": [ - "index.js", - "index.d.ts" - ], + "files": ["index.js", "index.d.ts", "secretspec.node"], "scripts": { + "build": "bash scripts/build-addon.sh", "test": "node --test" - }, - "dependencies": { - "koffi": "^2.9.0" - }, - "devDependencies": { - "typescript": "^6.0.3" } } diff --git a/secretspec-node/scripts/build-addon.sh b/secretspec-node/scripts/build-addon.sh new file mode 100644 index 0000000..f6bf341 --- /dev/null +++ b/secretspec-node/scripts/build-addon.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Build the napi-rs addon (release) and place it as secretspec.node next to +# index.js. A napi cdylib is itself a valid Node addon, so this is just a +# cargo build plus a rename. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +cargo build -p secretspec-node-native --release --manifest-path "$repo_root/Cargo.toml" + +target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +case "$(uname -s)" in + Darwin) src="libsecretspec_node_native.dylib" ;; + MINGW* | MSYS* | CYGWIN*) src="secretspec_node_native.dll" ;; + *) src="libsecretspec_node_native.so" ;; +esac + +cp "$target_dir/release/$src" "$pkg_dir/secretspec.node" +echo "built secretspec.node" diff --git a/secretspec-node/src/lib.rs b/secretspec-node/src/lib.rs new file mode 100644 index 0000000..09d148c --- /dev/null +++ b/secretspec-node/src/lib.rs @@ -0,0 +1,21 @@ +//! napi-rs Node addon for SecretSpec. +//! +//! A thin wrapper over `secretspec::resolve_json`, the same JSON-in/JSON-out +//! boundary the C ABI uses, so the Node binding shares one envelope contract +//! with every other language. The JS layer (index.js) does the request/response +//! marshaling and exposes the builder API. + +use napi_derive::napi; + +/// Resolve secrets from a JSON request string, returning the JSON response +/// envelope (`{"ok": true, "response": ...}` or `{"ok": false, "error": ...}`). +#[napi] +pub fn resolve(request_json: String) -> String { + secretspec::resolve_json(&request_json) +} + +/// The addon (ABI) version. +#[napi] +pub fn abi_version() -> String { + env!("CARGO_PKG_VERSION").to_string() +} diff --git a/secretspec-node/test/conformance.test.js b/secretspec-node/test/conformance.test.js index 85b356c..50beb55 100644 --- a/secretspec-node/test/conformance.test.js +++ b/secretspec-node/test/conformance.test.js @@ -9,19 +9,15 @@ const fs = require('node:fs'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); -function ensureLib() { - if (process.env.SECRETSPEC_FFI_LIB) return; - const repo = path.resolve(__dirname, '..', '..'); - execFileSync('cargo', ['build', '-p', 'secretspec-ffi'], { cwd: repo, stdio: 'inherit' }); - const meta = JSON.parse( - execFileSync('cargo', ['metadata', '--no-deps', '--format-version', '1'], { cwd: repo }), - ); - const name = - process.platform === 'darwin' ? 'libsecretspec_ffi.dylib' : 'libsecretspec_ffi.so'; - process.env.SECRETSPEC_FFI_LIB = path.join(meta.target_directory, 'debug', name); +function ensureAddon() { + const addon = path.resolve(__dirname, '..', 'secretspec.node'); + if (fs.existsSync(addon)) return; + execFileSync('bash', [path.resolve(__dirname, '..', 'scripts', 'build-addon.sh')], { + stdio: 'inherit', + }); } -ensureLib(); +ensureAddon(); const { SecretSpec } = require('../index.js'); const FIXTURES = path.resolve(__dirname, '..', '..', 'conformance', 'fixtures'); diff --git a/secretspec-node/test/resolve.test.js b/secretspec-node/test/resolve.test.js index 29933ee..9383874 100644 --- a/secretspec-node/test/resolve.test.js +++ b/secretspec-node/test/resolve.test.js @@ -7,21 +7,16 @@ const os = require('node:os'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); -// Build the secretspec-ffi cdylib and point the SDK at it, unless -// SECRETSPEC_FFI_LIB is already set. -function ensureLib() { - if (process.env.SECRETSPEC_FFI_LIB) return; - const repo = path.resolve(__dirname, '..', '..'); - execFileSync('cargo', ['build', '-p', 'secretspec-ffi'], { cwd: repo, stdio: 'inherit' }); - const meta = JSON.parse( - execFileSync('cargo', ['metadata', '--no-deps', '--format-version', '1'], { cwd: repo }), - ); - const name = - process.platform === 'darwin' ? 'libsecretspec_ffi.dylib' : 'libsecretspec_ffi.so'; - process.env.SECRETSPEC_FFI_LIB = path.join(meta.target_directory, 'debug', name); +// Build the napi addon (secretspec.node) unless it is already present. +function ensureAddon() { + const addon = path.resolve(__dirname, '..', 'secretspec.node'); + if (fs.existsSync(addon)) return; + execFileSync('bash', [path.resolve(__dirname, '..', 'scripts', 'build-addon.sh')], { + stdio: 'inherit', + }); } -ensureLib(); +ensureAddon(); const { SecretSpec, MissingRequiredError, diff --git a/secretspec/README.md b/secretspec/README.md index a93f535..f505cf9 100644 --- a/secretspec/README.md +++ b/secretspec/README.md @@ -188,7 +188,7 @@ profile, and generator works identically with no per-language resolution logic: - [Python](https://secretspec.dev/sdk/python) (via cffi) - [Go](https://secretspec.dev/sdk/go) (via purego, no cgo) - [Ruby](https://secretspec.dev/sdk/ruby) (via stdlib Fiddle) -- [Node.js / TypeScript](https://secretspec.dev/sdk/nodejs) (via koffi) +- [Node.js / TypeScript](https://secretspec.dev/sdk/nodejs) (napi-rs addon) ```python from secretspec import SecretSpec diff --git a/secretspec/src/lib.rs b/secretspec/src/lib.rs index f980c20..d30f689 100644 --- a/secretspec/src/lib.rs +++ b/secretspec/src/lib.rs @@ -76,7 +76,9 @@ pub use provider::Provider; pub use report::{ RESOLUTION_REPORT_SCHEMA_VERSION, ResolutionReport, ResolutionStatus, SecretResolution, }; -pub use resolve::{RESOLVE_SCHEMA_VERSION, ResolveResponse, ResolvedSecret, ResolvedSource}; +pub use resolve::{ + RESOLVE_SCHEMA_VERSION, ResolveResponse, ResolvedSecret, ResolvedSource, resolve_json, +}; pub use secrets::Secrets; pub use validation::ValidatedSecrets; diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs index b29ed89..499ce09 100644 --- a/secretspec/src/resolve.rs +++ b/secretspec/src/resolve.rs @@ -90,3 +90,99 @@ impl ResolveResponse { self } } + +#[derive(Debug, Default, Deserialize)] +struct JsonRequest { + #[serde(default)] + path: Option, + #[serde(default)] + provider: Option, + #[serde(default)] + profile: Option, + #[serde(default)] + reason: Option, + #[serde(default)] + no_values: bool, +} + +#[derive(Debug, Serialize)] +struct JsonError { + kind: String, + message: String, +} + +#[derive(Debug, Serialize)] +struct JsonEnvelope { + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + response: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +fn envelope_error(kind: &str, message: impl Into) -> JsonEnvelope { + JsonEnvelope { + ok: false, + response: None, + error: Some(JsonError { + kind: kind.to_string(), + message: message.into(), + }), + } +} + +fn resolve_envelope(request_json: &str) -> JsonEnvelope { + let request: JsonRequest = match serde_json::from_str(request_json) { + Ok(request) => request, + Err(e) => return envelope_error("invalid_request", format!("invalid request JSON: {e}")), + }; + + let loaded = match &request.path { + Some(path) => crate::Secrets::load_from(std::path::Path::new(path)), + None => crate::Secrets::load(), + }; + let mut app = match loaded { + Ok(app) => app, + Err(e) => return envelope_error(e.kind(), e.to_string()), + }; + + if let Some(provider) = request.provider { + app.set_provider(provider); + } + if let Some(profile) = request.profile { + app.set_profile(profile); + } + if let Some(reason) = request.reason { + app = app.with_reason(reason); + } + + match app.resolve() { + Ok(mut response) => { + if request.no_values { + response = response.without_values(); + } + JsonEnvelope { + ok: true, + response: Some(response), + error: None, + } + } + Err(e) => envelope_error(e.kind(), e.to_string()), + } +} + +/// Resolve secrets from a JSON request string and return the JSON response +/// envelope: `{"ok": true, "response": }` or +/// `{"ok": false, "error": {"kind", "message"}}`. +/// +/// This is the shared JSON boundary used by every native binding (the C ABI in +/// `secretspec-ffi` and the napi-rs Node addon), so the envelope contract is +/// defined in exactly one place. The request accepts optional `path`, +/// `provider`, `profile`, `reason`, and `no_values`. The response carries secret +/// values; treat its bytes as sensitive. +pub fn resolve_json(request_json: &str) -> String { + let envelope = resolve_envelope(request_json); + serde_json::to_string(&envelope).unwrap_or_else(|_| { + "{\"ok\":false,\"error\":{\"kind\":\"serialize\",\"message\":\"failed to serialize response\"}}".to_string() + }) +} From 792943faf68319e3f93c82a76f86257680f5bab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:22:59 -0400 Subject: [PATCH 20/56] ci: wire per-ecosystem release/publish pipelines Phase 6 / pillar A follow-up: turn the per-platform build workflows into release pipelines that produce portable artifacts and publish on a version tag, plus a RELEASE.md runbook. - Python (python-wheels.yml): build Linux wheels inside a manylinux_2_28 container and repair with auditwheel (vendors the cdylib's libdbus/glibc), native macOS/Windows wheels, and publish to PyPI via Trusted Publishing (OIDC). - Ruby (ruby-gems.yml): add a publish job that `gem push`es the platform gems (RUBYGEMS_API_KEY). Portable-Linux gem build noted as a follow-up. - Go: add secretspec-go/.gitattributes so the embedded libs are git-LFS tracked when a release commits them. - RELEASE.md documents each ecosystem's build approach, publish mechanism, required secrets, and known gaps (Ruby portable build; Go git-LFS + manual commit; Node multi-platform npm via @napi-rs/cli optional packages). UNVALIDATED: these are cross-platform CI + registry-credential pipelines that have not been run; they need a CI iteration and the documented repo secrets. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/python-wheels.yml | 86 ++++++++++++++++++----------- .github/workflows/ruby-gems.yml | 22 ++++++++ RELEASE.md | 57 +++++++++++++++++++ secretspec-go/.gitattributes | 5 ++ 4 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 RELEASE.md create mode 100644 secretspec-go/.gitattributes diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index f6758aa..285f9fb 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -1,15 +1,14 @@ name: "Python wheels" -# Builds platform wheels for the Python SDK that bundle the secretspec-ffi -# cdylib (into secretspec/_lib/), so `pip install secretspec` works without a -# separate native build. +# Builds portable wheels for the Python SDK that bundle the secretspec-ffi +# cdylib, and publishes them to PyPI on a version tag. # -# NOTE: the Linux wheels produced here are tagged `linux_*`, not `manylinux_*`. -# Publishing to PyPI requires repairing them against a manylinux toolchain -# (`auditwheel repair`) so the bundled cdylib's system dependencies (notably -# libdbus, pulled in by the keyring provider) are vendored and the glibc -# requirement is portable. That repair step is the remaining follow-up before a -# PyPI release; the build + bundling mechanism below is what is validated. +# Linux wheels are built inside a manylinux container (old glibc) and repaired +# with auditwheel so the bundled cdylib's system deps (notably libdbus from the +# keyring provider) are vendored. macOS/Windows build natively. +# +# UNVALIDATED: this pipeline has not been run; it needs a CI run to debug and, +# for publishing, PyPI Trusted Publishing configured for this repo/environment. on: workflow_dispatch: @@ -24,48 +23,73 @@ on: - ".github/workflows/python-wheels.yml" jobs: - wheels: - name: ${{ matrix.target }} + linux: + name: wheel ${{ matrix.arch }} (manylinux) + runs-on: ${{ matrix.runner }} + container: quay.io/pypa/manylinux_2_28_${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - { arch: x86_64, runner: ubuntu-latest } + - { arch: aarch64, runner: ubuntu-24.04-arm } + steps: + - uses: actions/checkout@v5 + - name: Install build deps (rust, dbus) + run: | + yum install -y dbus-devel + curl -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Build wheel and repair to manylinux + run: | + bash secretspec-py/scripts/stage-cdylib.sh + python3.11 -m build --wheel --outdir dist_raw secretspec-py + auditwheel repair --wheel-dir wheelhouse dist_raw/*.whl + - uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.arch }} + path: wheelhouse/*.whl + + native: + name: wheel ${{ matrix.target }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - - { target: linux-x86_64, runner: ubuntu-latest } - - { target: linux-aarch64, runner: ubuntu-24.04-arm } - { target: macos-x86_64, runner: macos-13 } - { target: macos-aarch64, runner: macos-14 } - { target: windows-x86_64, runner: windows-latest } - steps: - uses: actions/checkout@v5 - - - name: Install Linux system dependencies (dbus for keyring) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config - - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v5 with: python-version: "3.11" - - - name: Stage the cdylib and build the wheel + - name: Build wheel shell: bash run: | python -m pip install --upgrade build bash secretspec-py/scripts/stage-cdylib.sh ( cd secretspec-py && python -m build --wheel --outdir "$GITHUB_WORKSPACE/wheelhouse" ) - - - name: Smoke test the wheel - shell: bash - run: | - python -m pip install --find-links wheelhouse secretspec - # Run outside the repo with no SECRETSPEC_FFI_LIB so only the bundled - # library can satisfy the import. - cd "$RUNNER_TEMP" - python -c "import secretspec; print('abi', secretspec.abi_version())" - - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.target }} path: wheelhouse/*.whl + + publish: + name: publish to PyPI + if: startsWith(github.ref, 'refs/tags/v') + needs: [linux, native] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # PyPI Trusted Publishing (OIDC), no token needed + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml index 2922921..f13cecf 100644 --- a/.github/workflows/ruby-gems.yml +++ b/.github/workflows/ruby-gems.yml @@ -69,3 +69,25 @@ jobs: with: name: gem-${{ matrix.target }} path: secretspec-rb/*.gem + + publish: + name: publish to RubyGems + if: startsWith(github.ref, 'refs/tags/v') + needs: [gems] + runs-on: ubuntu-latest + steps: + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + - uses: actions/download-artifact@v4 + with: + path: gems + merge-multiple: true + - name: gem push + # Requires the RUBYGEMS_API_KEY secret (or RubyGems Trusted Publishing). + env: + GEM_HOST_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }} + run: | + for gem in gems/*.gem; do + gem push "$gem" + done diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..db0ecc5 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,57 @@ +# Releasing the language SDKs + +Each SDK is a thin client over the Rust core (the `secretspec-ffi` cdylib, or the +napi-rs addon for Node). A release builds the native artifact per platform and +publishes it through that ecosystem's registry, so users install with no native +build. The per-platform build workflows are drafted under `.github/workflows/`; +they have **not been run end to end** and need a CI iteration plus the secrets +below. + +Version tags are `vX.Y.Z`; the publish jobs trigger on them. + +## Python (PyPI) — `python-wheels.yml` + +- **Build:** Linux wheels are built inside a `manylinux_2_28` container (old + glibc) and repaired with `auditwheel`, which vendors the cdylib's system + dependencies (notably `libdbus`, pulled in by the keyring provider) and retags + to `manylinux`. macOS/Windows build natively. The wheel is `py3-none-` + and bundles the cdylib in `secretspec/_lib/`. +- **Publish:** `pypa/gh-action-pypi-publish` via **PyPI Trusted Publishing** + (OIDC). Configure a trusted publisher for this repo + a `pypi` environment; no + token needed. + +## Ruby (RubyGems) — `ruby-gems.yml` + +- **Build:** a platform gem (`Gem::Platform::CURRENT`) bundling the cdylib in + `vendor/`. +- **Publish:** `gem push` for each platform gem. +- **Secret:** `RUBYGEMS_API_KEY` (or configure RubyGems Trusted Publishing). +- **Gap:** the Linux gem currently links the runner's glibc; for a portable gem, + build the cdylib on an old-glibc baseline (e.g. a `manylinux` container, as the + Python job does, or `rake-compiler-dock`) and bundle that. Tracked follow-up. + +## Go (git-LFS) — `go-embed.yml` + +Go has no binary registry, so the per-platform cdylibs are committed into the +module and embedded via `go:embed` (behind the `embed_lib` build tag). + +- **Build:** `go-embed.yml` builds the per-platform libs and uploads them as + artifacts. +- **Release (manual):** stage all platforms' libs into `secretspec-go/lib/` + (from the CI artifacts), un-ignore them, `git lfs track` is already set via + `secretspec-go/.gitattributes`, commit them with LFS, and flip embedding on by + default (drop the `embed_lib` gate or document `-tags embed_lib`). The libs are + ~34 MB each, so plain git is unsuitable; **git-LFS must be enabled** for the + repo. + +## Node.js (npm) — `node-addon.yml` + +- **Build:** `node-addon.yml` builds the napi-rs addon (`secretspec.node`) per + platform and uploads it as an artifact. +- **Publish gap:** multi-platform npm distribution uses per-platform optional + packages (`@secretspec/-`) that the main package `optionalDependencies` + and loads at runtime. This is the pattern `@napi-rs/cli` automates. Adopting + the `@napi-rs/cli` project layout (so `napi build` / `napi prepublish` emit and + publish those packages) is the remaining follow-up; the current addon build is + what such a setup would publish. +- **Secret:** `NPM_TOKEN`. diff --git a/secretspec-go/.gitattributes b/secretspec-go/.gitattributes new file mode 100644 index 0000000..9481ffd --- /dev/null +++ b/secretspec-go/.gitattributes @@ -0,0 +1,5 @@ +# The embedded cdylibs are large (tens of MB); track them with git-LFS when a +# release commits them. They are gitignored during development. +lib/*.so filter=lfs diff=lfs merge=lfs -text +lib/*.dylib filter=lfs diff=lfs merge=lfs -text +lib/*.dll filter=lfs diff=lfs merge=lfs -text From db26b01e2569083dd88388d5fbbeb157a4575e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:47:35 -0400 Subject: [PATCH 21/56] test: e2e the schema/quicktype codegen pipeline in all four SDKs Closes the loose end where only Python had an automated codegen test (Go/Ruby/TS were verified by hand). Each SDK now runs the full pipeline in its native runner: secretspec schema -> quicktype -> typed deserializer over the SDK's fields(). - Schema emitter reworked to a single-root object (the union by default, or a profile's fields via `schema --profile`). quicktype only emits a converter for the ROOT type, so the previous Manifest wrapper / $ref root gave JS/Go no usable `toSecretSpec`/`UnmarshalSecretSpec` and mis-named the type. Pair with `quicktype --top-level SecretSpec`. - New e2e tests: Go (temp module, UnmarshalSecretSpec(FieldsJSON)); Ruby (dry-struct, from_dynamic!(fields)); Node (quicktype --lang javascript, toSecretSpec(fieldsJson())); Python updated to --top-level. All gated on npx. - ci-sdks.sh runs all Ruby test files; CLI docs + SDK pages + CHANGELOG updated for --top-level and --profile. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 21 ++-- docs/src/content/docs/reference/cli.md | 20 ++-- docs/src/content/docs/sdk/go.md | 2 +- docs/src/content/docs/sdk/nodejs.md | 2 +- docs/src/content/docs/sdk/python.md | 2 +- docs/src/content/docs/sdk/ruby.md | 2 +- scripts/ci-sdks.sh | 2 +- secretspec-go/codegen_test.go | 140 +++++++++++++++++++++++++ secretspec-node/test/codegen.test.js | 83 +++++++++++++++ secretspec-py/tests/test_codegen.py | 2 + secretspec-rb/test/test_codegen.rb | 79 ++++++++++++++ secretspec/src/cli/mod.rs | 21 ++-- secretspec/src/codegen.rs | 116 ++++++++++---------- 13 files changed, 399 insertions(+), 93 deletions(-) create mode 100644 secretspec-go/codegen_test.go create mode 100644 secretspec-node/test/codegen.test.js create mode 100644 secretspec-rb/test/test_codegen.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e2e00..eb5a87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,16 +62,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (union vs per-profile field sets, optionality, `as_path`, profile list). It is the single source of truth those decisions are computed in, so the Rust derive macro and the JSON Schema emitter cannot drift. `build_ir(&Config) -> CodegenIr`. -- New `secretspec schema` command: emits a JSON Schema for the manifest's typed - shapes (a `SecretSpec` union type plus one `Secrets` per profile). - Rather than hand-write a typed-accessor generator per language, feed this to - [quicktype](https://quicktype.io) to generate idiomatic types and - deserializers for any language, then hand the deserializer the flat - `{SECRET_NAME: value}` map from each SDK's new `fields()` helper (e.g. in - Python, `SecretSpec.from_dict(resolved.fields())`). This keeps the per-language - maintenance to the small `fields()` method. `fields()` (and a JSON variant for - Go/Node) is available on the resolved result in every SDK. Value-free: `schema` - reads only the manifest; `-o` writes to a file instead of stdout. +- New `secretspec schema` command: emits a single-root JSON Schema for the + manifest's typed shape (the union `SecretSpec` by default, or a profile's + fields with `--profile`). Rather than hand-write a typed-accessor generator per + language, feed this to [quicktype](https://quicktype.io) (`--top-level + SecretSpec`) to generate an idiomatic type and deserializer for any language, + then hand the deserializer the flat `{SECRET_NAME: value}` map from each SDK's + new `fields()` helper (e.g. in Python, `SecretSpec.from_dict(resolved.fields())`). + This keeps the per-language maintenance to the small `fields()` method. + `fields()` (and a JSON variant for Go/Node) is available on the resolved result + in every SDK, and the full `schema -> quicktype -> typed` pipeline is e2e-tested + in all four SDK suites. Value-free: `schema` reads only the manifest. ### Changed - The `secretspec-derive` macro now computes all of its typing decisions through diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 60fecbd..2603501 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -241,30 +241,32 @@ the serving provider's credential-free URI. The canonical JSON Schema is committed at `schema/resolve-response.schema.json`. ### schema -Emit a JSON Schema for the manifest's typed shapes: a `SecretSpec` type (the -union, safe for any profile) plus one `Secrets` type per profile. -Value-free: reads only the manifest, never a provider. +Emit a single-root JSON Schema for the manifest's typed shape: by default the +union `SecretSpec` (safe for any profile); with `--profile`, that profile's exact +fields. Value-free: reads only the manifest, never a provider. ```bash -secretspec schema [-o FILE] +secretspec schema [OPTIONS] ``` **Options:** +- `-P, --profile ` - Emit the schema for this profile's fields instead of the union - `-o, --output ` - Write to this file instead of stdout Rather than ship a typed-accessor generator per language, feed this schema to -[quicktype](https://quicktype.io), which generates idiomatic types **and** -deserializers for any language. At runtime, hand the generated deserializer the -flat `{SECRET_NAME: value}` map from the SDK's `fields()` helper: +[quicktype](https://quicktype.io), which generates an idiomatic type **and** +deserializer for any language. Name the type with `--top-level`. At runtime, hand +the generated deserializer the flat `{SECRET_NAME: value}` map from the SDK's +`fields()` helper: ```bash -$ secretspec schema | quicktype -s schema --lang python -o secrets_gen.py +$ secretspec schema | quicktype -s schema --top-level SecretSpec --lang python -o secrets_gen.py ``` ```python from secretspec import SecretSpec from secrets_gen import SecretSpec as Secrets # quicktype-generated, typed -resolved = SecretSpec.builder().with_profile("production").with_reason("boot").load() +resolved = SecretSpec.builder().with_reason("boot").load() s = Secrets.from_dict(resolved.fields()) print(s.database_url) # typed str ``` diff --git a/docs/src/content/docs/sdk/go.md b/docs/src/content/docs/sdk/go.md index dbb0e6a..e80a583 100644 --- a/docs/src/content/docs/sdk/go.md +++ b/docs/src/content/docs/sdk/go.md @@ -37,7 +37,7 @@ Generate typed structs with `secretspec schema` plus [quicktype](https://quicktype.io), then unmarshal `resolved.FieldsJSON()`: ```bash -secretspec schema | quicktype -s schema --lang go -o secrets_gen.go +secretspec schema | quicktype -s schema --top-level SecretSpec --lang go -o secrets_gen.go ``` ```go diff --git a/docs/src/content/docs/sdk/nodejs.md b/docs/src/content/docs/sdk/nodejs.md index 86f2458..ef104f5 100644 --- a/docs/src/content/docs/sdk/nodejs.md +++ b/docs/src/content/docs/sdk/nodejs.md @@ -35,7 +35,7 @@ Generate typed interfaces with `secretspec schema` plus [quicktype](https://quicktype.io), then convert `resolved.fieldsJson()`: ```bash -secretspec schema | quicktype -s schema --lang typescript -o secrets_gen.ts +secretspec schema | quicktype -s schema --top-level SecretSpec --lang typescript -o secrets_gen.ts ``` ```ts diff --git a/docs/src/content/docs/sdk/python.md b/docs/src/content/docs/sdk/python.md index 959ccf9..246f6f1 100644 --- a/docs/src/content/docs/sdk/python.md +++ b/docs/src/content/docs/sdk/python.md @@ -36,7 +36,7 @@ Generate typed classes with `secretspec schema` plus [quicktype](https://quicktype.io), then build them from `resolved.fields()`: ```bash -secretspec schema | quicktype -s schema --lang python -o secrets_gen.py +secretspec schema | quicktype -s schema --top-level SecretSpec --lang python -o secrets_gen.py ``` ```python diff --git a/docs/src/content/docs/sdk/ruby.md b/docs/src/content/docs/sdk/ruby.md index 2033ca5..0f71546 100644 --- a/docs/src/content/docs/sdk/ruby.md +++ b/docs/src/content/docs/sdk/ruby.md @@ -34,7 +34,7 @@ Generate typed classes with `secretspec schema` plus [quicktype](https://quicktype.io), then build them from `resolved.fields`: ```bash -secretspec schema | quicktype -s schema --lang ruby -o secrets_gen.rb +secretspec schema | quicktype -s schema --top-level SecretSpec --lang ruby -o secrets_gen.rb ``` ```ruby diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh index 4602cf8..68f306e 100755 --- a/scripts/ci-sdks.sh +++ b/scripts/ci-sdks.sh @@ -30,7 +30,7 @@ echo "==> Go" ( cd secretspec-go && go test ./... ) echo "==> Ruby" -( cd secretspec-rb && ruby test/test_resolve.rb ) +( cd secretspec-rb && ruby -e 'Dir["test/test_*.rb"].sort.each { |f| require File.expand_path(f) }' ) echo "==> Node" # The Node SDK uses a napi-rs addon (built by its test harness), not the cdylib, diff --git a/secretspec-go/codegen_test.go b/secretspec-go/codegen_test.go new file mode 100644 index 0000000..f1aa6ac --- /dev/null +++ b/secretspec-go/codegen_test.go @@ -0,0 +1,140 @@ +package secretspec + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func hasNpx() bool { + return exec.Command("bash", "-lc", "command -v npx").Run() == nil +} + +func run(t *testing.T, dir, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("%s %v: %v", name, args, err) + } +} + +// TestCodegen drives the full pipeline: secretspec schema -> quicktype --lang go +// -> UnmarshalSecretSpec(resolved.FieldsJSON()), compiling the generated code +// against this SDK. +func TestCodegen(t *testing.T) { + if !hasNpx() { + t.Skip("npx (quicktype) not available") + } + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + goSDK := wd + repo := filepath.Dir(wd) + + // Build + locate the secretspec CLI. + build := exec.Command("cargo", "build", "-p", "secretspec") + build.Dir = repo + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatal(err) + } + metaOut, err := func() ([]byte, error) { + c := exec.Command("cargo", "metadata", "--no-deps", "--format-version", "1") + c.Dir = repo + return c.Output() + }() + if err != nil { + t.Fatal(err) + } + var meta struct { + TargetDirectory string `json:"target_directory"` + } + if err := json.Unmarshal(metaOut, &meta); err != nil { + t.Fatal(err) + } + bin := filepath.Join(meta.TargetDirectory, "debug", "secretspec") + + dir := t.TempDir() + manifest := filepath.Join(dir, "secretspec.toml") + env := filepath.Join(dir, ".env") + os.WriteFile(manifest, []byte(` +[project] +name = "go-codegen" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { required = true } +LOG_LEVEL = { required = false, default = "info" } +`), 0o600) + os.WriteFile(env, []byte("DATABASE_URL=postgres://db\n"), 0o600) + + schema := filepath.Join(dir, "schema.json") + run(t, dir, bin, "-f", manifest, "schema", "-o", schema) + + os.MkdirAll(filepath.Join(dir, "secrets"), 0o755) + run(t, dir, "npx", "--yes", "quicktype", "-s", "schema", schema, + "--top-level", "SecretSpec", "--lang", "go", "--package", "secrets", + "-o", filepath.Join(dir, "secrets", "secrets.go")) + + main := `package main + +import ( + "encoding/json" + "fmt" + + secretspec "github.com/cachix/secretspec/secretspec-go" + "tmpcg/secrets" +) + +func main() { + r, err := secretspec.New(). + WithPath(` + jsonString(manifest) + `). + WithProvider("dotenv://" + ` + jsonString(env) + `). + WithReason("go codegen"). + Load() + if err != nil { + panic(err) + } + data, err := r.FieldsJSON() + if err != nil { + panic(err) + } + s, err := secrets.UnmarshalSecretSpec(data) + if err != nil { + panic(err) + } + out, _ := json.Marshal(s) + fmt.Println(string(out)) +} +` + os.WriteFile(filepath.Join(dir, "main.go"), []byte(main), 0o600) + os.WriteFile(filepath.Join(dir, "go.mod"), []byte( + "module tmpcg\n\ngo 1.23\n\nrequire github.com/cachix/secretspec/secretspec-go v0.0.0\n\nreplace github.com/cachix/secretspec/secretspec-go => "+goSDK+"\n", + ), 0o600) + + run(t, dir, "go", "mod", "tidy") + cmd := exec.Command("go", "run", ".") + cmd.Dir = dir + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + got := string(out) + if !strings.Contains(got, "postgres://db") || !strings.Contains(got, "info") { + t.Fatalf("unexpected generated-code output: %s", got) + } +} + +// jsonString renders s as a Go double-quoted string literal. +func jsonString(s string) string { + b, _ := json.Marshal(s) + return string(b) +} diff --git a/secretspec-node/test/codegen.test.js b/secretspec-node/test/codegen.test.js new file mode 100644 index 0000000..6d23f0a --- /dev/null +++ b/secretspec-node/test/codegen.test.js @@ -0,0 +1,83 @@ +'use strict'; + +// End-to-end codegen pipeline: +// secretspec schema -> quicktype -> toSecretSpec(resolved.fieldsJson()) +// Proves the schema we emit drives quicktype to a typed deserializer that +// consumes the runtime SDK's flat fields map. + +const assert = require('node:assert'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const REPO = path.resolve(__dirname, '..', '..'); + +function hasNpx() { + try { + execFileSync('bash', ['-lc', 'command -v npx'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function ensureAddon() { + if (fs.existsSync(path.resolve(__dirname, '..', 'secretspec.node'))) return; + execFileSync('bash', [path.resolve(__dirname, '..', 'scripts', 'build-addon.sh')], { + stdio: 'inherit', + }); +} + +function secretspecBin() { + execFileSync('cargo', ['build', '-p', 'secretspec'], { cwd: REPO, stdio: 'inherit' }); + const meta = JSON.parse( + execFileSync('cargo', ['metadata', '--no-deps', '--format-version', '1'], { cwd: REPO }), + ); + return path.join(meta.target_directory, 'debug', 'secretspec'); +} + +const MANIFEST = ` +[project] +name = "node-codegen" +revision = "1.0" + +[profiles.default] +DATABASE_URL = { required = true } +LOG_LEVEL = { required = false, default = "info" } +SENTRY_DSN = { required = false } +`; + +test('quicktype-generated converter consumes fieldsJson()', { skip: !hasNpx() }, () => { + ensureAddon(); + const { SecretSpec } = require('../index.js'); + const bin = secretspecBin(); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ss-node-cg-')); + const manifest = path.join(dir, 'secretspec.toml'); + const env = path.join(dir, '.env'); + fs.writeFileSync(manifest, MANIFEST); + fs.writeFileSync(env, 'DATABASE_URL=postgres://db\n'); + + const schema = path.join(dir, 'schema.json'); + execFileSync(bin, ['-f', manifest, 'schema', '-o', schema]); + + const generated = path.join(dir, 'gen.js'); + execFileSync('npx', [ + '--yes', 'quicktype', '-s', 'schema', schema, + '--top-level', 'SecretSpec', '--lang', 'javascript', '-o', generated, + ]); + + const { toSecretSpec } = require(generated); + + const resolved = SecretSpec.builder() + .withPath(manifest) + .withProvider(`dotenv://${env}`) + .withReason('node codegen') + .load(); + + const typed = toSecretSpec(resolved.fieldsJson()); + assert.equal(typed.DATABASE_URL, 'postgres://db'); + assert.equal(typed.LOG_LEVEL, 'info'); // from default +}); diff --git a/secretspec-py/tests/test_codegen.py b/secretspec-py/tests/test_codegen.py index ec00743..a09417d 100644 --- a/secretspec-py/tests/test_codegen.py +++ b/secretspec-py/tests/test_codegen.py @@ -52,6 +52,8 @@ def _generate_types(tmp_path: pathlib.Path, name: str): "-s", "schema", str(schema), + "--top-level", + "SecretSpec", "--lang", "python", "--python-version", diff --git a/secretspec-rb/test/test_codegen.rb b/secretspec-rb/test/test_codegen.rb new file mode 100644 index 0000000..0105449 --- /dev/null +++ b/secretspec-rb/test/test_codegen.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# End-to-end codegen pipeline: +# secretspec schema -> quicktype --lang ruby -> SecretSpec.from_dynamic!(resolved.fields) +# Proves the schema drives quicktype to a typed class that consumes the runtime +# SDK's flat fields hash. + +require "json" +require "rbconfig" +require "tmpdir" +require "minitest/autorun" + +REPO = File.expand_path("../..", __dir__) + +def npx? + system("bash", "-lc", "command -v npx", out: File::NULL, err: File::NULL) +end + +def build_artifacts + unless system("cargo", "build", "-p", "secretspec-ffi", "-p", "secretspec", chdir: REPO) + raise "cargo build failed" + end + meta = JSON.parse(`cd #{REPO} && cargo metadata --no-deps --format-version 1`) + target = meta["target_directory"] + lib = RbConfig::CONFIG["host_os"] =~ /darwin/ ? "libsecretspec_ffi.dylib" : "libsecretspec_ffi.so" + [File.join(target, "debug", lib), File.join(target, "debug", "secretspec")] +end + +class CodegenTest < Minitest::Test + def test_quicktype_ruby_consumes_fields + skip "npx (quicktype) not available" unless npx? + + lib, bin = build_artifacts + ENV["SECRETSPEC_FFI_LIB"] = lib + + Dir.mktmpdir do |dir| + manifest = File.join(dir, "secretspec.toml") + env_path = File.join(dir, ".env") + File.write(manifest, <<~TOML) + [project] + name = "rb-codegen" + revision = "1.0" + + [profiles.default] + DATABASE_URL = { required = true } + LOG_LEVEL = { required = false, default = "info" } + TOML + File.write(env_path, "DATABASE_URL=postgres://db\n") + + schema = File.join(dir, "schema.json") + assert system(bin, "-f", manifest, "schema", "-o", schema), "schema failed" + + gen = File.join(dir, "gen.rb") + assert system("npx", "--yes", "quicktype", "-s", "schema", schema, + "--top-level", "SecretSpec", "--lang", "ruby", "-o", gen), + "quicktype failed" + + # quicktype's Ruby output needs dry-struct/dry-types; install them into a + # throwaway gem home and make them requireable in-process. + gemhome = File.join(dir, "gems") + assert system("gem", "install", "--no-document", "--install-dir", gemhome, + "dry-struct", "dry-types", out: File::NULL), + "gem install failed" + Gem.use_paths(gemhome, [gemhome] + Gem.path) + + require gen + require_relative "../lib/secretspec" + + resolved = Secretspec::SecretSpec.builder + .with_path(manifest) + .with_provider("dotenv://#{env_path}") + .with_reason("rb codegen") + .load + typed = SecretSpec.from_dynamic!(resolved.fields) + assert_equal "postgres://db", typed.database_url + assert_equal "info", typed.log_level + end + end +end diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index 43cf7b6..6eaa5f3 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -103,15 +103,19 @@ enum Commands { #[arg(long)] explain: bool, }, - /// Emit a JSON Schema for the manifest's typed shapes. + /// Emit a JSON Schema for the manifest's typed shape. /// - /// Feed this to [quicktype](https://quicktype.io) to generate idiomatic - /// typed accessors (plus deserializers) for any language, then hand the - /// deserializer the flat map from each SDK's `fields()` helper. Value-free: - /// reads only the manifest, never a provider. + /// Feed this to [quicktype](https://quicktype.io) to generate an idiomatic + /// typed accessor (plus a deserializer) for any language, then hand the + /// deserializer the flat map from each SDK's `fields()` helper. By default it + /// describes the union `SecretSpec` (safe for any profile); `--profile` gives + /// that profile's exact fields. Value-free: reads only the manifest. /// - /// Example: `secretspec schema | quicktype -s schema --lang typescript` + /// Example: `secretspec schema | quicktype -s schema --top-level SecretSpec --lang typescript` Schema { + /// Emit the schema for this profile's fields instead of the union + #[arg(short = 'P', long)] + profile: Option, /// Write to this file instead of stdout #[arg(short, long)] output: Option, @@ -706,10 +710,11 @@ pub fn main() -> Result<()> { Ok(()) } // Generate typed accessors for another language (value-free) - Commands::Schema { output } => { + Commands::Schema { profile, output } => { let app = load_secrets(&cli.file, &cli.reason)?; let ir = crate::codegen::build_ir(app.config()); - let schema = crate::codegen::schema::emit(&ir); + let schema = crate::codegen::schema::emit(&ir, profile.as_deref()) + .map_err(|e| miette!("{e}"))?; match output { Some(path) => fs::write(&path, schema) .into_diagnostic() diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs index b0abcd4..fc433eb 100644 --- a/secretspec/src/codegen.rs +++ b/secretspec/src/codegen.rs @@ -2,11 +2,11 @@ //! //! Every typed-accessor generator computes the *same* decisions from a manifest: //! which secrets exist, whether a field is optional, whether it is a file path, -//! how profiles map to types. If each generator (the Rust derive macro plus the -//! eventual TypeScript/Python/Go/Ruby emitters) recomputed those decisions, they -//! would drift. This module is the single brain: a manifest is reduced to a -//! language-neutral [`CodegenIr`] once, and every emitter is a thin template -//! over it. +//! how profiles map to types. If each generator (the Rust derive macro and the +//! JSON Schema emitter that drives quicktype for other languages) recomputed +//! those decisions, they would drift. This module is the single brain: a +//! manifest is reduced to a language-neutral [`CodegenIr`] once, and each +//! emitter is a thin template over it. //! //! The IR deliberately mirrors the two shapes the derive crate exposes: //! - a **union** field set (`SecretSpec`) safe to use without knowing the @@ -176,14 +176,16 @@ pub fn build_ir(config: &Config) -> CodegenIr { /// JSON Schema emitter. /// /// Rather than hand-write typed accessors per language, we emit a JSON Schema -/// describing the manifest's types and let [quicktype](https://quicktype.io) -/// generate the idiomatic types and deserializers for any target language. We -/// then maintain only the small generic `fields()` helper in each runtime SDK, -/// which hands quicktype's deserializer a flat `{SECRET_NAME: value}` map. +/// describing one manifest shape and let [quicktype](https://quicktype.io) +/// generate the idiomatic type and deserializer for any target language. We then +/// maintain only the small generic `fields()` helper in each runtime SDK, which +/// hands quicktype's deserializer a flat `{SECRET_NAME: value}` map. /// -/// The schema defines one type per shape the derive crate exposes: `SecretSpec` -/// (the union, safe for any profile) and one `Secrets` per profile. A -/// `Manifest` wrapper references them all so quicktype emits every type. +/// The schema is a single-root object so quicktype emits a properly named type +/// with a converter in every language (a wrapper or `$ref` root makes quicktype +/// drop the converter or rename the type). By default it describes the union +/// `SecretSpec` (safe for any profile); with a profile it describes that +/// profile's exact fields. Pair it with `quicktype --top-level `. pub mod schema { use super::{CodegenIr, IrField}; use serde_json::{Map, Value, json}; @@ -198,7 +200,7 @@ pub mod schema { } } - fn object_schema(fields: &[IrField]) -> Value { + fn object_schema(title: &str, fields: &[IrField]) -> Value { let mut properties = Map::new(); let mut required = Vec::new(); for field in fields { @@ -208,8 +210,10 @@ pub mod schema { } } json!({ + "$schema": "http://json-schema.org/draft-06/schema#", "type": "object", "additionalProperties": false, + "title": title, "properties": Value::Object(properties), "required": required, }) @@ -224,44 +228,29 @@ pub mod schema { } /// Emit the JSON Schema (draft-06, the dialect quicktype consumes) for the - /// manifest's types. - pub fn emit(ir: &CodegenIr) -> String { - let mut definitions = Map::new(); - let mut wrapper_props = Map::new(); - - definitions.insert("SecretSpec".to_string(), object_schema(&ir.union)); - wrapper_props.insert( - "SecretSpec".to_string(), - json!({ "$ref": "#/definitions/SecretSpec" }), - ); - - for profile in &ir.profile_fields { - let name = format!("{}Secrets", capitalize(&profile.name)); - definitions.insert(name.clone(), object_schema(&profile.fields)); - wrapper_props.insert( - name.clone(), - json!({ "$ref": format!("#/definitions/{name}") }), - ); - } - - // A wrapper that references every type so quicktype emits all of them. - definitions.insert( - "Manifest".to_string(), - json!({ - "type": "object", - "additionalProperties": false, - "properties": Value::Object(wrapper_props), - }), - ); - - let schema = json!({ - "$schema": "http://json-schema.org/draft-06/schema#", - "$ref": "#/definitions/Manifest", - "title": ir.project, - "definitions": Value::Object(definitions), - }); - - format!("{}\n", serde_json::to_string_pretty(&schema).unwrap()) + /// union (`profile = None`) or one profile's fields. Returns an error if the + /// named profile does not exist. + pub fn emit(ir: &CodegenIr, profile: Option<&str>) -> Result { + let schema = match profile { + None => object_schema("SecretSpec", &ir.union), + Some(name) => { + let found = ir + .profile_fields + .iter() + .find(|p| p.name == name) + .ok_or_else(|| { + format!( + "unknown profile '{name}'; available: {}", + ir.profiles.join(", ") + ) + })?; + object_schema(&format!("{}Secrets", capitalize(name)), &found.fields) + } + }; + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&schema).unwrap() + )) } } @@ -419,19 +408,14 @@ mod tests { ), ])); - let rendered = schema::emit(&ir); - let value: serde_json::Value = serde_json::from_str(&rendered).unwrap(); - let defs = &value["definitions"]; - - // The union type and one type per profile, plus the Manifest wrapper. - assert!(defs["SecretSpec"].is_object()); - assert!(defs["DevelopmentSecrets"].is_object()); - assert!(defs["ProductionSecrets"].is_object()); - assert_eq!(value["$ref"], "#/definitions/Manifest"); + // Union schema: single-root object titled SecretSpec. + let union: serde_json::Value = + serde_json::from_str(&schema::emit(&ir, None).unwrap()).unwrap(); + assert_eq!(union["type"], "object"); + assert_eq!(union["title"], "SecretSpec"); // Required vs nullable: DATABASE_URL required everywhere; API_KEY optional // in development, so optional in the union and nullable in the schema. - let union = &defs["SecretSpec"]; assert_eq!(union["properties"]["DATABASE_URL"]["type"], "string"); assert_eq!( union["properties"]["API_KEY"]["type"], @@ -445,6 +429,16 @@ mod tests { .collect(); assert!(required.contains(&"DATABASE_URL")); assert!(!required.contains(&"API_KEY")); + + // A profile schema is titled Secrets with that profile's fields. + let prod: serde_json::Value = + serde_json::from_str(&schema::emit(&ir, Some("production")).unwrap()).unwrap(); + assert_eq!(prod["title"], "ProductionSecrets"); + assert!(prod["properties"]["DATABASE_URL"].is_object()); + assert!(prod["properties"]["API_KEY"].is_null()); // not in production + + // An unknown profile is an error. + assert!(schema::emit(&ir, Some("nope")).is_err()); } #[test] From 0bd9ca8467ebfb2d8ce9a434d340b7f9d042147b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:51:03 -0400 Subject: [PATCH 22/56] docs: add an SDK overview page The SDK section jumped straight into per-language pages with no explanation of how the polyglot stack works. Adds an Overview page (one Rust resolver, thin clients over the C ABI / napi addon, the shared runtime API and error model, typed access via schema+quicktype, and the bundled-library distribution model) and wires it as the first item in the SDK sidebar. Docs site builds clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/astro.config.mjs | 1 + docs/src/content/docs/sdk/overview.md | 69 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/src/content/docs/sdk/overview.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 983dcbb..dcf937e 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -190,6 +190,7 @@ Secrets can be stored in: keyring (default), dotenv files, environment variables { label: "SDK", items: [ + { label: "Overview", slug: "sdk/overview" }, { label: "Rust SDK", slug: "sdk/rust" }, { label: "Python SDK", slug: "sdk/python" }, { label: "Go SDK", slug: "sdk/go" }, diff --git a/docs/src/content/docs/sdk/overview.md b/docs/src/content/docs/sdk/overview.md new file mode 100644 index 0000000..3b91436 --- /dev/null +++ b/docs/src/content/docs/sdk/overview.md @@ -0,0 +1,69 @@ +--- +title: SDK Overview +description: How the SecretSpec language SDKs work +--- + +SecretSpec ships SDKs for Rust, Python, Go, Ruby, and Node.js/TypeScript. They +all resolve secrets from the same declarative `secretspec.toml`, and they all +behave identically, because they share one resolver. + +## One resolver, thin clients + +Resolution (providers, fallback chains, profiles, generation, `as_path` +materialization) lives in a single Rust core. Each SDK is a thin client over +that core rather than a reimplementation: + +- **Rust** uses the library directly, with a compile-time derive macro for + strongly-typed access. +- **Python** (cffi), **Go** (purego), and **Ruby** (Fiddle) load the + `secretspec-ffi` C ABI and exchange a small JSON request/response with it. +- **Node.js/TypeScript** uses a [napi-rs](https://napi.rs/) native addon that + embeds the same resolver. + +Because resolution happens in one place, every provider, chain, profile, and +generator works the same in every language, and a new provider added to the core +is immediately available everywhere with no per-SDK change. A cross-language +conformance suite asserts that all the SDKs reduce the same inputs to the same +result. + +## The runtime API + +Each SDK mirrors the Rust derive crate's vocabulary: a builder that takes a +provider, profile, and an access reason, and a `load`/`resolve` that returns the +resolved secrets plus the provider and profile used. A missing required secret +is a typed error, distinct from a transport failure (which carries a stable +`kind`). Secrets exposed `as_path` come back as a readable file path. + +```python +from secretspec import SecretSpec + +resolved = SecretSpec.builder().with_provider("keyring://").with_reason("boot").load() +print(resolved.secrets["DATABASE_URL"].get) +``` + +See each language's page for the idiomatic spelling: [Rust](/sdk/rust), +[Python](/sdk/python), [Go](/sdk/go), [Ruby](/sdk/ruby), and +[Node.js](/sdk/nodejs). + +## Typed access + +Beyond the Rust derive macro, typed accessors for the other languages are +generated from the manifest. `secretspec schema` emits a JSON Schema for the +secret shape; [quicktype](https://quicktype.io) turns it into an idiomatic type +and deserializer for any language, which you build from the SDK's `fields()` +map: + +```bash +secretspec schema | quicktype -s schema --top-level SecretSpec --lang +``` + +This keeps the per-language surface tiny: the SDK only provides `fields()`, and +quicktype owns the type generation. + +## Distribution + +The SDKs are designed to install with no native build: the C ABI library is +bundled in the Python wheel and the Ruby gem, embedded in the Go module, and +shipped as a napi-rs addon for Node. The native library is otherwise discovered +from the `SECRETSPEC_FFI_LIB` environment variable or a Cargo `target` +directory, which is how it works from a source checkout. From ee404ed88408b51c0f81ac7a193472f13fab629b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:53:48 -0400 Subject: [PATCH 23/56] docs: add a Language SDKs block to the landing page The landing page only showcased the Rust SDK. Adds a "Use it from any language" showcase section after it, with the shared builder API in Python, Node.js, Go, and Ruby, plus the one-resolver/thin-client framing and links to the SDK overview and the schema+quicktype typed-access path. Landing page builds clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/src/pages/index.astro | 73 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro index decfc64..e257da8 100644 --- a/docs/src/pages/index.astro +++ b/docs/src/pages/index.astro @@ -492,6 +492,79 @@ secretspec_derive::declare_se + +
+
+
+ Language SDKs +

Use it from any language.

+

+ Rust, Python, Go, Ruby, and Node.js/TypeScript all resolve from the same secretspec.toml through one Rust core — every provider, chain, and profile behaves identically, with no per-language logic. How the SDKs work → +

+
+ +
+ +
+
+
+ Python +
+ 
from secretspec import SecretSpec
+
+s = SecretSpec.builder() \
+    .with_provider("keyring://") \
+    .with_reason("boot").load()
+print(s.secrets["DATABASE_URL"].get)
+
+ + +
+
+
+ Node.js +
+ 
const { SecretSpec } = require("secretspec");
+
+const s = SecretSpec.builder()
+  .withProvider("keyring://")
+  .withReason("boot").load();
+console.log(s.secrets.DATABASE_URL.get());
+
+
+ +
+ +
+
+
+ Go +
+ 
s, _ := secretspec.New().
+    WithProvider("keyring://").
+    WithReason("boot").Load()
+fmt.Println(s.Secrets["DATABASE_URL"].Get())
+
+ + +
+
+
+ Ruby +
+ 
s = Secretspec::SecretSpec.builder
+      .with_provider("keyring://")
+      .with_reason("boot").load
+puts s.secrets["DATABASE_URL"].get
+
+
+ +

+ Each SDK is a thin client over one native core — the secretspec-ffi library (or a napi-rs addon for Node) — so a new provider is available everywhere at once. Need typed access? secretspec schema feeds quicktype to generate typed classes for any language, and the Rust SDK checks them at compile time. SDK overview → +

+
+
+
From 415605087af4bd3547b19bfb9b4d963347ec5153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 04:58:23 -0400 Subject: [PATCH 24/56] docs: merge the Rust and Language SDKs landing sections into one The landing page had a standalone "Compile-time secrets in Rust" section adjacent to the new "Language SDKs" section, so Rust read as separate from the SDKs when it is one of them. Folded the Rust main.rs example into the single Language SDKs section as its compile-time highlight, after the Python/Node/Go/ Ruby snippets. One coherent SDK section; landing page builds clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/src/pages/index.astro | 73 +++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro index e257da8..10d0294 100644 --- a/docs/src/pages/index.astro +++ b/docs/src/pages/index.astro @@ -452,46 +452,6 @@ $ secretspec audit -n 1
- -
-
-
- Type-safe SDK -

Compile-time secrets in Rust.

-

- The proc macro reads secretspec.toml at compile time and generates strongly-typed structs. Misspelling a secret name fails the build, not your deploy. SDK reference → -

-
- -
-
-
- main.rs -
- 
// Generate typed structs from secretspec.toml
-secretspec_derive::declare_secrets!("secretspec.toml");
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let secrets = Secrets::builder()
-        .with_provider("keyring")
-        .with_profile(Profile::Production)
-        .load()?;
-
-    // DATABASE_URL → database_url (compile-time checked)
-    println!("Database: {}", secrets.secrets.database_url);
-
-    // Optional secrets are Option<String>
-    if let Some(redis) = &secrets.secrets.redis_url {
-        println!("Redis: {}", redis);
-    }
-
-    secrets.secrets.set_as_env_vars();
-    Ok(())
-}
-
-
-
-
@@ -559,8 +519,39 @@ puts s.secrets["DATABASE_URL"].get
+

+ And in Rust, a proc macro reads secretspec.toml at compile time and generates strongly-typed structs — misspelling a secret fails the build, not your deploy. Rust SDK → +

+ +
+
+
+ main.rs +
+ 
// Generate typed structs from secretspec.toml
+secretspec_derive::declare_secrets!("secretspec.toml");
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let secrets = Secrets::builder()
+        .with_provider("keyring")
+        .with_profile(Profile::Production)
+        .load()?;
+
+    // DATABASE_URL → database_url (compile-time checked)
+    println!("Database: {}", secrets.secrets.database_url);
+
+    // Optional secrets are Option<String>
+    if let Some(redis) = &secrets.secrets.redis_url {
+        println!("Redis: {}", redis);
+    }
+
+    secrets.secrets.set_as_env_vars();
+    Ok(())
+}
+
+

- Each SDK is a thin client over one native core — the secretspec-ffi library (or a napi-rs addon for Node) — so a new provider is available everywhere at once. Need typed access? secretspec schema feeds quicktype to generate typed classes for any language, and the Rust SDK checks them at compile time. SDK overview → + Every SDK is a thin client over one native core — the secretspec-ffi library (or a napi-rs addon for Node) — so a new provider is available everywhere at once. For typed access in the dynamic languages, secretspec schema feeds quicktype to generate typed classes. SDK overview →

From 082a59ead17dd5d6336989c33279b93c7bdd99d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:34:57 -0400 Subject: [PATCH 25/56] fix(core): align per-profile schema, harden resolve_json, dedupe codegen - Per-profile JSON Schema (`schema --profile

`) now allows additional properties: `resolve --profile

` returns the profile's own secrets plus those inherited from `default`, which the per-profile type intentionally does not list (matching the derive macro), so a strict quicktype deserializer would otherwise reject a valid resolve result. The union schema stays exhaustive. - `resolve_json` now catches panics itself, so both native boundaries that funnel through it (the C ABI and the napi-rs Node addon) return the same `{"ok":false,"error":...}` envelope on an internal panic. - `secretspec::codegen` exposes one shared `capitalize`, used by both the schema emitter and the derive macro (was a byte-identical copy in each), so profile type-name casing can never drift. - `build_ir` computes the union field set in a single pass instead of re-scanning every profile per field; `validate`/`resolve` resolve each secret's merged config once per pass instead of twice. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 39 +++++++++ secretspec-derive/src/lib.rs | 33 +------- secretspec-derive/src/tests.rs | 14 ++-- secretspec/src/codegen.rs | 140 +++++++++++++++++++-------------- secretspec/src/resolve.rs | 12 ++- secretspec/src/secrets.rs | 24 ++++-- 6 files changed, 157 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb5a87b..7c94ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `ValidationErrors::report()`, returning the new public `ResolutionReport`, `SecretResolution`, and `ResolutionStatus` types. +### Fixed +- `secretspec::resolve_json` now catches panics itself, so both native + boundaries that funnel through it (the `secretspec-ffi` C ABI and the napi-rs + Node addon) return the same `{"ok": false, "error": {...}}` envelope on an + internal panic. Previously only the C ABI caught panics, so a panic in the + Node addon surfaced as an opaque thrown error instead of the documented + envelope. +- A per-profile JSON Schema (`secretspec schema --profile

`) now allows + additional properties. `secretspec resolve --profile

` returns the + profile's own secrets plus those inherited from the `default` profile (the + runtime resolver merges them; the per-profile type intentionally does not, + matching the derive macro), so a strict quicktype-generated deserializer would + otherwise reject a valid resolve result over the inherited keys. The union + schema stays exhaustive (`additionalProperties: false`). +- `secretspec resolve --profile

` and the SDKs no longer export an empty or + literal-`"null"` environment variable for a secret with no usable value + (e.g. under `no_values`): the Go, Node, and Ruby SDKs now skip such secrets in + `set_as_env`, matching Python. Ruby previously *deleted* the variable + (`ENV[name] = nil`); Node set the string `"null"`; Go set `""`. +- The Go, Python, Ruby, and Node SDKs now validate the response + `schema_version` against the version they were built for and surface a clear + error on mismatch, instead of silently misparsing a skewed `secretspec-ffi` + library. They also no longer panic / raise an opaque error when a successful + envelope is missing its `response` object. +- The Go SDK extracts the embedded `cdylib` into an owner-only (`0o700`) temp + directory and re-extracts when the cached file's content hash (not just its + size) differs, closing a predictable-path load and a stale-file reuse. + +### Changed +- `secretspec::codegen` exposes a single `capitalize` helper now shared by both + the JSON Schema emitter and the `secretspec-derive` macro (previously a + byte-identical copy in each), so profile type-name casing can never drift. +- `secretspec::codegen::build_ir` computes the union field set in one pass over + all profiles instead of re-scanning every profile per field, and + `validate`/`resolve` resolve each secret's merged config once per pass instead + of twice. +- The Python, Ruby, and Node SDK builders gained `with_no_values` / + `withNoValues` for parity with the Go SDK and the underlying request contract. + ## [0.12.0] - 2026-06-08 ### Added diff --git a/secretspec-derive/src/lib.rs b/secretspec-derive/src/lib.rs index 093f088..19a109f 100644 --- a/secretspec-derive/src/lib.rs +++ b/secretspec-derive/src/lib.rs @@ -22,7 +22,7 @@ use proc_macro::TokenStream; use quote::{format_ident, quote}; use secretspec::Config; -use secretspec::codegen::{CodegenIr, IrField, build_ir}; +use secretspec::codegen::{CodegenIr, IrField, build_ir, capitalize}; use std::collections::{BTreeMap, HashSet}; use syn::{LitStr, parse_macro_input}; @@ -226,7 +226,7 @@ impl ProfileVariant { /// // variant.capitalized == "Production" /// ``` fn new(name: String) -> Self { - let capitalized = capitalize_first(&name); + let capitalized = capitalize(&name); Self { name, capitalized } } @@ -456,7 +456,7 @@ fn is_valid_rust_identifier(s: &str) -> bool { /// - Profile names with invalid characters (e.g., "prod-env") fn validate_profile_identifiers(config: &Config, errors: &mut Vec) { for profile_name in config.profiles.keys() { - let variant_name = capitalize_first(profile_name); + let variant_name = capitalize(profile_name); if !is_valid_rust_identifier(&variant_name) { errors.push(format!( "Profile '{}' produces invalid Rust enum variant '{}'", @@ -1425,33 +1425,6 @@ fn generate_secret_spec_code(config: Config) -> proc_macro2::TokenStream { } } -/// Capitalize the first character of a string. -/// -/// Used to convert profile names to enum variant names. -/// -/// # Arguments -/// -/// * `s` - The string to capitalize -/// -/// # Returns -/// -/// A new string with the first character capitalized -/// -/// # Examples -/// -/// ```ignore -/// assert_eq!(capitalize_first("production"), "Production"); -/// assert_eq!(capitalize_first("test_env"), "Test_env"); -/// assert_eq!(capitalize_first(""), ""); -/// ``` -fn capitalize_first(s: &str) -> String { - let mut chars = s.chars(); - match chars.next() { - None => String::new(), - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - } -} - #[cfg(test)] #[path = "tests.rs"] mod tests; diff --git a/secretspec-derive/src/tests.rs b/secretspec-derive/src/tests.rs index acc0629..eb14974 100644 --- a/secretspec-derive/src/tests.rs +++ b/secretspec-derive/src/tests.rs @@ -1,15 +1,15 @@ #[cfg(test)] mod tests { - use crate::capitalize_first; use secretspec::Config; + use secretspec::codegen::capitalize; #[test] - fn test_capitalize_first() { - assert_eq!(capitalize_first("development"), "Development"); - assert_eq!(capitalize_first("production"), "Production"); - assert_eq!(capitalize_first("test"), "Test"); - assert_eq!(capitalize_first(""), ""); - assert_eq!(capitalize_first("a"), "A"); + fn test_capitalize() { + assert_eq!(capitalize("development"), "Development"); + assert_eq!(capitalize("production"), "Production"); + assert_eq!(capitalize("test"), "Test"); + assert_eq!(capitalize(""), ""); + assert_eq!(capitalize("a"), "A"); } #[test] diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs index fc433eb..4e5dd72 100644 --- a/secretspec/src/codegen.rs +++ b/secretspec/src/codegen.rs @@ -25,7 +25,7 @@ use crate::config::{Config, Secret}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, HashMap}; /// One field in a generated type. `name` is the canonical `UPPER_SNAKE` env key /// and the source of truth; each emitter applies its own casing. @@ -71,59 +71,72 @@ fn is_secret_optional(secret: &Secret) -> bool { secret.required != Some(true) } -/// For the union type a field is optional if it is optional in, or absent from, -/// any profile, so the union can safely represent secrets from any profile. -fn is_field_optional_across_profiles(name: &str, config: &Config) -> bool { - for profile in config.profiles.values() { - match profile.secrets.get(name) { - Some(secret) if !is_secret_optional(secret) => {} - // Optional in this profile, or missing from it. - _ => return true, - } - } - false -} - -/// For the union type a field is a path if any profile declares it `as_path`. -fn is_field_as_path_across_profiles(name: &str, config: &Config) -> bool { - config - .profiles - .values() - .any(|profile| profile.secrets.get(name).and_then(|s| s.as_path) == Some(true)) -} - -/// Pick a description for a union field: the first profile (by sorted name) that -/// declares one. -fn union_description(name: &str, config: &Config) -> Option { - let mut profile_names: Vec<&String> = config.profiles.keys().collect(); - profile_names.sort(); - profile_names.into_iter().find_map(|profile_name| { - config.profiles[profile_name] - .secrets - .get(name) - .and_then(|s| s.description.clone()) - }) -} - /// Build the union field set: every unique secret across all profiles, sorted. +/// +/// Computed in a single pass over every `(profile, secret)` rather than +/// re-scanning all profiles per field. A union field is: +/// - optional if it is optional in, or absent from, *any* profile (equivalently, +/// required only when present and `required = true` in **every** profile); +/// - a path if *any* profile declares it `as_path`; +/// - described by the first profile, in sorted name order, that declares a +/// description. fn build_union(config: &Config) -> Vec { - let names: BTreeSet<&String> = config - .profiles - .values() - .flat_map(|profile| profile.secrets.keys()) - .collect(); + let total_profiles = config.profiles.len(); - names - .into_iter() - .map(|name| IrField { - name: name.clone(), - optional: is_field_optional_across_profiles(name, config), - as_path: is_field_as_path_across_profiles(name, config), - description: union_description(name, config), + // Sorted once so "first description wins" is deterministic. + let mut sorted_profiles: Vec<&String> = config.profiles.keys().collect(); + sorted_profiles.sort(); + + struct Acc { + /// Profiles where the secret is present and `required = true`. + required_count: usize, + as_path: bool, + description: Option, + } + let mut acc: BTreeMap = BTreeMap::new(); + + for profile_name in sorted_profiles { + for (name, secret) in &config.profiles[profile_name].secrets { + let entry = acc.entry(name.clone()).or_insert(Acc { + required_count: 0, + as_path: false, + description: None, + }); + if !is_secret_optional(secret) { + entry.required_count += 1; + } + if secret.as_path == Some(true) { + entry.as_path = true; + } + if entry.description.is_none() { + entry.description = secret.description.clone(); + } + } + } + + acc.into_iter() + .map(|(name, a)| IrField { + name, + // Required only if present and `required = true` in every profile; + // optional if optional in, or missing from, any profile. + optional: a.required_count != total_profiles, + as_path: a.as_path, + description: a.description, }) .collect() } +/// Capitalize the first character, leaving the rest unchanged. Shared by the +/// JSON Schema emitter (for `Secrets` titles) and the derive macro (for +/// `SecretSpecProfile::` names) so the two never disagree on casing. +pub fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + } +} + /// Build the exact field set for one profile's raw secrets, sorted by name. fn build_profile_fields(secrets: &HashMap) -> Vec { let mut fields: Vec = secrets @@ -187,7 +200,7 @@ pub fn build_ir(config: &Config) -> CodegenIr { /// `SecretSpec` (safe for any profile); with a profile it describes that /// profile's exact fields. Pair it with `quicktype --top-level `. pub mod schema { - use super::{CodegenIr, IrField}; + use super::{CodegenIr, IrField, capitalize}; use serde_json::{Map, Value, json}; fn property_type(field: &IrField) -> Value { @@ -200,7 +213,7 @@ pub mod schema { } } - fn object_schema(title: &str, fields: &[IrField]) -> Value { + fn object_schema(title: &str, fields: &[IrField], additional_properties: bool) -> Value { let mut properties = Map::new(); let mut required = Vec::new(); for field in fields { @@ -212,27 +225,29 @@ pub mod schema { json!({ "$schema": "http://json-schema.org/draft-06/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": additional_properties, "title": title, "properties": Value::Object(properties), "required": required, }) } - fn capitalize(s: &str) -> String { - let mut chars = s.chars(); - match chars.next() { - None => String::new(), - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - } - } - /// Emit the JSON Schema (draft-06, the dialect quicktype consumes) for the /// union (`profile = None`) or one profile's fields. Returns an error if the /// named profile does not exist. + /// + /// The union lists every secret across every profile, so it is **exhaustive** + /// (`additionalProperties: false`): a runtime `fields()` map can never carry a + /// key the union does not declare. A per-profile schema lists only the + /// secrets that profile declares, but `secretspec resolve --profile

` + /// returns those **plus** secrets inherited from the `default` profile (the + /// runtime resolver merges them; the per-profile type intentionally does not, + /// matching the derive macro). So per-profile schemas allow additional + /// properties — otherwise a strict quicktype deserializer would reject a valid + /// resolve result over the inherited keys. pub fn emit(ir: &CodegenIr, profile: Option<&str>) -> Result { let schema = match profile { - None => object_schema("SecretSpec", &ir.union), + None => object_schema("SecretSpec", &ir.union, false), Some(name) => { let found = ir .profile_fields @@ -244,7 +259,7 @@ pub mod schema { ir.profiles.join(", ") ) })?; - object_schema(&format!("{}Secrets", capitalize(name)), &found.fields) + object_schema(&format!("{}Secrets", capitalize(name)), &found.fields, true) } }; Ok(format!( @@ -413,6 +428,8 @@ mod tests { serde_json::from_str(&schema::emit(&ir, None).unwrap()).unwrap(); assert_eq!(union["type"], "object"); assert_eq!(union["title"], "SecretSpec"); + // The union is exhaustive across every profile, so it is strict. + assert_eq!(union["additionalProperties"], false); // Required vs nullable: DATABASE_URL required everywhere; API_KEY optional // in development, so optional in the union and nullable in the schema. @@ -436,6 +453,9 @@ mod tests { assert_eq!(prod["title"], "ProductionSecrets"); assert!(prod["properties"]["DATABASE_URL"].is_object()); assert!(prod["properties"]["API_KEY"].is_null()); // not in production + // A per-profile schema must tolerate the default-inherited secrets that + // `resolve --profile production` adds beyond production's own fields. + assert_eq!(prod["additionalProperties"], true); // An unknown profile is an error. assert!(schema::emit(&ir, Some("nope")).is_err()); diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs index 499ce09..078672f 100644 --- a/secretspec/src/resolve.rs +++ b/secretspec/src/resolve.rs @@ -181,7 +181,17 @@ fn resolve_envelope(request_json: &str) -> JsonEnvelope { /// `provider`, `profile`, `reason`, and `no_values`. The response carries secret /// values; treat its bytes as sensitive. pub fn resolve_json(request_json: &str) -> String { - let envelope = resolve_envelope(request_json); + // Catch panics here, at the one place both native boundaries funnel through + // (the C ABI in `secretspec-ffi` and the napi-rs Node addon). Unwinding across + // either is undefined behavior, and turning a panic into the same + // `{"ok":false,"error":...}` envelope every binding already parses means all + // bindings behave identically — the C ABI no longer needs to be the only one + // guarding the boundary. + let envelope = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + resolve_envelope(request_json) + })) + .unwrap_or_else(|_| envelope_error("internal", "internal panic during resolve")); + serde_json::to_string(&envelope).unwrap_or_else(|_| { "{\"ok\":false,\"error\":{\"kind\":\"serialize\",\"message\":\"failed to serialize response\"}}".to_string() }) diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index eb6004d..468dce2 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -2124,15 +2124,27 @@ impl Secrets { keys }; + // Resolve each secret's effective config once. Both the + // provider-grouping pass and the processing pass below need it, + // and the field-level merge (current profile over `default`) it + // performs is not free — resolving it twice per secret was waste. + let secret_configs: HashMap = all_secrets + .iter() + .map(|(name, _)| { + let cfg = self + .resolve_secret_config(name, Some(&profile_name)) + .expect("Secret should exist in config since we're iterating over it"); + (name.clone(), cfg) + }) + .collect(); + let override_uri = self.resolve_provider_override(None); let mut provider_groups: HashMap, Vec> = HashMap::new(); let mut secret_primary_uris: HashMap> = HashMap::new(); for (name, _) in &all_secrets { - let secret_config = self - .resolve_secret_config(name, Some(&profile_name)) - .expect("Secret should exist in config since we're iterating over it"); + let secret_config = &secret_configs[name]; let provider_uri = match (&override_uri, secret_config.providers.as_deref()) { (Some(uri), _) => Some(uri.clone()), @@ -2199,9 +2211,7 @@ impl Secrets { // resolution report (status, which provider answered, generated, // defaulted). for (name, _) in all_secrets { - let secret_config = self - .resolve_secret_config(&name, Some(&profile_name)) - .expect("Secret should exist in config since we're iterating over it"); + let secret_config = &secret_configs[&name]; let required = secret_config.required.unwrap_or(true); let default = secret_config.default.clone(); let as_path = secret_config.as_path.unwrap_or(false); @@ -2268,7 +2278,7 @@ impl Secrets { )?; status = ResolutionStatus::Resolved; } else if let Some(generated_value) = - self.try_generate_secret(&name, &secret_config, &profile_name)? + self.try_generate_secret(&name, secret_config, &profile_name)? { generated = true; self.insert_resolved( From a4ed138be71f923a6cf878a25b04cf0b0e5082c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:35:55 -0400 Subject: [PATCH 26/56] fix(go): validate response, skip null env, verify embedded cdylib - Load nil-checks the response and validates `schema_version` against the version this SDK was built for, so a skewed library is reported rather than nil-panicked or silently misparsed. - SetAsEnv skips secrets with no usable value (e.g. under no_values) instead of exporting an empty string, via a new `usable()` helper. - extractEmbedded uses an owner-only (0o700) temp dir and reuses the cached cdylib only when its content hash matches, not just its size, closing a predictable-path load and a stale-file reuse. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-go/embedded.go | 12 +++++++--- secretspec-go/secretspec.go | 44 ++++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/secretspec-go/embedded.go b/secretspec-go/embedded.go index 80c5717..8454d5f 100644 --- a/secretspec-go/embedded.go +++ b/secretspec-go/embedded.go @@ -14,13 +14,19 @@ import ( // embedded_unsupported.go). func extractEmbedded() (string, error) { sum := sha256.Sum256(embeddedLib) - dir := filepath.Join(os.TempDir(), "secretspec-ffi-"+hex.EncodeToString(sum[:8])) - if err := os.MkdirAll(dir, 0o755); err != nil { + // Content-addressed by the full digest, in an owner-only directory: a + // different library never collides, and the path is not world-writable. + dir := filepath.Join(os.TempDir(), "secretspec-ffi-"+hex.EncodeToString(sum[:])) + if err := os.MkdirAll(dir, 0o700); err != nil { return "", err } path := filepath.Join(dir, embeddedLibName) - if info, err := os.Stat(path); err == nil && info.Size() == int64(len(embeddedLib)) { + // Reuse the cached file only if its contents hash to the embedded library's + // digest. A size-only check would reuse a truncated/corrupted or + // attacker-planted same-length file; verifying the content rejects those and + // re-extracts the genuine bytes below. + if existing, err := os.ReadFile(path); err == nil && sha256.Sum256(existing) == sum { return path, nil } diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 4515876..d5a897e 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -24,6 +24,12 @@ import ( "github.com/ebitengine/purego" ) +// resolveSchemaVersion is the response wire-format version this SDK understands. +// It tracks secretspec-ffi's RESOLVE_SCHEMA_VERSION; a mismatch means the loaded +// library is incompatible with this SDK, so Load reports it rather than silently +// misparsing. +const resolveSchemaVersion = 1 + var ( loadOnce sync.Once loadErr error @@ -140,18 +146,27 @@ type ResolvedSecret struct { SourceProvider *string } -// Get returns the usable string: the file path for as_path secrets, else the value. -func (s ResolvedSecret) Get() string { +// usable returns the secret's usable string and whether one is present: the file +// path for as_path secrets, otherwise the value. Both are absent when a +// value-less response (e.g. no_values) strips them, in which case ok is false. +func (s ResolvedSecret) usable() (string, bool) { if s.AsPath { if s.Path != nil { - return *s.Path + return *s.Path, true } - return "" + return "", false } if s.Value != nil { - return *s.Value + return *s.Value, true } - return "" + return "", false +} + +// Get returns the usable string: the file path for as_path secrets, else the +// value. It is the empty string when no usable value is present (see usable). +func (s ResolvedSecret) Get() string { + v, _ := s.usable() + return v } // Resolved is a successful resolution, mirroring the Rust Resolved wrapper. @@ -163,10 +178,14 @@ type Resolved struct { } // SetAsEnv exports each resolved secret into the process environment by name. +// Secrets with no usable value (e.g. under no_values) are skipped rather than +// exported as an empty string. func (r *Resolved) SetAsEnv() error { for name, secret := range r.Secrets { - if err := os.Setenv(name, secret.Get()); err != nil { - return err + if value, ok := secret.usable(); ok { + if err := os.Setenv(name, value); err != nil { + return err + } } } return nil @@ -270,6 +289,15 @@ func (b *Builder) Load() (*Resolved, error) { } resp := env.Response + if resp == nil { + return nil, &Error{Kind: "ffi", Message: "secretspec_resolve reported ok with no response"} + } + if resp.SchemaVersion != resolveSchemaVersion { + return nil, &Error{Kind: "version", Message: fmt.Sprintf( + "unsupported resolve schema version %d (expected %d); the secretspec-ffi library and this SDK are out of sync", + resp.SchemaVersion, resolveSchemaVersion, + )} + } if len(resp.MissingRequired) > 0 { return nil, &MissingRequiredError{Missing: resp.MissingRequired} } From d7828f1ffebecda2711e64cf78ac863877dc6caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:36:06 -0400 Subject: [PATCH 27/56] fix(python): validate response, lock dlopen, builder parity - _resolve_response checks the response is present and validates schema_version, raising SecretSpecError on mismatch instead of KeyError / silent misparse. - _load uses double-checked locking so concurrent first callers do not race to dlopen. - Dropped the divergent `source = "provider"` default (other SDKs pass it through); added with_no_values to the builder for parity. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-py/secretspec/__init__.py | 31 +++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py index c5aeba9..59f1239 100644 --- a/secretspec-py/secretspec/__init__.py +++ b/secretspec-py/secretspec/__init__.py @@ -16,12 +16,17 @@ import json import os import sys +import threading from dataclasses import dataclass, field from pathlib import Path from typing import Optional from cffi import FFI +# Response wire-format version this SDK understands. Tracks secretspec-ffi's +# RESOLVE_SCHEMA_VERSION; a mismatch means the loaded library is incompatible. +_RESOLVE_SCHEMA_VERSION = 1 + __all__ = [ "SecretSpec", "Resolved", @@ -84,12 +89,16 @@ def _find_library() -> str: _lib = None +_lib_lock = threading.Lock() def _load() -> object: + # Double-checked locking so concurrent first callers do not race to dlopen. global _lib if _lib is None: - _lib = _ffi.dlopen(_find_library()) + with _lib_lock: + if _lib is None: + _lib = _ffi.dlopen(_find_library()) return _lib @@ -179,7 +188,18 @@ def _resolve_response(request: dict) -> dict: if not envelope.get("ok", False): err = envelope.get("error", {}) raise SecretSpecError(err.get("kind", "unknown"), err.get("message", "")) - return envelope["response"] + response = envelope.get("response") + if response is None: + raise SecretSpecError("ffi", "secretspec_resolve reported ok with no response") + version = response.get("schema_version") + if version != _RESOLVE_SCHEMA_VERSION: + raise SecretSpecError( + "version", + f"unsupported resolve schema version {version} (expected " + f"{_RESOLVE_SCHEMA_VERSION}); the secretspec-ffi library and this SDK " + "are out of sync", + ) + return response def resolve( @@ -231,6 +251,11 @@ def with_reason(self, reason: Optional[str]) -> "_Builder": self._request["reason"] = reason return self + def with_no_values(self, no_values: bool = True) -> "_Builder": + """Omit secret values, returning only structure and provenance.""" + self._request["no_values"] = no_values + return self + def load(self) -> Resolved: response = _resolve_response(self._request) @@ -243,7 +268,7 @@ def load(self) -> Resolved: value=entry.get("value"), path=entry.get("path"), as_path=entry.get("as_path", False), - source=entry.get("source", "provider"), + source=entry.get("source", ""), source_provider=entry.get("source_provider"), ) for name, entry in response.get("secrets", {}).items() From b9a994b6dfca67cefb0f89967d628c30d585725f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:36:15 -0400 Subject: [PATCH 28/56] fix(ruby): skip null env, validate response, lock dlopen, builder parity - set_as_env! skips secrets with no usable value instead of `ENV[name] = nil`, which would delete the variable. - load nil-checks the response and validates schema_version, raising Secretspec::Error on mismatch. - ensure_loaded guards the one-time dlopen with a Mutex and re-checks @loaded inside the lock. - Added with_no_values to the builder for parity. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-rb/lib/secretspec.rb | 60 ++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index ee7efb1..f434fb5 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -15,6 +15,10 @@ require "rbconfig" module Secretspec + # Response wire-format version this SDK understands. Tracks secretspec-ffi's + # RESOLVE_SCHEMA_VERSION; a mismatch means the loaded library is incompatible. + RESOLVE_SCHEMA_VERSION = 1 + # A resolution failure (bad manifest, provider error, reason policy). class Error < StandardError attr_reader :kind @@ -45,9 +49,14 @@ def get # A successful resolution, mirroring the Rust Resolved wrapper. Resolved = Struct.new(:provider, :profile, :secrets, :missing_optional) do - # Export each resolved secret into ENV by its declared name. + # Export each resolved secret into ENV by its declared name. Secrets with no + # usable value (e.g. under no_values) are skipped rather than deleted from + # ENV (assigning nil would remove the variable). def set_as_env! - secrets.each { |name, secret| ENV[name] = secret.get } + secrets.each do |name, secret| + value = secret.get + ENV[name] = value unless value.nil? + end end # Flat { "SECRET_NAME" => value } hash (the file path for as_path). Feed this @@ -60,6 +69,9 @@ def fields # The narrow C ABI, loaded lazily via Fiddle. module Native + # Guards the one-time dlopen so concurrent first callers do not race. + @load_mutex = Mutex.new + class << self def resolve(request_json) ensure_loaded @@ -83,17 +95,23 @@ def abi_version def ensure_loaded return if @loaded - handle = Fiddle.dlopen(find_library) - @resolve = Fiddle::Function.new( - handle["secretspec_resolve"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOIDP - ) - @free = Fiddle::Function.new( - handle["secretspec_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID - ) - @abi = Fiddle::Function.new( - handle["secretspec_abi_version"], [], Fiddle::TYPE_VOIDP - ) - @loaded = true + @load_mutex.synchronize do + # Re-check inside the lock: another thread may have loaded while we + # waited, and @loaded is only set after every function is registered. + next if @loaded + + handle = Fiddle.dlopen(find_library) + @resolve = Fiddle::Function.new( + handle["secretspec_resolve"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOIDP + ) + @free = Fiddle::Function.new( + handle["secretspec_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID + ) + @abi = Fiddle::Function.new( + handle["secretspec_abi_version"], [], Fiddle::TYPE_VOIDP + ) + @loaded = true + end end def lib_names @@ -166,6 +184,12 @@ def with_reason(reason) self end + # Omit secret values, returning only structure and provenance. + def with_no_values(no_values = true) + @request["no_values"] = no_values + self + end + # Resolve the secrets. Raises MissingRequiredError if a required secret is # missing, and Error for any other failure. def load @@ -177,6 +201,16 @@ def load end response = envelope["response"] + raise Error.new("ffi", "secretspec_resolve reported ok with no response") if response.nil? + + version = response["schema_version"] + unless version == RESOLVE_SCHEMA_VERSION + raise Error.new("version", + "unsupported resolve schema version #{version} " \ + "(expected #{RESOLVE_SCHEMA_VERSION}); the secretspec-ffi " \ + "library and this SDK are out of sync") + end + missing = response["missing_required"] || [] raise MissingRequiredError.new(missing) unless missing.empty? From 458a6e9b1284324935d623ff57fda83be27c4477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:36:25 -0400 Subject: [PATCH 29/56] fix(node): validate response, skip null env, builder parity - load nil-checks the response and validates schema_version, throwing SecretSpecError on mismatch. - setAsEnv skips secrets with no usable value instead of coercing null to the string "null". - Added withNoValues to the builder (and index.d.ts) for parity. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-node/index.d.ts | 2 ++ secretspec-node/index.js | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/secretspec-node/index.d.ts b/secretspec-node/index.d.ts index a6a0590..246e2b5 100644 --- a/secretspec-node/index.d.ts +++ b/secretspec-node/index.d.ts @@ -36,6 +36,8 @@ export class Builder { withProvider(provider: string): this; withProfile(profile: string): this; withReason(reason: string): this; + /** Omit secret values, returning only structure and provenance. */ + withNoValues(noValues?: boolean): this; /** * Resolve the secrets. Throws MissingRequiredError if a required secret is * missing, and SecretSpecError for any other failure. diff --git a/secretspec-node/index.js b/secretspec-node/index.js index 14e47a4..da2e638 100644 --- a/secretspec-node/index.js +++ b/secretspec-node/index.js @@ -20,6 +20,10 @@ try { ); } +// Response wire-format version this SDK understands. Tracks secretspec-ffi's +// RESOLVE_SCHEMA_VERSION; a mismatch means the native addon is out of sync. +const RESOLVE_SCHEMA_VERSION = 1; + class SecretSpecError extends Error { constructor(kind, message) { super(`${message} (kind: ${kind})`); @@ -62,10 +66,17 @@ class Resolved { this.missingOptional = response.missing_optional || []; } - /** Export each resolved secret into process.env by its declared name. */ + /** + * Export each resolved secret into process.env by its declared name. Secrets + * with no usable value (e.g. under no_values) are skipped rather than coerced + * to the string "null". + */ setAsEnv() { for (const [name, secret] of Object.entries(this.secrets)) { - process.env[name] = secret.get(); + const value = secret.get(); + if (value != null) { + process.env[name] = value; + } } } @@ -96,6 +107,7 @@ class Builder { withProvider(p) { if (p != null) this._request.provider = p; return this; } withProfile(p) { if (p != null) this._request.profile = p; return this; } withReason(r) { if (r != null) this._request.reason = r; return this; } + withNoValues(v = true) { this._request.no_values = v; return this; } /** * Resolve the secrets. Throws MissingRequiredError if a required secret is @@ -108,6 +120,16 @@ class Builder { throw new SecretSpecError(err.kind || 'unknown', err.message || ''); } const response = envelope.response; + if (response == null) { + throw new SecretSpecError('ffi', 'secretspec_resolve reported ok with no response'); + } + if (response.schema_version !== RESOLVE_SCHEMA_VERSION) { + throw new SecretSpecError( + 'version', + `unsupported resolve schema version ${response.schema_version} (expected ` + + `${RESOLVE_SCHEMA_VERSION}); the native addon and this SDK are out of sync`, + ); + } const missing = response.missing_required || []; if (missing.length) { throw new MissingRequiredError(missing); From 49dd2a601c649bfb8aaedbfcdf03df92bfab852c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 15:36:35 -0400 Subject: [PATCH 30/56] fix(conformance): map the Windows cdylib name in the runner uname under git-bash/msys reports MINGW*/MSYS*/CYGWIN*; map those to secretspec_ffi.dll so the cross-language conformance gate can run on Windows, where the FFI artifact already ships. Co-Authored-By: Claude Opus 4.8 (1M context) --- conformance/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conformance/run.sh b/conformance/run.sh index 820d294..6021755 100755 --- a/conformance/run.sh +++ b/conformance/run.sh @@ -21,8 +21,9 @@ cargo build -p secretspec-ffi || exit 1 target_dir="$(cargo metadata --no-deps --format-version 1 \ | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" case "$(uname -s)" in - Darwin) lib_name="libsecretspec_ffi.dylib" ;; - *) lib_name="libsecretspec_ffi.so" ;; + Darwin) lib_name="libsecretspec_ffi.dylib" ;; + MINGW*|MSYS*|CYGWIN*) lib_name="secretspec_ffi.dll" ;; + *) lib_name="libsecretspec_ffi.so" ;; esac export SECRETSPEC_FFI_LIB="$target_dir/debug/$lib_name" echo "==> SECRETSPEC_FFI_LIB=$SECRETSPEC_FFI_LIB" From cbe4633d46e3d6311c8bfc0d582fef3d4513bf0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 16:50:00 -0400 Subject: [PATCH 31/56] fix(core): redact credentials from the reported provider URI validation_report_provider_uri returned the override and per-secret alias URIs verbatim, and this branch newly serializes that into the provider field of the resolution report (check --json/--explain) and the resolve response (resolve --json, every SDK's response.provider). A user-authored alias or --provider override embedding a credential (vault+token:s3cr3t@host, vault://host?token=...) therefore leaked into machine-readable output and across the FFI boundary, even though the sibling source_provider and the warn path already redact it. Route both raw returns through redact_uri_strict; add a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec/src/secrets.rs | 57 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index 468dce2..ba43089 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -885,13 +885,20 @@ impl Secrets { /// Returns a provider URI for validation result metadata without forcing a /// user-global default when every secret used an explicit or per-secret provider. + /// + /// The returned URI lands in the `provider` field of the resolution report and + /// the resolve response, which `check --explain` prints, `--json` emits, and the + /// other-language SDKs read over the FFI boundary. A user-authored alias or + /// override may embed a credential (`vault+token:s3cr3t@host`, + /// `vault://host?token=...`), so raw URIs are run through `redact_uri_strict` + /// first. The `provider.uri()` paths below are already credential-free. fn validation_report_provider_uri( &self, override_uri: Option<&str>, secret_primary_uris: &HashMap>, ) -> Result { if let Some(uri) = override_uri { - return Ok(uri.to_string()); + return Ok(crate::audit::redact_uri_strict(uri)); } if secret_primary_uris.values().any(Option::is_none) { @@ -905,7 +912,7 @@ impl Secrets { provider_uris.sort(); if let Some(uri) = provider_uris.first() { - return Ok((*uri).clone()); + return Ok(crate::audit::redact_uri_strict(uri.as_str())); } self.get_provider(None).map(|provider| provider.uri()) @@ -2556,3 +2563,49 @@ mod policy_tests { assert_eq!(env.len(), 1); } } + +#[cfg(test)] +mod report_provider_tests { + use super::*; + + /// The `provider` field of the resolution report / resolve response must not + /// echo a credential embedded in a user-authored override or alias URI. That + /// field is shown by `check --explain`, emitted by `--json`, and crosses the + /// SDK boundary, so `validation_report_provider_uri` runs raw URIs through + /// `redact_uri_strict` (the `provider.uri()` paths are already credential-free). + #[test] + fn report_provider_uri_redacts_credentials() { + let spec = Secrets::new( + Config { + project: crate::config::Project { + name: "redact-test".to_string(), + ..Default::default() + }, + profiles: HashMap::new(), + providers: None, + }, + None, + None, + None, + ); + + // Override branch: userinfo and query token are stripped. + let got = spec + .validation_report_provider_uri( + Some("vault+token:s3cr3t@host/db?token=abc"), + &HashMap::new(), + ) + .unwrap(); + assert_eq!(got, "vault+token:host/db"); + assert!(!got.contains("s3cr3t") && !got.contains("abc")); + + // Per-secret alias branch: the first sorted primary URI is redacted too. + let mut primaries = HashMap::new(); + primaries.insert("DB".to_string(), Some("vault://host?token=zzz".to_string())); + let got = spec + .validation_report_provider_uri(None, &primaries) + .unwrap(); + assert_eq!(got, "vault://host"); + assert!(!got.contains("zzz")); + } +} From ce5ad93da755b401722b9cdf951dba515c22add5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 16:50:46 -0400 Subject: [PATCH 32/56] perf(node): resolve off the event loop via loadAsync The napi resolve binding was synchronous, so resolving from a network-backed provider (1Password, LastPass) blocked the Node event loop for the whole round-trip. Add a resolveAsync binding that runs resolve_json on the libuv threadpool (napi AsyncTask) and a Builder.loadAsync() that awaits it. The synchronous load() is unchanged; loadAsync() reports a clear error against an older addon that lacks the binding. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-node/index.d.ts | 9 ++++++++- secretspec-node/index.js | 25 ++++++++++++++++++++++++- secretspec-node/src/lib.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/secretspec-node/index.d.ts b/secretspec-node/index.d.ts index 246e2b5..5b753d1 100644 --- a/secretspec-node/index.d.ts +++ b/secretspec-node/index.d.ts @@ -40,9 +40,16 @@ export class Builder { withNoValues(noValues?: boolean): this; /** * Resolve the secrets. Throws MissingRequiredError if a required secret is - * missing, and SecretSpecError for any other failure. + * missing, and SecretSpecError for any other failure. Synchronous: runs on the + * Node main thread. */ load(): Resolved; + /** + * Like load(), but resolves on the libuv threadpool so a provider doing + * network I/O does not block the Node event loop. Rejects with the same error + * types load() throws. + */ + loadAsync(): Promise; } export const SecretSpec: { diff --git a/secretspec-node/index.js b/secretspec-node/index.js index da2e638..55bf3df 100644 --- a/secretspec-node/index.js +++ b/secretspec-node/index.js @@ -112,9 +112,32 @@ class Builder { /** * Resolve the secrets. Throws MissingRequiredError if a required secret is * missing, and SecretSpecError for any other failure. + * + * Synchronous: the native resolve runs on the Node main thread. Prefer + * loadAsync() when a provider may do network I/O (1Password, LastPass). */ load() { - const envelope = JSON.parse(native.resolve(JSON.stringify(this._request))); + return this._parse(native.resolve(JSON.stringify(this._request))); + } + + /** + * Like load(), but resolves on the libuv threadpool so a provider doing + * network I/O does not block the Node event loop. Returns a Promise + * and rejects with the same error types load() throws. + */ + async loadAsync() { + if (typeof native.resolveAsync !== 'function') { + throw new SecretSpecError( + 'addon', + 'the loaded native addon predates resolveAsync; rebuild it with scripts/build-addon.sh', + ); + } + return this._parse(await native.resolveAsync(JSON.stringify(this._request))); + } + + /** Parse a JSON response envelope string into a Resolved (or throw). */ + _parse(raw) { + const envelope = JSON.parse(raw); if (!envelope.ok) { const err = envelope.error || {}; throw new SecretSpecError(err.kind || 'unknown', err.message || ''); diff --git a/secretspec-node/src/lib.rs b/secretspec-node/src/lib.rs index 09d148c..d769884 100644 --- a/secretspec-node/src/lib.rs +++ b/secretspec-node/src/lib.rs @@ -5,15 +5,46 @@ //! with every other language. The JS layer (index.js) does the request/response //! marshaling and exposes the builder API. +use napi::{Env, Result, Task, bindgen_prelude::AsyncTask}; use napi_derive::napi; /// Resolve secrets from a JSON request string, returning the JSON response /// envelope (`{"ok": true, "response": ...}` or `{"ok": false, "error": ...}`). +/// +/// This is synchronous and runs on the Node main thread; prefer [`resolve_async`] +/// when a provider may do network I/O. #[napi] pub fn resolve(request_json: String) -> String { secretspec::resolve_json(&request_json) } +/// Runs `resolve_json` on the libuv threadpool (via [`AsyncTask`]) so it never +/// runs on the JS thread. +pub struct ResolveTask { + request_json: String, +} + +impl Task for ResolveTask { + type Output = String; + type JsValue = String; + + fn compute(&mut self) -> Result { + Ok(secretspec::resolve_json(&self.request_json)) + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(output) + } +} + +/// Async variant of [`resolve`]: resolves on the libuv threadpool so a provider +/// doing network I/O (1Password, LastPass) does not block the Node event loop. +/// Returns a Promise of the same JSON response envelope string. +#[napi] +pub fn resolve_async(request_json: String) -> AsyncTask { + AsyncTask::new(ResolveTask { request_json }) +} + /// The addon (ABI) version. #[napi] pub fn abi_version() -> String { From e78ef49b8e4d49945a158cd31fb558d1b959ab3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 16:50:47 -0400 Subject: [PATCH 33/56] fix(sdks): load the most recently built cdylib, not always release When SECRETSPEC_FFI_LIB is unset, the Go, Python, and Ruby SDKs walk up to a Cargo target/ directory to find the library. They preferred release over debug, so a stale release build silently shadowed the debug build a developer had just produced (surfacing later as a confusing schema-version mismatch). Within the nearest target/, pick the candidate with the newest mtime instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-go/secretspec.go | 14 ++++++++++++-- secretspec-py/secretspec/__init__.py | 16 +++++++++++----- secretspec-rb/lib/secretspec.rb | 14 ++++++++------ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index d5a897e..2802dd6 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -63,14 +63,24 @@ func findLibrary() (string, error) { return "", err } for { + // Within the nearest ancestor target/, pick the most recently built + // library rather than always preferring release: a stale release build + // must not shadow the debug build the developer just produced. + var bestPath string + var best os.FileInfo for _, profile := range []string{"release", "debug"} { for _, name := range libNames() { candidate := filepath.Join(dir, "target", profile, name) - if _, err := os.Stat(candidate); err == nil { - return candidate, nil + if info, err := os.Stat(candidate); err == nil { + if best == nil || info.ModTime().After(best.ModTime()) { + best, bestPath = info, candidate + } } } } + if bestPath != "" { + return bestPath, nil + } parent := filepath.Dir(dir) if parent == dir { break diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py index 59f1239..65e0821 100644 --- a/secretspec-py/secretspec/__init__.py +++ b/secretspec-py/secretspec/__init__.py @@ -72,14 +72,20 @@ def _find_library() -> str: return str(bundled) # 3. A Cargo target directory, searching up from the current directory. + # Within the nearest target/, pick the most recently built library rather + # than always preferring release, so a stale release build does not shadow + # the debug build just produced. for base in [Path.cwd(), *Path.cwd().parents]: target = base / "target" if target.is_dir(): - for profile in ("release", "debug"): - for name in names: - candidate = target / profile / name - if candidate.exists(): - return str(candidate) + existing = [ + target / profile / name + for profile in ("release", "debug") + for name in names + if (target / profile / name).exists() + ] + if existing: + return str(max(existing, key=lambda p: p.stat().st_mtime)) raise SecretSpecError( "load", diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index f434fb5..f70066e 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -134,12 +134,14 @@ def find_library dir = Dir.pwd loop do - %w[release debug].each do |profile| - lib_names.each do |name| - candidate = File.join(dir, "target", profile, name) - return candidate if File.exist?(candidate) - end - end + # Within the nearest target/, pick the most recently built library + # rather than always preferring release, so a stale release build does + # not shadow the debug build just produced. + candidates = %w[release debug].flat_map do |profile| + lib_names.map { |name| File.join(dir, "target", profile, name) } + end.select { |c| File.exist?(c) } + return candidates.max_by { |c| File.mtime(c) } unless candidates.empty? + parent = File.dirname(dir) break if parent == dir From 54f0a603c796c73dcc4ece92c978f42ae70521d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 9 Jun 2026 16:50:47 -0400 Subject: [PATCH 34/56] docs: correct the Node "no native build" claim; changelog the fixes package.json ships no prebuilt addon and has no per-platform publish wiring (the CI workflow flags this as a follow-up), so the "npm install needs no native build" claim in the changelog and SDK docs was unbacked. Reword to say the addon is built from the Rust core via scripts/build-addon.sh and that prebuilt per-platform npm packages are a follow-up. Also record the credential redaction, loadAsync, and cdylib-discovery fixes under Unreleased. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 21 +++++++++++++++++++-- docs/src/content/docs/sdk/nodejs.md | 5 +++-- docs/src/content/docs/sdk/overview.md | 7 ++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c94ee5..08ab8e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 contract in exactly one place. `secretspec-ffi` is now a thin wrapper over it. - New `secretspec` Node.js / TypeScript SDK (`secretspec-node`): a thin wrapper over a napi-rs native addon that embeds the resolver, so Node apps inherit - every provider with no JS-side resolution logic and `npm install` needs no - native build. Mirrors the derive crate's vocabulary + every provider with no JS-side resolution logic. The native addon is built from + the Rust core with `scripts/build-addon.sh`; prebuilt per-platform npm packages + are a follow-up. Mirrors the derive crate's vocabulary (`SecretSpec.builder().withProvider(...).withProfile(...).withReason(...).load()` returning a `Resolved` with `provider`/`profile`/`secrets`, plus `setAsEnv()`). A missing required secret throws `MissingRequiredError`; other failures throw @@ -120,6 +121,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `SecretResolution`, and `ResolutionStatus` types. ### Fixed +- The `provider` field of the resolution report (`check --json`/`--explain`) and + the resolve response (`resolve --json`, every SDK's `response.provider`) is now + run through `redact_uri_strict`, so a user-authored provider alias or + `--provider` override that embeds a credential + (`vault+token:s3cr3t@host`, `vault://host?token=...`) no longer leaks that + credential into machine-readable output or across the FFI boundary. The + per-secret `source_provider` was already credential-free; this aligns the + top-level field with it. +- The Node SDK gains `Builder.loadAsync()` (backed by a new `resolveAsync` napi + binding that runs on the libuv threadpool), so resolving from a network-backed + provider no longer blocks the Node event loop. The synchronous `load()` is + unchanged. +- The Go, Python, and Ruby SDKs now load the most recently built `cdylib` when + walking up to a Cargo `target/` directory, instead of always preferring + `release`, so a stale release build no longer shadows the debug build a + developer just produced. - `secretspec::resolve_json` now catches panics itself, so both native boundaries that funnel through it (the `secretspec-ffi` C ABI and the napi-rs Node addon) return the same `{"ok": false, "error": {...}}` envelope on an diff --git a/docs/src/content/docs/sdk/nodejs.md b/docs/src/content/docs/sdk/nodejs.md index ef104f5..0e568c3 100644 --- a/docs/src/content/docs/sdk/nodejs.md +++ b/docs/src/content/docs/sdk/nodejs.md @@ -6,8 +6,9 @@ description: Resolve SecretSpec secrets from Node.js and TypeScript The Node.js / TypeScript SDK (`secretspec`) is a thin wrapper over a [napi-rs](https://napi.rs/) native addon that embeds the resolver. Resolution happens in the Rust core, so the SDK inherits every provider with no JS-side -logic, and `npm install` needs no native build. TypeScript declarations ship in -`index.d.ts`. +logic. The addon is built from the Rust core with `scripts/build-addon.sh`; +prebuilt per-platform npm packages are a follow-up. TypeScript declarations ship +in `index.d.ts`. ## Quick start diff --git a/docs/src/content/docs/sdk/overview.md b/docs/src/content/docs/sdk/overview.md index 3b91436..36d40a4 100644 --- a/docs/src/content/docs/sdk/overview.md +++ b/docs/src/content/docs/sdk/overview.md @@ -64,6 +64,7 @@ quicktype owns the type generation. The SDKs are designed to install with no native build: the C ABI library is bundled in the Python wheel and the Ruby gem, embedded in the Go module, and -shipped as a napi-rs addon for Node. The native library is otherwise discovered -from the `SECRETSPEC_FFI_LIB` environment variable or a Cargo `target` -directory, which is how it works from a source checkout. +built as a napi-rs addon for Node (prebuilt per-platform npm packages are a +follow-up). The native library is otherwise discovered from the +`SECRETSPEC_FFI_LIB` environment variable or a Cargo `target` directory, which +is how it works from a source checkout. From 8c6c6c1528c97bc446cb6f68166ec4ccdfdfe774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 10 Jun 2026 17:50:55 +0200 Subject: [PATCH 35/56] fix(core): resolve no_values without materializing values; add report() no_values now routes through a new Secrets::resolve_without_values, which never exposes a secret value or persists an as_path temp file, so no secret byte crosses the boundary and as_path resolution leaves nothing on disk. Previously the resolver fully materialized every value (and persisted every as_path temp file) and only then stripped them. Adds Secrets::report() and a mode:"report" request on the shared resolve_json boundary: a value-free ResolutionReport (per-secret status and provenance) that, unlike resolve, reports a missing required secret as a status rather than failing the call. This is the inventory/preflight view the CLI exposes as check --json, now reachable from every language SDK. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 44 ++++++++++++++++++ secretspec/src/cli/mod.rs | 12 ++--- secretspec/src/resolve.rs | 94 ++++++++++++++++++++++----------------- secretspec/src/secrets.rs | 72 +++++++++++++++++++++++++----- secretspec/src/tests.rs | 80 +++++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ab8e2..7a2f798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New `secretspec` Haskell SDK (`secretspec-hs`): a thin client over the + `secretspec-ffi` C ABI, linked at build time via the GHC FFI, so Haskell apps + inherit every provider with no Haskell-side resolution logic. Mirrors the + derive crate's vocabulary (`builder` with `withProvider`/`withProfile`/ + `withReason`/`withNoValues`, then `load`/`report`), returning a `Resolved` + (`get`/`fields`/`fieldsJson`/`setAsEnv`/`close`) or a value-free `Report`. A + missing required secret throws `MissingRequiredError`; other failures throw + `SecretSpecError` with a stable `errorKind`. `as_path` secrets come back as a + readable file path. Wired into the cross-language conformance suite (all three + dimensions) and a `Haskell SDK` CI workflow. The `secretspec-ffi` `cdylib` is + linked at build time (`--extra-lib-dirs`) and must be on the runtime loader + path. - New cross-language conformance suite (`conformance/`): shared fixtures (manifest + `.env` + a canonical `expected.json`) that every SDK resolves and must reduce to the identical canonical result, guaranteeing the Python, Go, @@ -74,6 +86,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `fields()` (and a JSON variant for Go/Node) is available on the resolved result in every SDK, and the full `schema -> quicktype -> typed` pipeline is e2e-tested in all four SDK suites. Value-free: `schema` reads only the manifest. +- New `Secrets::report()` and a `mode: "report"` request on the shared + `resolve_json` boundary: a value-free `ResolutionReport` (per-secret status and + provenance, never a value) that, unlike resolve, reports a missing required + secret as a `missing_required` status instead of failing the whole call. This + is the inventory/preflight view the CLI already exposes as + `check --json`/`--explain`, now reachable from every language SDK via a + `report()` builder method (Node also has `reportAsync()`) returning a `Report` + of `SecretReport` entries. Exercised by a new `report` conformance dimension so + the four SDKs stay byte-identical. +- Every language SDK gains a cleanup affordance for the temp files that back + `as_path` secrets, which the resolver persists (mode 0400) so their paths stay + valid after resolve returns: Go `Resolved.Close()`, Python `Resolved.close()` + (and `with` context-manager support), Ruby `Resolved#close` (and a block form + of `Builder#load` that closes automatically), and Node `Resolved.dispose()` + (plus `Symbol.dispose` for `using`). Previously a long-running SDK consumer + that resolved `as_path` secrets in a loop leaked one owner-readable file per + resolve into the temp dir with no way to clean them up. ### Changed - The `secretspec-derive` macro now computes all of its typing decisions through @@ -143,6 +172,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 internal panic. Previously only the C ABI caught panics, so a panic in the Node addon surfaced as an opaque thrown error instead of the documented envelope. +- A `no_values` resolution (`secretspec resolve --json --no-values` and every + SDK's no-values path) no longer copies secret values into the response or + persists `as_path` temp files. It now routes through a new + `Secrets::resolve_without_values`, which never calls `expose_secret` and never + keeps a temp file, so no secret byte crosses the boundary and an `as_path` + secret leaves nothing on disk. Previously the resolver fully materialized every + value (and persisted every `as_path` temp file) and only then stripped them. +- The Go SDK's `Resolved.Fields()`/`FieldsJSON()` now emit JSON `null` for a + value-less secret (e.g. under `no_values`) instead of the empty string `""`, + matching the `null` the Python, Ruby, and Node SDKs already emit; `Fields()` + now returns `map[string]*string` and a new `Usable()` accessor distinguishes an + absent value from an empty one. Node's `index.d.ts` types `fields()` as + `Record` (was `Record`, which a + `no_values` result violated). A new `no_values` conformance dimension asserts + all four SDKs produce the identical all-null fields map. - A per-profile JSON Schema (`secretspec schema --profile

`) now allows additional properties. `secretspec resolve --profile

` returns the profile's own secrets plus those inherited from the `default` profile (the diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index 6eaa5f3..b5f91b5 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -736,13 +736,13 @@ pub fn main() -> Result<()> { if let Some(p) = profile { app.set_profile(p); } - let mut response = app - .resolve() - .into_diagnostic() - .wrap_err("Failed to resolve secrets")?; - if no_values { - response = response.without_values(); + let response = if no_values { + app.resolve_without_values() + } else { + app.resolve() } + .into_diagnostic() + .wrap_err("Failed to resolve secrets")?; let rendered = serde_json::to_string_pretty(&response) .into_diagnostic() .wrap_err("Failed to serialize resolve response")?; diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs index 078672f..816841c 100644 --- a/secretspec/src/resolve.rs +++ b/secretspec/src/resolve.rs @@ -91,6 +91,20 @@ impl ResolveResponse { } } +/// Which resolution shape a request asks for. +#[derive(Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RequestMode { + /// The value-carrying [`ResolveResponse`] (the default). + #[default] + Resolve, + /// The value-free [`crate::report::ResolutionReport`]: per-secret status and + /// provenance, never a value, and a missing required secret is reported as a + /// status rather than failing the call. This is the inventory/preflight view + /// the CLI exposes as `check --json`. + Report, +} + #[derive(Debug, Default, Deserialize)] struct JsonRequest { #[serde(default)] @@ -103,38 +117,25 @@ struct JsonRequest { reason: Option, #[serde(default)] no_values: bool, + #[serde(default)] + mode: RequestMode, } -#[derive(Debug, Serialize)] -struct JsonError { - kind: String, - message: String, -} - -#[derive(Debug, Serialize)] -struct JsonEnvelope { - ok: bool, - #[serde(skip_serializing_if = "Option::is_none")] - response: Option, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, +fn error_envelope(kind: &str, message: impl Into) -> serde_json::Value { + serde_json::json!({ + "ok": false, + "error": { "kind": kind, "message": message.into() }, + }) } -fn envelope_error(kind: &str, message: impl Into) -> JsonEnvelope { - JsonEnvelope { - ok: false, - response: None, - error: Some(JsonError { - kind: kind.to_string(), - message: message.into(), - }), - } +fn ok_envelope(response: impl Serialize) -> serde_json::Value { + serde_json::json!({ "ok": true, "response": response }) } -fn resolve_envelope(request_json: &str) -> JsonEnvelope { +fn dispatch(request_json: &str) -> serde_json::Value { let request: JsonRequest = match serde_json::from_str(request_json) { Ok(request) => request, - Err(e) => return envelope_error("invalid_request", format!("invalid request JSON: {e}")), + Err(e) => return error_envelope("invalid_request", format!("invalid request JSON: {e}")), }; let loaded = match &request.path { @@ -143,7 +144,7 @@ fn resolve_envelope(request_json: &str) -> JsonEnvelope { }; let mut app = match loaded { Ok(app) => app, - Err(e) => return envelope_error(e.kind(), e.to_string()), + Err(e) => return error_envelope(e.kind(), e.to_string()), }; if let Some(provider) = request.provider { @@ -156,30 +157,40 @@ fn resolve_envelope(request_json: &str) -> JsonEnvelope { app = app.with_reason(reason); } - match app.resolve() { - Ok(mut response) => { - if request.no_values { - response = response.without_values(); - } - JsonEnvelope { - ok: true, - response: Some(response), - error: None, + match request.mode { + // Value-free report: never fails on a missing required secret, so an + // inventory/preflight consumer always gets the shape back. + RequestMode::Report => match app.report() { + Ok(report) => ok_envelope(report), + Err(e) => error_envelope(e.kind(), e.to_string()), + }, + // Value-carrying resolve. `no_values` takes the path that never copies a + // secret value into the response (and persists no temp file). + RequestMode::Resolve => { + let resolved = if request.no_values { + app.resolve_without_values() + } else { + app.resolve() + }; + match resolved { + Ok(response) => ok_envelope(response), + Err(e) => error_envelope(e.kind(), e.to_string()), } } - Err(e) => envelope_error(e.kind(), e.to_string()), } } /// Resolve secrets from a JSON request string and return the JSON response -/// envelope: `{"ok": true, "response": }` or +/// envelope: `{"ok": true, "response": }` or /// `{"ok": false, "error": {"kind", "message"}}`. /// /// This is the shared JSON boundary used by every native binding (the C ABI in /// `secretspec-ffi` and the napi-rs Node addon), so the envelope contract is /// defined in exactly one place. The request accepts optional `path`, -/// `provider`, `profile`, `reason`, and `no_values`. The response carries secret -/// values; treat its bytes as sensitive. +/// `provider`, `profile`, `reason`, `no_values`, and `mode` (`"resolve"` by +/// default, or `"report"` for the value-free [`crate::report::ResolutionReport`]). +/// A `resolve` response carries secret values; treat its bytes as sensitive. A +/// `report` response never does. pub fn resolve_json(request_json: &str) -> String { // Catch panics here, at the one place both native boundaries funnel through // (the C ABI in `secretspec-ffi` and the napi-rs Node addon). Unwinding across @@ -187,10 +198,9 @@ pub fn resolve_json(request_json: &str) -> String { // `{"ok":false,"error":...}` envelope every binding already parses means all // bindings behave identically — the C ABI no longer needs to be the only one // guarding the boundary. - let envelope = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - resolve_envelope(request_json) - })) - .unwrap_or_else(|_| envelope_error("internal", "internal panic during resolve")); + let envelope = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dispatch(request_json))) + .unwrap_or_else(|_| error_envelope("internal", "internal panic during resolve")); serde_json::to_string(&envelope).unwrap_or_else(|_| { "{\"ok\":false,\"error\":{\"kind\":\"serialize\",\"message\":\"failed to serialize response\"}}".to_string() diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index ba43089..bafeadc 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -4,7 +4,7 @@ use crate::audit::{AuditAction, AuditContext, AuditLogger, AuditOutcome}; use crate::config::{Config, GlobalConfig, Profile, RequireReason, Resolved}; use crate::error::{Result, SecretSpecError}; use crate::provider::Provider as ProviderTrait; -use crate::report::{ResolutionStatus, SecretResolution}; +use crate::report::{ResolutionReport, ResolutionStatus, SecretResolution}; use crate::resolve::{RESOLVE_SCHEMA_VERSION, ResolveResponse, ResolvedSecret, ResolvedSource}; use crate::validation::{ValidatedSecrets, ValidationErrors}; use colored::Colorize; @@ -2003,23 +2003,43 @@ impl Secrets { /// the caller; this is a one-shot boundary and the caller owns their /// lifetime thereafter. pub fn resolve(&self) -> Result { + self.resolve_impl(true) + } + + /// Like [`Self::resolve`], but never materializes secret values: every + /// `value`/`path` in the response is `None`, no `as_path` temp file is + /// persisted, and no secret byte is ever copied into the response. Structure + /// and provenance (`as_path`, `source`, `source_provider`, + /// `missing_optional`) are still populated. This backs the `no_values` + /// request path, so a policy/preflight consumer that wants only the resolve + /// shape never pulls secret values into its process, and `as_path` + /// resolution leaves no temp file behind. Resolution still runs (providers + /// are still queried) so provenance can be reported; a missing required + /// secret still fails the same way as [`Self::resolve`]. For a value-free + /// view that tolerates missing required secrets, use [`Self::report`]. + pub fn resolve_without_values(&self) -> Result { + self.resolve_impl(false) + } + + /// Shared core of [`Self::resolve`]/[`Self::resolve_without_values`]. + /// `include_values` gates whether resolved secret values are copied into the + /// response (and whether `as_path` temp files are persisted past the call). + fn resolve_impl(&self, include_values: bool) -> Result { match self.validate()? { Ok(mut validated) => { // Persist as_path temp files so returned paths outlive this call. - validated.keep_temp_files()?; + // Only needed when we actually return paths: under + // `!include_values` the temp files are dropped (auto-deleted) + // together with `validated`, so nothing is left on disk. + if include_values { + validated.keep_temp_files()?; + } let mut secrets = BTreeMap::new(); for entry in &validated.resolution { if entry.status != ResolutionStatus::Resolved { continue; } - let raw = validated - .resolved - .secrets - .get(&entry.name) - .expect("a Resolved entry always has a value") - .expose_secret() - .to_string(); let source = if entry.generated { ResolvedSource::Generated } else if entry.default_applied { @@ -2027,10 +2047,23 @@ impl Secrets { } else { ResolvedSource::Provider }; - let (value, path) = if entry.as_path { - (None, Some(raw)) + // Only copy the secret value out when the caller wants it; + // otherwise the bytes never enter the response. + let (value, path) = if !include_values { + (None, None) } else { - (Some(raw), None) + let raw = validated + .resolved + .secrets + .get(&entry.name) + .expect("a Resolved entry always has a value") + .expose_secret() + .to_string(); + if entry.as_path { + (None, Some(raw)) + } else { + (Some(raw), None) + } }; secrets.insert( entry.name.clone(), @@ -2073,6 +2106,21 @@ impl Secrets { } } + /// Resolve every declared secret into a value-free [`ResolutionReport`]: + /// per-secret status (resolved / missing-required / missing-optional) plus + /// provenance, never a value. Unlike [`Self::resolve`], a missing required + /// secret is reported as a `MissingRequired` status rather than failing the + /// call, so this is the inventory/preflight view: it answers "what is + /// declared and how would each secret resolve" even for a profile whose + /// secrets the caller cannot fully provide. It is the same report the CLI + /// surfaces as `check --json` / `check --explain`, exposed to the SDKs. + pub fn report(&self) -> Result { + Ok(match self.validate()? { + Ok(validated) => validated.report(), + Err(errors) => errors.report(), + }) + } + /// Resolves all secrets. `emit_check` controls whether this pass records a /// `Check` audit event. /// diff --git a/secretspec/src/tests.rs b/secretspec/src/tests.rs index 9625f08..9f49a2b 100644 --- a/secretspec/src/tests.rs +++ b/secretspec/src/tests.rs @@ -556,6 +556,86 @@ fn test_resolve_as_path_returns_persisted_path() { fs::remove_file(path).ok(); } +#[test] +fn test_resolve_without_values_keeps_structure_but_no_value_or_path() { + use crate::resolve::ResolvedSource; + + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write( + &env_path, + "DATABASE_URL=postgres://localhost/db\nTLS_CERT=----cert----\n", + ) + .unwrap(); + + let secret = |as_path: bool| Secret { + description: Some("t".to_string()), + required: Some(true), + as_path: Some(as_path), + ..Default::default() + }; + let mut secrets = HashMap::new(); + secrets.insert("DATABASE_URL".to_string(), secret(false)); + secrets.insert("TLS_CERT".to_string(), secret(true)); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + let response = spec.resolve_without_values().unwrap(); + assert!(response.is_ok()); + + // Plain secret: no value materialized, but structure + provenance preserved. + let db = &response.secrets["DATABASE_URL"]; + assert!(db.value.is_none()); + assert!(db.path.is_none()); + assert!(!db.as_path); + assert_eq!(db.source, ResolvedSource::Provider); + + // as_path secret: no value AND no path, so no temp file is persisted; the + // as_path flag is still reported so the shape is intact. + let cert = &response.secrets["TLS_CERT"]; + assert!(cert.value.is_none()); + assert!(cert.path.is_none()); + assert!(cert.as_path); +} + +#[test] +fn test_report_lists_missing_required_without_failing() { + use crate::report::ResolutionStatus; + + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "PRESENT=here\n").unwrap(); + + let secret = || Secret { + description: Some("t".to_string()), + required: Some(true), + ..Default::default() + }; + let mut secrets = HashMap::new(); + secrets.insert("PRESENT".to_string(), secret()); + secrets.insert("MISSING".to_string(), secret()); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + // resolve() fails the whole call when a required secret is missing. + assert!(!spec.resolve().unwrap().is_ok()); + + // report() instead lists every secret with a status and never a value, so an + // inventory/preflight consumer still gets the shape back. + let report = spec.report().unwrap(); + let status = |name: &str| { + report + .secrets + .iter() + .find(|s| s.name == name) + .map(|s| s.status.clone()) + }; + assert_eq!(status("PRESENT"), Some(ResolutionStatus::Resolved)); + assert_eq!(status("MISSING"), Some(ResolutionStatus::MissingRequired)); +} + #[test] fn test_secretspec_new() { let config = Config { From f11fbae006c147315b0715b28fc5a4c3b189ee1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 10 Jun 2026 17:51:55 +0200 Subject: [PATCH 36/56] fix(sdks): null fields() consistency, as_path cleanup, and report() - Go Fields()/FieldsJSON() now emit JSON null for a value-less secret instead of the empty string "", matching Python/Ruby/Node; Fields() returns map[string]*string and a new Usable() distinguishes absent from empty. Node index.d.ts types fields() as Record. - Every SDK gains a cleanup affordance for the persisted as_path temp files: Go Resolved.Close(), Python close()/context manager, Ruby close()/load block, Node dispose()/Symbol.dispose. - Every SDK gains report() (Node also reportAsync()) over the value-free report, which never fails on a missing required secret. - Conformance gains no_values and report dimensions (the latter asserts source_provider presence), locking the cross-language contract so a divergence like the Go ""-vs-null one cannot ship again. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fixtures/as_path/expected_no_values.json | 3 + .../fixtures/as_path/expected_report.json | 6 + .../fixtures/basic/expected_no_values.json | 4 + .../fixtures/basic/expected_report.json | 8 + secretspec-go/README.md | 20 +++ secretspec-go/conformance_test.go | 97 +++++++++++ secretspec-go/secretspec.go | 153 ++++++++++++++++-- secretspec-node/README.md | 18 +++ secretspec-node/index.d.ts | 40 ++++- secretspec-node/index.js | 98 +++++++++++ secretspec-node/test/conformance.test.js | 54 ++++++- secretspec-py/README.md | 19 +++ secretspec-py/secretspec/__init__.py | 123 +++++++++++++- secretspec-py/tests/test_conformance.py | 61 ++++++- secretspec-rb/README.md | 17 ++ secretspec-rb/lib/secretspec.rb | 81 +++++++++- secretspec-rb/test/test_resolve.rb | 50 +++++- 17 files changed, 816 insertions(+), 36 deletions(-) create mode 100644 conformance/fixtures/as_path/expected_no_values.json create mode 100644 conformance/fixtures/as_path/expected_report.json create mode 100644 conformance/fixtures/basic/expected_no_values.json create mode 100644 conformance/fixtures/basic/expected_report.json diff --git a/conformance/fixtures/as_path/expected_no_values.json b/conformance/fixtures/as_path/expected_no_values.json new file mode 100644 index 0000000..1a2046a --- /dev/null +++ b/conformance/fixtures/as_path/expected_no_values.json @@ -0,0 +1,3 @@ +{ + "TLS_CERT": null +} diff --git a/conformance/fixtures/as_path/expected_report.json b/conformance/fixtures/as_path/expected_report.json new file mode 100644 index 0000000..8d3131e --- /dev/null +++ b/conformance/fixtures/as_path/expected_report.json @@ -0,0 +1,6 @@ +{ + "profile": "default", + "secrets": { + "TLS_CERT": { "status": "resolved", "required": true, "as_path": true, "generated": false, "default_applied": false, "source_provider": true } + } +} diff --git a/conformance/fixtures/basic/expected_no_values.json b/conformance/fixtures/basic/expected_no_values.json new file mode 100644 index 0000000..9c0a99f --- /dev/null +++ b/conformance/fixtures/basic/expected_no_values.json @@ -0,0 +1,4 @@ +{ + "DATABASE_URL": null, + "LOG_LEVEL": null +} diff --git a/conformance/fixtures/basic/expected_report.json b/conformance/fixtures/basic/expected_report.json new file mode 100644 index 0000000..5aa238e --- /dev/null +++ b/conformance/fixtures/basic/expected_report.json @@ -0,0 +1,8 @@ +{ + "profile": "default", + "secrets": { + "DATABASE_URL": { "status": "resolved", "required": true, "as_path": false, "generated": false, "default_applied": false, "source_provider": true }, + "LOG_LEVEL": { "status": "resolved", "required": false, "as_path": false, "generated": false, "default_applied": true, "source_provider": false }, + "SENTRY_DSN": { "status": "missing_optional", "required": false, "as_path": false, "generated": false, "default_applied": false, "source_provider": false } + } +} diff --git a/secretspec-go/README.md b/secretspec-go/README.md index 1d7a3dc..67e22f7 100644 --- a/secretspec-go/README.md +++ b/secretspec-go/README.md @@ -36,6 +36,26 @@ func main() { A missing required secret returns `*MissingRequiredError`; any other failure returns `*Error` (with a stable `.Kind`). +## Cleanup + +`as_path` secrets are materialized to temp files that outlive the call. Call +`resolved.Close()` (e.g. `defer resolved.Close()`) when done so the secret files +do not accumulate in the temp dir. + +## Value-free report + +`Report()` returns the inventory/preflight view: per-secret status and +provenance, never a value. Unlike `Load()`, it does not fail when a required +secret is missing — it appears as a `SecretReport` with `Status` +`"missing_required"`. + +```go +report, _ := secretspec.New().WithProfile("production").Report() +for _, s := range report.Secrets { + fmt.Println(s.Name, s.Status, s.Required) +} +``` + ## Library discovery The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, diff --git a/secretspec-go/conformance_test.go b/secretspec-go/conformance_test.go index 9102066..a34c81c 100644 --- a/secretspec-go/conformance_test.go +++ b/secretspec-go/conformance_test.go @@ -98,3 +98,100 @@ func canonical(t *testing.T, resolved *Resolved) map[string]any { "missing_optional": missingOptional, } } + +// TestConformanceNoValues asserts that under no_values every SDK emits the same +// all-null fields map: a value-less secret must serialize to JSON null, not "". +func TestConformanceNoValues(t *testing.T) { + forEachFixture(t, func(t *testing.T, dir string) { + resolved, err := New(). + WithPath(filepath.Join(dir, "secretspec.toml")). + WithProvider("dotenv://" + filepath.Join(dir, ".env")). + WithReason("conformance"). + WithNoValues(true). + Load() + if err != nil { + t.Fatal(err) + } + defer resolved.Close() + + fieldsBytes, err := resolved.FieldsJSON() + if err != nil { + t.Fatal(err) + } + assertJSONEqualsFile(t, fieldsBytes, filepath.Join(dir, "expected_no_values.json")) + }) +} + +// TestConformanceReport asserts the value-free report (status + provenance, +// including whether a source_provider is present) is identical across SDKs. +func TestConformanceReport(t *testing.T) { + forEachFixture(t, func(t *testing.T, dir string) { + report, err := New(). + WithPath(filepath.Join(dir, "secretspec.toml")). + WithProvider("dotenv://" + filepath.Join(dir, ".env")). + WithReason("conformance"). + Report() + if err != nil { + t.Fatal(err) + } + actualBytes, err := json.Marshal(canonicalReport(report)) + if err != nil { + t.Fatal(err) + } + assertJSONEqualsFile(t, actualBytes, filepath.Join(dir, "expected_report.json")) + }) +} + +func canonicalReport(report *Report) map[string]any { + secrets := map[string]any{} + for _, s := range report.Secrets { + secrets[s.Name] = map[string]any{ + "status": s.Status, + "required": s.Required, + "as_path": s.AsPath, + "generated": s.Generated, + "default_applied": s.DefaultApplied, + // Present-or-not (not the path-dependent value) so the vector is + // machine-independent yet still catches a dropped source_provider. + "source_provider": s.SourceProvider != nil, + } + } + return map[string]any{"profile": report.Profile, "secrets": secrets} +} + +func forEachFixture(t *testing.T, fn func(*testing.T, string)) { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + fixtures := filepath.Join(filepath.Dir(wd), "conformance", "fixtures") + entries, err := os.ReadDir(fixtures) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + t.Run(entry.Name(), func(t *testing.T) { fn(t, filepath.Join(fixtures, entry.Name())) }) + } +} + +func assertJSONEqualsFile(t *testing.T, actual []byte, file string) { + t.Helper() + expectedBytes, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var expected, actualGeneric any + if err := json.Unmarshal(expectedBytes, &expected); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(actual, &actualGeneric); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actualGeneric, expected) { + t.Fatalf("mismatch for %s\n got: %s\nwant: %s", filepath.Base(file), actual, expectedBytes) + } +} diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 2802dd6..8172270 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -156,10 +156,12 @@ type ResolvedSecret struct { SourceProvider *string } -// usable returns the secret's usable string and whether one is present: the file +// Usable returns the secret's usable string and whether one is present: the file // path for as_path secrets, otherwise the value. Both are absent when a // value-less response (e.g. no_values) strips them, in which case ok is false. -func (s ResolvedSecret) usable() (string, bool) { +// This is the null-aware accessor; the other SDKs express the same thing as a +// get() that returns null/None/nil. +func (s ResolvedSecret) Usable() (string, bool) { if s.AsPath { if s.Path != nil { return *s.Path, true @@ -173,9 +175,10 @@ func (s ResolvedSecret) usable() (string, bool) { } // Get returns the usable string: the file path for as_path secrets, else the -// value. It is the empty string when no usable value is present (see usable). +// value. It is the empty string when no usable value is present; use Usable to +// distinguish an absent value from a genuinely empty one. func (s ResolvedSecret) Get() string { - v, _ := s.usable() + v, _ := s.Usable() return v } @@ -192,7 +195,7 @@ type Resolved struct { // exported as an empty string. func (r *Resolved) SetAsEnv() error { for name, secret := range r.Secrets { - if value, ok := secret.usable(); ok { + if value, ok := secret.Usable(); ok { if err := os.Setenv(name, value); err != nil { return err } @@ -202,20 +205,47 @@ func (r *Resolved) SetAsEnv() error { } // Fields returns a flat map of SECRET_NAME -> value (the file path for as_path). -func (r *Resolved) Fields() map[string]string { - out := make(map[string]string, len(r.Secrets)) +// A secret with no usable value (e.g. under no_values) maps to a nil pointer, +// which marshals to JSON null, matching the null the Python, Ruby, and Node SDKs +// emit; the value is a non-nil pointer otherwise. +func (r *Resolved) Fields() map[string]*string { + out := make(map[string]*string, len(r.Secrets)) for name, secret := range r.Secrets { - out[name] = secret.Get() + if v, ok := secret.Usable(); ok { + val := v + out[name] = &val + } else { + out[name] = nil + } } return out } -// FieldsJSON marshals Fields() to JSON, the input for a quicktype-generated -// deserializer (e.g. UnmarshalSecretSpec). See `secretspec schema`. +// FieldsJSON marshals Fields() to JSON (a `{SECRET_NAME: value-or-null}` object), +// the input for a quicktype-generated deserializer (e.g. UnmarshalSecretSpec). +// See `secretspec schema`. func (r *Resolved) FieldsJSON() ([]byte, error) { return json.Marshal(r.Fields()) } +// Close removes the temp files backing any as_path secrets in this result. The +// resolver persists those files (mode 0400) so their paths stay valid after +// resolve returns; the caller owns their lifetime. Call it (e.g. +// `defer resolved.Close()`) when done so secret files do not accumulate in the +// temp dir. Non-as_path secrets and a no_values result hold no path and are +// skipped, and a file already gone is not an error. +func (r *Resolved) Close() error { + var firstErr error + for _, secret := range r.Secrets { + if secret.AsPath && secret.Path != nil { + if err := os.Remove(*secret.Path); err != nil && !os.IsNotExist(err) && firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + // ABIVersion returns the version reported by the loaded library. func ABIVersion() (string, error) { if err := ensureLoaded(); err != nil { @@ -329,3 +359,106 @@ func (b *Builder) Load() (*Resolved, error) { MissingOptional: resp.MissingOptional, }, nil } + +// reportSchemaVersion is the value-free report wire-format version this SDK +// understands; it tracks secretspec's RESOLUTION_REPORT_SCHEMA_VERSION. +const reportSchemaVersion = 1 + +// SecretReport is the value-free resolution outcome for one declared secret: +// how it would resolve and from where, never the value itself. +type SecretReport struct { + Name string + Status string // "resolved" | "missing_required" | "missing_optional" + Required bool + SourceProvider *string + DefaultApplied bool + Generated bool + AsPath bool +} + +// Report is a value-free resolution snapshot: every declared secret and how it +// would resolve, never a value. Unlike Load, a missing required secret is +// reported as a SecretReport with Status "missing_required" rather than an +// error, so it describes a profile even when its secrets are not all available. +type Report struct { + Provider string + Profile string + Secrets []SecretReport +} + +type secretReportJSON struct { + Name string `json:"name"` + Status string `json:"status"` + Required bool `json:"required"` + SourceProvider *string `json:"source_provider"` + DefaultApplied bool `json:"default_applied"` + Generated bool `json:"generated"` + AsPath bool `json:"as_path"` +} + +type reportResponseJSON struct { + SchemaVersion int `json:"schema_version"` + Provider string `json:"provider"` + Profile string `json:"profile"` + Secrets []secretReportJSON `json:"secrets"` +} + +type reportEnvelopeJSON struct { + OK bool `json:"ok"` + Response *reportResponseJSON `json:"response"` + Error *errorJSON `json:"error"` +} + +// Report resolves the value-free report (the inventory/preflight view, the same +// one the CLI exposes as `check --json`). It never returns +// *MissingRequiredError: a missing required secret appears as a SecretReport +// with Status "missing_required". It returns *Error for a genuine failure. +func (b *Builder) Report() (*Report, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + req := make(map[string]any, len(b.req)+1) + for k, v := range b.req { + req[k] = v + } + req["mode"] = "report" + payload, err := json.Marshal(req) + if err != nil { + return nil, err + } + + ptr := cResolve(string(payload)) + if ptr == 0 { + return nil, &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + } + raw := goString(ptr) + cFree(ptr) + + var env reportEnvelopeJSON + if err := json.Unmarshal([]byte(raw), &env); err != nil { + return nil, err + } + if !env.OK { + kind, message := "unknown", "" + if env.Error != nil { + kind, message = env.Error.Kind, env.Error.Message + } + return nil, &Error{Kind: kind, Message: message} + } + resp := env.Response + if resp == nil { + return nil, &Error{Kind: "ffi", Message: "secretspec_resolve reported ok with no response"} + } + if resp.SchemaVersion != reportSchemaVersion { + return nil, &Error{Kind: "version", Message: fmt.Sprintf( + "unsupported report schema version %d (expected %d); the secretspec-ffi library and this SDK are out of sync", + resp.SchemaVersion, reportSchemaVersion, + )} + } + + secrets := make([]SecretReport, len(resp.Secrets)) + for i, s := range resp.Secrets { + secrets[i] = SecretReport(s) + } + return &Report{Provider: resp.Provider, Profile: resp.Profile, Secrets: secrets}, nil +} diff --git a/secretspec-node/README.md b/secretspec-node/README.md index 4ad49b7..efe0415 100644 --- a/secretspec-node/README.md +++ b/secretspec-node/README.md @@ -24,6 +24,24 @@ A missing required secret throws `MissingRequiredError`; any other failure throws `SecretSpecError` (with a stable `.kind`). TypeScript declarations ship in `index.d.ts`. +## Cleanup + +`as_path` secrets are materialized to temp files that outlive the call. Call +`resolved.dispose()` (or `using resolved = builder.load()`) when done so the +secret files do not accumulate in the temp dir. + +## Value-free report + +`report()` (and `reportAsync()`) returns the inventory/preflight view: per-secret +status and provenance, never a value. Unlike `load()`, it does not throw when a +required secret is missing — it appears as a `SecretReport` with status +`"missing_required"`. + +```js +const report = SecretSpec.builder().withProfile('production').report(); +for (const s of report.secrets) console.log(s.name, s.status, s.required); +``` + ## Library discovery The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, diff --git a/secretspec-node/index.d.ts b/secretspec-node/index.d.ts index 5b753d1..366f775 100644 --- a/secretspec-node/index.d.ts +++ b/secretspec-node/index.d.ts @@ -25,10 +25,38 @@ export class Resolved { missingOptional: string[]; /** Export each resolved secret into process.env by its declared name. */ setAsEnv(): void; - /** Flat { SECRET_NAME: value } object (the file path for as_path secrets). */ - fields(): Record; + /** + * Flat { SECRET_NAME: value } object (the file path for as_path secrets). A + * secret with no usable value (e.g. under no_values) maps to null, matching + * the other SDKs. + */ + fields(): Record; /** fields() as a JSON string, the input for a quicktype-generated deserializer. */ fieldsJson(): string; + /** + * Remove the temp files backing any as_path secrets in this result. Call when + * done (or use `using` for automatic disposal) so secret files do not + * accumulate in the temp dir. + */ + dispose(): void; + [Symbol.dispose](): void; +} + +export class SecretReport { + name: string; + /** "resolved" | "missing_required" | "missing_optional" */ + status: string; + required: boolean; + sourceProvider: string | null; + defaultApplied: boolean; + generated: boolean; + asPath: boolean; +} + +export class Report { + provider: string; + profile: string; + secrets: SecretReport[]; } export class Builder { @@ -50,6 +78,14 @@ export class Builder { * types load() throws. */ loadAsync(): Promise; + /** + * Resolve a value-free Report (the inventory/preflight view). Unlike load(), + * never throws MissingRequiredError: a missing required secret appears as a + * SecretReport with status "missing_required". + */ + report(): Report; + /** Like report(), but resolves on the libuv threadpool. */ + reportAsync(): Promise; } export const SecretSpec: { diff --git a/secretspec-node/index.js b/secretspec-node/index.js index 55bf3df..35d0f0e 100644 --- a/secretspec-node/index.js +++ b/secretspec-node/index.js @@ -1,5 +1,7 @@ 'use strict'; +const fs = require('node:fs'); + // Node.js SDK for SecretSpec, a declarative secrets manager. // // A thin wrapper over the native napi-rs addon (secretspec.node), which embeds @@ -24,6 +26,10 @@ try { // RESOLVE_SCHEMA_VERSION; a mismatch means the native addon is out of sync. const RESOLVE_SCHEMA_VERSION = 1; +// Wire-format version of the value-free report. Tracks secretspec's +// RESOLUTION_REPORT_SCHEMA_VERSION. +const REPORT_SCHEMA_VERSION = 1; + class SecretSpecError extends Error { constructor(kind, message) { super(`${message} (kind: ${kind})`); @@ -96,6 +102,49 @@ class Resolved { fieldsJson() { return JSON.stringify(this.fields()); } + + /** + * Remove the temp files backing any as_path secrets in this result. The + * resolver persists those files (mode 0400) so their paths stay valid after + * resolve returns; the caller owns their lifetime. Call dispose() when done + * (or use `using resolved = builder.load()` for automatic disposal) so secret + * files do not accumulate in the temp dir. A file already gone is not an error. + */ + dispose() { + for (const secret of Object.values(this.secrets)) { + if (secret.asPath && secret.path != null) { + try { + fs.unlinkSync(secret.path); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } + } + } + } + + [Symbol.dispose]() { + this.dispose(); + } +} + +class SecretReport { + constructor(entry) { + this.name = entry.name; + this.status = entry.status; + this.required = entry.required ?? false; + this.sourceProvider = entry.source_provider ?? null; + this.defaultApplied = entry.default_applied ?? false; + this.generated = entry.generated ?? false; + this.asPath = entry.as_path ?? false; + } +} + +class Report { + constructor(response) { + this.provider = response.provider; + this.profile = response.profile; + this.secrets = (response.secrets || []).map((s) => new SecretReport(s)); + } } class Builder { @@ -159,6 +208,53 @@ class Builder { } return new Resolved(response); } + + /** + * Resolve a value-free Report (the inventory/preflight view, the same one the + * CLI exposes as `check --json`). Unlike load(), never throws + * MissingRequiredError: a missing required secret appears as a SecretReport + * with status "missing_required". Synchronous; prefer reportAsync() for + * network-backed providers. + */ + report() { + return this._parseReport( + native.resolve(JSON.stringify({ ...this._request, mode: 'report' })), + ); + } + + /** Like report(), but resolves on the libuv threadpool. */ + async reportAsync() { + if (typeof native.resolveAsync !== 'function') { + throw new SecretSpecError( + 'addon', + 'the loaded native addon predates resolveAsync; rebuild it with scripts/build-addon.sh', + ); + } + return this._parseReport( + await native.resolveAsync(JSON.stringify({ ...this._request, mode: 'report' })), + ); + } + + /** Parse a JSON report envelope string into a Report (or throw). */ + _parseReport(raw) { + const envelope = JSON.parse(raw); + if (!envelope.ok) { + const err = envelope.error || {}; + throw new SecretSpecError(err.kind || 'unknown', err.message || ''); + } + const response = envelope.response; + if (response == null) { + throw new SecretSpecError('ffi', 'secretspec_resolve reported ok with no response'); + } + if (response.schema_version !== REPORT_SCHEMA_VERSION) { + throw new SecretSpecError( + 'version', + `unsupported report schema version ${response.schema_version} (expected ` + + `${REPORT_SCHEMA_VERSION}); the native addon and this SDK are out of sync`, + ); + } + return new Report(response); + } } const SecretSpec = { @@ -176,6 +272,8 @@ module.exports = { Builder, Resolved, ResolvedSecret, + Report, + SecretReport, SecretSpecError, MissingRequiredError, abiVersion, diff --git a/secretspec-node/test/conformance.test.js b/secretspec-node/test/conformance.test.js index 50beb55..3be9d1d 100644 --- a/secretspec-node/test/conformance.test.js +++ b/secretspec-node/test/conformance.test.js @@ -42,12 +42,56 @@ for (const fixture of fs.readdirSync(FIXTURES).sort()) { test(`conformance: ${fixture}`, () => { const expected = JSON.parse(fs.readFileSync(path.join(dir, 'expected.json'), 'utf8')); - const resolved = SecretSpec.builder() - .withPath(path.join(dir, 'secretspec.toml')) - .withProvider(`dotenv://${path.join(dir, '.env')}`) - .withReason('conformance') - .load(); + const resolved = builder(dir).load(); assert.deepStrictEqual(canonical(resolved), expected); }); + + test(`conformance no_values: ${fixture}`, () => { + // Under no_values every SDK must emit the same all-null fields map: a + // value-less secret serializes to null, not an empty string. + const expected = JSON.parse( + fs.readFileSync(path.join(dir, 'expected_no_values.json'), 'utf8'), + ); + const resolved = builder(dir).withNoValues().load(); + try { + assert.deepStrictEqual(JSON.parse(resolved.fieldsJson()), expected); + } finally { + resolved.dispose(); + } + }); + + test(`conformance report: ${fixture}`, () => { + // The value-free report (status + provenance) is identical across SDKs. + const expected = JSON.parse( + fs.readFileSync(path.join(dir, 'expected_report.json'), 'utf8'), + ); + const report = builder(dir).report(); + + assert.deepStrictEqual(canonicalReport(report), expected); + }); +} + +function builder(dir) { + return SecretSpec.builder() + .withPath(path.join(dir, 'secretspec.toml')) + .withProvider(`dotenv://${path.join(dir, '.env')}`) + .withReason('conformance'); +} + +function canonicalReport(report) { + const secrets = {}; + for (const s of report.secrets) { + secrets[s.name] = { + status: s.status, + required: s.required, + as_path: s.asPath, + generated: s.generated, + default_applied: s.defaultApplied, + // Present-or-not (not the path-dependent value) so the vector is + // machine-independent yet still catches a dropped source_provider. + source_provider: s.sourceProvider != null, + }; + } + return { profile: report.profile, secrets }; } diff --git a/secretspec-py/README.md b/secretspec-py/README.md index 60c8814..b8a1cf1 100644 --- a/secretspec-py/README.md +++ b/secretspec-py/README.md @@ -25,6 +25,25 @@ resolved.set_as_env() # export everything into os.environ A missing required secret raises `MissingRequiredError`; any other failure raises `SecretSpecError` (with a stable `.kind`). +## Cleanup + +`as_path` secrets are materialized to temp files that outlive the call. Use the +result as a context manager (`with SecretSpec.builder()...load() as resolved:`) +or call `resolved.close()` when done so the secret files do not accumulate. + +## Value-free report + +`report()` returns the inventory/preflight view: per-secret status and +provenance, never a value. Unlike `load()`, it does not raise when a required +secret is missing — it appears as a `SecretReport` with status +`"missing_required"`. + +```python +report = SecretSpec.builder().with_profile("production").report() +for s in report.secrets: + print(s.name, s.status, s.required) +``` + ## Library discovery The SDK loads the native library from, in order: the `SECRETSPEC_FFI_LIB` diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py index 65e0821..d407706 100644 --- a/secretspec-py/secretspec/__init__.py +++ b/secretspec-py/secretspec/__init__.py @@ -27,13 +27,20 @@ # RESOLVE_SCHEMA_VERSION; a mismatch means the loaded library is incompatible. _RESOLVE_SCHEMA_VERSION = 1 +# Wire-format version of the value-free report. Tracks secretspec's +# RESOLUTION_REPORT_SCHEMA_VERSION. +_REPORT_SCHEMA_VERSION = 1 + __all__ = [ "SecretSpec", "Resolved", "ResolvedSecret", + "Report", + "SecretReport", "SecretSpecError", "MissingRequiredError", "resolve", + "report", "abi_version", ] @@ -160,15 +167,64 @@ def set_as_env(self) -> None: if usable is not None: os.environ[name] = usable - def fields(self) -> dict: + def fields(self) -> dict[str, Optional[str]]: """Flat ``{SECRET_NAME: value}`` map (the file path for ``as_path``). + A secret with no usable value (e.g. under ``no_values``) maps to + ``None``, matching the null the other SDKs emit. + This is the input for a quicktype-generated deserializer: feed it to the generated type's ``from_dict`` to get a typed object. See ``secretspec schema``. """ return {name: secret.get for name, secret in self.secrets.items()} + def close(self) -> None: + """Remove the temp files backing any ``as_path`` secrets in this result. + + The resolver persists those files (mode 0400) so their paths stay valid + after resolve returns; the caller owns their lifetime. Call ``close()`` + (or use this object as a context manager) when done so secret files do + not accumulate in the temp dir. A file already gone is not an error. + """ + for secret in self.secrets.values(): + if secret.as_path and secret.path is not None: + try: + os.remove(secret.path) + except FileNotFoundError: + pass + + def __enter__(self) -> "Resolved": + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +@dataclass(frozen=True) +class SecretReport: + """Value-free resolution outcome for one declared secret: how it would + resolve and from where, never the value itself.""" + + name: str + status: str # "resolved" | "missing_required" | "missing_optional" + required: bool + source_provider: Optional[str] + default_applied: bool + generated: bool + as_path: bool + + +@dataclass(frozen=True) +class Report: + """A value-free resolution snapshot. Unlike :class:`Resolved`, a missing + required secret is a ``missing_required`` status here, not an error, so a + report describes a profile even when its secrets are not all available.""" + + provider: str + profile: str + secrets: list[SecretReport] + def abi_version() -> str: """The ABI version reported by the loaded library.""" @@ -208,6 +264,25 @@ def _resolve_response(request: dict) -> dict: return response +def _report_response(request: dict) -> dict: + envelope = _resolve_envelope(request) + if not envelope.get("ok", False): + err = envelope.get("error", {}) + raise SecretSpecError(err.get("kind", "unknown"), err.get("message", "")) + response = envelope.get("response") + if response is None: + raise SecretSpecError("ffi", "secretspec_resolve reported ok with no response") + version = response.get("schema_version") + if version != _REPORT_SCHEMA_VERSION: + raise SecretSpecError( + "version", + f"unsupported report schema version {version} (expected " + f"{_REPORT_SCHEMA_VERSION}); the secretspec-ffi library and this SDK " + "are out of sync", + ) + return response + + def resolve( *, path: Optional[str] = None, @@ -225,6 +300,24 @@ def resolve( ).with_reason(reason).load() +def report( + *, + path: Optional[str] = None, + provider: Optional[str] = None, + profile: Optional[str] = None, + reason: Optional[str] = None, +) -> Report: + """Resolve a value-free :class:`Report` (the inventory/preflight view). + + Unlike :func:`resolve`, never raises :class:`MissingRequiredError`: a missing + required secret appears as a :class:`SecretReport` with status + ``"missing_required"``. + """ + return SecretSpec.builder().with_path(path).with_provider(provider).with_profile( + profile + ).with_reason(reason).report() + + class SecretSpec: """Entry point mirroring the derive crate's ``SecretSpec::builder()``.""" @@ -285,3 +378,31 @@ def load(self) -> Resolved: secrets=secrets, missing_optional=response.get("missing_optional", []), ) + + def report(self) -> Report: + """Resolve a value-free :class:`Report` (the inventory/preflight view). + + Unlike :meth:`load`, never raises :class:`MissingRequiredError`: a missing + required secret appears as a :class:`SecretReport` with status + ``"missing_required"``. + """ + request = dict(self._request) + request["mode"] = "report" + response = _report_response(request) + secrets = [ + SecretReport( + name=s["name"], + status=s["status"], + required=s.get("required", False), + source_provider=s.get("source_provider"), + default_applied=s.get("default_applied", False), + generated=s.get("generated", False), + as_path=s.get("as_path", False), + ) + for s in response.get("secrets", []) + ] + return Report( + provider=response["provider"], + profile=response["profile"], + secrets=secrets, + ) diff --git a/secretspec-py/tests/test_conformance.py b/secretspec-py/tests/test_conformance.py index b542a5f..9d424bf 100644 --- a/secretspec-py/tests/test_conformance.py +++ b/secretspec-py/tests/test_conformance.py @@ -30,21 +30,68 @@ def _canonical(resolved) -> dict: } +def _canonical_report(report) -> dict: + return { + "profile": report.profile, + "secrets": { + s.name: { + "status": s.status, + "required": s.required, + "as_path": s.as_path, + "generated": s.generated, + "default_applied": s.default_applied, + # Present-or-not (not the path-dependent value) so the vector is + # machine-independent yet still catches a dropped source_provider. + "source_provider": s.source_provider is not None, + } + for s in report.secrets + }, + } + + def _fixtures(): return sorted(p.name for p in FIXTURES.iterdir() if p.is_dir()) -@pytest.mark.parametrize("fixture", _fixtures()) -def test_conformance(fixture): - directory = FIXTURES / fixture - expected = json.loads((directory / "expected.json").read_text()) - - resolved = ( +def _builder(directory): + return ( SecretSpec.builder() .with_path(str(directory / "secretspec.toml")) .with_provider(f"dotenv://{directory / '.env'}") .with_reason("conformance") - .load() ) + +@pytest.mark.parametrize("fixture", _fixtures()) +def test_conformance(fixture): + directory = FIXTURES / fixture + expected = json.loads((directory / "expected.json").read_text()) + + resolved = _builder(directory).load() + assert _canonical(resolved) == expected + + +@pytest.mark.parametrize("fixture", _fixtures()) +def test_conformance_no_values(fixture): + """Under no_values every SDK must emit the same all-null fields map: a + value-less secret serializes to null, not an empty string.""" + directory = FIXTURES / fixture + expected = json.loads((directory / "expected_no_values.json").read_text()) + + resolved = _builder(directory).with_no_values().load() + try: + assert resolved.fields() == expected + finally: + resolved.close() + + +@pytest.mark.parametrize("fixture", _fixtures()) +def test_conformance_report(fixture): + """The value-free report (status + provenance) is identical across SDKs.""" + directory = FIXTURES / fixture + expected = json.loads((directory / "expected_report.json").read_text()) + + report = _builder(directory).report() + + assert _canonical_report(report) == expected diff --git a/secretspec-rb/README.md b/secretspec-rb/README.md index d3c26ec..c4482f3 100644 --- a/secretspec-rb/README.md +++ b/secretspec-rb/README.md @@ -24,6 +24,23 @@ resolved.set_as_env! # export everything into ENV A missing required secret raises `Secretspec::MissingRequiredError`; any other failure raises `Secretspec::Error` (with a stable `#kind`). +## Cleanup + +`as_path` secrets are materialized to temp files that outlive the call. Pass a +block to `load` (which closes automatically) or call `resolved.close` when done +so the secret files do not accumulate in the temp dir. + +## Value-free report + +`report` returns the inventory/preflight view: per-secret status and provenance, +never a value. Unlike `load`, it does not raise when a required secret is missing +— it appears as a `SecretReport` with status `"missing_required"`. + +```ruby +report = Secretspec::SecretSpec.builder.with_profile("production").report +report.secrets.each { |s| puts [s.name, s.status, s.required].join(" ") } +``` + ## Library discovery The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index f70066e..12eca11 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -19,6 +19,10 @@ module Secretspec # RESOLVE_SCHEMA_VERSION; a mismatch means the loaded library is incompatible. RESOLVE_SCHEMA_VERSION = 1 + # Wire-format version of the value-free report. Tracks secretspec's + # RESOLUTION_REPORT_SCHEMA_VERSION. + REPORT_SCHEMA_VERSION = 1 + # A resolution failure (bad manifest, provider error, reason policy). class Error < StandardError attr_reader :kind @@ -59,14 +63,40 @@ def set_as_env! end end - # Flat { "SECRET_NAME" => value } hash (the file path for as_path). Feed this - # to a quicktype-generated deserializer (e.g. from_dynamic!). See - # `secretspec schema`. + # Flat { "SECRET_NAME" => value } hash (the file path for as_path). A secret + # with no usable value (e.g. under no_values) maps to nil, matching the null + # the other SDKs emit. Feed this to a quicktype-generated deserializer (e.g. + # from_dynamic!). See `secretspec schema`. def fields secrets.transform_values(&:get) end + + # Remove the temp files backing any as_path secrets in this result. The + # resolver persists those files (mode 0400) so their paths stay valid after + # resolve returns; the caller owns their lifetime. Call #close (or pass a + # block to Builder#load, which closes automatically) when done so secret + # files do not accumulate in the temp dir. A file already gone is not an + # error. + def close + secrets.each_value do |secret| + next unless secret.as_path && secret.path + + File.delete(secret.path) if File.exist?(secret.path) + end + nil + end end + # Value-free resolution outcome for one declared secret: how it would resolve + # and from where, never the value itself. + SecretReport = Struct.new(:name, :status, :required, :source_provider, + :default_applied, :generated, :as_path) + + # A value-free resolution snapshot. Unlike Resolved, a missing required secret + # is a "missing_required" status here, not an error, so a report describes a + # profile even when its secrets are not all available. + Report = Struct.new(:provider, :profile, :secrets) + # The narrow C ABI, loaded lazily via Fiddle. module Native # Guards the one-time dlopen so concurrent first callers do not race. @@ -194,6 +224,10 @@ def with_no_values(no_values = true) # Resolve the secrets. Raises MissingRequiredError if a required secret is # missing, and Error for any other failure. + # + # Without a block, returns the Resolved (the caller should #close it when + # done to clean up any as_path temp files). With a block, yields the Resolved + # and closes it afterwards, returning the block's value. def load envelope = JSON.parse(Native.resolve(JSON.generate(@request))) @@ -224,10 +258,49 @@ def load ) end - Resolved.new( + resolved = Resolved.new( response["provider"], response["profile"], secrets, response["missing_optional"] || [] ) + return resolved unless block_given? + + begin + yield resolved + ensure + resolved.close + end + end + + # Resolve a value-free Report (the inventory/preflight view, the same one the + # CLI exposes as `check --json`). Unlike #load, never raises + # MissingRequiredError: a missing required secret appears as a SecretReport + # with status "missing_required". + def report + request = @request.merge("mode" => "report") + envelope = JSON.parse(Native.resolve(JSON.generate(request))) + + unless envelope["ok"] + err = envelope["error"] || {} + raise Error.new(err["kind"] || "unknown", err["message"] || "") + end + + response = envelope["response"] + raise Error.new("ffi", "secretspec_resolve reported ok with no response") if response.nil? + + version = response["schema_version"] + unless version == REPORT_SCHEMA_VERSION + raise Error.new("version", + "unsupported report schema version #{version} " \ + "(expected #{REPORT_SCHEMA_VERSION}); the secretspec-ffi " \ + "library and this SDK are out of sync") + end + + secrets = (response["secrets"] || []).map do |s| + SecretReport.new(s["name"], s["status"], s["required"], + s["source_provider"], s["default_applied"], + s["generated"], s["as_path"]) + end + Report.new(response["provider"], response["profile"], secrets) end end diff --git a/secretspec-rb/test/test_resolve.rb b/secretspec-rb/test/test_resolve.rb index 7909b4b..7deea0d 100644 --- a/secretspec-rb/test/test_resolve.rb +++ b/secretspec-rb/test/test_resolve.rb @@ -156,15 +156,51 @@ def canonical(resolved) } end + def canonical_report(report) + secrets = {} + report.secrets.each do |s| + secrets[s.name] = { + "status" => s.status, + "required" => s.required, + "as_path" => s.as_path, + "generated" => s.generated, + "default_applied" => s.default_applied, + # Present-or-not (not the path-dependent value) so the vector is + # machine-independent yet still catches a dropped source_provider. + "source_provider" => !s.source_provider.nil? + } + end + { "profile" => report.profile, "secrets" => secrets } + end + + def conformance_builder(dir) + Secretspec::SecretSpec.builder + .with_path(File.join(dir, "secretspec.toml")) + .with_provider("dotenv://#{File.join(dir, '.env')}") + .with_reason("conformance") + end + Dir.glob(File.join(FIXTURES, "*")).select { |p| File.directory?(p) }.sort.each do |dir| - define_method("test_conformance_#{File.basename(dir)}") do + name = File.basename(dir) + + define_method("test_conformance_#{name}") do expected = JSON.parse(File.read(File.join(dir, "expected.json"))) - resolved = Secretspec::SecretSpec.builder - .with_path(File.join(dir, "secretspec.toml")) - .with_provider("dotenv://#{File.join(dir, '.env')}") - .with_reason("conformance") - .load - assert_equal expected, canonical(resolved) + assert_equal expected, canonical(conformance_builder(dir).load) + end + + # Under no_values every SDK must emit the same all-null fields map: a + # value-less secret serializes to null, not an empty string. + define_method("test_conformance_no_values_#{name}") do + expected = JSON.parse(File.read(File.join(dir, "expected_no_values.json"))) + conformance_builder(dir).with_no_values.load do |resolved| + assert_equal expected, resolved.fields + end + end + + # The value-free report (status + provenance) is identical across SDKs. + define_method("test_conformance_report_#{name}") do + expected = JSON.parse(File.read(File.join(dir, "expected_report.json"))) + assert_equal expected, canonical_report(conformance_builder(dir).report) end end end From 79ab0e302af5a1ab877ac6409071443b2eddaed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 10 Jun 2026 17:52:49 +0200 Subject: [PATCH 37/56] feat(haskell): add the secretspec-hs SDK A thin client over the secretspec-ffi C ABI, linked at build time via the GHC FFI. Mirrors the other SDKs: a builder (withPath/withProvider/withProfile/ withReason/withNoValues) plus load/report, returning a Resolved (get/fields/fieldsJson/setAsEnv/close) or a value-free Report. A missing required secret throws MissingRequiredError; other failures throw SecretSpecError with a stable errorKind. as_path secrets come back as a readable file path. Wires GHC into devenv, the cross-language conformance runner (all three dimensions), and ci-sdks.sh; adds a schema -> quicktype -> typed codegen e2e test (quicktype's Haskell target); and a Haskell SDK CI workflow that builds/tests on PR and publishes to Hackage on a version tag. Adds docs (SDK page, sidebar, overview, landing) and README entries. The cdylib is linked at build time (--extra-lib-dirs) and must be on the runtime loader path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/haskell-build.yml | 76 ++++++ CHANGELOG.md | 11 +- conformance/run.sh | 17 +- devenv.nix | 2 + docs/astro.config.mjs | 1 + docs/src/content/docs/sdk/haskell.md | 72 ++++++ docs/src/content/docs/sdk/overview.md | 15 +- docs/src/pages/index.astro | 16 +- scripts/ci-sdks.sh | 14 ++ secretspec-hs/.gitignore | 6 + secretspec-hs/LICENSE | 201 +++++++++++++++ secretspec-hs/README.md | 71 ++++++ secretspec-hs/secretspec.cabal | 48 ++++ secretspec-hs/src/SecretSpec.hs | 348 ++++++++++++++++++++++++++ secretspec-hs/test/Main.hs | 268 ++++++++++++++++++++ secretspec/README.md | 1 + 16 files changed, 1152 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/haskell-build.yml create mode 100644 docs/src/content/docs/sdk/haskell.md create mode 100644 secretspec-hs/.gitignore create mode 100644 secretspec-hs/LICENSE create mode 100644 secretspec-hs/README.md create mode 100644 secretspec-hs/secretspec.cabal create mode 100644 secretspec-hs/src/SecretSpec.hs create mode 100644 secretspec-hs/test/Main.hs diff --git a/.github/workflows/haskell-build.yml b/.github/workflows/haskell-build.yml new file mode 100644 index 0000000..699054e --- /dev/null +++ b/.github/workflows/haskell-build.yml @@ -0,0 +1,76 @@ +name: "Haskell SDK" + +# Builds and tests the Haskell SDK (secretspec-hs) against a freshly built +# cdylib. The SDK links the secretspec-ffi C ABI at build time via the FFI, so +# the cdylib's directory goes on both the linker path (--extra-lib-dirs) and the +# runtime loader path. + +on: + workflow_dispatch: + pull_request: + paths: + - "secretspec-hs/**" + - "secretspec-ffi/**" + - "secretspec/**" + - ".github/workflows/haskell-build.yml" + push: + branches: [main] + tags: + - v** + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: cachix/install-nix-action@v31 + - uses: cachix/cachix-action@v16 + with: + name: devenv + - name: Install devenv.sh + run: nix profile install nixpkgs#devenv + - name: Build cdylib + CLI and run the Haskell SDK test-suite + run: | + devenv shell -- bash -c ' + set -euo pipefail + cargo build -p secretspec-ffi -p secretspec + target_dir="$(cargo metadata --no-deps --format-version 1 \ + | grep -o "\"target_directory\":\"[^\"]*\"" | head -1 | sed "s/.*:\"\(.*\)\"/\1/")" + lib_dir="$target_dir/debug" + export SECRETSPEC_BIN="$lib_dir/secretspec" + cd secretspec-hs + cabal update + # --write-ghc-environment-files lets the codegen test compile the + # quicktype-generated module; SECRETSPEC_BIN lets it run the CLI. + LD_LIBRARY_PATH="$lib_dir:${LD_LIBRARY_PATH:-}" \ + cabal test --extra-lib-dirs="$lib_dir" \ + --write-ghc-environment-files=always --test-show-details=streaming + ' + + publish: + name: publish to Hackage + if: startsWith(github.ref, 'refs/tags/v') + needs: [build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: cachix/install-nix-action@v31 + - uses: cachix/cachix-action@v16 + with: + name: devenv + - name: Install devenv.sh + run: nix profile install nixpkgs#devenv + - name: sdist and upload to Hackage + # Requires the HACKAGE_TOKEN secret. The package links secretspec-ffi at + # build time, so Hackage's build bots cannot compile it (no cdylib); the + # upload still succeeds and the README documents the link requirement. + env: + HACKAGE_TOKEN: ${{ secrets.HACKAGE_TOKEN }} + run: | + devenv shell -- bash -c ' + set -euo pipefail + cd secretspec-hs + cabal sdist + cabal upload --publish --token="$HACKAGE_TOKEN" \ + dist-newstyle/sdist/secretspec-*.tar.gz + ' diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2f798..e16c397 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,9 +17,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 missing required secret throws `MissingRequiredError`; other failures throw `SecretSpecError` with a stable `errorKind`. `as_path` secrets come back as a readable file path. Wired into the cross-language conformance suite (all three - dimensions) and a `Haskell SDK` CI workflow. The `secretspec-ffi` `cdylib` is - linked at build time (`--extra-lib-dirs`) and must be on the runtime loader - path. + dimensions), the `schema -> quicktype -> typed` codegen e2e test (using + quicktype's Haskell target), and a `Haskell SDK` CI workflow that also publishes + to Hackage on a version tag. The `secretspec-ffi` `cdylib` is linked at build + time (`--extra-lib-dirs`) and must be on the runtime loader path. - New cross-language conformance suite (`conformance/`): shared fixtures (manifest + `.env` + a canonical `expected.json`) that every SDK resolves and must reduce to the identical canonical result, guaranteeing the Python, Go, @@ -85,7 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 This keeps the per-language maintenance to the small `fields()` method. `fields()` (and a JSON variant for Go/Node) is available on the resolved result in every SDK, and the full `schema -> quicktype -> typed` pipeline is e2e-tested - in all four SDK suites. Value-free: `schema` reads only the manifest. + in all five SDK suites. Value-free: `schema` reads only the manifest. - New `Secrets::report()` and a `mode: "report"` request on the shared `resolve_json` boundary: a value-free `ResolutionReport` (per-secret status and provenance, never a value) that, unlike resolve, reports a missing required @@ -186,7 +187,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 absent value from an empty one. Node's `index.d.ts` types `fields()` as `Record` (was `Record`, which a `no_values` result violated). A new `no_values` conformance dimension asserts - all four SDKs produce the identical all-null fields map. + all five SDKs produce the identical all-null fields map. - A per-profile JSON Schema (`secretspec schema --profile

`) now allows additional properties. `secretspec resolve --profile

` returns the profile's own secrets plus those inherited from the `default` profile (the diff --git a/conformance/run.sh b/conformance/run.sh index 6021755..a1556a1 100755 --- a/conformance/run.sh +++ b/conformance/run.sh @@ -54,11 +54,20 @@ run_node() { ( [ -d node_modules ] || npm install --no-audit --no-fund >/dev/null node --test test/conformance.test.js ); } +run_haskell() { ( + cd secretspec-hs + # The Haskell SDK links the cdylib at build time, so its directory must be on + # both the linker path (--extra-lib-dirs) and the runtime loader path. + lib_dir="$(dirname "$SECRETSPEC_FFI_LIB")" + export LD_LIBRARY_PATH="$lib_dir:${LD_LIBRARY_PATH:-}" + cabal test --extra-lib-dirs="$lib_dir" --test-show-details=streaming +); } -run "Python" python run_python -run "Go" go run_go -run "Ruby" ruby run_ruby -run "Node" node run_node +run "Python" python run_python +run "Go" go run_go +run "Ruby" ruby run_ruby +run "Node" node run_node +run "Haskell" cabal run_haskell echo echo "==> Conformance summary" diff --git a/devenv.nix b/devenv.nix index d8c2bf0..5486016 100644 --- a/devenv.nix +++ b/devenv.nix @@ -28,6 +28,8 @@ languages.go.enable = true; # Ruby SDK (secretspec-rb) binds the C ABI via stdlib Fiddle (dlopen). languages.ruby.enable = true; + # Haskell SDK (secretspec-hs) links the C ABI at build time via the FFI. + languages.haskell.enable = true; packages = [ # keyring diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index dcf937e..fb21366 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -196,6 +196,7 @@ Secrets can be stored in: keyring (default), dotenv files, environment variables { label: "Go SDK", slug: "sdk/go" }, { label: "Ruby SDK", slug: "sdk/ruby" }, { label: "Node.js SDK", slug: "sdk/nodejs" }, + { label: "Haskell SDK", slug: "sdk/haskell" }, ], }, { diff --git a/docs/src/content/docs/sdk/haskell.md b/docs/src/content/docs/sdk/haskell.md new file mode 100644 index 0000000..0415cfc --- /dev/null +++ b/docs/src/content/docs/sdk/haskell.md @@ -0,0 +1,72 @@ +--- +title: Haskell SDK +description: Resolve SecretSpec secrets from Haskell +--- + +The Haskell SDK (`secretspec-hs`) is a thin client over the `secretspec-ffi` C +ABI, linked at build time via the Haskell FFI. Resolution happens in the Rust +core, so the SDK inherits every provider with no Haskell-side logic. + +## Quick start + +```haskell +import qualified SecretSpec as S +import qualified Data.Map.Strict as Map +import Data.Function ((&)) + +main :: IO () +main = do + resolved <- + S.load + ( S.builder + & S.withProvider "keyring://" + & S.withProfile "production" + & S.withReason "boot web app" + ) + + print (S.resolvedProvider resolved, S.resolvedProfile resolved) + case Map.lookup "DATABASE_URL" (S.resolvedSecrets resolved) of + Just db -> print (S.get db) -- the value, or the file path for as_path secrets + Nothing -> pure () + S.setAsEnv resolved -- export everything into the process environment +``` + +A missing required secret throws `MissingRequiredError`; any other failure +throws `SecretSpecError` (with a stable `errorKind`). + +`as_path` secrets are materialized to temp files that outlive the call; call +`S.close resolved` when done so they do not accumulate in the temp dir. + +## Value-free report + +`S.report` returns the inventory/preflight view: per-secret status and +provenance, never a value. Unlike `load`, it does not throw when a required +secret is missing — that secret appears as a `SecretReport` with `srStatus` +`"missing_required"`. + +```haskell +rep <- S.report (S.builder & S.withProfile "production") +mapM_ (\s -> print (S.srName s, S.srStatus s, S.srRequired s)) (S.reportSecrets rep) +``` + +## Typed access (codegen) + +Generate a typed record with `secretspec schema` plus +[quicktype](https://quicktype.io), then decode `S.fieldsJson resolved`: + +```bash +secretspec schema | quicktype -s schema --top-level SecretSpec --lang haskell -o Secrets.hs +``` + +## Building + +The native `secretspec-ffi` library is linked at build time, so point cabal at +the built `cdylib` and put the same directory on the runtime loader path: + +```bash +cargo build -p secretspec-ffi +TARGET="$(cargo metadata --no-deps --format-version 1 \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +cabal build --extra-lib-dirs="$TARGET/debug" +LD_LIBRARY_PATH="$TARGET/debug" cabal test --extra-lib-dirs="$TARGET/debug" +``` diff --git a/docs/src/content/docs/sdk/overview.md b/docs/src/content/docs/sdk/overview.md index 36d40a4..9a7622d 100644 --- a/docs/src/content/docs/sdk/overview.md +++ b/docs/src/content/docs/sdk/overview.md @@ -3,9 +3,9 @@ title: SDK Overview description: How the SecretSpec language SDKs work --- -SecretSpec ships SDKs for Rust, Python, Go, Ruby, and Node.js/TypeScript. They -all resolve secrets from the same declarative `secretspec.toml`, and they all -behave identically, because they share one resolver. +SecretSpec ships SDKs for Rust, Python, Go, Ruby, Node.js/TypeScript, and +Haskell. They all resolve secrets from the same declarative `secretspec.toml`, +and they all behave identically, because they share one resolver. ## One resolver, thin clients @@ -17,6 +17,7 @@ that core rather than a reimplementation: strongly-typed access. - **Python** (cffi), **Go** (purego), and **Ruby** (Fiddle) load the `secretspec-ffi` C ABI and exchange a small JSON request/response with it. +- **Haskell** links the same C ABI at build time via the GHC FFI. - **Node.js/TypeScript** uses a [napi-rs](https://napi.rs/) native addon that embeds the same resolver. @@ -42,8 +43,8 @@ print(resolved.secrets["DATABASE_URL"].get) ``` See each language's page for the idiomatic spelling: [Rust](/sdk/rust), -[Python](/sdk/python), [Go](/sdk/go), [Ruby](/sdk/ruby), and -[Node.js](/sdk/nodejs). +[Python](/sdk/python), [Go](/sdk/go), [Ruby](/sdk/ruby), +[Node.js](/sdk/nodejs), and [Haskell](/sdk/haskell). ## Typed access @@ -68,3 +69,7 @@ built as a napi-rs addon for Node (prebuilt per-platform npm packages are a follow-up). The native library is otherwise discovered from the `SECRETSPEC_FFI_LIB` environment variable or a Cargo `target` directory, which is how it works from a source checkout. + +The Haskell SDK is the exception: it links the C ABI at build time, so the +`secretspec-ffi` `cdylib` must be on the linker path (`--extra-lib-dirs`) and the +runtime loader path when building and running it. diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro index 10d0294..a936ff1 100644 --- a/docs/src/pages/index.astro +++ b/docs/src/pages/index.astro @@ -459,7 +459,7 @@ $ secretspec audit -n 1 Language SDKs

Use it from any language.

- Rust, Python, Go, Ruby, and Node.js/TypeScript all resolve from the same secretspec.toml through one Rust core — every provider, chain, and profile behaves identically, with no per-language logic. How the SDKs work → + Rust, Python, Go, Ruby, Node.js/TypeScript, and Haskell all resolve from the same secretspec.toml through one Rust core — every provider, chain, and profile behaves identically, with no per-language logic. How the SDKs work →

@@ -519,6 +519,20 @@ puts s.secrets["DATABASE_URL"].get +
+ +
+
+
+ Haskell +
+ 
s <- S.load (S.builder
+      & S.withProvider "keyring://"
+      & S.withReason "boot")
+print (S.get =<< Map.lookup "DATABASE_URL" (S.resolvedSecrets s))
+
+
+

And in Rust, a proc macro reads secretspec.toml at compile time and generates strongly-typed structs — misspelling a secret fails the build, not your deploy. Rust SDK →

diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh index 68f306e..64c570b 100755 --- a/scripts/ci-sdks.sh +++ b/scripts/ci-sdks.sh @@ -37,4 +37,18 @@ echo "==> Node" # and has no npm dependencies. ( cd secretspec-node && node --test ) +echo "==> Haskell" +# The Haskell SDK links the cdylib at build time, so its directory goes on both +# the linker path (--extra-lib-dirs) and the runtime loader path. +( + cd secretspec-hs + hs_lib_dir="$(dirname "$SECRETSPEC_FFI_LIB")" + cabal update + # --write-ghc-environment-files lets the codegen test's runghc see aeson and + # the quicktype-generated module's transitive imports; SECRETSPEC_BIN (set + # above) lets it run `secretspec schema`. + LD_LIBRARY_PATH="$hs_lib_dir:${LD_LIBRARY_PATH:-}" \ + cabal test --extra-lib-dirs="$hs_lib_dir" --write-ghc-environment-files=always +) + echo "==> All SDK suites passed" diff --git a/secretspec-hs/.gitignore b/secretspec-hs/.gitignore new file mode 100644 index 0000000..1453b6a --- /dev/null +++ b/secretspec-hs/.gitignore @@ -0,0 +1,6 @@ +dist-newstyle/ +dist/ +*.hi +*.o +.ghc.environment.* +cabal.project.local diff --git a/secretspec-hs/LICENSE b/secretspec-hs/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/secretspec-hs/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/secretspec-hs/README.md b/secretspec-hs/README.md new file mode 100644 index 0000000..2e16fbd --- /dev/null +++ b/secretspec-hs/README.md @@ -0,0 +1,71 @@ +# secretspec (Haskell SDK) + +Haskell bindings for [SecretSpec](https://secretspec.dev/), a declarative secrets +manager. A thin client over the `secretspec-ffi` C ABI, linked at build time via +the Haskell FFI. Resolution happens in the Rust core, so the SDK inherits every +provider with no Haskell-side logic. + +```haskell +import qualified SecretSpec as S +import qualified Data.Map.Strict as Map +import Data.Function ((&)) + +main :: IO () +main = do + resolved <- + S.load + ( S.builder + & S.withProvider "keyring://" + & S.withProfile "production" + & S.withReason "boot web app" + ) + + print (S.resolvedProvider resolved, S.resolvedProfile resolved) + case Map.lookup "DATABASE_URL" (S.resolvedSecrets resolved) of + Just db -> print (S.get db) -- the value, or the file path for as_path secrets + Nothing -> pure () + S.setAsEnv resolved -- export everything into the process environment +``` + +A missing required secret throws `MissingRequiredError`; any other failure +throws `SecretSpecError` (with a stable `errorKind`). + +## Cleanup + +`as_path` secrets are materialized to temp files that outlive the call. Call +`SecretSpec.close resolved` when done so the secret files do not accumulate in +the temp dir. + +## Value-free report + +`SecretSpec.report` returns the inventory/preflight view: per-secret status and +provenance, never a value. Unlike `load`, it does not throw when a required +secret is missing — it appears as a `SecretReport` with `srStatus` +`"missing_required"`. + +```haskell +rep <- S.report (S.builder & S.withProfile "production") +mapM_ (\s -> print (S.srName s, S.srStatus s, S.srRequired s)) (S.reportSecrets rep) +``` + +## Typed access (codegen) + +Generate a typed record with `secretspec schema` plus +[quicktype](https://quicktype.io), then decode `SecretSpec.fieldsJson resolved`: + +```bash +secretspec schema | quicktype -s schema --top-level SecretSpec --lang haskell -o Secrets.hs +``` + +## Building + +The native `secretspec-ffi` library is linked at build time. Point cabal at the +built `cdylib` and put the same directory on the runtime loader path: + +```bash +cargo build -p secretspec-ffi +TARGET="$(cargo metadata --no-deps --format-version 1 \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +cabal build --extra-lib-dirs="$TARGET/debug" +LD_LIBRARY_PATH="$TARGET/debug" cabal test --extra-lib-dirs="$TARGET/debug" +``` diff --git a/secretspec-hs/secretspec.cabal b/secretspec-hs/secretspec.cabal new file mode 100644 index 0000000..eb25702 --- /dev/null +++ b/secretspec-hs/secretspec.cabal @@ -0,0 +1,48 @@ +cabal-version: 2.4 +name: secretspec +version: 0.12.0 +synopsis: Haskell SDK for SecretSpec, a declarative secrets manager +description: + A thin client over the @secretspec-ffi@ C ABI (linked at build time). + Resolution (providers, chains, profiles, generation, @as_path@) happens in the + Rust core; this package marshals a JSON request to the native library and + parses the response, mirroring the Rust derive crate's vocabulary. +homepage: https://secretspec.dev/ +license: Apache-2.0 +license-file: LICENSE +author: Domen Kožar +maintainer: domen@enlambda.com +category: Configuration +build-type: Simple + +library + exposed-modules: SecretSpec + hs-source-dirs: src + build-depends: base >=4.14 && <5 + , aeson >=2.0 && <3 + , bytestring <1 + , containers <1 + , directory <2 + , text <3 + -- The native resolver, linked at build time. Pass its location with + -- `cabal build --extra-lib-dirs=/debug` (and put the same + -- directory on the runtime loader path, e.g. LD_LIBRARY_PATH). + extra-libraries: secretspec_ffi + default-language: Haskell2010 + ghc-options: -Wall + +test-suite secretspec-test + type: exitcode-stdio-1.0 + main-is: Main.hs + hs-source-dirs: test + build-depends: base + , secretspec + , aeson >=2.0 + , bytestring + , containers + , directory + , filepath + , process + , text + default-language: Haskell2010 + ghc-options: -Wall -threaded diff --git a/secretspec-hs/src/SecretSpec.hs b/secretspec-hs/src/SecretSpec.hs new file mode 100644 index 0000000..119a2b7 --- /dev/null +++ b/secretspec-hs/src/SecretSpec.hs @@ -0,0 +1,348 @@ +{-# LANGUAGE ForeignFunctionInterface #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Haskell SDK for SecretSpec, a declarative secrets manager. +-- +-- A thin client over the @secretspec-ffi@ C ABI, linked at build time. +-- Resolution (providers, fallback chains, profiles, generation, @as_path@) +-- happens entirely in the Rust core; this module marshals a JSON request to +-- @secretspec_resolve@, parses the response envelope, and exposes it with the +-- same vocabulary as the Rust derive crate. +-- +-- > import qualified SecretSpec as S +-- > import Data.Function ((&)) +-- > +-- > main = do +-- > resolved <- S.load (S.builder & S.withProvider "keyring://" & S.withReason "boot") +-- > print (S.get =<< Data.Map.lookup "DATABASE_URL" (S.resolvedSecrets resolved)) +-- > S.setAsEnv resolved +module SecretSpec + ( -- * Builder + Builder + , builder + , withPath + , withProvider + , withProfile + , withReason + , withNoValues + -- * Resolve (value-carrying) + , Resolved(..) + , ResolvedSecret(..) + , load + , get + , fields + , fieldsJson + , setAsEnv + , close + -- * Report (value-free) + , Report(..) + , SecretReport(..) + , report + -- * Errors + , SecretSpecError(..) + , MissingRequiredError(..) + -- * Misc + , abiVersion + ) where + +import Control.Exception (Exception, throwIO) +import Control.Monad (forM_, unless, when) +import Data.Aeson (FromJSON (..), Value, eitherDecodeStrict, encode, + object, withObject, (.!=), (.:), (.:?), (.=)) +import Data.Aeson.Types (parseEither) +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Foreign.C.String (CString, peekCString) +import Foreign.Ptr (nullPtr) +import System.Directory (doesFileExist, removeFile) +import System.Environment (setEnv) + +-- The three C ABI functions, linked at build time (-lsecretspec_ffi). They are +-- declared @safe@ because @secretspec_resolve@ may block on provider I/O +-- (1Password, LastPass), and a @safe@ call lets other Haskell threads run. +foreign import ccall safe "secretspec_resolve" + c_secretspec_resolve :: CString -> IO CString + +foreign import ccall safe "secretspec_free" + c_secretspec_free :: CString -> IO () + +foreign import ccall safe "secretspec_abi_version" + c_secretspec_abi_version :: IO CString + +-- | Wire-format version of the value-carrying resolve response this SDK +-- understands. Tracks @secretspec@'s @RESOLVE_SCHEMA_VERSION@. +resolveSchemaVersion :: Int +resolveSchemaVersion = 1 + +-- | Wire-format version of the value-free report. Tracks @secretspec@'s +-- @RESOLUTION_REPORT_SCHEMA_VERSION@. +reportSchemaVersion :: Int +reportSchemaVersion = 1 + +-- | A resolution failure (bad manifest, provider error, reason policy). Carries +-- a stable @kind@. +data SecretSpecError = SecretSpecError + { errorKind :: Text + , errorMessage :: Text + } deriving (Show, Eq) + +instance Exception SecretSpecError + +-- | One or more required secrets were not found anywhere. +newtype MissingRequiredError = MissingRequiredError + { missing :: [Text] + } deriving (Show, Eq) + +instance Exception MissingRequiredError + +-- | One resolved secret. Exactly one of 'secretValue' \/ 'secretPath' is set: +-- the path for @as_path@ secrets, the value otherwise. Both are 'Nothing' for a +-- value-less ('withNoValues') response. +data ResolvedSecret = ResolvedSecret + { secretValue :: Maybe Text + , secretPath :: Maybe Text + , secretAsPath :: Bool + , secretSource :: Text + , secretSourceProvider :: Maybe Text + } deriving (Show, Eq) + +instance FromJSON ResolvedSecret where + parseJSON = withObject "ResolvedSecret" $ \o -> + ResolvedSecret + <$> o .:? "value" + <*> o .:? "path" + <*> o .:? "as_path" .!= False + <*> o .: "source" + <*> o .:? "source_provider" + +-- | A successful resolution, mirroring the Rust @Resolved@ wrapper. +data Resolved = Resolved + { resolvedProvider :: Text + , resolvedProfile :: Text + , resolvedSecrets :: Map Text ResolvedSecret + , resolvedMissingOptional :: [Text] + } deriving (Show, Eq) + +-- | The value-free resolution outcome for one declared secret: how it would +-- resolve and from where, never the value itself. +data SecretReport = SecretReport + { srName :: Text + , srStatus :: Text -- ^ @"resolved"@, @"missing_required"@, or @"missing_optional"@. + , srRequired :: Bool + , srSourceProvider :: Maybe Text + , srDefaultApplied :: Bool + , srGenerated :: Bool + , srAsPath :: Bool + } deriving (Show, Eq) + +instance FromJSON SecretReport where + parseJSON = withObject "SecretReport" $ \o -> + SecretReport + <$> o .: "name" + <*> o .: "status" + <*> o .:? "required" .!= False + <*> o .:? "source_provider" + <*> o .:? "default_applied" .!= False + <*> o .:? "generated" .!= False + <*> o .:? "as_path" .!= False + +-- | A value-free resolution snapshot. Unlike 'Resolved', a missing required +-- secret is a @"missing_required"@ status here, not an error. +data Report = Report + { reportProvider :: Text + , reportProfile :: Text + , reportSecrets :: [SecretReport] + } deriving (Show, Eq) + +-- | A resolution request. Build it from 'builder' with the @withX@ setters and +-- pass it to 'load' or 'report'. +data Builder = Builder + { bPath :: Maybe Text + , bProvider :: Maybe Text + , bProfile :: Maybe Text + , bReason :: Maybe Text + , bNoValues :: Bool + } + +-- | A builder with no options set. +builder :: Builder +builder = Builder Nothing Nothing Nothing Nothing False + +-- | Resolve from a manifest at this path instead of walking up from the working +-- directory. +withPath :: Text -> Builder -> Builder +withPath v b = b { bPath = Just v } + +-- | Override the provider (a @keyring:\/\/@-style URI or a configured alias). +withProvider :: Text -> Builder -> Builder +withProvider v b = b { bProvider = Just v } + +-- | Override the profile. +withProfile :: Text -> Builder -> Builder +withProfile v b = b { bProfile = Just v } + +-- | Set a human-readable reason for this access (for audited providers). +withReason :: Text -> Builder -> Builder +withReason v b = b { bReason = Just v } + +-- | Omit secret values, returning only structure and provenance. +withNoValues :: Bool -> Builder -> Builder +withNoValues v b = b { bNoValues = v } + +-- | The usable string: the file path for @as_path@ secrets, else the value. +-- 'Nothing' when no usable value is present (e.g. under 'withNoValues'). +get :: ResolvedSecret -> Maybe Text +get s = if secretAsPath s then secretPath s else secretValue s + +-- | Flat @name -> usable value@ map ('Nothing' encodes to JSON @null@), the +-- input for a quicktype-generated deserializer. See @secretspec schema@. +fields :: Resolved -> Map Text (Maybe Text) +fields = Map.map get . resolvedSecrets + +-- | 'fields' as a JSON byte string (a @{SECRET_NAME: value-or-null}@ object). +fieldsJson :: Resolved -> BL.ByteString +fieldsJson = encode . fields + +-- | Export each resolved secret into the process environment by its declared +-- name. Secrets with no usable value (e.g. under 'withNoValues') are skipped. +setAsEnv :: Resolved -> IO () +setAsEnv r = + forM_ (Map.toList (resolvedSecrets r)) $ \(name, secret) -> + case get secret of + Just v -> setEnv (T.unpack name) (T.unpack v) + Nothing -> pure () + +-- | Remove the temp files backing any @as_path@ secrets in this result. The +-- resolver persists those files (mode 0400) so their paths stay valid after +-- resolve returns; the caller owns their lifetime. Call 'close' when done so the +-- secret files do not accumulate in the temp dir. A file already gone is ignored. +close :: Resolved -> IO () +close r = + forM_ (Map.elems (resolvedSecrets r)) $ \secret -> + case (secretAsPath secret, secretPath secret) of + (True, Just p) -> do + let fp = T.unpack p + exists <- doesFileExist fp + when exists (removeFile fp) + _ -> pure () + +-- | The ABI version reported by the loaded native library. +abiVersion :: IO Text +abiVersion = do + -- A static, library-owned string; do not free it. + c <- c_secretspec_abi_version + T.pack <$> peekCString c + +-- | Resolve the secrets. Throws 'MissingRequiredError' if a required secret is +-- missing, and 'SecretSpecError' for any other failure. +load :: Builder -> IO Resolved +load b = do + resp <- callNative (requestBytes b Nothing) + value <- responseValue resp resolveSchemaVersion "resolve" + (prov, prof, secs, mreq, mopt) <- fromResult (parseEither pResolve value) + case mreq of + [] -> pure (Resolved prov prof secs mopt) + xs -> throwIO (MissingRequiredError xs) + where + pResolve = withObject "response" $ \o -> + (,,,,) + <$> o .: "provider" + <*> o .: "profile" + <*> o .:? "secrets" .!= Map.empty + <*> o .:? "missing_required" .!= [] + <*> o .:? "missing_optional" .!= [] + +-- | Resolve a value-free 'Report' (the inventory\/preflight view, the same one +-- the CLI exposes as @check --json@). Unlike 'load', it does not throw when a +-- required secret is missing: that secret appears as a 'SecretReport' with +-- status @"missing_required"@. +report :: Builder -> IO Report +report b = do + resp <- callNative (requestBytes b (Just "report")) + value <- responseValue resp reportSchemaVersion "report" + (prov, prof, secs) <- fromResult (parseEither pReport value) + pure (Report prov prof secs) + where + pReport = withObject "response" $ \o -> + (,,) + <$> o .: "provider" + <*> o .: "profile" + <*> o .:? "secrets" .!= [] + +-- Build the request JSON for a resolve (@mode = Nothing@) or report +-- (@mode = Just "report"@), omitting unset options. +requestBytes :: Builder -> Maybe Text -> BL.ByteString +requestBytes b mode = + encode . object $ + catMaybes + [ ("path" .=) <$> bPath b + , ("provider" .=) <$> bProvider b + , ("profile" .=) <$> bProfile b + , ("reason" .=) <$> bReason b + ] + ++ ["no_values" .= True | bNoValues b] + ++ ["mode" .= m | Just m <- [mode]] + +-- Marshal a request to secretspec_resolve and copy the response out before +-- freeing the native allocation. +callNative :: BL.ByteString -> IO BS.ByteString +callNative reqLazy = + BS.useAsCString (BL.toStrict reqLazy) $ \creq -> do + cresp <- c_secretspec_resolve creq + if cresp == nullPtr + then throwIO (SecretSpecError "ffi" "secretspec_resolve returned null") + else do + resp <- BS.packCString cresp + c_secretspec_free cresp + pure resp + +-- Decode the envelope, unwrap @ok@/@error@, and check the schema version, +-- returning the response object as a 'Value' for the caller to project. +responseValue :: BS.ByteString -> Int -> Text -> IO Value +responseValue resp expectVer kind = do + env <- case eitherDecodeStrict resp :: Either String (Envelope Value) of + Left e -> throwIO (SecretSpecError "parse" (T.pack e)) + Right v -> pure v + if not (envOk env) + then case envError env of + Just (ErrInfo k m) -> throwIO (SecretSpecError k m) + Nothing -> throwIO (SecretSpecError "unknown" "") + else case envResponse env of + Nothing -> throwIO (SecretSpecError "ffi" "secretspec_resolve reported ok with no response") + Just value -> do + ver <- fromResult (parseEither (withObject "response" (.: "schema_version")) value) + unless (ver == expectVer) (throwIO (versionError ver expectVer kind)) + pure value + +versionError :: Int -> Int -> Text -> SecretSpecError +versionError got expected kind = + SecretSpecError "version" $ + T.concat + [ "unsupported ", kind, " schema version ", T.pack (show got) + , " (expected ", T.pack (show expected) + , "); the secretspec-ffi library and this SDK are out of sync" + ] + +fromResult :: Either String a -> IO a +fromResult = either (throwIO . SecretSpecError "parse" . T.pack) pure + +-- The response envelope shared by every native binding. +data Envelope a = Envelope + { envOk :: Bool + , envResponse :: Maybe a + , envError :: Maybe ErrInfo + } + +instance FromJSON a => FromJSON (Envelope a) where + parseJSON = withObject "Envelope" $ \o -> + Envelope <$> o .: "ok" <*> o .:? "response" <*> o .:? "error" + +data ErrInfo = ErrInfo Text Text + +instance FromJSON ErrInfo where + parseJSON = withObject "error" $ \o -> ErrInfo <$> o .: "kind" <*> o .: "message" diff --git a/secretspec-hs/test/Main.hs b/secretspec-hs/test/Main.hs new file mode 100644 index 0000000..f3d4c14 --- /dev/null +++ b/secretspec-hs/test/Main.hs @@ -0,0 +1,268 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | Unit + cross-language conformance tests for the Haskell SDK. Run with the +-- cdylib on the linker and loader paths: +-- +-- > cabal test --extra-lib-dirs="$TARGET/debug" # with LD_LIBRARY_PATH set +module Main (main) where + +import Control.Exception (SomeException, try) +import Control.Monad (filterM, forM) +import Data.Aeson (Value, eitherDecode, eitherDecodeStrict, object, + (.=)) +import qualified Data.Aeson.Key as Key +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import Data.Function ((&)) +import Data.List (isInfixOf, isPrefixOf, sort) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe, isJust) +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.IO as TIO +import qualified SecretSpec as S +import System.Directory (canonicalizePath, createDirectoryIfMissing, + doesDirectoryExist, findExecutable, + getTemporaryDirectory, listDirectory) +import System.Environment (lookupEnv) +import System.Exit (exitFailure, exitSuccess) +import System.FilePath (()) +import System.Process (callProcess, readProcess) + +main :: IO () +main = do + fixturesDir <- canonicalizePath ("." ".." "conformance" "fixtures") + names <- listDirectory fixturesDir + fixtures <- filterM doesDirectoryExist (map (fixturesDir ) (sort names)) + + let tests = + [ ("abi_version_nonempty", testAbiVersion) + , ("missing_required_throws", testMissingRequired) + , ("codegen", testCodegen) + ] + ++ concatMap conformanceTests fixtures + + results <- mapM runOne tests + let failed = [name | (name, ok) <- results, not ok] + putStrLn "" + putStrLn (show (length results - length failed) ++ "/" ++ show (length results) ++ " passed") + if null failed + then exitSuccess + else putStrLn ("FAILED: " ++ unwords failed) >> exitFailure + +runOne :: (String, IO ()) -> IO (String, Bool) +runOne (name, act) = do + r <- try act :: IO (Either SomeException ()) + case r of + Right () -> putStrLn ("ok " ++ name) >> pure (name, True) + Left e -> putStrLn ("FAIL " ++ name ++ ": " ++ show e) >> pure (name, False) + +expect :: Bool -> String -> IO () +expect True _ = pure () +expect False msg = ioError (userError msg) + +-- Unit tests -------------------------------------------------------------- + +testAbiVersion :: IO () +testAbiVersion = do + v <- S.abiVersion + expect (not (T.null v)) "abi version was empty" + +testMissingRequired :: IO () +testMissingRequired = do + tmp <- getTemporaryDirectory + let dir = tmp "secretspec-hs-missing" + createDirectoryIfMissing True dir + writeFile (dir "secretspec.toml") $ + unlines + [ "[project]" + , "name = \"hs-missing\"" + , "revision = \"1.0\"" + , "" + , "[profiles.default]" + , "NEEDED = { description = \"x\", required = true }" + ] + writeFile (dir ".env") "" + r <- + try (S.load (fixtureBuilder dir)) :: + IO (Either S.MissingRequiredError S.Resolved) + case r of + Left _ -> pure () + Right _ -> ioError (userError "expected MissingRequiredError") + +-- End-to-end codegen: secretspec schema -> quicktype --lang haskell -> compile +-- the generated module and decode the SDK's own fieldsJson output with it, so +-- the schema -> fields() linkage is exercised, not just hand-written JSON. Skips +-- unless SECRETSPEC_BIN, npx, runghc, and a cabal-written GHC environment file +-- (so runghc sees aeson and the generated module's transitive imports) are all +-- present -- i.e. when run via `cabal test --write-ghc-environment-files=always` +-- with SECRETSPEC_BIN set, as scripts/ci-sdks.sh does. +testCodegen :: IO () +testCodegen = do + mbin <- lookupEnv "SECRETSPEC_BIN" + npx <- findExecutable "npx" + rghc <- findExecutable "runghc" + hasEnv <- any (isPrefixOf ".ghc.environment.") <$> listDirectory "." + case (mbin, npx, rghc, hasEnv) of + (Just bin, Just _, Just _, True) -> runCodegen bin + _ -> putStrLn " (skipped: needs SECRETSPEC_BIN, npx, runghc, and a ghc env file)" + +runCodegen :: FilePath -> IO () +runCodegen bin = do + tmp <- getTemporaryDirectory + let dir = tmp "secretspec-hs-codegen" + createDirectoryIfMissing True dir + writeFile (dir "secretspec.toml") $ + unlines + [ "[project]" + , "name = \"hs-codegen\"" + , "revision = \"1.0\"" + , "" + , "[profiles.default]" + , "DATABASE_URL = { required = true }" + , "LOG_LEVEL = { required = false, default = \"info\" }" + ] + writeFile (dir ".env") "DATABASE_URL=postgres://db\n" + + -- The SDK itself produces the flat fields JSON the generated decoder consumes, + -- so this exercises the real schema -> fields() linkage. + resolved <- + S.load + ( S.builder + & S.withPath (T.pack (dir "secretspec.toml")) + & S.withProvider (T.pack ("dotenv://" ++ (dir ".env"))) + & S.withReason "codegen" + ) + BL.writeFile (dir "fields.json") (S.fieldsJson resolved) + + callProcess bin ["-f", dir "secretspec.toml", "schema", "-o", dir "schema.json"] + callProcess + "npx" + [ "--yes", "quicktype", "-s", "schema", dir "schema.json" + , "--top-level", "SecretSpec", "--lang", "haskell" + , "--module", "Secrets", "-o", dir "Secrets.hs" + ] + writeFile (dir "Driver.hs") driverSource + + out <- readProcess "runghc" ["-i" ++ dir, dir "Driver.hs", dir "fields.json"] "" + expect ("codegen OK" `isInfixOf` out) ("unexpected driver output: " ++ out) + +-- A standalone program that decodes the SDK's fields JSON with the +-- quicktype-generated SecretSpec type, proving the generated code is usable. +driverSource :: String +driverSource = + unlines + [ "{-# LANGUAGE OverloadedStrings #-}" + , "import Secrets (SecretSpec(..), decodeTopLevel)" + , "import qualified Data.ByteString.Lazy as BL" + , "import System.Environment (getArgs)" + , "import System.Exit (exitFailure)" + , "main :: IO ()" + , "main = do" + , " [f] <- getArgs" + , " bytes <- BL.readFile f" + , " case decodeTopLevel bytes of" + , " Just s | databaseURLSecretSpec s == \"postgres://db\" -> putStrLn \"codegen OK\"" + , " _ -> exitFailure" + ] + +-- Conformance ------------------------------------------------------------- + +conformanceTests :: FilePath -> [(String, IO ())] +conformanceTests dir = + [ ("conformance:" ++ base, testConformance dir) + , ("conformance_no_values:" ++ base, testNoValues dir) + , ("conformance_report:" ++ base, testReport dir) + ] + where + base = lastSegment dir + +testConformance :: FilePath -> IO () +testConformance dir = do + resolved <- S.load (fixtureBuilder dir) + actual <- canonical resolved + expected <- readJson (dir "expected.json") + expect (actual == expected) (mismatch "expected.json" actual expected) + +-- Under no_values every SDK must emit the same all-null fields map: a +-- value-less secret serializes to null, not an empty string. +testNoValues :: FilePath -> IO () +testNoValues dir = do + resolved <- S.load (S.withNoValues True (fixtureBuilder dir)) + let actual = either error id (eitherDecode (S.fieldsJson resolved)) :: Value + expected <- readJson (dir "expected_no_values.json") + expect (actual == expected) (mismatch "expected_no_values.json" actual expected) + S.close resolved + +-- The value-free report (status + provenance) is identical across SDKs. +testReport :: FilePath -> IO () +testReport dir = do + rep <- S.report (fixtureBuilder dir) + let actual = canonicalReport rep + expected <- readJson (dir "expected_report.json") + expect (actual == expected) (mismatch "expected_report.json" actual expected) + +fixtureBuilder :: FilePath -> S.Builder +fixtureBuilder dir = + S.builder + & S.withPath (T.pack (dir "secretspec.toml")) + & S.withProvider (T.pack ("dotenv://" ++ (dir ".env"))) + & S.withReason "conformance" + +canonical :: S.Resolved -> IO Value +canonical r = do + entries <- + forM (Map.toList (S.resolvedSecrets r)) $ \(name, secret) -> do + value <- + if S.secretAsPath secret + then TIO.readFile (T.unpack (fromMaybe "" (S.secretPath secret))) + else pure (fromMaybe "" (S.secretValue secret)) + pure + ( Key.fromText name + .= object + [ "value" .= value + , "source" .= S.secretSource secret + , "as_path" .= S.secretAsPath secret + ] + ) + pure $ + object + [ "profile" .= S.resolvedProfile r + , "secrets" .= object entries + , "missing_required" .= ([] :: [Text]) + , "missing_optional" .= sort (S.resolvedMissingOptional r) + ] + +canonicalReport :: S.Report -> Value +canonicalReport rep = + object + [ "profile" .= S.reportProfile rep + , "secrets" + .= object + [ Key.fromText (S.srName s) + .= object + [ "status" .= S.srStatus s + , "required" .= S.srRequired s + , "as_path" .= S.srAsPath s + , "generated" .= S.srGenerated s + , "default_applied" .= S.srDefaultApplied s + , -- Present-or-not (not the path-dependent value) so the vector + -- is machine-independent yet still catches a dropped provider. + "source_provider" .= isJust (S.srSourceProvider s) + ] + | s <- S.reportSecrets rep + ] + ] + +readJson :: FilePath -> IO Value +readJson p = do + bytes <- BS.readFile p + either (ioError . userError) pure (eitherDecodeStrict bytes) + +mismatch :: String -> Value -> Value -> String +mismatch name actual expected = + name ++ " mismatch\n got: " ++ show actual ++ "\nwant: " ++ show expected + +lastSegment :: FilePath -> String +lastSegment = reverse . takeWhile (/= '/') . reverse diff --git a/secretspec/README.md b/secretspec/README.md index f505cf9..42087c9 100644 --- a/secretspec/README.md +++ b/secretspec/README.md @@ -189,6 +189,7 @@ profile, and generator works identically with no per-language resolution logic: - [Go](https://secretspec.dev/sdk/go) (via purego, no cgo) - [Ruby](https://secretspec.dev/sdk/ruby) (via stdlib Fiddle) - [Node.js / TypeScript](https://secretspec.dev/sdk/nodejs) (napi-rs addon) +- [Haskell](https://secretspec.dev/sdk/haskell) (build-time FFI link) ```python from secretspec import SecretSpec From 6af14bb240b76921f6a04e6710a46eac4413f386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 12:23:43 -0400 Subject: [PATCH 38/56] fix(core): make value-free resolve/report side-effect-free and surface chain errors The value-free surfaces (Secrets::report(), resolve_without_values(), the FFI no_values/report requests, and check --json/--explain) ran the full resolution, so they minted and stored a brand new secret in the provider for any declared generate secret that was not yet set, and failed outright on a read-only provider. Thread a Materialize flag through validate_audited so those entry points share the identical resolution logic but skip its two side effects: a generatable-but-absent secret is reported as it would resolve (generated) without being created, and no as_path secret is written to a temp file. Also: a per-secret provider chain whose primary provider errors and whose fallback chain has no value now surfaces that provider error, exactly as a single-provider failure already did, instead of silently reporting the secret as missing_required, so machine consumers can tell an outage from an unprovisioned secret. Route check --json/--explain through report() (removing a duplicated inline copy of its construction). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 39 ++++++++ secretspec/src/cli/mod.rs | 13 ++- secretspec/src/resolve.rs | 5 + secretspec/src/secrets.rs | 192 +++++++++++++++++++++++++++----------- secretspec/src/tests.rs | 151 ++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e16c397..9f92ec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `SecretResolution`, and `ResolutionStatus` types. ### Fixed +- The value-free resolution surfaces — `Secrets::report()`, `resolve_without_values()` + / the FFI `{"no_values": true}` and `{"mode": "report"}` requests, and + `check --json`/`--explain` (now routed through `report()`) — are side-effect-free. + Previously they ran the full resolution, which **minted and stored a brand-new + secret in the provider** for any declared `generate` secret that was not yet + set (and failed outright on a read-only provider like `env://`). A read-only + preflight no longer writes to a provider: a generatable-but-absent secret is + reported as it *would* resolve (`generated`) without being created. +- The value-free path no longer writes `as_path` secrets to a temp file. It used + to materialize every secret to disk (a 0400 file in the temp dir) only to delete + it on drop, contrary to its documented "leaves no temp file behind" contract; + a `no_values` resolve now never touches disk. +- A per-secret provider chain whose **primary provider errors** (e.g. an + unreachable vault) and whose fallback chain has no value now surfaces that + provider error, exactly as a single-provider failure already did, instead of + silently reporting the secret as `missing_required`. Machine consumers + (`check --json`, the SDKs) can again distinguish a provider outage from an + unprovisioned secret. +- The Haskell SDK no longer leaks the native (secret-bearing) response buffer + when an asynchronous exception — e.g. a `System.Timeout.timeout` around + `load`/`report` — arrives mid-resolve: the free is now installed under `mask` + and runs via `finally`. +- The Go SDK no longer panics when the loaded library is missing a symbol (an + incompatible/old cdylib): `purego.RegisterLibFunc` panics are recovered and + returned as a `load` `*Error`, instead of escaping `sync.Once` and leaving the + loader permanently poisoned with nil function pointers. +- A zero-value Go `Builder` (`var b secretspec.Builder` / `&Builder{}`, not via + `New()`) no longer panics with a nil-map write in its `WithX` setters; the + request map is initialized lazily. +- The Go SDK extracts an embedded (`-tags embed_lib`) cdylib into a per-user, + owner-only cache directory (under `os.UserCacheDir()`) and verifies the + directory is private before use, instead of a predictably-named directory under + the shared system temp dir. This closes a local-attacker file-swap (TOCTOU) + that could run attacker code in the process on a shared host, and avoids + `noexec` temp mounts. An embedded git-LFS pointer (from a botched release) is + now rejected with a clear error rather than fed to `dlopen`. Distribution + switched to the system-library model (provide the cdylib via + `SECRETSPEC_FFI_LIB`, or vendor it for an `embed_lib` build); git-LFS + module + proxy cannot ship a working library, so it is no longer prescribed. - The `provider` field of the resolution report (`check --json`/`--explain`) and the resolve response (`resolve --json`, every SDK's `response.provider`) is now run through `redact_uri_strict`, so a user-authored provider alias or diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index b5f91b5..e970df3 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -674,14 +674,13 @@ pub fn main() -> Result<()> { // on every declared secret (including missing required ones) and // exit non-zero when a required secret is missing, so CI can gate. if json || explain { - let report = match app - .validate() + // Value-free report: never mints, stores, or writes a secret as + // a side effect of this read-only preflight (unlike `validate()`, + // which is the value-injecting path). + let report = app + .report() .into_diagnostic() - .wrap_err("Failed to resolve secrets")? - { - Ok(validated) => validated.report(), - Err(errors) => errors.report(), - }; + .wrap_err("Failed to resolve secrets")?; if json { let rendered = serde_json::to_string_pretty(&report) diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs index 816841c..371ff99 100644 --- a/secretspec/src/resolve.rs +++ b/secretspec/src/resolve.rs @@ -14,6 +14,11 @@ //! is empty and `missing_required` is populated, mirroring the derive crate's //! `load()` which fails rather than returning partial secrets. //! +//! The `no_values` request variant (and [`crate::Secrets::resolve_without_values`]) +//! produces the same shape with every `value`/`path` set to `None`, and is +//! additionally side-effect-free: it never mints a generated secret and never +//! writes an `as_path` temp file. +//! //! The shape is versioned via [`RESOLVE_SCHEMA_VERSION`]. The canonical JSON //! Schema lives at `schema/resolve-response.schema.json`. diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index bafeadc..34d4e12 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -54,6 +54,25 @@ fn warn_primary_provider_failure(display_uri: Option<&str>, err: &SecretSpecErro ); } +/// Whether a resolution pass may produce side effects and persist secrets. +/// +/// A resolution pass always queries providers to learn what is present, but the +/// two value-free entry points ([`Secrets::report`], [`Secrets::resolve_without_values`]) +/// must not change anything as a side effect of reading. This flag gates the two +/// mutating steps of a pass so those entry points can share the exact same +/// resolution logic without inheriting its side effects. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Materialize { + /// Full pass: mint-and-store a missing generatable secret and write each + /// `as_path` secret to a temp file. Backs `validate()`/`resolve()`/`check`. + Values, + /// Value-free pass: never write a generated secret back to a provider and + /// never persist a secret to disk. A generatable-but-absent secret is still + /// reported as it *would* resolve, without minting it. Backs `report()` and + /// `resolve_without_values()`. + None, +} + /// Walks up from the current directory looking for `secretspec.toml`. fn find_config_file() -> Result { let mut dir = std::env::current_dir()?; @@ -1321,8 +1340,9 @@ impl Secrets { // First validate to see what's missing. Use the non-auditing variant: // the caller that owns this operation (`check`, `run`) records its own - // audit event, so re-validating here must not emit another `Check`. - let validation_result = self.validate_audited(false)?; + // audit event, so re-validating here must not emit another `Check`. This + // is the value-injecting path (`run`), so it materializes fully. + let validation_result = self.validate_audited(false, Materialize::Values)?; match validation_result { Ok(valid_secrets) => Ok(valid_secrets), @@ -1410,7 +1430,7 @@ impl Secrets { // Re-validate to get the updated results // Re-validate after prompting; still part of the same // operation, so do not emit another `Check` event. - match self.validate_audited(false)? { + match self.validate_audited(false, Materialize::Values)? { Ok(valid_secrets) => Ok(valid_secrets), Err(still_errors) => Err(SecretSpecError::RequiredSecretMissing( still_errors.missing_required.join(", "), @@ -1850,6 +1870,17 @@ impl Secrets { /// Returns `Ok(Some(value))` if generation succeeded, /// `Ok(None)` if generation is not configured, /// or `Err` if generation was configured but failed. + /// Whether an absent secret would be auto-generated on a full resolve: it + /// declares an enabled `generate` config. Lets the value-free pass report + /// that a missing secret *would* resolve via generation without minting and + /// storing it (the side effect [`Self::try_generate_secret`] performs). A + /// `generate`-enabled secret with no declared type still counts here; the + /// resulting "no type" error is raised only on the full pass that actually + /// generates, keeping the value-free preflight failure-free. + fn would_generate(secret_config: &crate::config::Secret) -> bool { + matches!(&secret_config.generate, Some(g) if g.is_enabled()) + } + fn try_generate_secret( &self, name: &str, @@ -1986,7 +2017,7 @@ impl Secrets { /// and by `secretspec-derive`-generated code — so it records exactly one /// `Check` audit event per call. pub fn validate(&self) -> Result> { - self.validate_audited(true) + self.validate_audited(true, Materialize::Values) } /// Resolve every declared secret into a value-carrying [`ResolveResponse`], @@ -2006,31 +2037,38 @@ impl Secrets { self.resolve_impl(true) } - /// Like [`Self::resolve`], but never materializes secret values: every - /// `value`/`path` in the response is `None`, no `as_path` temp file is - /// persisted, and no secret byte is ever copied into the response. Structure + /// Like [`Self::resolve`], but value-free and side-effect-free: every + /// `value`/`path` in the response is `None`, no `as_path` temp file is ever + /// written, and no missing generatable secret is minted or stored. Structure /// and provenance (`as_path`, `source`, `source_provider`, /// `missing_optional`) are still populated. This backs the `no_values` - /// request path, so a policy/preflight consumer that wants only the resolve - /// shape never pulls secret values into its process, and `as_path` - /// resolution leaves no temp file behind. Resolution still runs (providers - /// are still queried) so provenance can be reported; a missing required - /// secret still fails the same way as [`Self::resolve`]. For a value-free - /// view that tolerates missing required secrets, use [`Self::report`]. + /// request path, so a policy/preflight consumer gets the resolve shape + /// without persisting a secret to disk or mutating a provider. Resolution + /// still queries providers so provenance can be reported — a value may + /// transit memory transiently to learn whether it is present — but nothing + /// is materialized; a missing required secret still fails the same way as + /// [`Self::resolve`]. For a value-free view that tolerates missing required + /// secrets, use [`Self::report`]. pub fn resolve_without_values(&self) -> Result { self.resolve_impl(false) } /// Shared core of [`Self::resolve`]/[`Self::resolve_without_values`]. /// `include_values` gates whether resolved secret values are copied into the - /// response (and whether `as_path` temp files are persisted past the call). + /// response and, in turn, whether the underlying pass mints generated + /// secrets and writes `as_path` temp files at all. fn resolve_impl(&self, include_values: bool) -> Result { - match self.validate()? { + let materialize = if include_values { + Materialize::Values + } else { + Materialize::None + }; + match self.validate_audited(true, materialize)? { Ok(mut validated) => { // Persist as_path temp files so returned paths outlive this call. - // Only needed when we actually return paths: under - // `!include_values` the temp files are dropped (auto-deleted) - // together with `validated`, so nothing is left on disk. + // Only the full pass writes any: under `Materialize::None` no + // temp file is ever created, so there is nothing to persist and + // nothing is left on disk. if include_values { validated.keep_temp_files()?; } @@ -2114,8 +2152,14 @@ impl Secrets { /// declared and how would each secret resolve" even for a profile whose /// secrets the caller cannot fully provide. It is the same report the CLI /// surfaces as `check --json` / `check --explain`, exposed to the SDKs. + /// + /// This pass is value-free and side-effect-free: it never mints or stores a + /// generatable secret and never writes an `as_path` temp file. A secret that + /// *would* be generated on a real resolve is reported as resolved + /// (`generated`), so the report still answers "would this resolve" without + /// mutating any provider or touching disk. pub fn report(&self) -> Result { - Ok(match self.validate()? { + Ok(match self.validate_audited(true, Materialize::None)? { Ok(validated) => validated.report(), Err(errors) => errors.report(), }) @@ -2131,9 +2175,16 @@ impl Secrets { /// not also recorded as a `Check`. The trade-off: a direct /// `ensure_secrets` call (rare; not the path `secretspec-derive` uses) does /// not emit a `Check` read event, though any writes it performs are audited. + /// + /// `materialize` gates the pass's two side effects (minting+storing a + /// generated secret, and writing `as_path` temp files). [`Materialize::None`] + /// runs the identical resolution but skips both, so the value-free entry + /// points reach the same per-secret status without mutating a provider or + /// touching disk; see [`Materialize`]. fn validate_audited( &self, emit_check: bool, + materialize: Materialize, ) -> Result> { // Enforce the reason policy. For the top-level read (`emit_check`) a denial // is itself audited; internal re-validations (emit_check=false) re-check the @@ -2282,13 +2333,18 @@ impl Secrets { match fetched_values.remove(&name) { Some(value) => { source_provider = group_uris.get(&secret_primary_uris[&name]).cloned(); - self.insert_resolved( - &mut secrets, - &mut temp_files, - name, - value, - as_path, - )?; + // Copy the value into the response only on a full + // pass; a value-free pass has the status it needs and + // never materializes a value or writes a temp file. + if materialize == Materialize::Values { + self.insert_resolved( + &mut secrets, + &mut temp_files, + name, + value, + as_path, + )?; + } status = ResolutionStatus::Resolved; } None => { @@ -2301,13 +2357,31 @@ impl Secrets { (None, Some(providers)) if providers.len() > 1 => { let fallback_uris = self.resolve_provider_aliases(Some(&providers[1..]))?; - self.get_secret_from_providers( + let resolved = self.get_secret_from_providers( &self.config.project.name, &name, &profile_name, fallback_uris.as_deref(), None, - )? + )?; + // A primary that *errored* (not merely + // lacked the secret) plus an empty + // fallback chain is not "missing": the + // authoritative provider is unreachable + // and might hold the value. Surface the + // primary error, exactly as the + // single-provider arm below does, instead + // of silently downgrading to + // missing_required — a machine consumer + // must be able to tell an outage from an + // unprovisioned secret. + if resolved.0.is_none() && primary_failed { + let err = failed_primary_uris + .remove(primary_uri) + .expect("primary_failed implies entry present"); + return Err(err); + } + resolved } // No alternative chain to try and the primary failed: surface the // original error rather than reporting the secret as merely @@ -2324,36 +2398,48 @@ impl Secrets { if let Some(value) = fallback_value { source_provider = fallback_uri; - self.insert_resolved( - &mut secrets, - &mut temp_files, - name, - value, - as_path, - )?; + if materialize == Materialize::Values { + self.insert_resolved( + &mut secrets, + &mut temp_files, + name, + value, + as_path, + )?; + } status = ResolutionStatus::Resolved; - } else if let Some(generated_value) = - self.try_generate_secret(&name, secret_config, &profile_name)? - { + } else if Self::would_generate(secret_config) { + // The secret would be auto-generated. A full pass + // mints and stores it (writing a temp file when + // `as_path`); a value-free pass reports that it + // would resolve without minting or storing + // anything. generated = true; - self.insert_resolved( - &mut secrets, - &mut temp_files, - name, - generated_value, - as_path, - )?; + if materialize == Materialize::Values { + let generated_value = self + .try_generate_secret(&name, secret_config, &profile_name)? + .expect("would_generate implies a generated value"); + self.insert_resolved( + &mut secrets, + &mut temp_files, + name, + generated_value, + as_path, + )?; + } status = ResolutionStatus::Resolved; } else if let Some(default_value) = default { default_applied = true; - self.insert_resolved( - &mut secrets, - &mut temp_files, - name.clone(), - SecretString::new(default_value.clone().into()), - as_path, - )?; - with_defaults.push((name, default_value)); + if materialize == Materialize::Values { + self.insert_resolved( + &mut secrets, + &mut temp_files, + name.clone(), + SecretString::new(default_value.clone().into()), + as_path, + )?; + with_defaults.push((name, default_value)); + } status = ResolutionStatus::Resolved; } else if required { missing_required.push(name); diff --git a/secretspec/src/tests.rs b/secretspec/src/tests.rs index 9f49a2b..f663435 100644 --- a/secretspec/src/tests.rs +++ b/secretspec/src/tests.rs @@ -636,6 +636,157 @@ fn test_report_lists_missing_required_without_failing() { assert_eq!(status("MISSING"), Some(ResolutionStatus::MissingRequired)); } +/// A generatable secret with no stored value must be reported by the value-free +/// surfaces (`report()`, `resolve_without_values()`) as *would-generate* without +/// actually minting and storing it — a read-only preflight must not mutate the +/// provider. The full `resolve()` still generates and writes. +#[test] +fn test_value_free_surfaces_do_not_generate_or_store() { + use crate::config::GenerateConfig; + use crate::report::ResolutionStatus; + use crate::resolve::ResolvedSource; + + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + fs::write(&env_path, "").unwrap(); + + let mut secrets = HashMap::new(); + secrets.insert( + "SESSION_KEY".to_string(), + Secret { + description: Some("generated".to_string()), + required: Some(true), + secret_type: Some("hex".to_string()), + generate: Some(GenerateConfig::Bool(true)), + ..Default::default() + }, + ); + + let provider = format!("dotenv://{}", env_path.display()); + let spec = Secrets::new(resolve_test_config(secrets), None, Some(provider), None); + + // report(): the secret would resolve via generation, but nothing is written. + let report = spec.report().unwrap(); + let entry = report + .secrets + .iter() + .find(|s| s.name == "SESSION_KEY") + .expect("SESSION_KEY in report"); + assert_eq!(entry.status, ResolutionStatus::Resolved); + assert!(entry.generated); + assert_eq!( + fs::read_to_string(&env_path).unwrap(), + "", + "report() must not store a generated secret" + ); + + // resolve_without_values(): same — provenance says generated, no value, no write. + let response = spec.resolve_without_values().unwrap(); + let resolved = &response.secrets["SESSION_KEY"]; + assert_eq!(resolved.source, ResolvedSource::Generated); + assert!(resolved.value.is_none()); + assert_eq!( + fs::read_to_string(&env_path).unwrap(), + "", + "resolve_without_values() must not store a generated secret" + ); + + // The full resolve still generates and persists the value. + let full = spec.resolve().unwrap(); + assert!(full.is_ok()); + assert!(full.secrets["SESSION_KEY"].value.is_some()); + assert!( + fs::read_to_string(&env_path) + .unwrap() + .contains("SESSION_KEY"), + "resolve() generates and stores the secret" + ); +} + +/// The value-free `report()` over a read-only provider must succeed (a missing +/// generatable secret is reported as would-generate) rather than failing because +/// a generated value cannot be stored. Regression: the value-free path used to +/// reach the provider write and error on `env://`. +#[test] +fn test_value_free_report_tolerates_read_only_provider() { + use crate::config::GenerateConfig; + use crate::report::ResolutionStatus; + + let mut secrets = HashMap::new(); + secrets.insert( + "SESSION_KEY".to_string(), + Secret { + description: Some("generated".to_string()), + required: Some(true), + secret_type: Some("hex".to_string()), + generate: Some(GenerateConfig::Bool(true)), + ..Default::default() + }, + ); + + let spec = Secrets::new( + resolve_test_config(secrets), + None, + Some("env://".to_string()), + None, + ); + + let report = spec.report().expect("report() must not fail on env://"); + let entry = report + .secrets + .iter() + .find(|s| s.name == "SESSION_KEY") + .expect("SESSION_KEY in report"); + assert_eq!(entry.status, ResolutionStatus::Resolved); + assert!(entry.generated); +} + +/// When a per-secret provider chain's primary provider *errors* (not merely +/// lacks the secret) and the fallback chain has no value, the resolution must +/// surface the provider error — exactly like a single-provider failure — instead +/// of silently downgrading to `missing_required`, so a machine consumer can tell +/// an outage from an unprovisioned secret. +#[test] +fn test_chain_primary_error_surfaces_instead_of_missing() { + let temp_dir = TempDir::new().unwrap(); + let env_path = temp_dir.path().join(".env"); + // Fallback provider is reachable but does not hold the secret. + fs::write(&env_path, "").unwrap(); + + let mut secrets = HashMap::new(); + secrets.insert( + "DB_PASSWORD".to_string(), + Secret { + description: Some("db".to_string()), + required: Some(true), + providers: Some(vec!["primary".to_string(), "fallback".to_string()]), + ..Default::default() + }, + ); + + // Primary alias resolves to an unbuildable provider (the "outage"); fallback + // is a healthy dotenv that simply lacks the key. + let mut provider_aliases = HashMap::new(); + provider_aliases.insert("primary".to_string(), "bogus://unreachable".to_string()); + provider_aliases.insert( + "fallback".to_string(), + format!("dotenv://{}", env_path.display()), + ); + + let mut config = resolve_test_config(secrets); + config.providers = Some(provider_aliases); + + // No explicit provider override, so the per-secret chain is used. + let spec = Secrets::new(config, None, None, None); + + // The primary provider error must propagate, not be reported as missing. + assert!( + spec.resolve().is_err(), + "a primary provider outage with an empty fallback must surface the error" + ); + assert!(spec.report().is_err()); +} + #[test] fn test_secretspec_new() { let config = Config { From 65c0f7fc77684c6bd3cd6286c6c807ace6b315d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 12:25:21 -0400 Subject: [PATCH 39/56] fix(go): harden the SDK and switch to the system library distribution model Three robustness fixes plus a distribution correction: * Embedded (-tags embed_lib) cdylibs are extracted into a per-user, owner-only cache directory (os.UserCacheDir) whose privacy is verified before use, instead of a predictably named directory under the shared system temp dir. This closes a local attacker file swap (TOCTOU) that could run attacker code in the process on a shared host, and avoids noexec temp mounts. An embedded git LFS pointer (from a botched release) is rejected with a clear error rather than fed to dlopen. * A missing symbol in the loaded library no longer panics: purego.RegisterLibFunc panics are recovered and returned as a load error, so an incompatible cdylib does not escape sync.Once and leave the loader poisoned with nil pointers. * A zero-value Builder (var b Builder, not via New()) no longer panics with a nil-map write in its WithX setters; the request map initializes lazily. Distribution moves to the system library model: git LFS plus the module proxy cannot ship a working library (the proxy serves LFS pointer text), so it is no longer prescribed. Consumers provide the cdylib via SECRETSPEC_FFI_LIB or vendor it for an embed_lib build. RELEASE.md, .gitattributes, .gitignore, the workflow, README, and docs updated accordingly. Tests release as_path temp files so repeated runs leave no secret files behind. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/go-embed.yml | 6 ++- RELEASE.md | 36 ++++++++----- docs/src/content/docs/sdk/go.md | 15 +++++- secretspec-go/.gitattributes | 10 ++-- secretspec-go/.gitignore | 4 +- secretspec-go/README.md | 20 ++++++- secretspec-go/conformance_test.go | 3 ++ secretspec-go/embedded.go | 81 ++++++++++++++++++++++++++--- secretspec-go/embedded_nonunix.go | 17 ++++++ secretspec-go/embedded_unix.go | 38 ++++++++++++++ secretspec-go/embedded_unix_test.go | 63 ++++++++++++++++++++++ secretspec-go/secretspec.go | 34 ++++++++++-- secretspec-go/secretspec_test.go | 13 +++++ 13 files changed, 302 insertions(+), 38 deletions(-) create mode 100644 secretspec-go/embedded_nonunix.go create mode 100644 secretspec-go/embedded_unix.go create mode 100644 secretspec-go/embedded_unix_test.go diff --git a/.github/workflows/go-embed.yml b/.github/workflows/go-embed.yml index c499798..88807c7 100644 --- a/.github/workflows/go-embed.yml +++ b/.github/workflows/go-embed.yml @@ -2,8 +2,10 @@ name: "Go embedded lib" # Builds the per-platform cdylib the Go SDK embeds (go:embed, behind the # `embed_lib` build tag) and verifies the embedded build works with no -# SECRETSPEC_FFI_LIB set. The libraries are uploaded as artifacts; a release -# commits them into the module via git-LFS (they are tens of MB each). +# SECRETSPEC_FFI_LIB set. The libraries are uploaded as artifacts and attached to +# GitHub releases for users who want a self-contained `-tags embed_lib` build; +# they are never committed to the repo (the Go module proxy does not carry binary +# assets — see RELEASE.md). on: workflow_dispatch: diff --git a/RELEASE.md b/RELEASE.md index db0ecc5..f07425a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -30,19 +30,29 @@ Version tags are `vX.Y.Z`; the publish jobs trigger on them. build the cdylib on an old-glibc baseline (e.g. a `manylinux` container, as the Python job does, or `rake-compiler-dock`) and bundle that. Tracked follow-up. -## Go (git-LFS) — `go-embed.yml` - -Go has no binary registry, so the per-platform cdylibs are committed into the -module and embedded via `go:embed` (behind the `embed_lib` build tag). - -- **Build:** `go-embed.yml` builds the per-platform libs and uploads them as - artifacts. -- **Release (manual):** stage all platforms' libs into `secretspec-go/lib/` - (from the CI artifacts), un-ignore them, `git lfs track` is already set via - `secretspec-go/.gitattributes`, commit them with LFS, and flip embedding on by - default (drop the `embed_lib` gate or document `-tags embed_lib`). The libs are - ~34 MB each, so plain git is unsuitable; **git-LFS must be enabled** for the - repo. +## Go (system library) — `go-embed.yml` + +Go has no binary registry, and the module proxy (`proxy.golang.org`) builds +module zips from raw git objects — it does **not** run git-LFS smudge filters, so +LFS-tracked files reach consumers as ~130-byte pointer text, not libraries. +`go:embed` over LFS therefore cannot ship a working library through `go get`. +(Committing the ~34 MB-per-platform libs to *plain* git would work but bloats +history permanently and ships every platform's lib in the module zip.) + +So the Go SDK follows the purego norm: the cdylib is provided at runtime, not +shipped through the module. Consumers either set `SECRETSPEC_FFI_LIB` to an +installed/built `libsecretspec_ffi`, or build with `-tags embed_lib` after +staging the per-platform library into `secretspec-go/lib/` themselves (a +self-contained, vendored build — not a module-proxy install). + +- **Build:** `go-embed.yml` builds the per-platform libs, uploads them as + artifacts, and smoke-tests an `-tags embed_lib` build with a staged lib. +- **Release:** nothing to publish to a registry. Attach the per-platform cdylibs + to the GitHub release so users who want a self-contained build can download and + stage them. Do **not** commit binaries to the repo (plain git or LFS). + +> The loader rejects an embedded git-LFS pointer with a clear error, so a botched +> LFS-based build fails loudly instead of feeding pointer text to `dlopen`. ## Node.js (npm) — `node-addon.yml` diff --git a/docs/src/content/docs/sdk/go.md b/docs/src/content/docs/sdk/go.md index e80a583..f483992 100644 --- a/docs/src/content/docs/sdk/go.md +++ b/docs/src/content/docs/sdk/go.md @@ -48,5 +48,16 @@ fmt.Println(typed.DatabaseURL) ## Library discovery -The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, -or a Cargo `target` directory found by searching up from the working directory. +The native `secretspec-ffi` cdylib is resolved at runtime, in order: + +1. The `SECRETSPEC_FFI_LIB` environment variable (an explicit path). +2. A library embedded at build time with `-tags embed_lib`. +3. A Cargo `target` directory found by searching up from the working directory + (the development path). + +The SDK uses [purego](https://github.com/ebitengine/purego), so the cdylib is +loaded at runtime, not linked. Either install/build `libsecretspec_ffi` and set +`SECRETSPEC_FFI_LIB`, or stage the per-platform library into `lib/` and build +with `-tags embed_lib` for a self-contained binary. The embedded library is +extracted to a per-user, owner-only cache directory at first use, and is not +distributed through the Go module proxy. diff --git a/secretspec-go/.gitattributes b/secretspec-go/.gitattributes index 9481ffd..6970c7b 100644 --- a/secretspec-go/.gitattributes +++ b/secretspec-go/.gitattributes @@ -1,5 +1,5 @@ -# The embedded cdylibs are large (tens of MB); track them with git-LFS when a -# release commits them. They are gitignored during development. -lib/*.so filter=lfs diff=lfs merge=lfs -text -lib/*.dylib filter=lfs diff=lfs merge=lfs -text -lib/*.dll filter=lfs diff=lfs merge=lfs -text +# Per-platform cdylibs are never committed to this repo: the Go module proxy +# does not run git-LFS smudge filters, so LFS-tracked binaries reach `go get` +# consumers as pointer files, and plain git would bloat history. They are staged +# locally for `-tags embed_lib` builds (see lib/ in .gitignore) and attached to +# GitHub releases instead. See RELEASE.md. diff --git a/secretspec-go/.gitignore b/secretspec-go/.gitignore index 2d6ce01..0ba7666 100644 --- a/secretspec-go/.gitignore +++ b/secretspec-go/.gitignore @@ -1,3 +1,3 @@ -# Embedded cdylibs are large (tens of MB); staged at build time, committed for a -# release via git-LFS rather than plain git. +# Per-platform cdylibs are staged here for local `-tags embed_lib` builds and +# never committed (see .gitattributes / RELEASE.md). lib/ diff --git a/secretspec-go/README.md b/secretspec-go/README.md index 67e22f7..21aec91 100644 --- a/secretspec-go/README.md +++ b/secretspec-go/README.md @@ -58,5 +58,21 @@ for _, s := range report.Secrets { ## Library discovery -The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, -or a Cargo `target` directory found by searching up from the working directory. +The native `secretspec-ffi` cdylib is resolved at runtime, in order: + +1. The `SECRETSPEC_FFI_LIB` environment variable (an explicit path). +2. A library embedded at build time with `-tags embed_lib` (see below). +3. A Cargo `target` directory found by searching up from the working directory + (the development path). + +This SDK uses [purego](https://github.com/ebitengine/purego), so the cdylib is +loaded at runtime rather than linked. Provide it one of two ways: + +- **System library:** install/build `libsecretspec_ffi` and point + `SECRETSPEC_FFI_LIB` at it (or run inside a Cargo checkout, which the search in + step 3 finds automatically). +- **Vendored/embedded:** stage the per-platform library into `lib/` and build + with `go build -tags embed_lib`. The library is then embedded via `go:embed` + and extracted to a per-user, owner-only cache directory at first use. This is + an opt-in for self-contained builds; it is **not** shipped through the Go + module proxy (which does not carry binary assets). diff --git a/secretspec-go/conformance_test.go b/secretspec-go/conformance_test.go index a34c81c..fff4bca 100644 --- a/secretspec-go/conformance_test.go +++ b/secretspec-go/conformance_test.go @@ -37,6 +37,9 @@ func TestConformance(t *testing.T) { if err != nil { t.Fatal(err) } + // Remove any as_path temp files this value-carrying resolve + // materialized, so repeated runs do not accumulate secret files. + defer resolved.Close() actual := canonical(t, resolved) diff --git a/secretspec-go/embedded.go b/secretspec-go/embedded.go index 8454d5f..85d3bdd 100644 --- a/secretspec-go/embedded.go +++ b/secretspec-go/embedded.go @@ -1,31 +1,63 @@ package secretspec import ( + "bytes" "crypto/sha256" "encoding/hex" + "fmt" "os" "path/filepath" + "strconv" ) // extractEmbedded writes the build-time-embedded cdylib to a content-addressed -// temp file (reused across runs and processes) and returns its path, so purego +// file under a per-user, owner-only directory and returns its path, so purego // can dlopen it. The per-platform `embeddedLib` and `embeddedLibName` are // defined in the build-tagged embedded__.go files (or zeroed by // embedded_unsupported.go). +// +// Security: the extraction directory must be one no other user can write to, +// otherwise a local attacker on a shared host could pre-create the +// predictably-named directory or swap the file between extraction and dlopen +// (a TOCTOU that yields code execution in this process). We therefore extract +// under the per-user cache directory (inside $HOME), create the leaf 0700, and +// verify it is genuinely private — owned by us, not a symlink, no group/other +// access — before trusting anything inside it. func extractEmbedded() (string, error) { + // A git-LFS pointer (or any non-library blob) embedded by a botched release + // is not a loadable library; fail loudly here instead of handing pointer + // text to dlopen and getting a cryptic "invalid ELF header". + if isLFSPointer(embeddedLib) { + return "", &Error{ + Kind: "load", + Message: "embedded library is a git-LFS pointer, not a shared library; " + + "the build embedded an unresolved LFS file — set SECRETSPEC_FFI_LIB " + + "to a real cdylib or rebuild without git-LFS", + } + } + sum := sha256.Sum256(embeddedLib) - // Content-addressed by the full digest, in an owner-only directory: a - // different library never collides, and the path is not world-writable. - dir := filepath.Join(os.TempDir(), "secretspec-ffi-"+hex.EncodeToString(sum[:])) + + base, err := extractBaseDir() + if err != nil { + return "", err + } + // Content-addressed by the full digest: a different library never collides. + dir := filepath.Join(base, "secretspec-ffi", hex.EncodeToString(sum[:])) if err := os.MkdirAll(dir, 0o700); err != nil { return "", err } + // MkdirAll is a no-op on a pre-existing directory regardless of its owner or + // mode, so verify the leaf is private before writing into or reading from it. + if err := verifyPrivateDir(dir); err != nil { + return "", err + } path := filepath.Join(dir, embeddedLibName) // Reuse the cached file only if its contents hash to the embedded library's - // digest. A size-only check would reuse a truncated/corrupted or - // attacker-planted same-length file; verifying the content rejects those and - // re-extracts the genuine bytes below. + // digest. A size-only check would reuse a truncated/corrupted file; verifying + // the content rejects those and re-extracts the genuine bytes below. (The + // directory is private, so the file cannot have been planted by another user.) if existing, err := os.ReadFile(path); err == nil && sha256.Sum256(existing) == sum { return path, nil } @@ -55,3 +87,38 @@ func extractEmbedded() (string, error) { } return path, nil } + +// extractBaseDir returns the per-user base directory to extract under. The user +// cache directory ($XDG_CACHE_HOME or ~/.cache, %LocalAppData% on Windows) is +// inside the user's own home, so no other user can write to it — unlike the +// shared system temp dir. It is also usually not mounted `noexec`, which the +// system temp dir sometimes is. Falls back to a euid-scoped temp directory only +// when no cache dir is resolvable; `verifyPrivateDir` still guards either way. +func extractBaseDir() (string, error) { + if cache, err := os.UserCacheDir(); err == nil && cache != "" { + return cache, nil + } + tmp := os.TempDir() + if tmp == "" { + return "", &Error{ + Kind: "load", + Message: "no user cache or temp directory available to extract the embedded library", + } + } + // euid-scope the fallback name so a foreign-owned squat on a shared temp dir + // does not permanently block us; verifyPrivateDir rejects the foreign dir. + return filepath.Join(tmp, "secretspec-"+strconv.Itoa(geteuid())), nil +} + +// isLFSPointer reports whether b is a git-LFS pointer file rather than a real +// library. Pointer files are small text blobs that begin with this version line. +func isLFSPointer(b []byte) bool { + return bytes.HasPrefix(b, []byte("version https://git-lfs.github.com/spec/")) +} + +func cacheDirError(dir string, reason string) error { + return &Error{ + Kind: "load", + Message: fmt.Sprintf("refusing to use library cache directory %q: %s", dir, reason), + } +} diff --git a/secretspec-go/embedded_nonunix.go b/secretspec-go/embedded_nonunix.go new file mode 100644 index 0000000..e52692e --- /dev/null +++ b/secretspec-go/embedded_nonunix.go @@ -0,0 +1,17 @@ +//go:build !unix + +package secretspec + +import "os" + +func geteuid() int { + if uid := os.Getuid(); uid >= 0 { + return uid + } + return 0 +} + +// verifyPrivateDir is a no-op off unix: the per-user cache directory +// (%LocalAppData% on Windows) is already user-scoped and the POSIX ownership +// model checked on unix does not apply. +func verifyPrivateDir(string) error { return nil } diff --git a/secretspec-go/embedded_unix.go b/secretspec-go/embedded_unix.go new file mode 100644 index 0000000..2dd22cf --- /dev/null +++ b/secretspec-go/embedded_unix.go @@ -0,0 +1,38 @@ +//go:build unix + +package secretspec + +import ( + "os" + "syscall" +) + +func geteuid() int { return os.Geteuid() } + +// verifyPrivateDir ensures dir is a real directory (not a symlink) owned by the +// current user with no group/other access, so no other user on the host can +// plant or swap a file inside it between extraction and dlopen. +func verifyPrivateDir(dir string) error { + info, err := os.Lstat(dir) + if err != nil { + return err + } + mode := info.Mode() + if mode&os.ModeSymlink != 0 { + return cacheDirError(dir, "is a symlink") + } + if !mode.IsDir() { + return cacheDirError(dir, "is not a directory") + } + if mode.Perm()&0o077 != 0 { + return cacheDirError(dir, "is writable or readable by group or others") + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return cacheDirError(dir, "ownership could not be determined") + } + if int(st.Uid) != os.Geteuid() { + return cacheDirError(dir, "is owned by another user") + } + return nil +} diff --git a/secretspec-go/embedded_unix_test.go b/secretspec-go/embedded_unix_test.go new file mode 100644 index 0000000..86bcf98 --- /dev/null +++ b/secretspec-go/embedded_unix_test.go @@ -0,0 +1,63 @@ +//go:build unix + +package secretspec + +import ( + "os" + "path/filepath" + "testing" +) + +func TestVerifyPrivateDirAcceptsOwnerOnlyDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := verifyPrivateDir(dir); err != nil { + t.Fatalf("owner-only 0700 dir should be accepted, got: %v", err) + } +} + +func TestVerifyPrivateDirRejectsGroupOrWorldAccess(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cache") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + // A directory other users can write to is exactly the TOCTOU/file-swap risk + // the check exists to reject. + if err := os.Chmod(dir, 0o777); err != nil { + t.Fatal(err) + } + if err := verifyPrivateDir(dir); err == nil { + t.Fatal("world-writable dir must be rejected") + } +} + +func TestVerifyPrivateDirRejectsSymlink(t *testing.T) { + base := t.TempDir() + target := filepath.Join(base, "real") + if err := os.MkdirAll(target, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(base, "link") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := verifyPrivateDir(link); err == nil { + t.Fatal("a symlinked cache dir must be rejected") + } +} + +func TestIsLFSPointer(t *testing.T) { + pointer := []byte("version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 34000000\n") + if !isLFSPointer(pointer) { + t.Fatal("git-LFS pointer text should be detected") + } + // A real ELF starts with 0x7f 'E' 'L' 'F'; never a pointer. + if isLFSPointer([]byte{0x7f, 'E', 'L', 'F', 0, 0, 0, 0}) { + t.Fatal("an ELF header must not be treated as an LFS pointer") + } + if isLFSPointer(nil) { + t.Fatal("empty embedded lib is not an LFS pointer") + } +} diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 8172270..25bf8f7 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -95,6 +95,19 @@ func findLibrary() (string, error) { func ensureLoaded() error { loadOnce.Do(func() { + // purego.RegisterLibFunc panics (it does not return an error) when a + // symbol is missing. Recover so an incompatible library yields a returned + // *Error instead of a panic that escapes the Once — which would otherwise + // mark it done with loadErr nil and the function pointers nil, turning + // every later call into a nil-pointer panic for the process lifetime. + defer func() { + if r := recover(); r != nil { + loadErr = &Error{ + Kind: "load", + Message: fmt.Sprintf("failed to bind secretspec-ffi symbols (incompatible library?): %v", r), + } + } + }() path, err := findLibrary() if err != nil { loadErr = err @@ -264,11 +277,22 @@ func New() *Builder { return &Builder{req: map[string]any{}} } -func (b *Builder) WithPath(path string) *Builder { b.req["path"] = path; return b } -func (b *Builder) WithProvider(p string) *Builder { b.req["provider"] = p; return b } -func (b *Builder) WithProfile(p string) *Builder { b.req["profile"] = p; return b } -func (b *Builder) WithReason(reason string) *Builder { b.req["reason"] = reason; return b } -func (b *Builder) WithNoValues(v bool) *Builder { b.req["no_values"] = v; return b } +// set lazily initializes the request map so a zero-value Builder (e.g. +// `var b Builder` or `&Builder{}`, not just New()) does not panic with a +// nil-map write in the setters below. +func (b *Builder) set(key string, value any) *Builder { + if b.req == nil { + b.req = map[string]any{} + } + b.req[key] = value + return b +} + +func (b *Builder) WithPath(path string) *Builder { return b.set("path", path) } +func (b *Builder) WithProvider(p string) *Builder { return b.set("provider", p) } +func (b *Builder) WithProfile(p string) *Builder { return b.set("profile", p) } +func (b *Builder) WithReason(reason string) *Builder { return b.set("reason", reason) } +func (b *Builder) WithNoValues(v bool) *Builder { return b.set("no_values", v) } type envelopeJSON struct { OK bool `json:"ok"` diff --git a/secretspec-go/secretspec_test.go b/secretspec-go/secretspec_test.go index ea10294..c48a3ba 100644 --- a/secretspec-go/secretspec_test.go +++ b/secretspec-go/secretspec_test.go @@ -161,6 +161,9 @@ TLS_CERT = { description = "cert", required = true, as_path = true } if err != nil { t.Fatal(err) } + // as_path materializes a 0400 temp file the caller owns; remove it so the + // test does not leave secret-bearing files behind in the temp dir. + defer resolved.Close() cert := resolved.Secrets["TLS_CERT"] if !cert.AsPath || cert.Value != nil { @@ -175,6 +178,16 @@ TLS_CERT = { description = "cert", required = true, as_path = true } } } +// A zero-value Builder (not constructed via New) must not panic on a nil-map +// write in the setters. +func TestZeroValueBuilderDoesNotPanic(t *testing.T) { + var b Builder + got := b.WithPath("x").WithProvider("env://").WithProfile("p") + if got.req["path"] != "x" || got.req["provider"] != "env://" || got.req["profile"] != "p" { + t.Fatalf("zero-value builder did not record fields: %+v", got.req) + } +} + func TestInvalidManifest(t *testing.T) { _, err := New(). WithPath("/definitely/does/not/exist/secretspec.toml"). From 9fbd7dd9b9f4e9ce02182954437aa9b9c208b83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 12:25:33 -0400 Subject: [PATCH 40/56] fix(haskell): free the native response under mask/finally callNative copied the response out and freed it in straight-line IO, so an asynchronous exception (e.g. a System.Timeout.timeout around load/report) arriving between the call returning and the free leaked the native, secret bearing response buffer. Install the free under mask and run it via finally so it always executes. The conformance test now closes its value-carrying Resolved so as_path temp files do not accumulate. Co-Authored-By: Claude Opus 4.8 --- secretspec-hs/src/SecretSpec.hs | 22 +++++++++++++--------- secretspec-hs/test/Main.hs | 3 +++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/secretspec-hs/src/SecretSpec.hs b/secretspec-hs/src/SecretSpec.hs index 119a2b7..8882d84 100644 --- a/secretspec-hs/src/SecretSpec.hs +++ b/secretspec-hs/src/SecretSpec.hs @@ -45,7 +45,7 @@ module SecretSpec , abiVersion ) where -import Control.Exception (Exception, throwIO) +import Control.Exception (Exception, finally, mask, throwIO) import Control.Monad (forM_, unless, when) import Data.Aeson (FromJSON (..), Value, eitherDecodeStrict, encode, object, withObject, (.!=), (.:), (.:?), (.=)) @@ -290,16 +290,20 @@ requestBytes b mode = -- Marshal a request to secretspec_resolve and copy the response out before -- freeing the native allocation. +-- +-- The response is a Rust allocation the caller must free, and it carries secret +-- values. @mask@ keeps an async exception (e.g. a 'System.Timeout.timeout' +-- around 'load') from landing between the call returning and the free being +-- installed, and @finally@ guarantees the free runs whether @packCString@ +-- succeeds, throws, or is interrupted — so the secret-bearing buffer never leaks. callNative :: BL.ByteString -> IO BS.ByteString callNative reqLazy = - BS.useAsCString (BL.toStrict reqLazy) $ \creq -> do - cresp <- c_secretspec_resolve creq - if cresp == nullPtr - then throwIO (SecretSpecError "ffi" "secretspec_resolve returned null") - else do - resp <- BS.packCString cresp - c_secretspec_free cresp - pure resp + BS.useAsCString (BL.toStrict reqLazy) $ \creq -> + mask $ \restore -> do + cresp <- c_secretspec_resolve creq + if cresp == nullPtr + then throwIO (SecretSpecError "ffi" "secretspec_resolve returned null") + else restore (BS.packCString cresp) `finally` c_secretspec_free cresp -- Decode the envelope, unwrap @ok@/@error@, and check the schema version, -- returning the response object as a 'Value' for the caller to project. diff --git a/secretspec-hs/test/Main.hs b/secretspec-hs/test/Main.hs index f3d4c14..200f9c0 100644 --- a/secretspec-hs/test/Main.hs +++ b/secretspec-hs/test/Main.hs @@ -184,6 +184,9 @@ testConformance dir = do actual <- canonical resolved expected <- readJson (dir "expected.json") expect (actual == expected) (mismatch "expected.json" actual expected) + -- Remove any as_path temp files this value-carrying resolve materialized, so + -- repeated runs do not leave secret-bearing files behind in the temp dir. + S.close resolved -- Under no_values every SDK must emit the same all-null fields map: a -- value-less secret serializes to null, not an empty string. From ee59238bdf326ab17a5cde9935ea92fa18fe784f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 12:25:44 -0400 Subject: [PATCH 41/56] test(sdks): release as_path temp files in value-carrying tests The value-carrying as_path and conformance tests in the Python, Ruby, and Node suites resolved the as_path fixture but never disposed the result, so each run left another 0400 secret-bearing temp file behind (only the no_values variants cleaned up). Close/dispose the result, matching the no_values tests: Python via try/finally close(), Ruby via the block form of load that closes in ensure, and Node via try/finally dispose(). Co-Authored-By: Claude Opus 4.8 --- secretspec-node/test/conformance.test.js | 8 +++++-- secretspec-node/test/resolve.test.js | 14 ++++++++---- secretspec-py/tests/test_conformance.py | 8 +++++-- secretspec-py/tests/test_resolve.py | 13 ++++++++---- secretspec-rb/test/test_resolve.rb | 27 ++++++++++++++---------- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/secretspec-node/test/conformance.test.js b/secretspec-node/test/conformance.test.js index 3be9d1d..e632535 100644 --- a/secretspec-node/test/conformance.test.js +++ b/secretspec-node/test/conformance.test.js @@ -43,8 +43,12 @@ for (const fixture of fs.readdirSync(FIXTURES).sort()) { test(`conformance: ${fixture}`, () => { const expected = JSON.parse(fs.readFileSync(path.join(dir, 'expected.json'), 'utf8')); const resolved = builder(dir).load(); - - assert.deepStrictEqual(canonical(resolved), expected); + try { + assert.deepStrictEqual(canonical(resolved), expected); + } finally { + // Remove any as_path temp files this value-carrying resolve materialized. + resolved.dispose(); + } }); test(`conformance no_values: ${fixture}`, () => { diff --git a/secretspec-node/test/resolve.test.js b/secretspec-node/test/resolve.test.js index 9383874..6b2f9fa 100644 --- a/secretspec-node/test/resolve.test.js +++ b/secretspec-node/test/resolve.test.js @@ -116,10 +116,16 @@ TLS_CERT = { description = "cert", required = true, as_path = true } .withReason('node test') .load(); - const cert = resolved.secrets.TLS_CERT; - assert.equal(cert.asPath, true); - assert.equal(cert.value, null); - assert.equal(fs.readFileSync(cert.get(), 'utf8'), '----cert----'); + try { + const cert = resolved.secrets.TLS_CERT; + assert.equal(cert.asPath, true); + assert.equal(cert.value, null); + assert.equal(fs.readFileSync(cert.get(), 'utf8'), '----cert----'); + } finally { + // as_path materializes a 0400 temp file the caller owns; remove it so the + // test leaves no secret-bearing file behind in the temp dir. + resolved.dispose(); + } }); test('invalid manifest throws SecretSpecError (not MissingRequired)', () => { diff --git a/secretspec-py/tests/test_conformance.py b/secretspec-py/tests/test_conformance.py index 9d424bf..53c0ca6 100644 --- a/secretspec-py/tests/test_conformance.py +++ b/secretspec-py/tests/test_conformance.py @@ -68,8 +68,12 @@ def test_conformance(fixture): expected = json.loads((directory / "expected.json").read_text()) resolved = _builder(directory).load() - - assert _canonical(resolved) == expected + try: + assert _canonical(resolved) == expected + finally: + # Remove any as_path temp files this value-carrying resolve materialized, + # so repeated runs do not accumulate secret files in the temp dir. + resolved.close() @pytest.mark.parametrize("fixture", _fixtures()) diff --git a/secretspec-py/tests/test_resolve.py b/secretspec-py/tests/test_resolve.py index 8cc7acb..e9b8e23 100644 --- a/secretspec-py/tests/test_resolve.py +++ b/secretspec-py/tests/test_resolve.py @@ -112,10 +112,15 @@ def test_as_path_returns_readable_file(tmp_path): .load() ) - cert = resolved.secrets["TLS_CERT"] - assert cert.as_path - assert cert.value is None - assert pathlib.Path(cert.get).read_text() == "----cert-bytes----" + try: + cert = resolved.secrets["TLS_CERT"] + assert cert.as_path + assert cert.value is None + assert pathlib.Path(cert.get).read_text() == "----cert-bytes----" + finally: + # as_path materializes a 0400 temp file the caller owns; remove it so + # the test leaves no secret-bearing file behind in the temp dir. + resolved.close() def test_invalid_manifest_raises_secretspec_error(tmp_path): diff --git a/secretspec-rb/test/test_resolve.rb b/secretspec-rb/test/test_resolve.rb index 7deea0d..547d6c0 100644 --- a/secretspec-rb/test/test_resolve.rb +++ b/secretspec-rb/test/test_resolve.rb @@ -112,16 +112,18 @@ def test_as_path_returns_readable_file TOML manifest_path, provider = project(dir, "TLS_CERT=----cert----\n", manifest: manifest) - resolved = Secretspec::SecretSpec.builder - .with_path(manifest_path) - .with_provider(provider) - .with_reason("rb test") - .load - - cert = resolved.secrets["TLS_CERT"] - assert cert.as_path - assert_nil cert.value - assert_equal "----cert----", File.read(cert.get) + # Block form closes the Resolved (removing the 0400 as_path temp file) so + # the test leaves no secret-bearing file behind in the temp dir. + Secretspec::SecretSpec.builder + .with_path(manifest_path) + .with_provider(provider) + .with_reason("rb test") + .load do |resolved| + cert = resolved.secrets["TLS_CERT"] + assert cert.as_path + assert_nil cert.value + assert_equal "----cert----", File.read(cert.get) + end end end @@ -185,7 +187,10 @@ def conformance_builder(dir) define_method("test_conformance_#{name}") do expected = JSON.parse(File.read(File.join(dir, "expected.json"))) - assert_equal expected, canonical(conformance_builder(dir).load) + # Block form closes the Resolved so as_path temp files do not accumulate. + conformance_builder(dir).load do |resolved| + assert_equal expected, canonical(resolved) + end end # Under no_values every SDK must emit the same all-null fields map: a From 0db108bb6cd89fcd928ab242c6acf0569f3de90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 13:51:02 -0400 Subject: [PATCH 42/56] refactor(cli): drop the resolve subcommand, keep the contract FFI-only The value-carrying ResolveResponse stays the SDK boundary, but only over the secretspec-ffi C ABI. Not shipping it as a CLI verb keeps the command surface verb-level auditable: check never prints a value, get prints exactly one (per-key audited and reason-gated), and bulk value extraction never becomes a pipeable plaintext artifact. Adding the subcommand back later is backwards compatible; removing it after people script against it would not be. Secrets::resolve()/resolve_without_values()/resolve_json(), the FFI crate, the SDKs, and the committed JSON Schema are unchanged. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 34 +++++++++---------- docs/src/content/docs/reference/cli.md | 41 ----------------------- secretspec/src/cli/mod.rs | 45 -------------------------- secretspec/src/codegen.rs | 2 +- secretspec/src/resolve.rs | 3 +- secretspec/src/secrets.rs | 3 +- 6 files changed, 20 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f92ec1..e505fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,19 +123,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the boundary; returned strings are owned by the caller and freed with `secretspec_free`. A C header ships at `secretspec-ffi/include/secretspec.h`. `SecretSpecError::kind()` is now public so SDKs can do typed error handling. -- `secretspec resolve --json` resolves every declared secret and prints a - versioned, value-carrying JSON object: the SDK boundary other-language - clients consume. Each entry reports the value (or, for `as_path` secrets, the - path to a persisted temp file), its `source` (`provider`, `generated`, - `default`), and the serving provider's credential-free URI. When a required - secret is missing the command exits non-zero with an empty `secrets` object - and a populated `missing_required` list, mirroring the derive crate's - `load()`. `--no-values` emits the same structure without secret values. Unlike - `check`, this command prints secret values to stdout and is meant to be piped, - not displayed. The same payload is available to the Rust SDK via - `Secrets::resolve()`, returning the new public `ResolveResponse`, - `ResolvedSecret`, and `ResolvedSource` types; its JSON Schema is committed at - `schema/resolve-response.schema.json`. +- `Secrets::resolve()` resolves every declared secret into a versioned, + value-carrying `ResolveResponse`: the SDK boundary other-language clients + consume over the `secretspec-ffi` C ABI. Each entry reports the value (or, + for `as_path` secrets, the path to a persisted temp file), its `source` + (`provider`, `generated`, `default`), and the serving provider's + credential-free URI. When a required secret is missing the resolution fails + with an empty `secrets` object and a populated `missing_required` list, + mirroring the derive crate's `load()`. The new `ResolveResponse`, + `ResolvedSecret`, and `ResolvedSource` types are public; the payload's JSON + Schema is committed at `schema/resolve-response.schema.json`. The contract is + deliberately not exposed as a CLI subcommand: `check` never prints a value, + `get` prints exactly one, and bulk value extraction stays in-process behind + the C ABI. - `secretspec check --json` and `secretspec check --explain` surface a value-free resolution report describing how every declared secret resolved for the active profile: its status (`resolved`, `missing_required`, @@ -212,8 +212,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 internal panic. Previously only the C ABI caught panics, so a panic in the Node addon surfaced as an opaque thrown error instead of the documented envelope. -- A `no_values` resolution (`secretspec resolve --json --no-values` and every - SDK's no-values path) no longer copies secret values into the response or +- A `no_values` resolution (every SDK's no-values path) no longer copies + secret values into the response or persists `as_path` temp files. It now routes through a new `Secrets::resolve_without_values`, which never calls `expose_secret` and never keeps a temp file, so no secret byte crosses the boundary and an `as_path` @@ -228,13 +228,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `no_values` result violated). A new `no_values` conformance dimension asserts all five SDKs produce the identical all-null fields map. - A per-profile JSON Schema (`secretspec schema --profile

`) now allows - additional properties. `secretspec resolve --profile

` returns the + additional properties. Resolving with a profile returns the profile's own secrets plus those inherited from the `default` profile (the runtime resolver merges them; the per-profile type intentionally does not, matching the derive macro), so a strict quicktype-generated deserializer would otherwise reject a valid resolve result over the inherited keys. The union schema stays exhaustive (`additionalProperties: false`). -- `secretspec resolve --profile

` and the SDKs no longer export an empty or +- The SDKs no longer export an empty or literal-`"null"` environment variable for a secret with no usable value (e.g. under `no_values`): the Go, Node, and Ruby SDKs now skip such secrets in `set_as_env`, matching Python. Ruby previously *deleted* the variable diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 2603501..4657c3c 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -199,47 +199,6 @@ $ secretspec get DATABASE_URL --profile production postgresql://prod.example.com/mydb ``` -### resolve -Resolve every declared secret and print it as JSON. This is the SDK boundary: -other-language clients consume this payload (over a subprocess or the C ABI) -rather than reimplementing resolution. - -```bash -secretspec resolve [OPTIONS] -``` - -Unlike `check`, `resolve` prints secret **values** to stdout. Pipe it into a -program; do not display it. Use `--no-values` for a value-free structural view. -When a required secret is missing, the command exits non-zero with an empty -`secrets` object and a populated `missing_required` list (mirroring the SDK's -`load()`). - -**Options:** -- `-p, --provider ` - Provider backend to use -- `-P, --profile ` - Profile to use -- `--no-values` - Omit secret values, emitting only structure and provenance - -**Example:** -```bash -$ secretspec resolve --profile production -{ - "schema_version": 1, - "provider": "keyring://", - "profile": "production", - "secrets": { - "DATABASE_URL": { "value": "postgresql://prod.example.com/mydb", "as_path": false, "source": "provider", "source_provider": "keyring://" }, - "TLS_CERT": { "path": "/tmp/.tmpAbc123", "as_path": true, "source": "provider", "source_provider": "keyring://" } - }, - "missing_required": [], - "missing_optional": [] -} -``` - -Each entry reports the value (or, for `as_path` secrets, the `path` to a -persisted temp file), its `source` (`provider`, `generated`, or `default`), and -the serving provider's credential-free URI. The canonical JSON Schema is -committed at `schema/resolve-response.schema.json`. - ### schema Emit a single-root JSON Schema for the manifest's typed shape: by default the union `SecretSpec` (safe for any profile); with `--profile`, that profile's exact diff --git a/secretspec/src/cli/mod.rs b/secretspec/src/cli/mod.rs index e970df3..6fb3089 100644 --- a/secretspec/src/cli/mod.rs +++ b/secretspec/src/cli/mod.rs @@ -120,22 +120,6 @@ enum Commands { #[arg(short, long)] output: Option, }, - /// Resolve all secrets and print them as JSON (the SDK boundary). - /// - /// Unlike `check`, this prints secret VALUES (to stdout). It is intended for - /// programmatic consumption by other-language SDKs and tooling; pipe it, do - /// not display it. Use `--no-values` for a value-free structural view. - Resolve { - /// Provider backend to use - #[arg(short, long, env = "SECRETSPEC_PROVIDER")] - provider: Option, - /// Profile to use - #[arg(short = 'P', long, env = "SECRETSPEC_PROFILE")] - profile: Option, - /// Omit secret values, emitting only structure and provenance - #[arg(long)] - no_values: bool, - }, /// Init or show ~/.config/secretspec/config.toml Config { #[command(subcommand)] @@ -722,35 +706,6 @@ pub fn main() -> Result<()> { } Ok(()) } - // Resolve all secrets to JSON (the SDK boundary; prints values) - Commands::Resolve { - provider, - profile, - no_values, - } => { - let mut app = load_secrets(&cli.file, &cli.reason)?; - if let Some(p) = provider { - app.set_provider(p); - } - if let Some(p) = profile { - app.set_profile(p); - } - let response = if no_values { - app.resolve_without_values() - } else { - app.resolve() - } - .into_diagnostic() - .wrap_err("Failed to resolve secrets")?; - let rendered = serde_json::to_string_pretty(&response) - .into_diagnostic() - .wrap_err("Failed to serialize resolve response")?; - println!("{}", rendered); - if !response.is_ok() { - std::process::exit(1); - } - Ok(()) - } // Import secrets from one provider to another Commands::Import { from_provider } => { let app = load_secrets(&cli.file, &cli.reason)?; diff --git a/secretspec/src/codegen.rs b/secretspec/src/codegen.rs index 4e5dd72..e32de8a 100644 --- a/secretspec/src/codegen.rs +++ b/secretspec/src/codegen.rs @@ -239,7 +239,7 @@ pub mod schema { /// The union lists every secret across every profile, so it is **exhaustive** /// (`additionalProperties: false`): a runtime `fields()` map can never carry a /// key the union does not declare. A per-profile schema lists only the - /// secrets that profile declares, but `secretspec resolve --profile

` + /// secrets that profile declares, but resolving with that profile /// returns those **plus** secrets inherited from the `default` profile (the /// runtime resolver merges them; the per-profile type intentionally does not, /// matching the derive macro). So per-profile schemas allow additional diff --git a/secretspec/src/resolve.rs b/secretspec/src/resolve.rs index 371ff99..14e62c4 100644 --- a/secretspec/src/resolve.rs +++ b/secretspec/src/resolve.rs @@ -3,8 +3,7 @@ //! Unlike the value-free [`crate::report::ResolutionReport`] (which powers //! `check --json` and must never expose a value), this payload **does** carry //! the resolved secret values. It is the single authoritative output that any -//! other-language SDK consumes, either over the C ABI (in-process) or via -//! `secretspec resolve --json` (subprocess). Producing it deliberately exposes +//! other-language SDK consumes over the C ABI. Producing it deliberately exposes //! secrets, so it is only built at an explicit resolve boundary and its bytes //! must be treated as sensitive by the caller. //! diff --git a/secretspec/src/secrets.rs b/secretspec/src/secrets.rs index 34d4e12..df71c45 100644 --- a/secretspec/src/secrets.rs +++ b/secretspec/src/secrets.rs @@ -2021,8 +2021,7 @@ impl Secrets { } /// Resolve every declared secret into a value-carrying [`ResolveResponse`], - /// the authoritative output other-language SDKs consume (over the C ABI or - /// `secretspec resolve --json`). + /// the authoritative output other-language SDKs consume over the C ABI. /// /// Unlike [`Self::validate`], the returned payload **carries secret /// values** (or, for `as_path` secrets, the path to a persisted temp file). From 0d4361409ed8da07a826638c754ec6a116067ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 14:20:12 -0400 Subject: [PATCH 43/56] ci: pin Rust via rust-toolchain.toml, scope artifact PR triggers, give review devenv Pin the Rust version in a single rust-toolchain.toml consumed by both devenv (languages.rust.toolchainFile) and the native CI runners via rustup, so released artifacts build with the same compiler CI tests against. Scope the artifact workflows (ffi, node, python, ruby, go) to PR changes in their own directories; core resolver changes are already verified on PRs by the devenv-based test.yml and sdks.yml, and the full matrices still run on tags and manual dispatch. Install devenv in the Claude review workflow and allowlist devenv commands so the reviewer can build and run tests instead of reviewing blind. Co-Authored-By: Claude Fable 5 --- .github/workflows/claude-code-review.yml | 19 +++++++++++++++---- .github/workflows/ffi-build.yml | 10 +++++----- .github/workflows/go-embed.yml | 9 ++++++--- .github/workflows/node-addon.yml | 7 +++++-- .github/workflows/python-wheels.yml | 13 +++++++++---- .github/workflows/ruby-gems.yml | 9 ++++++--- devenv.nix | 6 +++--- rust-toolchain.toml | 8 ++++++++ 8 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 85e5ae6..b2a46ea 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -31,6 +31,15 @@ jobs: with: fetch-depth: 1 + # Provide the project devenv so the reviewer can build and run tests + # (e.g. `devenv shell -- cargo test --all`) instead of reviewing blind. + - uses: cachix/install-nix-action@v31 + - uses: cachix/cachix-action@v16 + with: + name: devenv + - name: Install devenv.sh + run: nix profile install nixpkgs#devenv + - name: Run Claude Code Review id: claude-review uses: anthropics/claude-code-action@v1 @@ -51,7 +60,9 @@ jobs: Be constructive and helpful in your feedback. - # Optional: pass extra CLI arguments such as a model or tool allowlist - # claude_args: | - # --model claude-opus-4-20250514 - # --allowedTools "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)" + The project devenv is available; you can build and run tests with + e.g. `devenv shell -- cargo test --all` or `devenv test` to verify + suspected issues before reporting them. + + claude_args: | + --allowedTools "Bash(devenv:*)" diff --git a/.github/workflows/ffi-build.yml b/.github/workflows/ffi-build.yml index 4463b16..c18a892 100644 --- a/.github/workflows/ffi-build.yml +++ b/.github/workflows/ffi-build.yml @@ -15,10 +15,12 @@ on: push: tags: - v** + # PR runs are scoped to changes in the FFI crate or this workflow. Core + # resolver changes are verified on PRs by the devenv-based test.yml and + # sdks.yml; the full matrix here still runs on tags and manual dispatch. pull_request: paths: - "secretspec-ffi/**" - - "secretspec/**" - ".github/workflows/ffi-build.yml" jobs: @@ -52,10 +54,8 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} + - name: Install Rust (pinned by rust-toolchain.toml) + run: rustup toolchain install - name: Build cdylib run: cargo build -p secretspec-ffi --release --target ${{ matrix.target }} diff --git a/.github/workflows/go-embed.yml b/.github/workflows/go-embed.yml index 88807c7..1e8cebe 100644 --- a/.github/workflows/go-embed.yml +++ b/.github/workflows/go-embed.yml @@ -12,11 +12,13 @@ on: push: tags: - v** + # PR runs are scoped to changes in the Go SDK or this workflow. Core + # resolver and FFI changes are verified on PRs by the devenv-based test.yml, + # sdks.yml, and ffi-build.yml; the full matrix here still runs on tags and + # manual dispatch. pull_request: paths: - "secretspec-go/**" - - "secretspec-ffi/**" - - "secretspec/**" - ".github/workflows/go-embed.yml" jobs: @@ -40,7 +42,8 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config - - uses: dtolnay/rust-toolchain@stable + - name: Install Rust (pinned by rust-toolchain.toml) + run: rustup toolchain install - uses: actions/setup-go@v5 with: go-version: "1.23" diff --git a/.github/workflows/node-addon.yml b/.github/workflows/node-addon.yml index bf3176d..e83e6de 100644 --- a/.github/workflows/node-addon.yml +++ b/.github/workflows/node-addon.yml @@ -13,10 +13,12 @@ on: push: tags: - v** + # PR runs are scoped to changes in the Node SDK or this workflow. Core + # resolver changes are verified on PRs by the devenv-based test.yml and + # sdks.yml; the full matrix here still runs on tags and manual dispatch. pull_request: paths: - "secretspec-node/**" - - "secretspec/**" - ".github/workflows/node-addon.yml" jobs: @@ -40,7 +42,8 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config - - uses: dtolnay/rust-toolchain@stable + - name: Install Rust (pinned by rust-toolchain.toml) + run: rustup toolchain install - uses: actions/setup-node@v4 with: node-version: "22" diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 285f9fb..eec9bed 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -15,11 +15,13 @@ on: push: tags: - v** + # PR runs are scoped to changes in the Python SDK or this workflow. Core + # resolver and FFI changes are verified on PRs by the devenv-based test.yml, + # sdks.yml, and ffi-build.yml; the full matrix here still runs on tags and + # manual dispatch. pull_request: paths: - "secretspec-py/**" - - "secretspec-ffi/**" - - "secretspec/**" - ".github/workflows/python-wheels.yml" jobs: @@ -38,8 +40,10 @@ jobs: - name: Install build deps (rust, dbus) run: | yum install -y dbus-devel - curl -sSf https://sh.rustup.rs | sh -s -- -y + curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + # Installs the version pinned in rust-toolchain.toml + "$HOME/.cargo/bin/rustup" toolchain install - name: Build wheel and repair to manylinux run: | bash secretspec-py/scripts/stage-cdylib.sh @@ -62,7 +66,8 @@ jobs: - { target: windows-x86_64, runner: windows-latest } steps: - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - name: Install Rust (pinned by rust-toolchain.toml) + run: rustup toolchain install - uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml index f13cecf..2f37898 100644 --- a/.github/workflows/ruby-gems.yml +++ b/.github/workflows/ruby-gems.yml @@ -15,11 +15,13 @@ on: push: tags: - v** + # PR runs are scoped to changes in the Ruby SDK or this workflow. Core + # resolver and FFI changes are verified on PRs by the devenv-based test.yml, + # sdks.yml, and ffi-build.yml; the full matrix here still runs on tags and + # manual dispatch. pull_request: paths: - "secretspec-rb/**" - - "secretspec-ffi/**" - - "secretspec/**" - ".github/workflows/ruby-gems.yml" jobs: @@ -43,7 +45,8 @@ jobs: if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config - - uses: dtolnay/rust-toolchain@stable + - name: Install Rust (pinned by rust-toolchain.toml) + run: rustup toolchain install - uses: ruby/setup-ruby@v1 with: ruby-version: "3.3" diff --git a/devenv.nix b/devenv.nix index 5486016..9b7a6ed 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,9 +1,9 @@ { pkgs, ... }: { languages.rust = { enable = true; - # Pinned to >= 1.92 for the detect-coding-agent dependency's MSRV. - channel = "stable"; - version = "1.92.0"; + # The Rust version is pinned in rust-toolchain.toml, which the native CI + # runners (artifact workflows that cannot use devenv) read via rustup. + toolchainFile = ./rust-toolchain.toml; }; languages.javascript = { enable = true; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..c3ba728 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,8 @@ +# Single source of truth for the Rust version. Consumed by: +# - devenv (languages.rust.toolchainFile in devenv.nix), covering local +# shells and the devenv-based workflows +# - rustup on the native CI runners (artifact workflows that cannot use +# devenv because their outputs must not link the Nix store) +# Pinned to >= 1.92 for the detect-coding-agent dependency's MSRV. +[toolchain] +channel = "1.92.0" From d08f967e46c98861c8e0bed479226e9d32ec0b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 14:21:44 -0400 Subject: [PATCH 44/56] ci(sdks): free runner disk space before the workspace build The SDKs job fills the ~14GB free on a hosted ubuntu runner (Nix store + full debug build with the AWS/GCP/Bitwarden provider stacks) and dies with ENOSPC, so drop the preinstalled toolchains we never use. Co-Authored-By: Claude Fable 5 --- .github/workflows/sdks.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/sdks.yml b/.github/workflows/sdks.yml index 119745e..97c9702 100644 --- a/.github/workflows/sdks.yml +++ b/.github/workflows/sdks.yml @@ -14,6 +14,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + # The full workspace debug build (AWS/GCP/Bitwarden providers) plus the + # Nix store overflows the ~14GB free on a hosted runner. + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android \ + /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /opt/ghc \ + /usr/local/share/boost /usr/local/share/powershell + sudo docker image prune --all --force + df -h / - uses: cachix/install-nix-action@v31 - uses: cachix/cachix-action@v16 with: From d069295f2cef7d128741d929f66eb8b50f19a63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 14:42:01 -0400 Subject: [PATCH 45/56] ci(haskell): free runner disk space before the workspace build Same ENOSPC as the SDKs job: the cdylib + CLI debug build with the full provider stacks plus the Nix store overflows the hosted runner's disk, this time so badly the runner could not even write its own logs. Co-Authored-By: Claude Fable 5 --- .github/workflows/haskell-build.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/haskell-build.yml b/.github/workflows/haskell-build.yml index 699054e..0d9d6a9 100644 --- a/.github/workflows/haskell-build.yml +++ b/.github/workflows/haskell-build.yml @@ -23,6 +23,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + # The cdylib + CLI debug build (AWS/GCP/Bitwarden providers) plus the + # Nix store overflows the ~14GB free on a hosted runner. + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android \ + /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /opt/ghc \ + /usr/local/share/boost /usr/local/share/powershell + sudo docker image prune --all --force + df -h / - uses: cachix/install-nix-action@v31 - uses: cachix/cachix-action@v16 with: From eed82a87b304e7f9e09eba4062cd9556a270da87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 14:54:30 -0400 Subject: [PATCH 46/56] ci: replace retired macos-13 runners with macos-15-intel The macos-13 label is no longer provisioned by GitHub, so every x86_64-apple-darwin artifact job sat queued forever while the rest of the matrix passed. Co-Authored-By: Claude Fable 5 --- .github/workflows/ffi-build.yml | 2 +- .github/workflows/go-embed.yml | 2 +- .github/workflows/node-addon.yml | 2 +- .github/workflows/python-wheels.yml | 2 +- .github/workflows/ruby-gems.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ffi-build.yml b/.github/workflows/ffi-build.yml index c18a892..cba0f1c 100644 --- a/.github/workflows/ffi-build.yml +++ b/.github/workflows/ffi-build.yml @@ -38,7 +38,7 @@ jobs: runner: ubuntu-24.04-arm artifact: libsecretspec_ffi.so - target: x86_64-apple-darwin - runner: macos-13 + runner: macos-15-intel artifact: libsecretspec_ffi.dylib - target: aarch64-apple-darwin runner: macos-14 diff --git a/.github/workflows/go-embed.yml b/.github/workflows/go-embed.yml index 1e8cebe..0b3cd51 100644 --- a/.github/workflows/go-embed.yml +++ b/.github/workflows/go-embed.yml @@ -31,7 +31,7 @@ jobs: include: - { target: linux_amd64, runner: ubuntu-latest } - { target: linux_arm64, runner: ubuntu-24.04-arm } - - { target: darwin_amd64, runner: macos-13 } + - { target: darwin_amd64, runner: macos-15-intel } - { target: darwin_arm64, runner: macos-14 } - { target: windows_amd64, runner: windows-latest } diff --git a/.github/workflows/node-addon.yml b/.github/workflows/node-addon.yml index e83e6de..e95c6db 100644 --- a/.github/workflows/node-addon.yml +++ b/.github/workflows/node-addon.yml @@ -31,7 +31,7 @@ jobs: include: - { target: linux-x64, runner: ubuntu-latest } - { target: linux-arm64, runner: ubuntu-24.04-arm } - - { target: darwin-x64, runner: macos-13 } + - { target: darwin-x64, runner: macos-15-intel } - { target: darwin-arm64, runner: macos-14 } - { target: win32-x64, runner: windows-latest } diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index eec9bed..03e8904 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: include: - - { target: macos-x86_64, runner: macos-13 } + - { target: macos-x86_64, runner: macos-15-intel } - { target: macos-aarch64, runner: macos-14 } - { target: windows-x86_64, runner: windows-latest } steps: diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml index 2f37898..981aa05 100644 --- a/.github/workflows/ruby-gems.yml +++ b/.github/workflows/ruby-gems.yml @@ -34,7 +34,7 @@ jobs: include: - { target: x86_64-linux, runner: ubuntu-latest } - { target: aarch64-linux, runner: ubuntu-24.04-arm } - - { target: x86_64-darwin, runner: macos-13 } + - { target: x86_64-darwin, runner: macos-15-intel } - { target: arm64-darwin, runner: macos-14 } - { target: x64-mingw, runner: windows-latest } From 5e1be3e8cae0b83d501d4802ba58ef70cdf61c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 17:47:46 -0400 Subject: [PATCH 47/56] fix(windows): support drive-letter provider paths and Windows SDK builds Three Windows breakages surfaced by the first complete artifact CI runs: - Provider specs like dotenv://C:\path\.env failed with "invalid port number" because C: parsed as a URL host:port. Drive-letter paths are now carried in the URL path component (forward-slash separators) and the dotenv provider strips the URL's leading slash from them. - The Go SDK never compiled on Windows: purego.Dlopen and RTLD_* exist only on Unix. The library open is now split into build-tagged files, using syscall.LoadLibrary on Windows. - The Node codegen test spawned npx, which is npx.cmd on Windows and unreachable without a shell. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 6 +++++ secretspec-go/dlopen_unix.go | 11 +++++++++ secretspec-go/dlopen_windows.go | 13 +++++++++++ secretspec-go/secretspec.go | 2 +- secretspec-node/test/codegen.test.js | 3 ++- secretspec/src/provider/dotenv.rs | 15 ++++++++++++ secretspec/src/provider/mod.rs | 34 ++++++++++++++++++++++++++++ 7 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 secretspec-go/dlopen_unix.go create mode 100644 secretspec-go/dlopen_windows.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e505fdb..7060d9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `SecretResolution`, and `ResolutionStatus` types. ### Fixed +- Provider specifications containing Windows drive-letter paths (e.g. + `dotenv://C:\Users\me\.env`) now parse correctly. Previously `C:` was + interpreted as a URL `host:port` pair and rejected with "invalid port + number"; the path is now carried in the URL path component (with + forward-slash separators, which Windows accepts) and the dotenv provider + strips the URL's leading slash from drive-letter paths. - The value-free resolution surfaces — `Secrets::report()`, `resolve_without_values()` / the FFI `{"no_values": true}` and `{"mode": "report"}` requests, and `check --json`/`--explain` (now routed through `report()`) — are side-effect-free. diff --git a/secretspec-go/dlopen_unix.go b/secretspec-go/dlopen_unix.go new file mode 100644 index 0000000..a990fbd --- /dev/null +++ b/secretspec-go/dlopen_unix.go @@ -0,0 +1,11 @@ +//go:build unix + +package secretspec + +import "github.com/ebitengine/purego" + +// openLibrary loads the shared library at path and returns an opaque handle +// usable with purego.RegisterLibFunc. +func openLibrary(path string) (uintptr, error) { + return purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) +} diff --git a/secretspec-go/dlopen_windows.go b/secretspec-go/dlopen_windows.go new file mode 100644 index 0000000..7616eed --- /dev/null +++ b/secretspec-go/dlopen_windows.go @@ -0,0 +1,13 @@ +//go:build windows + +package secretspec + +import "syscall" + +// openLibrary loads the DLL at path and returns an opaque handle usable with +// purego.RegisterLibFunc (purego resolves symbols via GetProcAddress on +// Windows; purego.Dlopen only exists on Unix). +func openLibrary(path string) (uintptr, error) { + handle, err := syscall.LoadLibrary(path) + return uintptr(handle), err +} diff --git a/secretspec-go/secretspec.go b/secretspec-go/secretspec.go index 25bf8f7..84fcbed 100644 --- a/secretspec-go/secretspec.go +++ b/secretspec-go/secretspec.go @@ -113,7 +113,7 @@ func ensureLoaded() error { loadErr = err return } - handle, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) + handle, err := openLibrary(path) if err != nil { loadErr = err return diff --git a/secretspec-node/test/codegen.test.js b/secretspec-node/test/codegen.test.js index 6d23f0a..29e300a 100644 --- a/secretspec-node/test/codegen.test.js +++ b/secretspec-node/test/codegen.test.js @@ -64,10 +64,11 @@ test('quicktype-generated converter consumes fieldsJson()', { skip: !hasNpx() }, execFileSync(bin, ['-f', manifest, 'schema', '-o', schema]); const generated = path.join(dir, 'gen.js'); + // On Windows npx is npx.cmd, which spawn only reaches through a shell. execFileSync('npx', [ '--yes', 'quicktype', '-s', 'schema', schema, '--top-level', 'SecretSpec', '--lang', 'javascript', '-o', generated, - ]); + ], { shell: process.platform === 'win32' }); const { toSecretSpec } = require(generated); diff --git a/secretspec/src/provider/dotenv.rs b/secretspec/src/provider/dotenv.rs index d39cdfd..4997772 100644 --- a/secretspec/src/provider/dotenv.rs +++ b/secretspec/src/provider/dotenv.rs @@ -103,6 +103,14 @@ impl TryFrom<&ProviderUrl> for DotEnvConfig { ".env".to_string() }; + // URL paths are always absolute, so a Windows drive-letter path + // arrives as "/C:/...": drop the leading slash to make it a valid + // Windows path. + let path = match path.as_bytes() { + [b'/', drive, b':', b'/', ..] if drive.is_ascii_alphabetic() => path[1..].to_string(), + _ => path, + }; + Ok(Self { path: PathBuf::from(path), }) @@ -328,6 +336,13 @@ mod tests { let url = ProviderUrl::new(Url::parse("dotenv://foobar/custom/path/.env").unwrap()); let config: DotEnvConfig = (&url).try_into().unwrap(); assert_eq!(config.path.to_str().unwrap(), "foobar/custom/path/.env"); + + // Windows drive-letter path: TryFrom<&str> turns "dotenv://C:\a\.env" + // into "dotenv:///C:/a/.env"; the leading slash must not survive into + // the filesystem path. + let url = ProviderUrl::new(Url::parse("dotenv:///C:/Users/me/.env").unwrap()); + let config: DotEnvConfig = (&url).try_into().unwrap(); + assert_eq!(config.path.to_str().unwrap(), "C:/Users/me/.env"); } #[test] diff --git a/secretspec/src/provider/mod.rs b/secretspec/src/provider/mod.rs index 77adedc..9e287a6 100644 --- a/secretspec/src/provider/mod.rs +++ b/secretspec/src/provider/mod.rs @@ -621,6 +621,14 @@ impl TryFrom<&str> for Box { let url_string = match rest { // Just scheme name (e.g., "keyring") "" | ":" => format!("{}://", scheme), + // A Windows drive-letter path (e.g. "dotenv://C:\Users\me\.env") + // cannot travel in the URL authority: "C:" would parse as + // host:port. Carry it in the path component with an empty + // authority and forward-slash separators (valid in Windows paths). + s if is_windows_drive_path(s.strip_prefix("//").unwrap_or(s)) => { + let path = s.strip_prefix("//").unwrap_or(s).replace('\\', "/"); + format!("{}:///{}", scheme, path) + } // Standard URI format already has // (e.g., "onepassword://vault/path") s if s.starts_with("//") => format!("{}:{}", scheme, s), // Path only format (e.g., "dotenv:/path/to/.env") @@ -660,6 +668,13 @@ impl TryFrom<&Url> for Box { } } +/// Returns true for paths that start with a Windows drive designator followed +/// by a separator (e.g. `C:\...` or `C:/...`). +fn is_windows_drive_path(s: &str) -> bool { + let b = s.as_bytes(); + b.len() >= 3 && b[0].is_ascii_alphabetic() && b[1] == b':' && (b[2] == b'/' || b[2] == b'\\') +} + fn provider_from_url(url: &ProviderUrl) -> Result> { let scheme = url.scheme(); @@ -732,6 +747,25 @@ mod url_tests { assert_eq!(ProviderUrl::encode("Home Lab"), "Home%20Lab"); } + #[test] + fn windows_drive_paths_parse_as_provider_specs() { + // "C:" must not be treated as host:port ("invalid port number"). + for spec in [ + r"dotenv://C:\Users\me\.env", + r"dotenv://C:/Users/me/.env", + r"dotenv:C:\Users\me\.env", + ] { + assert!( + Box::::try_from(spec).is_ok(), + "should parse: {}", + spec + ); + } + // Unix and relative forms are unaffected. + assert!(Box::::try_from("dotenv:///tmp/.env").is_ok()); + assert!(Box::::try_from("dotenv://.env").is_ok()); + } + #[test] fn provider_info_display_with_and_without_examples() { let with = ProviderInfo { From de2713aa0bbf927fdb4371ba76bf844b493906bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 17:48:09 -0400 Subject: [PATCH 48/56] ci: drop Intel macOS artifacts, move arm macOS legs to macos-latest The Go SDK segfaults in the purego call path on x86_64 darwin and Apple has moved the platform on; stop building Intel macOS artifacts (FFI cdylib, wheels, gems, Node addon, Go embed lib) rather than debugging a dying target. Intel mac users can still build from source via the system-library path. Pin the remaining macOS runners to macos-latest instead of macos-14 so the next image retirement does not silently strand jobs in the queue like macos-13 did; artifact compatibility comes from rustc's deployment target, not the runner OS. Co-Authored-By: Claude Fable 5 --- .github/workflows/ffi-build.yml | 5 +---- .github/workflows/go-embed.yml | 3 +-- .github/workflows/node-addon.yml | 3 +-- .github/workflows/python-wheels.yml | 3 +-- .github/workflows/ruby-gems.yml | 3 +-- secretspec-go/embedded_darwin_amd64.go | 10 ---------- secretspec-go/embedded_unsupported.go | 2 +- 7 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 secretspec-go/embedded_darwin_amd64.go diff --git a/.github/workflows/ffi-build.yml b/.github/workflows/ffi-build.yml index cba0f1c..46d61e3 100644 --- a/.github/workflows/ffi-build.yml +++ b/.github/workflows/ffi-build.yml @@ -37,11 +37,8 @@ jobs: - target: aarch64-unknown-linux-gnu runner: ubuntu-24.04-arm artifact: libsecretspec_ffi.so - - target: x86_64-apple-darwin - runner: macos-15-intel - artifact: libsecretspec_ffi.dylib - target: aarch64-apple-darwin - runner: macos-14 + runner: macos-latest artifact: libsecretspec_ffi.dylib - target: x86_64-pc-windows-msvc runner: windows-latest diff --git a/.github/workflows/go-embed.yml b/.github/workflows/go-embed.yml index 0b3cd51..d11f2e3 100644 --- a/.github/workflows/go-embed.yml +++ b/.github/workflows/go-embed.yml @@ -31,8 +31,7 @@ jobs: include: - { target: linux_amd64, runner: ubuntu-latest } - { target: linux_arm64, runner: ubuntu-24.04-arm } - - { target: darwin_amd64, runner: macos-15-intel } - - { target: darwin_arm64, runner: macos-14 } + - { target: darwin_arm64, runner: macos-latest } - { target: windows_amd64, runner: windows-latest } steps: diff --git a/.github/workflows/node-addon.yml b/.github/workflows/node-addon.yml index e95c6db..6e5e912 100644 --- a/.github/workflows/node-addon.yml +++ b/.github/workflows/node-addon.yml @@ -31,8 +31,7 @@ jobs: include: - { target: linux-x64, runner: ubuntu-latest } - { target: linux-arm64, runner: ubuntu-24.04-arm } - - { target: darwin-x64, runner: macos-15-intel } - - { target: darwin-arm64, runner: macos-14 } + - { target: darwin-arm64, runner: macos-latest } - { target: win32-x64, runner: windows-latest } steps: diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index 03e8904..d02e059 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -61,8 +61,7 @@ jobs: fail-fast: false matrix: include: - - { target: macos-x86_64, runner: macos-15-intel } - - { target: macos-aarch64, runner: macos-14 } + - { target: macos-aarch64, runner: macos-latest } - { target: windows-x86_64, runner: windows-latest } steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml index 981aa05..460057c 100644 --- a/.github/workflows/ruby-gems.yml +++ b/.github/workflows/ruby-gems.yml @@ -34,8 +34,7 @@ jobs: include: - { target: x86_64-linux, runner: ubuntu-latest } - { target: aarch64-linux, runner: ubuntu-24.04-arm } - - { target: x86_64-darwin, runner: macos-15-intel } - - { target: arm64-darwin, runner: macos-14 } + - { target: arm64-darwin, runner: macos-latest } - { target: x64-mingw, runner: windows-latest } steps: diff --git a/secretspec-go/embedded_darwin_amd64.go b/secretspec-go/embedded_darwin_amd64.go deleted file mode 100644 index b194b80..0000000 --- a/secretspec-go/embedded_darwin_amd64.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build embed_lib && darwin && amd64 - -package secretspec - -import _ "embed" - -//go:embed lib/secretspec_ffi_darwin_amd64.dylib -var embeddedLib []byte - -const embeddedLibName = "libsecretspec_ffi.dylib" diff --git a/secretspec-go/embedded_unsupported.go b/secretspec-go/embedded_unsupported.go index 7602607..1f0bf50 100644 --- a/secretspec-go/embedded_unsupported.go +++ b/secretspec-go/embedded_unsupported.go @@ -1,4 +1,4 @@ -//go:build embed_lib && !(linux && amd64) && !(linux && arm64) && !(darwin && amd64) && !(darwin && arm64) && !(windows && amd64) +//go:build embed_lib && !(linux && amd64) && !(linux && arm64) && !(darwin && arm64) && !(windows && amd64) package secretspec From 21c7a3dfcb0bd36cb6233c859b86513523d6fcbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 11 Jun 2026 18:06:18 -0400 Subject: [PATCH 49/56] fix(node): install the addon atomically and prebuild it in the SDKs CI node --test runs the three test files in parallel processes; each one's ensureAddon() kicked off build-addon.sh when secretspec.node was absent, and the final `cp` truncated the addon in place while a sibling process could already have it mapped, killing it with SIGBUS (seen once in the SDKs workflow). Install via temp file + rename so an existing mapping keeps its inode, and build once up front in ci-sdks.sh so the test processes never race to build at all. Co-Authored-By: Claude Fable 5 --- scripts/ci-sdks.sh | 6 ++++-- secretspec-node/scripts/build-addon.sh | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh index 64c570b..166a6e5 100755 --- a/scripts/ci-sdks.sh +++ b/scripts/ci-sdks.sh @@ -33,8 +33,10 @@ echo "==> Ruby" ( cd secretspec-rb && ruby -e 'Dir["test/test_*.rb"].sort.each { |f| require File.expand_path(f) }' ) echo "==> Node" -# The Node SDK uses a napi-rs addon (built by its test harness), not the cdylib, -# and has no npm dependencies. +# The Node SDK uses a napi-rs addon (not the cdylib) and has no npm +# dependencies. Build the addon once up front: the test files each ensure it +# exists and would otherwise race to build it in parallel processes. +bash secretspec-node/scripts/build-addon.sh ( cd secretspec-node && node --test ) echo "==> Haskell" diff --git a/secretspec-node/scripts/build-addon.sh b/secretspec-node/scripts/build-addon.sh index f6bf341..4267cbe 100644 --- a/secretspec-node/scripts/build-addon.sh +++ b/secretspec-node/scripts/build-addon.sh @@ -18,5 +18,9 @@ case "$(uname -s)" in *) src="libsecretspec_node_native.so" ;; esac -cp "$target_dir/release/$src" "$pkg_dir/secretspec.node" +# Install atomically: node --test runs test files in parallel processes that +# may build concurrently, and overwriting in place SIGBUSes a process that has +# already mapped the addon. A rename keeps the old inode valid for them. +cp "$target_dir/release/$src" "$pkg_dir/secretspec.node.tmp.$$" +mv -f "$pkg_dir/secretspec.node.tmp.$$" "$pkg_dir/secretspec.node" echo "built secretspec.node" From 65a2cc2c6541f4fdfd51972b1b3a504d06e46b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 12 Jun 2026 12:48:53 -0400 Subject: [PATCH 50/56] ci(python): drop the stale UNVALIDATED note, the wheel pipeline has run green Co-Authored-By: Claude Fable 5 --- .github/workflows/python-wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index d02e059..f46fd76 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -7,8 +7,9 @@ name: "Python wheels" # with auditwheel so the bundled cdylib's system deps (notably libdbus from the # keyring provider) are vendored. macOS/Windows build natively. # -# UNVALIDATED: this pipeline has not been run; it needs a CI run to debug and, -# for publishing, PyPI Trusted Publishing configured for this repo/environment. +# The build/repair pipeline is validated in CI; the publish job additionally +# needs PyPI Trusted Publishing configured for this repo's "pypi" environment +# before the first version tag. on: workflow_dispatch: From d7c42c0dbed3771811c45ecd21072bb0adc4f55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:15:30 -0400 Subject: [PATCH 51/56] feat(sdks): add a build-time static-link contract for the FFI archive Export SECRETSPEC_FFI_STATICLIB / _INCLUDE / _NATIVE_LIBS (the archive's transitive system libs, captured from `rustc --print native-static-libs`, never hardcoded) from ci-sdks.sh and conformance/run.sh, so each SDK can statically link libsecretspec_ffi.a instead of dlopening the cdylib. Pin the musl targets in rust-toolchain.toml and wire the musl cross-toolchain into devenv by absolute path in env (NOT in packages, which would inject musl-static libdbus into the host NIX_LDFLAGS and corrupt the glibc build). Add setuptools to the Python venv (cffi needs it on 3.12+); drop the unused maturin. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 33 +++++++++++++++++++++++++++++++++ conformance/run.sh | 21 ++++++++++++++++----- devenv.nix | 34 ++++++++++++++++++++++++++++++---- rust-toolchain.toml | 4 ++++ scripts/ci-sdks.sh | 44 +++++++++++++++++++++++++++++++++++++------- 5 files changed, 120 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7060d9d..086802d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 resolve into the temp dir with no way to clean them up. ### Changed +- The language SDKs are migrating from dlopening the `secretspec-ffi` `cdylib` at + runtime to **statically linking** the `secretspec-ffi` `staticlib` + (`libsecretspec_ffi.a`), so the Rust resolver is embedded in each artifact with + no separate library to locate and no loader path. `scripts/ci-sdks.sh` and + `conformance/run.sh` now export a build-time link contract — + `SECRETSPEC_FFI_STATICLIB`, `SECRETSPEC_FFI_INCLUDE`, and + `SECRETSPEC_FFI_NATIVE_LIBS` (the archive's transitive system libs, captured + from `rustc --print native-static-libs`, never hardcoded). The **Haskell SDK** + is the first converted: it links the archive (staged alone on `--extra-lib-dirs`, + native deps passed via `--ghc-options=-optl…`) and no longer needs + `LD_LIBRARY_PATH` at build or run time. +- The **Python SDK** now statically links `libsecretspec_ffi.a` into a compiled + CPython extension (cffi out-of-line API mode) instead of dlopening the cdylib: + no bundled `_lib/*`, no `SECRETSPEC_FFI_LIB`, no runtime discovery. The + extension targets the limited API (`py_limited_api`), so one `cp39-abi3` + wheel per platform serves all CPython >= 3.9. Building a wheel now needs a Rust + toolchain and a C compiler (`cffi`/`setuptools` are build-system requires). +- The **Ruby SDK** now compiles an mkmf C extension (`secretspec_ext`) that + statically links `libsecretspec_ffi.a` instead of dlopening the cdylib via + Fiddle: nothing to locate at runtime, no `SECRETSPEC_FFI_LIB`. Distributed as a + platform gem bundling the prebuilt archive + header + native-libs manifest; + `gem install` compiles only the ~40-line C glue, so one platform gem serves all + Ruby ABIs (>= 3.0). Install now needs a C compiler + Ruby headers (+ libdbus). +- The **Node SDK** README is corrected to describe the napi-rs addon (which + already statically embeds the resolver); it never used the cdylib/dlopen path. +- The **Go SDK** gains an opt-in `-tags static` binding that uses cgo to + statically link `libsecretspec_ffi.a` into the Go binary (the resolver is + embedded, no runtime library to locate). On Linux this is built for a musl + target and combined with `-ldflags '-linkmode external -extldflags "-static"'` + for a fully-static executable. The default `go get` path is unchanged — + purego/dlopen, no cgo, toolchain-free — so the static build is strictly + additive (`scripts/stage-staticlib.sh` stages the archive + header + generated + cgo LDFLAGS). The `rust-toolchain.toml` pins the musl targets. - The `secretspec-derive` macro now computes all of its typing decisions through the shared `secretspec::codegen` IR instead of its own duplicated logic. The generated `SecretSpec`/`SecretSpecProfile`/`Profile` API and builder are diff --git a/conformance/run.sh b/conformance/run.sh index a1556a1..247e20b 100755 --- a/conformance/run.sh +++ b/conformance/run.sh @@ -26,7 +26,15 @@ case "$(uname -s)" in *) lib_name="libsecretspec_ffi.so" ;; esac export SECRETSPEC_FFI_LIB="$target_dir/debug/$lib_name" +# Static-link contract (see scripts/ci-sdks.sh): the .a plus the archive's +# transitive native deps, for SDKs that link statically instead of dlopening. +export SECRETSPEC_FFI_STATICLIB="$target_dir/debug/libsecretspec_ffi.a" +export SECRETSPEC_FFI_INCLUDE="$repo_root/secretspec-ffi/include" +SECRETSPEC_FFI_NATIVE_LIBS="$(cargo rustc -q -p secretspec-ffi --crate-type staticlib -- \ + --print native-static-libs 2>&1 | sed -n 's/^note: native-static-libs: //p' | tail -1)" +export SECRETSPEC_FFI_NATIVE_LIBS echo "==> SECRETSPEC_FFI_LIB=$SECRETSPEC_FFI_LIB" +echo "==> SECRETSPEC_FFI_STATICLIB=$SECRETSPEC_FFI_STATICLIB" names=() statuses=() @@ -56,11 +64,14 @@ run_node() { ( ); } run_haskell() { ( cd secretspec-hs - # The Haskell SDK links the cdylib at build time, so its directory must be on - # both the linker path (--extra-lib-dirs) and the runtime loader path. - lib_dir="$(dirname "$SECRETSPEC_FFI_LIB")" - export LD_LIBRARY_PATH="$lib_dir:${LD_LIBRARY_PATH:-}" - cabal test --extra-lib-dirs="$lib_dir" --test-show-details=streaming + # The Haskell SDK statically links the secretspec-ffi archive at build time, so + # there is no runtime loader path. Stage the .a alone (target/debug also holds + # the .so) and pass its transitive native deps as linker options. + hs_lib_dir="$(mktemp -d)" + cp "$SECRETSPEC_FFI_STATICLIB" "$hs_lib_dir/" + ghc_optl=() + for l in $SECRETSPEC_FFI_NATIVE_LIBS; do ghc_optl+=("--ghc-options=-optl$l"); done + cabal test --extra-lib-dirs="$hs_lib_dir" "${ghc_optl[@]}" --test-show-details=streaming ); } run "Python" python run_python diff --git a/devenv.nix b/devenv.nix index 9b7a6ed..ca7fbf1 100644 --- a/devenv.nix +++ b/devenv.nix @@ -3,6 +3,8 @@ enable = true; # The Rust version is pinned in rust-toolchain.toml, which the native CI # runners (artifact workflows that cannot use devenv) read via rustup. + # The musl targets for the fully-static Go binary are declared in + # rust-toolchain.toml (read automatically via toolchainFile). toolchainFile = ./rust-toolchain.toml; }; languages.javascript = { @@ -21,12 +23,18 @@ requirements = '' cffi pytest + # cffi's out-of-line compile (API mode) shells out to setuptools to build + # the extension that statically links libsecretspec_ffi.a. + setuptools ''; }; }; - # Go SDK (secretspec-go) binds the C ABI via purego (dlopen, no cgo). + # Go SDK (secretspec-go): default binding is purego (dlopen, no cgo); the + # `-tags static` binding uses cgo to statically link libsecretspec_ffi.a, and on + # Linux is built fully static against musl (see the env block below). languages.go.enable = true; - # Ruby SDK (secretspec-rb) binds the C ABI via stdlib Fiddle (dlopen). + # Ruby SDK (secretspec-rb) compiles an mkmf C extension that statically links + # libsecretspec_ffi.a. languages.ruby.enable = true; # Haskell SDK (secretspec-hs) links the C ABI at build time via the FFI. languages.haskell.enable = true; @@ -38,10 +46,28 @@ pkgs.cargo-tarpaulin # installers pkgs.cargo-dist - # packaging the Python SDK wheel that bundles the cdylib - pkgs.maturin ]; + # Fully-static musl build of the Go SDK (-tags static + -extldflags -static). + # The musl C cross-toolchain and static libdbus/libunwind are referenced HERE by + # absolute path only -- NOT added to `packages`, because devenv `packages` inject + # their lib dirs into the host NIX_LDFLAGS, which would make the ordinary glibc + # build pick up the musl-static libdbus (a libc ABI mismatch -> __register_atfork + # link errors). Referenced by path, they realise into the store without polluting + # the host build environment. The CC_/linker vars are musl-target-scoped, so host + # (glibc) cargo builds are unaffected; MUSL_CC / MUSL_STATIC_LDFLAGS feed the cgo + # step so the final binary statically links libdbus + libunwind. + env = + let + muslcc = "${pkgs.pkgsCross.musl64.stdenv.cc}/bin/x86_64-unknown-linux-musl-gcc"; + in + { + CC_x86_64_unknown_linux_musl = muslcc; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER = muslcc; + MUSL_CC = muslcc; + MUSL_STATIC_LDFLAGS = "-L${pkgs.pkgsStatic.dbus.lib}/lib -L${pkgs.pkgsStatic.libunwind}/lib"; + }; + git-hooks.hooks = { rustfmt.enable = true; clippy.enable = true; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c3ba728..ecb6b6c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -6,3 +6,7 @@ # Pinned to >= 1.92 for the detect-coding-agent dependency's MSRV. [toolchain] channel = "1.92.0" +# musl targets for the fully-static Go SDK binary (-tags static + -extldflags +# -static, built in the SDKs/go-static workflows). rustup reads these on the +# native runners; devenv reads them via toolchainFile. +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] diff --git a/scripts/ci-sdks.sh b/scripts/ci-sdks.sh index 166a6e5..c29553f 100755 --- a/scripts/ci-sdks.sh +++ b/scripts/ci-sdks.sh @@ -10,7 +10,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -echo "==> Building cdylib + CLI" +echo "==> Building cdylib + staticlib + CLI" cargo build -p secretspec-ffi -p secretspec target_dir="$(cargo metadata --no-deps --format-version 1 \ @@ -19,17 +19,41 @@ case "$(uname -s)" in Darwin) lib_name="libsecretspec_ffi.dylib" ;; *) lib_name="libsecretspec_ffi.so" ;; esac +# Runtime-dlopen contract (SDKs not yet migrated to static linking still use it). export SECRETSPEC_FFI_LIB="$target_dir/debug/$lib_name" export SECRETSPEC_BIN="$target_dir/debug/secretspec" + +# Static-link contract: SDKs link libsecretspec_ffi.a (the resolver compiled in) +# instead of dlopening the cdylib. A Rust staticlib does not carry its own native +# dependency closure, so capture the transitive system libs the archive needs and +# hand them to every consumer's linker. NEVER hardcode this list -- it drifts as +# providers change (today: -ldbus-1 -lgcc_s -lutil -lrt -lpthread -lm -ldl -lc). +export SECRETSPEC_FFI_STATICLIB="$target_dir/debug/libsecretspec_ffi.a" +export SECRETSPEC_FFI_INCLUDE="$repo_root/secretspec-ffi/include" +SECRETSPEC_FFI_NATIVE_LIBS="$(cargo rustc -q -p secretspec-ffi --crate-type staticlib -- \ + --print native-static-libs 2>&1 | sed -n 's/^note: native-static-libs: //p' | tail -1)" +export SECRETSPEC_FFI_NATIVE_LIBS echo "==> SECRETSPEC_FFI_LIB=$SECRETSPEC_FFI_LIB" +echo "==> SECRETSPEC_FFI_STATICLIB=$SECRETSPEC_FFI_STATICLIB" +echo "==> SECRETSPEC_FFI_NATIVE_LIBS=$SECRETSPEC_FFI_NATIVE_LIBS" echo "==> Python" ( cd secretspec-py && python -m pytest -q ) -echo "==> Go" +echo "==> Go (default purego/dlopen path)" ( cd secretspec-go && go test ./... ) +echo "==> Go (-tags static: cgo links the archive in)" +# Stage the debug archive + header + generated cgo LDFLAGS, then exercise the +# static binding. This is the glibc self-contained build; the fully-static musl +# binary is built in the go-static.yml artifact workflow. +( cd secretspec-go && SECRETSPEC_FFI_PROFILE=debug bash scripts/stage-staticlib.sh ) +( cd secretspec-go && CGO_ENABLED=1 go test -tags static ./... ) + echo "==> Ruby" +# The Ruby SDK compiles an mkmf C extension that statically links the archive +# (using the SECRETSPEC_FFI_* contract above); build it once up front. +bash secretspec-rb/scripts/build-ext.sh ( cd secretspec-rb && ruby -e 'Dir["test/test_*.rb"].sort.each { |f| require File.expand_path(f) }' ) echo "==> Node" @@ -40,17 +64,23 @@ bash secretspec-node/scripts/build-addon.sh ( cd secretspec-node && node --test ) echo "==> Haskell" -# The Haskell SDK links the cdylib at build time, so its directory goes on both -# the linker path (--extra-lib-dirs) and the runtime loader path. +# The Haskell SDK statically links the secretspec-ffi archive at build time: the +# Rust resolver is embedded in the test binary, so there is NO runtime loader path +# (no LD_LIBRARY_PATH). Stage libsecretspec_ffi.a alone into an isolated dir so +# -lsecretspec_ffi resolves to the archive (target/debug also holds the .so), and +# pass the archive's transitive native deps as linker options. ( cd secretspec-hs - hs_lib_dir="$(dirname "$SECRETSPEC_FFI_LIB")" + hs_lib_dir="$(mktemp -d)" + cp "$SECRETSPEC_FFI_STATICLIB" "$hs_lib_dir/" + ghc_optl=() + for l in $SECRETSPEC_FFI_NATIVE_LIBS; do ghc_optl+=("--ghc-options=-optl$l"); done cabal update # --write-ghc-environment-files lets the codegen test's runghc see aeson and # the quicktype-generated module's transitive imports; SECRETSPEC_BIN (set # above) lets it run `secretspec schema`. - LD_LIBRARY_PATH="$hs_lib_dir:${LD_LIBRARY_PATH:-}" \ - cabal test --extra-lib-dirs="$hs_lib_dir" --write-ghc-environment-files=always + cabal test --extra-lib-dirs="$hs_lib_dir" "${ghc_optl[@]}" \ + --write-ghc-environment-files=always ) echo "==> All SDK suites passed" From 87e7331d27b25c96d5a6df8a1c6e3ba4e1e92578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:15:42 -0400 Subject: [PATCH 52/56] feat(haskell): statically link the FFI archive, drop LD_LIBRARY_PATH Stage libsecretspec_ffi.a alone on --extra-lib-dirs (so -lsecretspec_ffi resolves to the archive, not the co-located .so) and pass its native deps via --ghc-options=-optl. The Rust resolver is embedded in the binary, so no runtime loader path is needed. SecretSpec.hs is unchanged (ccall safe is linkage-agnostic). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/haskell-build.yml | 32 ++++++++++++++++++----------- secretspec-hs/secretspec.cabal | 9 +++++--- secretspec-hs/src/SecretSpec.hs | 3 ++- secretspec-hs/test/Main.hs | 8 +++++--- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/haskell-build.yml b/.github/workflows/haskell-build.yml index 0d9d6a9..0b73c1f 100644 --- a/.github/workflows/haskell-build.yml +++ b/.github/workflows/haskell-build.yml @@ -1,9 +1,9 @@ name: "Haskell SDK" # Builds and tests the Haskell SDK (secretspec-hs) against a freshly built -# cdylib. The SDK links the secretspec-ffi C ABI at build time via the FFI, so -# the cdylib's directory goes on both the linker path (--extra-lib-dirs) and the -# runtime loader path. +# secretspec-ffi staticlib. The SDK statically links the C ABI archive at build +# time, so the Rust resolver is embedded in the binary and there is no runtime +# loader path (no LD_LIBRARY_PATH). on: workflow_dispatch: @@ -38,22 +38,29 @@ jobs: name: devenv - name: Install devenv.sh run: nix profile install nixpkgs#devenv - - name: Build cdylib + CLI and run the Haskell SDK test-suite + - name: Build staticlib + CLI and run the Haskell SDK test-suite run: | devenv shell -- bash -c ' set -euo pipefail cargo build -p secretspec-ffi -p secretspec target_dir="$(cargo metadata --no-deps --format-version 1 \ | grep -o "\"target_directory\":\"[^\"]*\"" | head -1 | sed "s/.*:\"\(.*\)\"/\1/")" - lib_dir="$target_dir/debug" - export SECRETSPEC_BIN="$lib_dir/secretspec" + export SECRETSPEC_BIN="$target_dir/debug/secretspec" + # Capture the staticlib archive plus its transitive native deps. Stage + # the .a alone so -lsecretspec_ffi resolves to the archive (target/debug + # also holds the .so) and the resolver is embedded with no loader path. + native_libs="$(cargo rustc -q -p secretspec-ffi --crate-type staticlib -- \ + --print native-static-libs 2>&1 | sed -n "s/^note: native-static-libs: //p" | tail -1)" + hs_lib_dir="$(mktemp -d)" + cp "$target_dir/debug/libsecretspec_ffi.a" "$hs_lib_dir/" + ghc_optl=() + for l in $native_libs; do ghc_optl+=("--ghc-options=-optl$l"); done cd secretspec-hs cabal update # --write-ghc-environment-files lets the codegen test compile the # quicktype-generated module; SECRETSPEC_BIN lets it run the CLI. - LD_LIBRARY_PATH="$lib_dir:${LD_LIBRARY_PATH:-}" \ - cabal test --extra-lib-dirs="$lib_dir" \ - --write-ghc-environment-files=always --test-show-details=streaming + cabal test --extra-lib-dirs="$hs_lib_dir" "${ghc_optl[@]}" \ + --write-ghc-environment-files=always --test-show-details=streaming ' publish: @@ -70,9 +77,10 @@ jobs: - name: Install devenv.sh run: nix profile install nixpkgs#devenv - name: sdist and upload to Hackage - # Requires the HACKAGE_TOKEN secret. The package links secretspec-ffi at - # build time, so Hackage's build bots cannot compile it (no cdylib); the - # upload still succeeds and the README documents the link requirement. + # Requires the HACKAGE_TOKEN secret. The package statically links + # secretspec-ffi at build time, so Hackage's build bots cannot compile it + # (no staticlib, no Rust toolchain); the upload still succeeds and the + # README documents the link requirement. env: HACKAGE_TOKEN: ${{ secrets.HACKAGE_TOKEN }} run: | diff --git a/secretspec-hs/secretspec.cabal b/secretspec-hs/secretspec.cabal index eb25702..f9b1f80 100644 --- a/secretspec-hs/secretspec.cabal +++ b/secretspec-hs/secretspec.cabal @@ -24,9 +24,12 @@ library , containers <1 , directory <2 , text <3 - -- The native resolver, linked at build time. Pass its location with - -- `cabal build --extra-lib-dirs=/debug` (and put the same - -- directory on the runtime loader path, e.g. LD_LIBRARY_PATH). + -- The native resolver, statically linked at build time. Point --extra-lib-dirs + -- at a directory containing ONLY libsecretspec_ffi.a (so -lsecretspec_ffi + -- resolves to the archive, not a co-located .so), and pass the archive's + -- transitive native deps via --ghc-options=-optl (capture them with + -- `cargo rustc -p secretspec-ffi --crate-type staticlib -- --print native-static-libs`). + -- The Rust code is embedded in the binary, so no LD_LIBRARY_PATH is needed. extra-libraries: secretspec_ffi default-language: Haskell2010 ghc-options: -Wall diff --git a/secretspec-hs/src/SecretSpec.hs b/secretspec-hs/src/SecretSpec.hs index 8882d84..a294d4e 100644 --- a/secretspec-hs/src/SecretSpec.hs +++ b/secretspec-hs/src/SecretSpec.hs @@ -62,7 +62,8 @@ import Foreign.Ptr (nullPtr) import System.Directory (doesFileExist, removeFile) import System.Environment (setEnv) --- The three C ABI functions, linked at build time (-lsecretspec_ffi). They are +-- The three C ABI functions, statically linked at build time (the archive +-- libsecretspec_ffi.a is embedded; -lsecretspec_ffi resolves to it). They are -- declared @safe@ because @secretspec_resolve@ may block on provider I/O -- (1Password, LastPass), and a @safe@ call lets other Haskell threads run. foreign import ccall safe "secretspec_resolve" diff --git a/secretspec-hs/test/Main.hs b/secretspec-hs/test/Main.hs index 200f9c0..cd8bb5d 100644 --- a/secretspec-hs/test/Main.hs +++ b/secretspec-hs/test/Main.hs @@ -1,10 +1,12 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} --- | Unit + cross-language conformance tests for the Haskell SDK. Run with the --- cdylib on the linker and loader paths: +-- | Unit + cross-language conformance tests for the Haskell SDK. The native +-- resolver is statically linked, so no loader path is needed -- stage the +-- archive alone and pass its transitive native deps: -- --- > cabal test --extra-lib-dirs="$TARGET/debug" # with LD_LIBRARY_PATH set +-- > cabal test --extra-lib-dirs="$DIR_WITH_ONLY_THE_A" \ +-- > --ghc-options=-optl-ldbus-1 ... # libs from `--print native-static-libs` module Main (main) where import Control.Exception (SomeException, try) From dc7fb921ee26c2ca806124d680d54114fee859ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:15:52 -0400 Subject: [PATCH 53/56] feat(python): statically link the resolver via cffi API mode Switch cffi from ABI/dlopen mode to out-of-line API mode (_build_ffi.py): compile a CPython extension that statically links libsecretspec_ffi.a, reusing the existing secretspec.h. No bundled cdylib, no SECRETSPEC_FFI_LIB, no runtime discovery. The extension targets the limited API (py_limited_api), so one cp39-abi3 wheel per platform serves all CPython >= 3.9. Building a wheel now needs a Rust toolchain and a C compiler. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/python-wheels.yml | 18 ++-- secretspec-py/.gitignore | 5 +- secretspec-py/README.md | 14 +-- secretspec-py/pyproject.toml | 9 +- secretspec-py/scripts/stage-cdylib.sh | 22 ----- secretspec-py/secretspec/__init__.py | 94 +++----------------- secretspec-py/secretspec/_build_ffi.py | 116 +++++++++++++++++++++++++ secretspec-py/setup.py | 26 +++--- secretspec-py/tests/conftest.py | 65 ++++++++------ 9 files changed, 205 insertions(+), 164 deletions(-) delete mode 100755 secretspec-py/scripts/stage-cdylib.sh create mode 100644 secretspec-py/secretspec/_build_ffi.py diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index f46fd76..42fa696 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -1,11 +1,14 @@ name: "Python wheels" -# Builds portable wheels for the Python SDK that bundle the secretspec-ffi -# cdylib, and publishes them to PyPI on a version tag. +# Builds portable abi3 wheels for the Python SDK and publishes them to PyPI on a +# version tag. The Rust resolver is statically linked into a compiled CPython +# extension (cffi API mode over libsecretspec_ffi.a) -- there is no separate +# cdylib bundled. The extension targets the limited API, so one cp39-abi3 wheel +# per platform serves all CPython >= 3.9. # # Linux wheels are built inside a manylinux container (old glibc) and repaired -# with auditwheel so the bundled cdylib's system deps (notably libdbus from the -# keyring provider) are vendored. macOS/Windows build natively. +# with auditwheel so the extension's dynamic system deps (notably libdbus from +# the keyring provider) are vendored. macOS/Windows build natively. # # The build/repair pipeline is validated in CI; the publish job additionally # needs PyPI Trusted Publishing configured for this repo's "pypi" environment @@ -47,7 +50,9 @@ jobs: "$HOME/.cargo/bin/rustup" toolchain install - name: Build wheel and repair to manylinux run: | - bash secretspec-py/scripts/stage-cdylib.sh + # build isolation installs setuptools + cffi (build-system.requires); + # cffi compiles the extension, statically linking libsecretspec_ffi.a + # (cargo builds the release archive). auditwheel vendors libdbus. python3.11 -m build --wheel --outdir dist_raw secretspec-py auditwheel repair --wheel-dir wheelhouse dist_raw/*.whl - uses: actions/upload-artifact@v4 @@ -75,7 +80,8 @@ jobs: shell: bash run: | python -m pip install --upgrade build - bash secretspec-py/scripts/stage-cdylib.sh + # cffi compiles the extension and statically links libsecretspec_ffi.a + # (cargo builds the release archive) during the isolated wheel build. ( cd secretspec-py && python -m build --wheel --outdir "$GITHUB_WORKSPACE/wheelhouse" ) - uses: actions/upload-artifact@v4 with: diff --git a/secretspec-py/.gitignore b/secretspec-py/.gitignore index 619d235..2ef3f20 100644 --- a/secretspec-py/.gitignore +++ b/secretspec-py/.gitignore @@ -4,4 +4,7 @@ __pycache__/ build/ dist/ *.egg-info/ -secretspec/_lib/ +# Compiled cffi extension + its generated C glue (built from libsecretspec_ffi.a). +secretspec/_secretspec_cffi* +*.so +*.o diff --git a/secretspec-py/README.md b/secretspec-py/README.md index b8a1cf1..cd10d25 100644 --- a/secretspec-py/README.md +++ b/secretspec-py/README.md @@ -44,9 +44,11 @@ for s in report.secrets: print(s.name, s.status, s.required) ``` -## Library discovery - -The SDK loads the native library from, in order: the `SECRETSPEC_FFI_LIB` -environment variable, a copy bundled in the installed wheel, or a Cargo `target` -directory found by searching up from the working directory (useful in a source -checkout). +## Native library + +The Rust resolver is statically linked into a compiled extension +(`secretspec._secretspec_cffi`) inside the installed wheel, so there is nothing +to locate at runtime and no `SECRETSPEC_FFI_LIB` to set. The prebuilt `abi3` +wheels are self-contained (`pip install secretspec`). From a source checkout the +extension is compiled on demand by the test harness, which needs a Rust +toolchain and a C compiler on `PATH`. diff --git a/secretspec-py/pyproject.toml b/secretspec-py/pyproject.toml index 3761aeb..593dc4b 100644 --- a/secretspec-py/pyproject.toml +++ b/secretspec-py/pyproject.toml @@ -1,5 +1,7 @@ [build-system] -requires = ["setuptools>=68"] +# cffi compiles the out-of-line extension (secretspec/_build_ffi.py) at build +# time, statically linking libsecretspec_ffi.a; a Rust toolchain must be on PATH. +requires = ["setuptools>=68", "cffi>=1.15"] build-backend = "setuptools.build_meta" [project] @@ -17,8 +19,3 @@ Repository = "https://github.com/cachix/secretspec" [tool.setuptools] packages = ["secretspec"] - -[tool.setuptools.package-data] -# The cdylib is bundled here by the packaging step; absent in a source checkout, -# where the SDK falls back to SECRETSPEC_FFI_LIB or a Cargo target directory. -secretspec = ["_lib/*"] diff --git a/secretspec-py/scripts/stage-cdylib.sh b/secretspec-py/scripts/stage-cdylib.sh deleted file mode 100755 index 0e9a4aa..0000000 --- a/secretspec-py/scripts/stage-cdylib.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Build the secretspec-ffi cdylib (release) and stage it into -# secretspec/_lib/ so a wheel build bundles it. Run before building the wheel. -set -euo pipefail - -pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -repo_root="$(cd "$pkg_dir/.." && pwd)" - -cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" - -target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ - | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" -case "$(uname -s)" in - Darwin) lib_name="libsecretspec_ffi.dylib" ;; - MINGW* | MSYS* | CYGWIN*) lib_name="secretspec_ffi.dll" ;; - *) lib_name="libsecretspec_ffi.so" ;; -esac - -mkdir -p "$pkg_dir/secretspec/_lib" -cp "$target_dir/release/$lib_name" "$pkg_dir/secretspec/_lib/$lib_name" -echo "staged $lib_name into secretspec/_lib/" diff --git a/secretspec-py/secretspec/__init__.py b/secretspec-py/secretspec/__init__.py index d407706..e6acc89 100644 --- a/secretspec-py/secretspec/__init__.py +++ b/secretspec-py/secretspec/__init__.py @@ -7,21 +7,23 @@ (a builder with ``with_provider``/``with_profile``/``with_reason`` and ``load``, returning a ``Resolved`` with ``.secrets``/``.provider``/``.profile``). -The library is loaded via cffi (dlopen) from, in order: the ``SECRETSPEC_FFI_LIB`` -environment variable, a copy bundled in the wheel, or a Cargo target directory. +The Rust resolver is statically linked into a compiled CPython extension +(``secretspec._secretspec_cffi``, built by ``_build_ffi.py`` via cffi API mode), +so there is no separate library to locate, no ``SECRETSPEC_FFI_LIB``, and no +runtime dlopen. """ from __future__ import annotations import json import os -import sys -import threading from dataclasses import dataclass, field -from pathlib import Path from typing import Optional -from cffi import FFI +# The compiled extension statically embeds the secretspec-ffi C ABI. ``_lib`` +# exposes secretspec_resolve / secretspec_free / secretspec_abi_version; ``_ffi`` +# provides the string/NULL helpers. +from secretspec._secretspec_cffi import ffi as _ffi, lib as _lib # Response wire-format version this SDK understands. Tracks secretspec-ffi's # RESOLVE_SCHEMA_VERSION; a mismatch means the loaded library is incompatible. @@ -44,77 +46,6 @@ "abi_version", ] -# The narrow C ABI. Mirrors secretspec-ffi/include/secretspec.h. -_ffi = FFI() -_ffi.cdef( - """ - char *secretspec_resolve(const char *request_json); - void secretspec_free(char *ptr); - const char *secretspec_abi_version(void); - """ -) - - -def _candidate_lib_names() -> list[str]: - if sys.platform == "darwin": - return ["libsecretspec_ffi.dylib"] - if sys.platform == "win32": - return ["secretspec_ffi.dll"] - return ["libsecretspec_ffi.so"] - - -def _find_library() -> str: - # 1. Explicit override (used in development and tests). - override = os.environ.get("SECRETSPEC_FFI_LIB") - if override: - return override - - names = _candidate_lib_names() - - # 2. Bundled next to this package (the wheel distribution layout). - here = Path(__file__).resolve().parent - for name in names: - bundled = here / "_lib" / name - if bundled.exists(): - return str(bundled) - - # 3. A Cargo target directory, searching up from the current directory. - # Within the nearest target/, pick the most recently built library rather - # than always preferring release, so a stale release build does not shadow - # the debug build just produced. - for base in [Path.cwd(), *Path.cwd().parents]: - target = base / "target" - if target.is_dir(): - existing = [ - target / profile / name - for profile in ("release", "debug") - for name in names - if (target / profile / name).exists() - ] - if existing: - return str(max(existing, key=lambda p: p.stat().st_mtime)) - - raise SecretSpecError( - "load", - "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB " - "to its path", - ) - - -_lib = None -_lib_lock = threading.Lock() - - -def _load() -> object: - # Double-checked locking so concurrent first callers do not race to dlopen. - global _lib - if _lib is None: - with _lib_lock: - if _lib is None: - _lib = _ffi.dlopen(_find_library()) - return _lib - - class SecretSpecError(Exception): """A resolution call failed (bad manifest, provider error, reason policy).""" @@ -227,21 +158,20 @@ class Report: def abi_version() -> str: - """The ABI version reported by the loaded library.""" - ptr = _load().secretspec_abi_version() + """The ABI version reported by the statically linked library.""" + ptr = _lib.secretspec_abi_version() return _ffi.string(ptr).decode() def _resolve_envelope(request: dict) -> dict: - lib = _load() payload = json.dumps(request).encode("utf-8") - ptr = lib.secretspec_resolve(payload) + ptr = _lib.secretspec_resolve(payload) if ptr == _ffi.NULL: raise SecretSpecError("ffi", "secretspec_resolve returned null") try: raw = _ffi.string(ptr).decode("utf-8") finally: - lib.secretspec_free(ptr) + _lib.secretspec_free(ptr) return json.loads(raw) diff --git a/secretspec-py/secretspec/_build_ffi.py b/secretspec-py/secretspec/_build_ffi.py new file mode 100644 index 0000000..8f07c09 --- /dev/null +++ b/secretspec-py/secretspec/_build_ffi.py @@ -0,0 +1,116 @@ +"""cffi out-of-line (API mode) build script for the secretspec Python SDK. + +Compiles a CPython extension (``secretspec._secretspec_cffi``) that STATICALLY +links the secretspec-ffi archive (``libsecretspec_ffi.a``), so the Rust resolver +is embedded in the extension. There is no separate cdylib to ship in the wheel, +no ``SECRETSPEC_FFI_LIB``, and no runtime dlopen discovery. + +The archive and its transitive native deps are taken, in order, from: + 1. the ``SECRETSPEC_FFI_STATICLIB`` / ``SECRETSPEC_FFI_NATIVE_LIBS`` / + ``SECRETSPEC_FFI_INCLUDE`` environment variables (the build-time link + contract exported by ``scripts/ci-sdks.sh``); otherwise + 2. a release build produced here with cargo, capturing the native deps from + ``rustc --print native-static-libs`` (wheel builds, local ``pip install``). + +The extension is built against the CPython limited API (``py_limited_api=True``), +so one ``abi3`` wheel serves every CPython >= 3.9 on a given platform. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import subprocess + +from cffi import FFI + +_PKG_DIR = pathlib.Path(__file__).resolve().parent # secretspec-py/secretspec +_REPO_ROOT = _PKG_DIR.parents[1] # repository root + +# The narrow C ABI. Mirrors secretspec-ffi/include/secretspec.h. +_CDEF = """ + char *secretspec_resolve(const char *request_json); + void secretspec_free(char *ptr); + const char *secretspec_abi_version(void); +""" + + +def _native_libs_from_note(text: str) -> list[str]: + marker = "native-static-libs: " + libs: list[str] = [] + for line in text.splitlines(): + if marker in line: + libs = line.split(marker, 1)[1].split() + return libs + + +def _cargo_target_dir() -> pathlib.Path: + meta = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return pathlib.Path(json.loads(meta.stdout)["target_directory"]) + + +def _resolve_link_inputs() -> tuple[str, list[str], str]: + """Return (staticlib_path, native_libs, include_dir).""" + env_lib = os.environ.get("SECRETSPEC_FFI_STATICLIB") + env_native = os.environ.get("SECRETSPEC_FFI_NATIVE_LIBS") + env_include = os.environ.get("SECRETSPEC_FFI_INCLUDE") + if env_lib and env_native is not None and env_include: + return env_lib, env_native.split(), env_include + + # No prebuilt contract in the environment: build the release archive and + # capture its native deps ourselves. + subprocess.run( + ["cargo", "build", "-p", "secretspec-ffi", "--release"], + cwd=_REPO_ROOT, + check=True, + ) + note = subprocess.run( + [ + "cargo", + "rustc", + "-q", + "-p", + "secretspec-ffi", + "--release", + "--crate-type", + "staticlib", + "--", + "--print", + "native-static-libs", + ], + cwd=_REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + native = _native_libs_from_note(note.stderr + note.stdout) + staticlib = _cargo_target_dir() / "release" / "libsecretspec_ffi.a" + include = _REPO_ROOT / "secretspec-ffi" / "include" + return str(staticlib), native, str(include) + + +_staticlib, _native_libs, _include = _resolve_link_inputs() + +ffibuilder = FFI() +ffibuilder.cdef(_CDEF) +ffibuilder.set_source( + "secretspec._secretspec_cffi", + '#include "secretspec.h"', + include_dirs=[_include], + # Link the whole archive in (the linker pulls the referenced objects); the + # archive's transitive system libs follow it on the link line. + extra_objects=[_staticlib], + extra_link_args=_native_libs, + py_limited_api=True, +) + + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/secretspec-py/setup.py b/secretspec-py/setup.py index 18d0923..5343d48 100644 --- a/secretspec-py/setup.py +++ b/secretspec-py/setup.py @@ -1,13 +1,10 @@ -"""Build a platform-specific wheel that bundles the secretspec-ffi cdylib. +"""Build a platform-specific ``abi3`` wheel with the Rust resolver linked in. -Project metadata lives in ``pyproject.toml``; this file only (a) forces a -platform (non-pure) wheel so pip installs the right native library per OS/arch, -and (b) tags it ``py3-none-`` because the package is pure Python apart -from the bundled library, which works on any Python 3. - -The native library must be staged into ``secretspec/_lib/`` before building it -(see ``scripts/stage-cdylib.sh``); it is declared as package data in -``pyproject.toml``. +Project metadata lives in ``pyproject.toml``. This file wires the cffi +out-of-line extension (``secretspec/_build_ffi.py``), which compiles a CPython +extension that statically links ``libsecretspec_ffi.a``. The extension targets +the CPython limited API (``py_limited_api=True``), so one ``cp39-abi3-`` +wheel serves every CPython >= 3.9 on that platform. """ from setuptools import setup @@ -20,7 +17,7 @@ class BinaryDistribution(Distribution): - """Marks the distribution as containing a native library, so the wheel is + """Marks the distribution as containing a native extension, so the wheel is platform-specific rather than ``any``.""" def has_ext_modules(self) -> bool: @@ -28,12 +25,15 @@ def has_ext_modules(self) -> bool: class PlatformWheel(bdist_wheel): - def get_tag(self): - _, _, platform = super().get_tag() - return "py3", "none", platform + def finalize_options(self) -> None: + # Emit a single abi3 wheel (cp39-abi3-) for the limited-API + # extension instead of one wheel per CPython minor version. + super().finalize_options() + self.py_limited_api = "cp39" setup( distclass=BinaryDistribution, cmdclass={"bdist_wheel": PlatformWheel}, + cffi_modules=["secretspec/_build_ffi.py:ffibuilder"], ) diff --git a/secretspec-py/tests/conftest.py b/secretspec-py/tests/conftest.py index 9c8e12f..1c33b61 100644 --- a/secretspec-py/tests/conftest.py +++ b/secretspec-py/tests/conftest.py @@ -1,7 +1,11 @@ -"""Ensure the secretspec-ffi cdylib is built and discoverable before tests run. - -If SECRETSPEC_FFI_LIB is not already set, build the crate and point the SDK at -the freshly built library in the Cargo target directory. +"""Compile the secretspec cffi extension before the tests import the package. + +The SDK now statically links ``libsecretspec_ffi.a`` into a compiled extension +(``secretspec._secretspec_cffi``) rather than dlopening a cdylib, so the +extension must exist before ``import secretspec``. Build the debug archive + +CLI, point the build-time link contract at the debug archive (fast), then +compile the extension in place. Mirrors the Node SDK's "ensure the addon exists" +harness. ``SECRETSPEC_BIN`` (the CLI) is still needed by the codegen test. """ import json @@ -11,42 +15,47 @@ import sys _REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] - - -def _lib_name() -> str: - if sys.platform == "darwin": - return "libsecretspec_ffi.dylib" - if sys.platform == "win32": - return "secretspec_ffi.dll" - return "libsecretspec_ffi.so" +_PKG_DIR = _REPO_ROOT / "secretspec-py" def _bin_name() -> str: return "secretspec.exe" if sys.platform == "win32" else "secretspec" -def _ensure_lib() -> None: - have_lib = bool(os.environ.get("SECRETSPEC_FFI_LIB")) - have_bin = bool(os.environ.get("SECRETSPEC_BIN")) - if have_lib and have_bin: - return +def _native_libs() -> str: + note = subprocess.run( + ["cargo", "rustc", "-q", "-p", "secretspec-ffi", "--crate-type", + "staticlib", "--", "--print", "native-static-libs"], + cwd=_REPO_ROOT, capture_output=True, text=True, check=True, + ) + marker = "native-static-libs: " + libs = "" + for line in (note.stderr + note.stdout).splitlines(): + if marker in line: + libs = line.split(marker, 1)[1].strip() + return libs - # Build the cdylib (for the runtime SDK) and the CLI (for codegen tests). + +def _ensure_extension_and_bin() -> None: subprocess.run( ["cargo", "build", "-p", "secretspec-ffi", "-p", "secretspec"], - cwd=_REPO_ROOT, - check=True, + cwd=_REPO_ROOT, check=True, ) meta = subprocess.run( ["cargo", "metadata", "--no-deps", "--format-version", "1"], - cwd=_REPO_ROOT, - check=True, - capture_output=True, - text=True, + cwd=_REPO_ROOT, check=True, capture_output=True, text=True, + ) + debug = pathlib.Path(json.loads(meta.stdout)["target_directory"]) / "debug" + os.environ.setdefault("SECRETSPEC_BIN", str(debug / _bin_name())) + # Point _build_ffi.py at the debug archive (fast) unless a contract is set. + os.environ.setdefault("SECRETSPEC_FFI_STATICLIB", str(debug / "libsecretspec_ffi.a")) + os.environ.setdefault("SECRETSPEC_FFI_INCLUDE", str(_REPO_ROOT / "secretspec-ffi" / "include")) + if "SECRETSPEC_FFI_NATIVE_LIBS" not in os.environ: + os.environ["SECRETSPEC_FFI_NATIVE_LIBS"] = _native_libs() + subprocess.run( + [sys.executable, str(_PKG_DIR / "secretspec" / "_build_ffi.py")], + cwd=_PKG_DIR, check=True, ) - debug_dir = pathlib.Path(json.loads(meta.stdout)["target_directory"]) / "debug" - os.environ.setdefault("SECRETSPEC_FFI_LIB", str(debug_dir / _lib_name())) - os.environ.setdefault("SECRETSPEC_BIN", str(debug_dir / _bin_name())) -_ensure_lib() +_ensure_extension_and_bin() From 7fef27cefafb049b6cb91a033d1129746eb92af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:16:03 -0400 Subject: [PATCH 54/56] feat(ruby): statically link the resolver via an mkmf C extension Replace the Fiddle/dlopen binding with a thin mkmf C extension (ext/secretspec) that statically links libsecretspec_ffi.a and exposes the C ABI to Ruby. Nothing to locate at runtime, no SECRETSPEC_FFI_LIB. Distributed as a platform gem that bundles the prebuilt archive + header + native-libs manifest; gem install compiles only the ~40-line C glue, so one platform gem serves every Ruby ABI. Install now needs a C compiler + Ruby headers (+ libdbus). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ruby-gems.yml | 22 ++-- secretspec-rb/.gitignore | 7 ++ secretspec-rb/ext/secretspec/extconf.rb | 61 ++++++++++ secretspec-rb/ext/secretspec/secretspec_ext.c | 46 ++++++++ secretspec-rb/lib/secretspec.rb | 104 ++++-------------- secretspec-rb/scripts/build-ext.sh | 29 +++++ secretspec-rb/scripts/stage-cdylib.sh | 22 ---- secretspec-rb/scripts/stage-staticlib.sh | 23 ++++ secretspec-rb/secretspec.gemspec | 20 ++-- secretspec-rb/test/test_codegen.rb | 14 ++- secretspec-rb/test/test_resolve.rb | 18 ++- 11 files changed, 224 insertions(+), 142 deletions(-) create mode 100644 secretspec-rb/ext/secretspec/extconf.rb create mode 100644 secretspec-rb/ext/secretspec/secretspec_ext.c create mode 100644 secretspec-rb/scripts/build-ext.sh delete mode 100755 secretspec-rb/scripts/stage-cdylib.sh create mode 100644 secretspec-rb/scripts/stage-staticlib.sh diff --git a/.github/workflows/ruby-gems.yml b/.github/workflows/ruby-gems.yml index 460057c..52ab5bd 100644 --- a/.github/workflows/ruby-gems.yml +++ b/.github/workflows/ruby-gems.yml @@ -1,14 +1,15 @@ name: "Ruby gems" -# Builds platform-specific gems for the Ruby SDK that bundle the secretspec-ffi -# cdylib (into vendor/), so `gem install secretspec` works without a separate -# native build. +# Builds platform-specific gems for the Ruby SDK. Each gem bundles the +# secretspec-ffi staticlib (into vendor/); at `gem install` mkmf compiles a tiny +# C glue and statically links that archive, so the resolver is embedded in the +# extension and one platform gem serves every Ruby ABI. # # NOTE: as with the Python wheels, the Linux gem here links the runner's glibc # and system libraries (notably libdbus, via the keyring provider). A portable -# release build should use a baseline toolchain (e.g. rake-compiler-dock) and -# vendor those deps. That is the remaining follow-up; the build + bundling -# mechanism below is what is validated. +# release build should use a baseline toolchain and vendor those deps. That is +# the remaining follow-up; the build + bundling mechanism below is what is +# validated. Install requires a C compiler + Ruby headers (+ libdbus-1-dev). on: workflow_dispatch: @@ -50,19 +51,20 @@ jobs: with: ruby-version: "3.3" - - name: Stage the cdylib and build the platform gem + - name: Stage the staticlib and build the platform gem shell: bash run: | - bash secretspec-rb/scripts/stage-cdylib.sh + bash secretspec-rb/scripts/stage-staticlib.sh ( cd secretspec-rb && gem build secretspec.gemspec ) - name: Smoke test the gem shell: bash run: | cd secretspec-rb + # Installing compiles the extension (spec.extensions) against the + # bundled vendor/libsecretspec_ffi.a -- no Rust toolchain needed here. gem install --no-document --install-dir "$RUNNER_TEMP/gemhome" secretspec-*.gem - # Run outside the repo with no SECRETSPEC_FFI_LIB so only the bundled - # library can satisfy the require. + # Run outside the repo so only the installed extension satisfies require. cd "$RUNNER_TEMP" GEM_HOME="$RUNNER_TEMP/gemhome" GEM_PATH="$RUNNER_TEMP/gemhome" \ ruby -e 'require "secretspec"; puts "abi " + Secretspec.abi_version' diff --git a/secretspec-rb/.gitignore b/secretspec-rb/.gitignore index f78b421..4ea5680 100644 --- a/secretspec-rb/.gitignore +++ b/secretspec-rb/.gitignore @@ -1,2 +1,9 @@ vendor/ *.gem +# Compiled extension + mkmf build artifacts. +lib/secretspec/secretspec_ext.* +ext/**/Makefile +ext/**/*.o +ext/**/*.so +ext/**/*.bundle +ext/**/mkmf.log diff --git a/secretspec-rb/ext/secretspec/extconf.rb b/secretspec-rb/ext/secretspec/extconf.rb new file mode 100644 index 0000000..839da3e --- /dev/null +++ b/secretspec-rb/ext/secretspec/extconf.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# Builds the secretspec native extension, statically linking the secretspec-ffi +# archive (libsecretspec_ffi.a) into the extension object. A Rust staticlib does +# not carry its own native dependency closure, so the archive's transitive system +# libs (captured from `rustc --print native-static-libs`, never hardcoded) are +# appended to the link line after it. + +require "mkmf" + +ext_dir = __dir__ +pkg_dir = File.expand_path("../..", ext_dir) # secretspec-rb +repo_root = File.expand_path("..", pkg_dir) # workspace root (dev checkout) +vendor = File.join(pkg_dir, "vendor") + +# The staticlib: explicit contract, the bundled platform-gem copy, or a Cargo +# target dir (dev checkout, newest of release/debug). +def find_staticlib(vendor, repo_root) + env = ENV["SECRETSPEC_FFI_STATICLIB"] + return env if env && !env.empty? && File.exist?(env) + + bundled = File.join(vendor, "libsecretspec_ffi.a") + return bundled if File.exist?(bundled) + + %w[release debug] + .map { |p| File.join(repo_root, "target", p, "libsecretspec_ffi.a") } + .select { |c| File.exist?(c) } + .max_by { |c| File.mtime(c) } +end + +# The archive's transitive native deps: explicit contract, the bundled manifest, +# or captured live from rustc (dev checkout). +def find_native_libs(vendor, repo_root) + env = ENV["SECRETSPEC_FFI_NATIVE_LIBS"] + return env if env && !env.empty? + + manifest = File.join(vendor, "native-static-libs.txt") + return File.read(manifest).strip if File.exist?(manifest) + + note = `cd #{repo_root} && cargo rustc -q -p secretspec-ffi --crate-type staticlib -- --print native-static-libs 2>&1` + note[/native-static-libs:\s*(.*)/, 1].to_s.strip +end + +staticlib = find_staticlib(vendor, repo_root) +abort("secretspec: could not locate libsecretspec_ffi.a; set SECRETSPEC_FFI_STATICLIB") unless staticlib + +# Header: the bundled vendor copy (platform gem) or the ffi crate's include dir. +include_dir = + if File.exist?(File.join(vendor, "secretspec.h")) + vendor + else + File.join(repo_root, "secretspec-ffi", "include") + end + +$INCFLAGS << " -I#{include_dir}" +# $LOCAL_LIBS is emitted before $libs on the link line, so the archive (pulled +# for the referenced symbols) precedes the system libs it depends on. +$LOCAL_LIBS << " #{staticlib}" +$libs = "#{$libs} #{find_native_libs(vendor, repo_root)}" + +create_makefile("secretspec/secretspec_ext") diff --git a/secretspec-rb/ext/secretspec/secretspec_ext.c b/secretspec-rb/ext/secretspec/secretspec_ext.c new file mode 100644 index 0000000..7d8dbc4 --- /dev/null +++ b/secretspec-rb/ext/secretspec/secretspec_ext.c @@ -0,0 +1,46 @@ +/* + * Native glue for the secretspec Ruby SDK. + * + * A thin C extension that statically links the secretspec-ffi archive + * (libsecretspec_ffi.a) and exposes its three C ABI functions to Ruby as + * Secretspec::Native.c_resolve / c_abi_version. The Rust resolver is embedded in + * this extension object, so there is no separate cdylib to ship or dlopen. + */ +#include +#include "secretspec.h" + +/* + * Secretspec::Native.c_resolve(request_json) -> String or nil + * + * Marshals the JSON request to the Rust resolver and copies the owned response + * into a Ruby String before freeing it. Returns nil if the resolver returns NULL + * (catastrophic allocation failure); the Ruby wrapper turns that into an Error. + */ +static VALUE +native_resolve(VALUE self, VALUE request_json) +{ + const char *request = StringValueCStr(request_json); + char *result = secretspec_resolve(request); + if (result == NULL) { + return Qnil; + } + VALUE out = rb_str_new_cstr(result); + secretspec_free(result); + return out; +} + +/* Secretspec::Native.c_abi_version -> String (static, not freed). */ +static VALUE +native_abi_version(VALUE self) +{ + return rb_str_new_cstr(secretspec_abi_version()); +} + +void +Init_secretspec_ext(void) +{ + VALUE mod = rb_define_module("Secretspec"); + VALUE native = rb_define_module_under(mod, "Native"); + rb_define_singleton_method(native, "c_resolve", native_resolve, 1); + rb_define_singleton_method(native, "c_abi_version", native_abi_version, 0); +} diff --git a/secretspec-rb/lib/secretspec.rb b/secretspec-rb/lib/secretspec.rb index 12eca11..b31f62d 100644 --- a/secretspec-rb/lib/secretspec.rb +++ b/secretspec-rb/lib/secretspec.rb @@ -2,17 +2,19 @@ # Ruby SDK for SecretSpec, a declarative secrets manager. # -# A thin client over the secretspec-ffi C ABI, loaded at runtime via the stdlib -# Fiddle (dlopen, no native gem). Resolution happens entirely in the Rust core, -# so the SDK inherits every provider with no Ruby-side logic. Mirrors the Rust -# derive crate's vocabulary. -# -# The native library is located via SECRETSPEC_FFI_LIB, or a Cargo target -# directory found by searching up from the working directory. +# A thin client over the secretspec-ffi C ABI. The Rust resolver is statically +# linked into a native extension (secretspec_ext), so the SDK inherits every +# provider with no Ruby-side logic and there is nothing to locate at runtime. +# Mirrors the Rust derive crate's vocabulary. -require "fiddle" require "json" -require "rbconfig" + +# The compiled extension lives next to this file in a source/dev checkout, but in +# an installed gem RubyGems places it in a separate extensions dir already on +# $LOAD_PATH. Put this file's dir on the path so the absolute require resolves in +# both layouts. +$LOAD_PATH.unshift(__dir__) unless $LOAD_PATH.include?(__dir__) +require "secretspec/secretspec_ext" module Secretspec # Response wire-format version this SDK understands. Tracks secretspec-ffi's @@ -97,88 +99,20 @@ def close # profile even when its secrets are not all available. Report = Struct.new(:provider, :profile, :secrets) - # The narrow C ABI, loaded lazily via Fiddle. + # The narrow C ABI, statically linked into the secretspec_ext extension. The + # Native.c_resolve / c_abi_version C functions are defined in + # ext/secretspec/secretspec_ext.c; these wrappers add the Ruby-side error type. module Native - # Guards the one-time dlopen so concurrent first callers do not race. - @load_mutex = Mutex.new - class << self def resolve(request_json) - ensure_loaded - ptr = @resolve.call(request_json) - raise Error.new("ffi", "secretspec_resolve returned null") if ptr.null? - - begin - ptr.to_s - ensure - @free.call(ptr) - end - end - - def abi_version - ensure_loaded - @abi.call.to_s - end + result = c_resolve(request_json) + raise Error.new("ffi", "secretspec_resolve returned null") if result.nil? - private - - def ensure_loaded - return if @loaded - - @load_mutex.synchronize do - # Re-check inside the lock: another thread may have loaded while we - # waited, and @loaded is only set after every function is registered. - next if @loaded - - handle = Fiddle.dlopen(find_library) - @resolve = Fiddle::Function.new( - handle["secretspec_resolve"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOIDP - ) - @free = Fiddle::Function.new( - handle["secretspec_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID - ) - @abi = Fiddle::Function.new( - handle["secretspec_abi_version"], [], Fiddle::TYPE_VOIDP - ) - @loaded = true - end + result end - def lib_names - case RbConfig::CONFIG["host_os"] - when /darwin/ then ["libsecretspec_ffi.dylib"] - when /mswin|mingw/ then ["secretspec_ffi.dll"] - else ["libsecretspec_ffi.so"] - end - end - - def find_library - env = ENV["SECRETSPEC_FFI_LIB"] - return env if env && !env.empty? - - # A copy bundled in a platform gem (staged into vendor/ at build time). - lib_names.each do |name| - bundled = File.expand_path("../../vendor/#{name}", __FILE__) - return bundled if File.exist?(bundled) - end - - dir = Dir.pwd - loop do - # Within the nearest target/, pick the most recently built library - # rather than always preferring release, so a stale release build does - # not shadow the debug build just produced. - candidates = %w[release debug].flat_map do |profile| - lib_names.map { |name| File.join(dir, "target", profile, name) } - end.select { |c| File.exist?(c) } - return candidates.max_by { |c| File.mtime(c) } unless candidates.empty? - - parent = File.dirname(dir) - break if parent == dir - - dir = parent - end - raise Error.new("load", - "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB") + def abi_version + c_abi_version end end end diff --git a/secretspec-rb/scripts/build-ext.sh b/secretspec-rb/scripts/build-ext.sh new file mode 100644 index 0000000..1d06279 --- /dev/null +++ b/secretspec-rb/scripts/build-ext.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Compile the secretspec native extension (statically linking +# libsecretspec_ffi.a) and place it on the SDK's load path for dev and tests. +# extconf.rb honors the SECRETSPEC_FFI_STATICLIB / SECRETSPEC_FFI_NATIVE_LIBS / +# SECRETSPEC_FFI_INCLUDE contract (exported by scripts/ci-sdks.sh); otherwise it +# builds and locates the debug archive from the Cargo target dir. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +if [ -z "${SECRETSPEC_FFI_STATICLIB:-}" ]; then + cargo build -p secretspec-ffi --manifest-path "$repo_root/Cargo.toml" +fi + +ext_dir="$pkg_dir/ext/secretspec" +( cd "$ext_dir" && ruby extconf.rb && make --silent ) + +# The build output lands in ext_dir (target_prefix only affects the install dir); +# copy it onto the SDK's load path so `require "secretspec/secretspec_ext"` finds it. +mkdir -p "$pkg_dir/lib/secretspec" +built="" +for f in "$ext_dir/secretspec_ext.so" "$ext_dir/secretspec_ext.bundle"; do + [ -f "$f" ] && built="$f" && break +done +[ -n "$built" ] || { echo "build-ext: no secretspec_ext.{so,bundle} produced" >&2; exit 1; } +cp "$built" "$pkg_dir/lib/secretspec/$(basename "$built")" +echo "built $(basename "$built") into lib/secretspec/" diff --git a/secretspec-rb/scripts/stage-cdylib.sh b/secretspec-rb/scripts/stage-cdylib.sh deleted file mode 100755 index 5beead0..0000000 --- a/secretspec-rb/scripts/stage-cdylib.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Build the secretspec-ffi cdylib (release) and stage it into vendor/ so a -# platform gem build bundles it. Run before `gem build`. -set -euo pipefail - -pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -repo_root="$(cd "$pkg_dir/.." && pwd)" - -cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" - -target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ - | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" -case "$(uname -s)" in - Darwin) lib_name="libsecretspec_ffi.dylib" ;; - MINGW* | MSYS* | CYGWIN*) lib_name="secretspec_ffi.dll" ;; - *) lib_name="libsecretspec_ffi.so" ;; -esac - -mkdir -p "$pkg_dir/vendor" -cp "$target_dir/release/$lib_name" "$pkg_dir/vendor/$lib_name" -echo "staged $lib_name into vendor/" diff --git a/secretspec-rb/scripts/stage-staticlib.sh b/secretspec-rb/scripts/stage-staticlib.sh new file mode 100644 index 0000000..3d25161 --- /dev/null +++ b/secretspec-rb/scripts/stage-staticlib.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Stage the secretspec-ffi staticlib (release) into vendor/ so a platform gem +# build bundles it: the archive, the C header, and the archive's transitive +# native deps. `gem install` then compiles only the tiny C glue and links the +# bundled archive. Run before `gem build`. +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +cargo build -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" + +target_dir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" + +mkdir -p "$pkg_dir/vendor" +cp "$target_dir/release/libsecretspec_ffi.a" "$pkg_dir/vendor/libsecretspec_ffi.a" +cp "$repo_root/secretspec-ffi/include/secretspec.h" "$pkg_dir/vendor/secretspec.h" +cargo rustc -q -p secretspec-ffi --release --manifest-path "$repo_root/Cargo.toml" \ + --crate-type staticlib -- --print native-static-libs 2>&1 \ + | sed -n 's/^note: native-static-libs: //p' | tail -1 > "$pkg_dir/vendor/native-static-libs.txt" +echo "staged libsecretspec_ffi.a + secretspec.h + native-static-libs.txt into vendor/" diff --git a/secretspec-rb/secretspec.gemspec b/secretspec-rb/secretspec.gemspec index 3e43f68..edc83ce 100644 --- a/secretspec-rb/secretspec.gemspec +++ b/secretspec-rb/secretspec.gemspec @@ -4,19 +4,21 @@ Gem::Specification.new do |spec| spec.name = "secretspec" spec.version = "0.12.0" spec.summary = "Declarative secrets, every environment, any provider (Ruby SDK)" - spec.description = "Ruby bindings for SecretSpec, a thin client over the " \ - "secretspec-ffi C ABI (loaded via stdlib Fiddle)." + spec.description = "Ruby bindings for SecretSpec: a native extension that " \ + "statically links the secretspec-ffi C ABI." spec.authors = ["Cachix"] spec.license = "Apache-2.0" spec.homepage = "https://secretspec.dev/" - spec.files = Dir["lib/**/*.rb"] + ["README.md"] + Dir["vendor/*"] + spec.files = Dir["lib/**/*.rb"] + Dir["ext/**/*.{c,rb}"] + + ["README.md"] + Dir["vendor/*"] + spec.extensions = ["ext/secretspec/extconf.rb"] spec.require_paths = ["lib"] spec.required_ruby_version = ">= 3.0" - # When the cdylib has been staged into vendor/ (see scripts/stage-cdylib.sh), - # build a platform-specific gem that bundles it, so `gem install` needs no - # native build. Without it, a pure-Ruby gem is built (the SDK then falls back - # to SECRETSPEC_FFI_LIB or a Cargo target directory). - staged = Dir["vendor/libsecretspec_ffi.*"] + Dir["vendor/secretspec_ffi.dll"] - spec.platform = Gem::Platform::CURRENT unless staged.empty? + # The extension compiles a tiny C glue at `gem install` and statically links + # the prebuilt libsecretspec_ffi.a staged into vendor/ (see + # scripts/stage-staticlib.sh). The archive is platform-specific, so build a + # platform gem when it is present; one such gem serves every Ruby ABI. + staged = File.exist?("vendor/libsecretspec_ffi.a") + spec.platform = Gem::Platform::CURRENT if staged end diff --git a/secretspec-rb/test/test_codegen.rb b/secretspec-rb/test/test_codegen.rb index 0105449..4d17323 100644 --- a/secretspec-rb/test/test_codegen.rb +++ b/secretspec-rb/test/test_codegen.rb @@ -16,22 +16,26 @@ def npx? system("bash", "-lc", "command -v npx", out: File::NULL, err: File::NULL) end +# Build the CLI (for `secretspec schema`) and compile the native extension (the +# resolver, statically linked). Returns the CLI path; the SDK loads the resolver +# from the compiled extension, not a runtime library. def build_artifacts unless system("cargo", "build", "-p", "secretspec-ffi", "-p", "secretspec", chdir: REPO) raise "cargo build failed" end + pkg = File.expand_path("..", __dir__) + if Dir[File.join(pkg, "lib", "secretspec", "secretspec_ext.{so,bundle}")].empty? + system("bash", File.join(pkg, "scripts", "build-ext.sh")) || raise("build-ext.sh failed") + end meta = JSON.parse(`cd #{REPO} && cargo metadata --no-deps --format-version 1`) - target = meta["target_directory"] - lib = RbConfig::CONFIG["host_os"] =~ /darwin/ ? "libsecretspec_ffi.dylib" : "libsecretspec_ffi.so" - [File.join(target, "debug", lib), File.join(target, "debug", "secretspec")] + File.join(meta["target_directory"], "debug", "secretspec") end class CodegenTest < Minitest::Test def test_quicktype_ruby_consumes_fields skip "npx (quicktype) not available" unless npx? - lib, bin = build_artifacts - ENV["SECRETSPEC_FFI_LIB"] = lib + bin = build_artifacts Dir.mktmpdir do |dir| manifest = File.join(dir, "secretspec.toml") diff --git a/secretspec-rb/test/test_resolve.rb b/secretspec-rb/test/test_resolve.rb index 547d6c0..98bfa00 100644 --- a/secretspec-rb/test/test_resolve.rb +++ b/secretspec-rb/test/test_resolve.rb @@ -1,23 +1,19 @@ # frozen_string_literal: true require "json" -require "rbconfig" require "tmpdir" require "minitest/autorun" -# Build the secretspec-ffi cdylib and point the SDK at it, unless -# SECRETSPEC_FFI_LIB is already set. -def ensure_lib - return if ENV["SECRETSPEC_FFI_LIB"] && !ENV["SECRETSPEC_FFI_LIB"].empty? +# Compile the native extension (statically linking libsecretspec_ffi.a) unless it +# is already built. ci-sdks.sh builds it explicitly; this covers standalone runs. +def ensure_ext + pkg = File.expand_path("..", __dir__) + return unless Dir[File.join(pkg, "lib", "secretspec", "secretspec_ext.{so,bundle}")].empty? - repo = File.expand_path("../..", __dir__) - system("cargo", "build", "-p", "secretspec-ffi", chdir: repo) || raise("cargo build failed") - meta = JSON.parse(`cd #{repo} && cargo metadata --no-deps --format-version 1`) - name = RbConfig::CONFIG["host_os"] =~ /darwin/ ? "libsecretspec_ffi.dylib" : "libsecretspec_ffi.so" - ENV["SECRETSPEC_FFI_LIB"] = File.join(meta["target_directory"], "debug", name) + system("bash", File.join(pkg, "scripts", "build-ext.sh")) || raise("build-ext.sh failed") end -ensure_lib +ensure_ext require_relative "../lib/secretspec" MANIFEST = <<~TOML From b1201f36a75e71fbb63fe91844efff93b33d85ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:16:11 -0400 Subject: [PATCH 55/56] docs(node): describe the napi addon, not the removed dlopen path The Node SDK already statically embeds the resolver in its napi-rs addon; the README still described the old koffi/dlopen/SECRETSPEC_FFI_LIB model. Co-Authored-By: Claude Opus 4.8 (1M context) --- secretspec-node/README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/secretspec-node/README.md b/secretspec-node/README.md index efe0415..af47f67 100644 --- a/secretspec-node/README.md +++ b/secretspec-node/README.md @@ -1,9 +1,10 @@ # secretspec (Node.js SDK) Node.js / TypeScript bindings for [SecretSpec](https://secretspec.dev/), a -declarative secrets manager. A thin client over the `secretspec-ffi` C ABI, -loaded at runtime via [koffi](https://koffi.dev/) (dlopen). Resolution happens -in the Rust core, so the SDK inherits every provider with no JS-side logic. +declarative secrets manager. A thin wrapper over a napi-rs native addon +(`secretspec.node`) that statically embeds the Rust resolver, so the SDK inherits +every provider with no JS-side logic and nothing to load at runtime. Resolution +happens in the Rust core. ```js const { SecretSpec } = require('secretspec'); @@ -42,7 +43,9 @@ const report = SecretSpec.builder().withProfile('production').report(); for (const s of report.secrets) console.log(s.name, s.status, s.required); ``` -## Library discovery +## Native addon -The native library is found via the `SECRETSPEC_FFI_LIB` environment variable, -or a Cargo `target` directory found by searching up from the working directory. +The resolver is compiled into the napi-rs addon (`secretspec.node`), so there is +no separate library to locate and no `SECRETSPEC_FFI_LIB` to set. Prebuilt +per-platform addons are published as npm packages (no install-time native build); +from a source checkout the addon is built by `scripts/build-addon.sh`. From 258c2939bac72a11b655d282b8811223f094026f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 16 Jun 2026 19:16:21 -0400 Subject: [PATCH 56/56] feat(go): add an opt-in -tags static cgo binding (fully static on musl) Refactor the binding behind hooks (ensureLoaded/nativeResolve/nativeABIVersion): binding_purego.go (!static) keeps the default dlopen path, binding_cgo.go (static) links libsecretspec_ffi.a in via cgo. On Linux the archive is built for musl and combined with -extldflags -static for a fully-static executable. The default go get path is unchanged (purego, no cgo, toolchain-free), so static is strictly additive. stage-staticlib.sh stages the archive + header + generated cgo LDFLAGS. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/go-static.yml | 95 +++++++++++++ secretspec-go/.gitignore | 8 +- secretspec-go/README.md | 60 +++++--- secretspec-go/binding_cgo.go | 42 ++++++ secretspec-go/binding_purego.go | 147 +++++++++++++++++++ secretspec-go/dlopen_unix.go | 2 +- secretspec-go/dlopen_windows.go | 2 +- secretspec-go/scripts/build-static-musl.sh | 34 +++++ secretspec-go/scripts/stage-staticlib.sh | 51 +++++++ secretspec-go/secretspec.go | 158 +++------------------ 10 files changed, 440 insertions(+), 159 deletions(-) create mode 100644 .github/workflows/go-static.yml create mode 100644 secretspec-go/binding_cgo.go create mode 100644 secretspec-go/binding_purego.go create mode 100644 secretspec-go/scripts/build-static-musl.sh create mode 100644 secretspec-go/scripts/stage-staticlib.sh diff --git a/.github/workflows/go-static.yml b/.github/workflows/go-static.yml new file mode 100644 index 0000000..d7547e7 --- /dev/null +++ b/.github/workflows/go-static.yml @@ -0,0 +1,95 @@ +name: "Go static lib" + +# Builds the fully-static (musl) Go binary for the `-tags static` binding: cgo +# links libsecretspec_ffi.a directly into the executable, so the Rust resolver is +# embedded and the binary has no dynamic dependencies at all. Built via devenv, +# which provides the musl C cross-toolchain (for the sqlite3/aws-lc-sys build +# scripts and the cgo link) and static libdbus/libunwind. +# +# The default `go get` path stays purego/dlopen (no cgo); this artifact is for +# users who want a self-contained static binary. The staticlib + header are +# uploaded for linking; they are never committed (the Go module proxy carries no +# binary assets -- see RELEASE.md). aarch64-musl and the macOS self-contained +# static build are follow-ups; the static binding itself is exercised on every PR +# by sdks.yml's `-tags static` leg. + +on: + workflow_dispatch: + push: + tags: + - v** + pull_request: + paths: + - "secretspec-go/**" + - "secretspec-ffi/**" + - ".github/workflows/go-static.yml" + +jobs: + static: + name: x86_64-linux-musl (fully static) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + # The musl rebuild of the full provider stack plus the Nix store overflows + # the ~14GB free on a hosted runner. + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android \ + /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /opt/ghc \ + /usr/local/share/boost /usr/local/share/powershell + sudo docker image prune --all --force + df -h / + - uses: cachix/install-nix-action@v31 + - uses: cachix/cachix-action@v16 + with: + name: devenv + - name: Install devenv.sh + run: nix profile install nixpkgs#devenv + + - name: Build and smoke test the fully-static binary + run: | + devenv shell -- bash -c ' + set -euo pipefail + # Stage the release musl archive + header + generated cgo LDFLAGS + # (cargo compiles the C deps with the musl cc via the CC_/linker env). + ( cd secretspec-go && SECRETSPEC_FFI_TARGET=x86_64-unknown-linux-musl \ + SECRETSPEC_FFI_PROFILE=release bash scripts/stage-staticlib.sh ) + + # A self-contained smoke program: build it fully static and assert it. + smoke="$RUNNER_TEMP/ssstatic" + mkdir -p "$smoke" + cat > "$smoke/main.go" < "$smoke/go.mod" < $GITHUB_WORKSPACE/secretspec-go + EOF + cd "$smoke" + GOFLAGS=-mod=mod CGO_ENABLED=1 CC="$MUSL_CC" CGO_LDFLAGS="$MUSL_STATIC_LDFLAGS" \ + go build -buildvcs=false -tags static \ + -ldflags "-linkmode external -extldflags \"-static\"" -o ssstatic . + file ssstatic + file ssstatic | grep -q "statically linked" + ldd ssstatic 2>&1 | grep -q "not a dynamic executable" + ./ssstatic + ' + + - uses: actions/upload-artifact@v4 + with: + name: go-static-x86_64-linux-musl + path: | + secretspec-go/lib/*.a + secretspec-go/include/secretspec.h + secretspec-go/cgo_ldflags_*.go diff --git a/secretspec-go/.gitignore b/secretspec-go/.gitignore index 0ba7666..e757e8b 100644 --- a/secretspec-go/.gitignore +++ b/secretspec-go/.gitignore @@ -1,3 +1,7 @@ -# Per-platform cdylibs are staged here for local `-tags embed_lib` builds and -# never committed (see .gitattributes / RELEASE.md). +# Per-platform native artifacts are staged here for local `-tags embed_lib` +# (cdylibs) and `-tags static` (.a archives, the vendored header, and the +# generated cgo LDFLAGS) builds, and never committed (see .gitattributes / +# RELEASE.md). lib/ +include/ +cgo_ldflags_*.go diff --git a/secretspec-go/README.md b/secretspec-go/README.md index 21aec91..4b1df3c 100644 --- a/secretspec-go/README.md +++ b/secretspec-go/README.md @@ -1,10 +1,11 @@ # secretspec (Go SDK) Go bindings for [SecretSpec](https://secretspec.dev/), a declarative secrets -manager. A thin client over the `secretspec-ffi` C ABI, loaded at runtime via -[purego](https://github.com/ebitengine/purego) (dlopen, no cgo). Resolution -happens in the Rust core, so the SDK inherits every provider with no Go-side -logic. +manager. A thin client over the `secretspec-ffi` C ABI. Resolution happens in the +Rust core, so the SDK inherits every provider with no Go-side logic. By default +the resolver is loaded at runtime via +[purego](https://github.com/ebitengine/purego) (dlopen, no cgo), keeping `go get` +toolchain-free; `-tags static` instead links it in statically (see below). ```go package main @@ -56,23 +57,46 @@ for _, s := range report.Secrets { } ``` -## Library discovery +## Binding the native resolver -The native `secretspec-ffi` cdylib is resolved at runtime, in order: +### Default: purego (dlopen, no cgo) + +The `secretspec-ffi` cdylib is resolved at runtime, in order: 1. The `SECRETSPEC_FFI_LIB` environment variable (an explicit path). -2. A library embedded at build time with `-tags embed_lib` (see below). +2. A library embedded at build time with `-tags embed_lib`. 3. A Cargo `target` directory found by searching up from the working directory (the development path). -This SDK uses [purego](https://github.com/ebitengine/purego), so the cdylib is -loaded at runtime rather than linked. Provide it one of two ways: - -- **System library:** install/build `libsecretspec_ffi` and point - `SECRETSPEC_FFI_LIB` at it (or run inside a Cargo checkout, which the search in - step 3 finds automatically). -- **Vendored/embedded:** stage the per-platform library into `lib/` and build - with `go build -tags embed_lib`. The library is then embedded via `go:embed` - and extracted to a per-user, owner-only cache directory at first use. This is - an opt-in for self-contained builds; it is **not** shipped through the Go - module proxy (which does not carry binary assets). +This keeps `go get` toolchain-free; the cdylib is loaded at runtime rather than +linked. Provide it via `SECRETSPEC_FFI_LIB` / a Cargo checkout, or stage the +per-platform library into `lib/` and build `-tags embed_lib` (embedded via +`go:embed`, extracted to a per-user, owner-only cache directory at first use). +Neither the cdylib nor the archive is shipped through the Go module proxy (which +does not carry binary assets); they are attached to GitHub releases. + +### `-tags static`: cgo, statically linked + +For a self-contained binary with no runtime library to locate, link the resolver +statically. This uses **cgo** (a C toolchain is required) and links +`libsecretspec_ffi.a` directly into the Go binary: + +```bash +# Stage the archive + header + generated cgo LDFLAGS, then build with cgo. +bash scripts/stage-staticlib.sh +CGO_ENABLED=1 go build -tags static ./... +``` + +On Linux this can be made **fully static** (no dynamic libraries at all) by +building the archive for a musl target and passing the static link flags: + +```bash +SECRETSPEC_FFI_TARGET=x86_64-unknown-linux-musl \ + SECRETSPEC_FFI_PROFILE=release bash scripts/stage-staticlib.sh +CGO_ENABLED=1 go build -tags static \ + -ldflags '-linkmode external -extldflags "-static"' ./... +``` + +macOS links the archive in but stays self-contained-except-system-frameworks (no +static libSystem). Windows stays on the default purego path. The prebuilt +archives are attached to GitHub releases (`go-static.yml`). diff --git a/secretspec-go/binding_cgo.go b/secretspec-go/binding_cgo.go new file mode 100644 index 0000000..45fdabf --- /dev/null +++ b/secretspec-go/binding_cgo.go @@ -0,0 +1,42 @@ +//go:build static + +package secretspec + +// Static binding: cgo links libsecretspec_ffi.a directly into the Go binary, so +// the Rust resolver is embedded (fully static on Linux/musl with +// `-ldflags '-linkmode external -extldflags "-static"'`). The archive path and +// its transitive native deps come from the generated, per-platform +// cgo_ldflags__.go (produced by scripts/stage-staticlib.sh); the header +// is vendored under include/. + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#include +#include "secretspec.h" +*/ +import "C" + +import "unsafe" + +// ensureLoaded is a no-op: the resolver is linked in, nothing to load. +func ensureLoaded() error { return nil } + +// nativeResolve calls secretspec_resolve and returns the owned response, freeing +// both the C request copy and the returned allocation. +func nativeResolve(payload string) (string, error) { + req := C.CString(payload) + defer C.free(unsafe.Pointer(req)) + + res := C.secretspec_resolve(req) + if res == nil { + return "", &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + } + out := C.GoString(res) + C.secretspec_free(res) + return out, nil +} + +// nativeABIVersion returns the ABI version string (a static C string, not freed). +func nativeABIVersion() (string, error) { + return C.GoString(C.secretspec_abi_version()), nil +} diff --git a/secretspec-go/binding_purego.go b/secretspec-go/binding_purego.go new file mode 100644 index 0000000..ad79472 --- /dev/null +++ b/secretspec-go/binding_purego.go @@ -0,0 +1,147 @@ +//go:build !static + +package secretspec + +// Default binding: purego (dlopen, no cgo). The Rust resolver lives in a shared +// library located at runtime via SECRETSPEC_FFI_LIB, an embedded copy, or a Cargo +// target directory, so `go get` needs no native toolchain. The `-tags static` +// build (binding_cgo.go) replaces this with a statically linked archive. + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "unsafe" + + "github.com/ebitengine/purego" +) + +var ( + loadOnce sync.Once + loadErr error + cResolve func(string) uintptr + cFree func(uintptr) + cABI func() uintptr +) + +func libNames() []string { + switch runtime.GOOS { + case "darwin": + return []string{"libsecretspec_ffi.dylib"} + case "windows": + return []string{"secretspec_ffi.dll"} + default: + return []string{"libsecretspec_ffi.so"} + } +} + +func findLibrary() (string, error) { + if p := os.Getenv("SECRETSPEC_FFI_LIB"); p != "" { + return p, nil + } + // A library embedded at build time (go:embed, per platform) is extracted to + // a temp file and used, so `go get` works with no native build. + if len(embeddedLib) > 0 { + return extractEmbedded() + } + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + // Within the nearest ancestor target/, pick the most recently built + // library rather than always preferring release: a stale release build + // must not shadow the debug build the developer just produced. + var bestPath string + var best os.FileInfo + for _, profile := range []string{"release", "debug"} { + for _, name := range libNames() { + candidate := filepath.Join(dir, "target", profile, name) + if info, err := os.Stat(candidate); err == nil { + if best == nil || info.ModTime().After(best.ModTime()) { + best, bestPath = info, candidate + } + } + } + } + if bestPath != "" { + return bestPath, nil + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", &Error{ + Kind: "load", + Message: "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB", + } +} + +func ensureLoaded() error { + loadOnce.Do(func() { + // purego.RegisterLibFunc panics (it does not return an error) when a + // symbol is missing. Recover so an incompatible library yields a returned + // *Error instead of a panic that escapes the Once — which would otherwise + // mark it done with loadErr nil and the function pointers nil, turning + // every later call into a nil-pointer panic for the process lifetime. + defer func() { + if r := recover(); r != nil { + loadErr = &Error{ + Kind: "load", + Message: fmt.Sprintf("failed to bind secretspec-ffi symbols (incompatible library?): %v", r), + } + } + }() + path, err := findLibrary() + if err != nil { + loadErr = err + return + } + handle, err := openLibrary(path) + if err != nil { + loadErr = err + return + } + purego.RegisterLibFunc(&cResolve, handle, "secretspec_resolve") + purego.RegisterLibFunc(&cFree, handle, "secretspec_free") + purego.RegisterLibFunc(&cABI, handle, "secretspec_abi_version") + }) + return loadErr +} + +// nativeResolve calls secretspec_resolve and returns the owned response, freeing +// the C allocation. +func nativeResolve(payload string) (string, error) { + ptr := cResolve(payload) + if ptr == 0 { + return "", &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + } + raw := goString(ptr) + cFree(ptr) + return raw, nil +} + +// nativeABIVersion returns the ABI version string (a static C string, not freed). +func nativeABIVersion() (string, error) { + return goString(cABI()), nil +} + +// goString copies a NUL-terminated C string at ptr into a Go string. The +// pointer comes from the C ABI (a Rust allocation), not Go's heap, so this is a +// legitimate FFI read; `go vet`'s unsafeptr check flags it as a false positive +// (it is not part of the `go test` vet subset). +func goString(ptr uintptr) string { + if ptr == 0 { + return "" + } + base := unsafe.Pointer(ptr) + length := 0 + for *(*byte)(unsafe.Add(base, length)) != 0 { + length++ + } + return string(unsafe.Slice((*byte)(base), length)) +} diff --git a/secretspec-go/dlopen_unix.go b/secretspec-go/dlopen_unix.go index a990fbd..f3509f2 100644 --- a/secretspec-go/dlopen_unix.go +++ b/secretspec-go/dlopen_unix.go @@ -1,4 +1,4 @@ -//go:build unix +//go:build unix && !static package secretspec diff --git a/secretspec-go/dlopen_windows.go b/secretspec-go/dlopen_windows.go index 7616eed..972a04c 100644 --- a/secretspec-go/dlopen_windows.go +++ b/secretspec-go/dlopen_windows.go @@ -1,4 +1,4 @@ -//go:build windows +//go:build windows && !static package secretspec diff --git a/secretspec-go/scripts/build-static-musl.sh b/secretspec-go/scripts/build-static-musl.sh new file mode 100644 index 0000000..ed7dc20 --- /dev/null +++ b/secretspec-go/scripts/build-static-musl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Build a fully-static (musl) Go binary that links the secretspec-ffi archive in +# via cgo. Run inside the project devenv shell, which provides the musl C +# cross-toolchain and static libdbus/libunwind via MUSL_CC / MUSL_STATIC_LDFLAGS +# (and the CC_/linker env so cargo compiles the C deps against musl): +# +# devenv shell -- bash secretspec-go/scripts/build-static-musl.sh [out] +# +# With no argument it builds the SDK's own packages (a compile/link check); +# `file ` on a produced binary reports "statically linked". +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +: "${MUSL_CC:?set by devenv; run inside 'devenv shell'}" +: "${MUSL_STATIC_LDFLAGS:?set by devenv; run inside 'devenv shell'}" + +target="${SECRETSPEC_FFI_TARGET:-x86_64-unknown-linux-musl}" +profile="${SECRETSPEC_FFI_PROFILE:-release}" + +# Stage the musl archive + header + generated cgo LDFLAGS (cargo uses the musl cc +# from the CC_/linker env for the C deps). +SECRETSPEC_FFI_TARGET="$target" SECRETSPEC_FFI_PROFILE="$profile" \ + bash "$pkg_dir/scripts/stage-staticlib.sh" + +what="${1:-./...}" +out="${2:-}" +build=(go build -buildvcs=false -tags static + -ldflags '-linkmode external -extldflags "-static"') +[ -n "$out" ] && build+=(-o "$out") + +cd "$pkg_dir" +CGO_ENABLED=1 CC="$MUSL_CC" CGO_LDFLAGS="$MUSL_STATIC_LDFLAGS" "${build[@]}" "$what" +echo "built fully-static ($target): ${out:-$what}" diff --git a/secretspec-go/scripts/stage-staticlib.sh b/secretspec-go/scripts/stage-staticlib.sh new file mode 100644 index 0000000..90a5369 --- /dev/null +++ b/secretspec-go/scripts/stage-staticlib.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Stage the secretspec-ffi staticlib for the `-tags static` cgo build: the +# per-platform archive (lib/), the C header (include/), and a generated +# cgo_ldflags__.go carrying the archive path + its transitive native +# deps (captured from `rustc --print native-static-libs`, never hardcoded). +# +# Honors: +# SECRETSPEC_FFI_PROFILE release|debug (default: debug) +# SECRETSPEC_FFI_TARGET a rust target triple, e.g. x86_64-unknown-linux-musl +# (default: host; required for a fully-static binary) +set -euo pipefail + +pkg_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$pkg_dir/.." && pwd)" + +goos="$(go env GOOS)" +goarch="$(go env GOARCH)" +profile="${SECRETSPEC_FFI_PROFILE:-debug}" +target="${SECRETSPEC_FFI_TARGET:-}" + +build=(-p secretspec-ffi --manifest-path "$repo_root/Cargo.toml") +[ "$profile" = release ] && build+=(--release) +[ -n "$target" ] && build+=(--target "$target") +cargo build "${build[@]}" + +native_libs="$(cargo rustc -q "${build[@]}" --crate-type staticlib -- \ + --print native-static-libs 2>&1 | sed -n 's/^note: native-static-libs: //p' | tail -1)" + +tdir="$(cargo metadata --no-deps --format-version 1 --manifest-path "$repo_root/Cargo.toml" \ + | grep -o '"target_directory":"[^"]*"' | head -1 | sed 's/.*:"\(.*\)"/\1/')" +a_path="$tdir/${target:+$target/}$profile/libsecretspec_ffi.a" + +mkdir -p "$pkg_dir/lib" "$pkg_dir/include" +cp "$a_path" "$pkg_dir/lib/libsecretspec_ffi_${goos}_${goarch}.a" +cp "$repo_root/secretspec-ffi/include/secretspec.h" "$pkg_dir/include/secretspec.h" + +# The cgo LDFLAGS live in a generated per-platform file (the wasmtime-go pattern): +# the archive is pulled for the referenced symbols, then its native deps follow. +cat > "$pkg_dir/cgo_ldflags_${goos}_${goarch}.go" < 0 { - return extractEmbedded() - } - dir, err := os.Getwd() - if err != nil { - return "", err - } - for { - // Within the nearest ancestor target/, pick the most recently built - // library rather than always preferring release: a stale release build - // must not shadow the debug build the developer just produced. - var bestPath string - var best os.FileInfo - for _, profile := range []string{"release", "debug"} { - for _, name := range libNames() { - candidate := filepath.Join(dir, "target", profile, name) - if info, err := os.Stat(candidate); err == nil { - if best == nil || info.ModTime().After(best.ModTime()) { - best, bestPath = info, candidate - } - } - } - } - if bestPath != "" { - return bestPath, nil - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - return "", &Error{ - Kind: "load", - Message: "could not locate the secretspec-ffi library; set SECRETSPEC_FFI_LIB", - } -} - -func ensureLoaded() error { - loadOnce.Do(func() { - // purego.RegisterLibFunc panics (it does not return an error) when a - // symbol is missing. Recover so an incompatible library yields a returned - // *Error instead of a panic that escapes the Once — which would otherwise - // mark it done with loadErr nil and the function pointers nil, turning - // every later call into a nil-pointer panic for the process lifetime. - defer func() { - if r := recover(); r != nil { - loadErr = &Error{ - Kind: "load", - Message: fmt.Sprintf("failed to bind secretspec-ffi symbols (incompatible library?): %v", r), - } - } - }() - path, err := findLibrary() - if err != nil { - loadErr = err - return - } - handle, err := openLibrary(path) - if err != nil { - loadErr = err - return - } - purego.RegisterLibFunc(&cResolve, handle, "secretspec_resolve") - purego.RegisterLibFunc(&cFree, handle, "secretspec_free") - purego.RegisterLibFunc(&cABI, handle, "secretspec_abi_version") - }) - return loadErr -} - -// goString copies a NUL-terminated C string at ptr into a Go string. The -// pointer comes from the C ABI (a Rust allocation), not Go's heap, so this is a -// legitimate FFI read; `go vet`'s unsafeptr check flags it as a false positive -// (it is not part of the `go test` vet subset). -func goString(ptr uintptr) string { - if ptr == 0 { - return "" - } - base := unsafe.Pointer(ptr) - length := 0 - for *(*byte)(unsafe.Add(base, length)) != 0 { - length++ - } - return string(unsafe.Slice((*byte)(base), length)) -} - // Error is a resolution failure (bad manifest, provider error, reason policy). type Error struct { Kind string @@ -259,12 +147,12 @@ func (r *Resolved) Close() error { return firstErr } -// ABIVersion returns the version reported by the loaded library. +// ABIVersion returns the version reported by the native resolver. func ABIVersion() (string, error) { if err := ensureLoaded(); err != nil { return "", err } - return goString(cABI()), nil + return nativeABIVersion() } // Builder configures a resolution, mirroring the derive crate's SecretSpec::builder(). @@ -333,12 +221,10 @@ func (b *Builder) Load() (*Resolved, error) { return nil, err } - ptr := cResolve(string(payload)) - if ptr == 0 { - return nil, &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + raw, err := nativeResolve(string(payload)) + if err != nil { + return nil, err } - raw := goString(ptr) - cFree(ptr) var env envelopeJSON if err := json.Unmarshal([]byte(raw), &env); err != nil { @@ -451,12 +337,10 @@ func (b *Builder) Report() (*Report, error) { return nil, err } - ptr := cResolve(string(payload)) - if ptr == 0 { - return nil, &Error{Kind: "ffi", Message: "secretspec_resolve returned null"} + raw, err := nativeResolve(string(payload)) + if err != nil { + return nil, err } - raw := goString(ptr) - cFree(ptr) var env reportEnvelopeJSON if err := json.Unmarshal([]byte(raw), &env); err != nil {