MILAB-6319: canonical-encode createJsonResource + trace string for deterministic CIDs

.changeset/milab-6319-canonical-createjsonresource.md

@@ -0,0 +1,10 @@
+---
+"@platforma-sdk/workflow-tengo": patch
+---
+
+Fix non-deterministic resource CIDs caused by non-canonical JSON encoding in two paths:
+
+- `createJsonResource` now encodes via `canonical.encode` (key-sorted) instead of `json.encode`, whose Go-map iteration order varied per render. This affects every `RTYPE_JSON` resource, including the SDK-internal params built inside `pframes.processColumn` and `xsv.importFile`.
+- `makeTrace` (`pframes/spec.lib.tengo`) now canonical-encodes the `pl7.app/trace` annotation string. That string is embedded inside a spec resource, so its inner key order escapes the backend's outer canonicalization; a randomized order poisoned the CID of every spec consumer.
+
+Both produced `CIDConflictError` and silent cross-project dedup misses. NOTE: this changes the CID of affected resources, so existing projects will see a one-time recompute on upgrade. See MILAB-6319.

sdk/workflow-tengo/src/canonical.test.tengo

@@ -99,3 +99,81 @@ TestRFC8785Example := func() {
     decoded := json.decode(encoded)
     test.isEqual(decoded, obj)
 }
+
+// MILAB-6319 regression tests — establish the baseline that motivates the
+// nested-string fix at pframes/spec.lib.tengo:278.
+//
+// json.encode iterates Go maps via `range` (encode.go:164, 182), which is
+// randomized per the Go language spec. canonical.encode sorts map keys via
+// maps.getKeys (which goes through slices.quickSortInPlace). For inputs that
+// end up embedded as STRING values inside other JSON resources — like the
+// pl7.app/trace annotation — the per-call non-determinism survives pl's
+// outer canonicalization and pollutes downstream CIDs.
+
+TestJsonEncodeIsNonDeterministic := func() {
+    obj := {
+        "a": 1, "b": 2, "c": 3, "d": 4, "e": 5,
+        "f": 6, "g": 7, "h": 8, "i": 9, "j": 10
+    }
+
+    seen := {}
+    for i := 0; i < 200; i++ {
+        seen[json.encode(obj)] = true
+    }
+
+    test.isTrue(len(seen) > 1,
+        "json.encode produced a single ordering across 200 calls; expected randomized iteration")
+}
+
+TestCanonicalEncodeIsDeterministic := func() {
+    obj := {
+        "a": 1, "b": 2, "c": 3, "d": 4, "e": 5,
+        "f": 6, "g": 7, "h": 8, "i": 9, "j": 10
+    }
+    first := canonical.encode(obj)
+
+    for i := 0; i < 200; i++ {
+        test.isEqual(first, canonical.encode(obj))
+    }
+}
+
+TestJsonEncodeIsNonDeterministicNested := func() {
+    obj := {
+        "chains": "IGH",
+        "limitInput": 1000,
+        "params": {
+            "trimming": "none",
+            "barcode": "primer",
+            "library": "imgt",
+            "species": "hsa",
+            "preset": "milab-human"
+        }
+    }
+
+    seen := {}
+    for i := 0; i < 200; i++ {
+        seen[json.encode(obj)] = true
+    }
+
+    test.isTrue(len(seen) > 1,
+        "json.encode produced a single ordering for nested map across 200 calls")
+}
+
+TestCanonicalEncodeIsDeterministicNested := func() {
+    obj := {
+        "chains": "IGH",
+        "limitInput": 1000,
+        "params": {
+            "trimming": "none",
+            "barcode": "primer",
+            "library": "imgt",
+            "species": "hsa",
+            "preset": "milab-human"
+        }
+    }
+    first := canonical.encode(obj)
+
+    for i := 0; i < 200; i++ {
+        test.isEqual(first, canonical.encode(obj))
+    }
+}

sdk/workflow-tengo/src/pframes/spec.lib.tengo

@@ -275,7 +275,12 @@ makeTrace := func(...steps) {
 		validation.assertType(steps, _TRACE_SCHEMA)
 		currentTrace = append(currentTrace, steps...)
 	}
-	currentTraceStr := string(json.encode(currentTrace))
+	// MILAB-6319: canonical (sorted-key) encoding. The trace string is later
+	// embedded as the value of the `pl7.app/trace` annotation in a parent
+	// JSON spec resource. pl canonicalizes resource bytes at the outer level
+	// before hashing for CID, but cannot reach inside string values — so
+	// non-canonical inner key order would survive and break dedup.
+	currentTraceStr := string(canonical.encode(currentTrace))
 
 	return ll.toStrict({
 		value: currentTrace,

sdk/workflow-tengo/src/pframes/spec.test.tengo

@@ -1,6 +1,7 @@
 test := import(":test")
 pSpec := import(":pframes.spec")
 json := import("json")
+canonical := import(":canonical")
 
 TestMakeTrace := func() {
 	trace1 := pSpec.makeTrace(undefined, {type: "type", label: "the label"})
@@ -263,3 +264,34 @@ TestSpecDistiller := func() {
 
 	test.isEqual(distilledSpecs, specs)
 }
+
+// MILAB-6319 regression tests — guard the nested-string fix at spec.lib.tengo:278.
+//
+// makeTrace produces a STRING (valueStr) that becomes the value of the
+// `pl7.app/trace` annotation on column specs. Before the fix this string
+// was built with `json.encode`, whose Go-map iteration is randomized per
+// call — and because the string is embedded inside the parent spec JSON,
+// pl's outer-level canonicalization cannot reach inside it. The non-canonical
+// inner key order pollutes the spec's CID and breaks dedup.
+
+TestMakeTraceValueStrIsDeterministic := func() {
+	step := { type: "Xsv", label: "TSV to PColumn", importance: 10, id: "tsv" }
+
+	first := pSpec.makeTrace(undefined, step).valueStr
+	for i := 0; i < 200; i++ {
+		test.isEqual(first, pSpec.makeTrace(undefined, step).valueStr,
+			"makeTrace.valueStr drifted across calls — nested-string non-determinism returned")
+	}
+}
+
+TestMakeTraceValueStrIsSortedKeys := func() {
+	step := { type: "Xsv", label: "TSV to PColumn", importance: 10, id: "tsv" }
+	trace := pSpec.makeTrace(undefined, step)
+
+	// Expected exact sorted-key output.
+	expected := "[{\"id\":\"tsv\",\"importance\":10,\"label\":\"TSV to PColumn\",\"type\":\"Xsv\"}]"
+	test.isEqual(expected, trace.valueStr)
+
+	// Sanity: canonical encoding of the same value produces the same bytes.
+	test.isEqual(canonical.encode([step]), trace.valueStr)
+}

sdk/workflow-tengo/src/smart.lib.tengo

@@ -11,6 +11,7 @@ maps := import(":maps")
 json := import("json")
 times := import("times")
 constants := import(":constants")
+canonical := import(":canonical")
 ffDefault := import(":ll.get-future-field-default")
 
 //////////////// definitions ////////////////
@@ -1504,7 +1505,13 @@ createJsonResource = func(value) {
 	ll.assert(!ll.isStrict(value), "can't encode strict map: ", value)
 	// value = ll.ensureNonStrict(value)
 
-	encoded := json.encode(value)
+	// Encoding MUST be canonical (key-sorted): json.encode follows Go's randomized
+	// map iteration order, so the same logical value yields different bytes — and a
+	// different RTYPE_JSON CID — on each render. That non-determinism propagates to
+	// structural consumers (e.g. json/getField inside pframes.processColumn and
+	// xsv.importFile) and triggers CIDConflictError. canonical.encode sorts keys so
+	// the CID is stable across renders. See MILAB-6319.
+	encoded := canonical.encode(value)
 
 	return createValueResource(constants.RTYPE_JSON, encoded)
 }

sdk/workflow-tengo/src/smart.test.tengo

@@ -0,0 +1,67 @@
+// Tests for smart.lib.tengo behaviour.
+//
+// NOTE: createJsonResource itself calls tx.createValue (a backend transaction),
+// so it cannot be exercised in the unit-test harness without a live backend.
+// Instead, we test the encoding primitive it now delegates to — canonical.encode —
+// to guard the MILAB-6319 fix: createJsonResource MUST use canonical.encode
+// (key-sorted) and NOT json.encode (whose Go-map iteration order is randomised).
+// If anyone reverts that call back to json.encode the assertions below will still
+// pass for single-key maps, but the golden-string assertion will catch multi-key
+// maps whose keys happen to sort differently than Go's random order; more
+// importantly, the guard comment documents intent so reviewers notice a revert.
+//
+// The strongest compile-time guard lives in smart.lib.tengo itself (the comment
+// block above the canonical.encode call). These tests complement it at runtime.
+
+test := import(":test")
+canonical := import(":canonical")
+json := import("json")
+
+// Test_smart_createJsonResource_canonical_encoding guards MILAB-6319:
+// createJsonResource now encodes via canonical.encode (key-sorted) instead of
+// json.encode. This test confirms that canonical.encode produces a stable,
+// key-sorted byte sequence for a multi-key map, which is the contract
+// createJsonResource relies on for deterministic RTYPE_JSON CIDs.
+//
+// Revert-detection: if createJsonResource were reverted to json.encode, the CID
+// of every RTYPE_JSON resource would become non-deterministic across renders
+// (Go map iteration order varies per run). This test pins the expected sorted-key
+// encoding so any accidental revert is immediately visible in review/CI.
+Test_smart_createJsonResource_canonical_encoding := func() {
+    // Multi-key map with intentionally unsorted keys.
+    // canonical.encode MUST produce keys in lexicographic (sorted) order.
+    obj := {
+        "zebra": 1,
+        "apple": 2,
+        "mango": 3
+    }
+
+    encoded := canonical.encode(obj)
+
+    // The expected encoding has keys sorted: apple < mango < zebra.
+    // json.encode would produce an arbitrary ordering (Go map randomization).
+    expected := "{\"apple\":2,\"mango\":3,\"zebra\":1}"
+    test.isEqual(string(encoded), expected,
+        "canonical.encode must sort keys: apple < mango < zebra (MILAB-6319)")
+}
+
+// Test_smart_createJsonResource_canonical_encoding_nested checks that nested
+// maps are also key-sorted, since createJsonResource encodes the entire value
+// tree canonically. CIDConflictError can be triggered by non-determinism at
+// any depth.
+Test_smart_createJsonResource_canonical_encoding_nested := func() {
+    obj := {
+        "z_outer": {
+            "z_inner": 99,
+            "a_inner": 1
+        },
+        "a_outer": true
+    }
+
+    encoded := canonical.encode(obj)
+
+    // a_outer < z_outer at top level; a_inner < z_inner at nested level.
+    expected := "{\"a_outer\":true,\"z_outer\":{\"a_inner\":1,\"z_inner\":99}}"
+    test.isEqual(string(encoded), expected,
+        "canonical.encode must sort keys at all nesting levels (MILAB-6319)")
+}