diff --git a/.changeset/calm-ravens-deny.md b/.changeset/calm-ravens-deny.md new file mode 100644 index 0000000000..5dfc6e40bf --- /dev/null +++ b/.changeset/calm-ravens-deny.md @@ -0,0 +1,12 @@ +--- +"@milaboratories/columns-collection-driver": minor +"@milaboratories/milaboratories.ui-examples.ui": minor +"@milaboratories/pl-middle-layer": minor +"@milaboratories/pf-driver": minor +"@milaboratories/pl-model-common": minor +"@milaboratories/pl-tree": minor +"@platforma-sdk/ui-vue": minor +"@platforma-sdk/model": minor +--- + +ColumnLazy / ColumnsCollection API that bring new way to work with columns in model part diff --git a/docs/column-identity.md b/docs/column-identity.md new file mode 100644 index 0000000000..386967446e --- /dev/null +++ b/docs/column-identity.md @@ -0,0 +1,141 @@ +# Column identity: physical vs logical + +This document defines the conventions for column identifiers across the SDK +and the host (`pl-middle-layer`). Two distinct identities are involved in a +table-rendering pipeline; mixing them up causes engine-level "column not +found" errors and silent variant deduplication. + +## The two identities + +### Physical identity — `PObjectId` + +A bare `PObjectId` (`LocalPObjectId | GlobalPObjectId`) names exactly one +**stored** column in the result pool. It is the key the host column +registry resolves against. + +Use it (and only it) at these boundaries: + +- Column registry lookup on the host (`resolvePColumnById` in + `lib/node/pl-middle-layer/src/js_render/column_registry.ts`). +- The id list passed to `ctx.createPFrame([...ids])` — PFrame is the + physical column registry, one entry per bare id, so duplicates must be + collapsed (use `extractPObjectId` + dedupe). +- Spec frame keys passed to `pframeSpec.createSpecFrame({...})` when + evaluating queries over already-resolved physical specs. + +### Logical identity — `ColumnUniversalId` + +`ColumnUniversalId` is the union of bare ids and rich wrapper ids +serialized as canonical JSON: + +- `LocalPObjectId` / `GlobalPObjectId` — bare physical id. +- `ColumnFilteredId` — `{source, axisFilters}`. +- `ColumnDiscoveredId` — `{column, path, qualifications}`. +- `ColumnOverridedId` — `{source, specOverrides}`. + +Definitions: `lib/model/common/src/drivers/pframe/spec/ids.ts`. + +A logical id describes a contextualised view of a physical column: which +linker chain you reached it through, which axes are sliced, which spec +overrides are applied. Two `ColumnDiscoveredId`s sharing the same +`column` but with different `path` are *different* logical columns even +though they point to the same stored data. + +Use it (and only it) everywhere else: + +- `ColumnRecipe.id` (`sdk/model/src/columns/column_recipes/types.ts`). +- The `column` field of every `SpecQuery` leaf (`{type: "column", column: }`). +- `PTableColumnId.id` in filter and sorting references — UI gets these + from the recipe and ships them back unchanged. +- `derivedLabels` / `hiddenColumnIds` keys: variants of the same physical + column have independent labels and independent hide/show state. + +### Crossing the boundary: `extractPObjectId` + +`extractPObjectId(id: ColumnUniversalId): PObjectId` +(`lib/model/common/src/drivers/pframe/spec/ids.ts:97`) is the **only** +sanctioned way to drop from a logical id to its physical counterpart. It +walks wrapper layers (`isColumnFilteredKey` → `source`, +`isColumnOverridedKey` → `source`, `isColumnDiscoveredKey` → `column`) +until it reaches a bare `PObjectId`. + +Call it only at the boundary points listed above. Recipe internals and +SpecQuery construction must never strip down to bare ids early. + +## Recipe contract + +Every concrete `ColumnRecipe` must satisfy: + +1. `recipe.id` is its full logical id (`ColumnUniversalId`). +2. `recipe.getQuery()` returns a `SpecQuery` whose terminal column leaf + carries `recipe.id` — not the inner recipe's id, not the bare id. + +For leaf recipes (`ColumnLazyImpl`) this is trivial: `id` is the bare +`PObjectId` and the query is `{type: "column", column: id}`. + +For wrapper recipes (Overrided / Filtered / Discovered) the inner recipe +already produced a query whose leaf carries the **inner's** id. The +wrapper must lift its own id onto that leaf while preserving everything +else (linker chains, sliceAxes, specOverride wrappers). Use the shared +helper: + +```ts +import { rebrandLeafId } from "./leaf_rebrand"; + +// inside Wrapper.getQuery(): +const wrapped = wrapWith(rebrandLeafId(this.inner.getQuery(), this.inner.id, this.id)); +``` + +`rebrandLeafId(node, fromId, toId)` +(`sdk/model/src/columns/column_recipes/leaf_rebrand.ts`) walks the full +`SpecQuery` tree via `mapSpecQueryColumns` and renames only those column +leaves whose id equals `fromId`. Linker leaves and unrelated refs are +left untouched. This is the only correct way to compose wrapper queries. + +## SpecQuery generics + +The default leaf type of `SpecQuery` was widened from `PObjectId` to +`ColumnUniversalId` +(`lib/model/common/src/drivers/pframe/query/query_spec.ts`). The same +default propagates to `SpecQueryColumn`, `SpecQueryJoinEntry`, +`SpecQueryLinkerJoin`, `SpecExprColumnRef`, `PTableColumnSpecColumn.id`, +`PTableColumnIdColumn.id`, `QueryColumnIdColumn.id`. Existing call sites +that pin `` keep their narrow contract; everything else +widens to accept rich ids. + +This makes the type system match the runtime: leaves and table-column +ids genuinely carry logical ids, and we no longer need to cast. + +## Why per-variant uniqueness matters + +`pframe-engine` deduplicates output columns by their leaf id. If two +`ColumnDiscoveredRecipe` instances reach the same physical hit through +different linker chains and both emit `{column: }` at the +leaf, the engine sees one column and the second variant disappears +silently. With every recipe lifting its own id to the leaf, the engine +sees two distinct leaves and produces two output columns — which is what +the table is asking for. + +The same logic applies to wrappers used as filter / sort targets: the +resolver returns `recipe.id` (logical), so the engine must find a column +with that logical id in the integrated table. Lifting wrapper ids onto +the leaves is what makes that lookup succeed. + +## Where these rules live in code + +- Type definitions: `lib/model/common/src/drivers/pframe/spec/ids.ts`, + `lib/model/common/src/drivers/pframe/query/query_spec.ts`, + `lib/model/common/src/drivers/pframe/table_common.ts`. +- Recipe implementations: + `sdk/model/src/columns/column_lazy.ts`, + `sdk/model/src/columns/column_recipes/column_overrided_recipe.ts`, + `sdk/model/src/columns/column_recipes/column_filtered_recipe.ts`, + `sdk/model/src/columns/column_recipes/column_discovered_recipe.ts`, + `sdk/model/src/columns/column_recipes/leaf_rebrand.ts`. +- Linker collection (logical→physical at the boundary): + `sdk/model/src/columns/utils.ts` (`collectLinkerIds`, + `collectLinkerColumns`). +- Resolver (filters/sorting): `sdk/model/src/components/PlDataTable/columnResolver.ts`. +- Table assembly: `sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts` + (notice `createPFrame(uniq([...].map(extractPObjectId)))`). +- Host-side physical resolver: `lib/node/pl-middle-layer/src/js_render/column_registry.ts`. diff --git a/etc/blocks/filter-column-test/test/src/wf.test.ts b/etc/blocks/filter-column-test/test/src/wf.test.ts index 3042917bd0..ac9053336b 100644 --- a/etc/blocks/filter-column-test/test/src/wf.test.ts +++ b/etc/blocks/filter-column-test/test/src/wf.test.ts @@ -1,10 +1,10 @@ import type { BlockData } from "@milaboratories/milaboratories.test-filter-column.model"; import type { platforma } from "@milaboratories/milaboratories.test-filter-column.model"; -import { blockSpec as tableTestBlockSpec } from "@milaboratories/milaboratories.test-block-table"; import type { InferBlockState, Platforma } from "@platforma-sdk/model"; import { createDatasetSelection, createPrimaryRef, wrapOutputs } from "@platforma-sdk/model"; import type { ML, RawHelpers } from "@platforma-sdk/test"; import { awaitStableState, blockTest } from "@platforma-sdk/test"; +import { blockSpec as tableTestBlockSpec } from "@milaboratories/milaboratories.test-block-table"; import { blockSpec } from "this-block"; import { assert } from "vitest"; diff --git a/etc/blocks/table-test/model/src/index.ts b/etc/blocks/table-test/model/src/index.ts index b05b472723..b74d44058f 100644 --- a/etc/blocks/table-test/model/src/index.ts +++ b/etc/blocks/table-test/model/src/index.ts @@ -1,10 +1,15 @@ import { BlockModelV3, + ColumnUniversalId, + ColumnsCollection, DataModelBuilder, PObjectId, PlDataTableFilters, createPlDataTableStateV2, createPlDataTableV3, + deriveAxisValuesLabels, + expandByPartition, + isColumnLazy, type InferHrefType, type InferOutputsType, type PlDataTableStateV2, @@ -13,11 +18,13 @@ import { export type BlockData = { label: string; tableState: PlDataTableStateV2; + tableSplitState: PlDataTableStateV2; }; const blockDataModel = new DataModelBuilder().from("v1").init(() => ({ label: "Table Test", tableState: createPlDataTableStateV2(), + tableSplitState: createPlDataTableStateV2(), })); export type BlockArgs = BlockData; @@ -32,6 +39,11 @@ export const platforma = BlockModelV3.create(blockDataModel) href: "/", label: "Table V3", }, + { + type: "link", + href: "/split", + label: "Table Split", + }, ]; }) @@ -69,8 +81,10 @@ export const platforma = BlockModelV3.create(blockDataModel) filters: [ { type: "greaterThan", - column: - '{"type":"column","id":"{\\"name\\":\\"value\\",\\"resolvePath\\":[\\"main\\",\\"tableFrame\\"]}"}', + column: { + type: "column", + id: '{"name":"value","resolvePath":["main","tableFrame"]}' as PObjectId, + }, x: 11, }, ], @@ -84,21 +98,66 @@ export const platforma = BlockModelV3.create(blockDataModel) displayOptions: { ordering: [ // "category" leftmost (highest priority) - { match: (spec) => spec.name === "category", priority: 20 }, + { match: { name: "^category$" }, priority: 20 }, // Then "value" - { match: (spec) => spec.name === "value", priority: 10 }, + { match: { name: "^value$" }, priority: 10 }, // Unmatched columns (score, note) keep their original order ], visibility: [ // "note" hidden by default (user can re-enable in UI) - { match: (spec) => spec.name === "note", visibility: "hidden" }, + { match: { name: "^note$" }, visibility: "hidden" }, // "score" optional — hidden by default but toggleable - { match: (spec) => spec.name === "score", visibility: "optional" }, + { match: { name: "^score$" }, visibility: "optional" }, ], }, }); }) + // Same `value`/`name` anchor as tableV3, but with extra columns produced by + // splitting the `count` (group, name) column on its `group` axis: each group + // value becomes its own ColumnOverridedRecipe in `columns`, joining onto the + // primary on the shared `name` axis. + .outputWithStatus("tableSplitV3", (ctx) => { + const valueAnchor = { name: "value", axes: [{ name: "name" }] }; + + // Use only leaf (ColumnLazy) hits as primary. `discover` with anchors can + // also surface multi-axis Discovered variants (e.g. count [group, name] + // reached via linker_name_group_alt); those belong in secondary, not in + // the join's primary side — mirrors what `discoverTableColumns` does + // internally for the selector-form path. + const primary = ColumnsCollection() + .discover({ anchors: { main: valueAnchor }, mode: "exact" }) + .getColumns() + .filter(isColumnLazy); + if (primary.length === 0) return undefined; + + const countLeaves = ColumnsCollection() + .filter({ include: { name: [{ type: "exact", value: "count" }] } }) + .getColumns() + .filter(isColumnLazy); + + const splitRecipes = expandByPartition(countLeaves, [{ idx: 0 }], { + axisValuesLabels: deriveAxisValuesLabels(), + }); + if (splitRecipes === undefined) return undefined; + + const primaryIds = new Set(primary.map((c) => c.id)); + const secondary = ColumnsCollection() + .discover({ anchors: { main: valueAnchor }, mode: "enrichment", maxHops: 4 }) + .getColumns() + .filter((c) => !primaryIds.has(c.id)); + + return createPlDataTableV3(ctx, { + primaryColumns: primary, + columns: [...secondary, ...splitRecipes], + + tableState: ctx.data.tableSplitState, + labelsOptions: { + formatters: { linker: (linkerLabels) => `[${linkerLabels.join(" > ")}]` }, + }, + }); + }) + .done(); export type BlockOutputs = InferOutputsType; diff --git a/etc/blocks/table-test/test/src/wf.test.ts b/etc/blocks/table-test/test/src/wf.test.ts index 4bff396662..d48db26feb 100644 --- a/etc/blocks/table-test/test/src/wf.test.ts +++ b/etc/blocks/table-test/test/src/wf.test.ts @@ -2,10 +2,9 @@ import type { PTableHandle } from "@platforma-sdk/model"; import { blockTest } from "@platforma-sdk/test"; import { blockSpec } from "this-block"; -type TableOutput = { - ok: boolean; - value: { visibleTableHandle: string; fullTableHandle: string } | undefined; -}; +type TableOutput = + | { ok: true; value: { visibleTableHandle: string; fullTableHandle: string }; stable: boolean } + | { ok: false; errors: { message?: string }[]; moreErrors: boolean }; blockTest( "V3 table resolves linked columns through multi-hop linker chain", @@ -17,8 +16,11 @@ blockTest( const state = await helpers.awaitBlockDoneAndGetStableBlockState(blockId, 50000); const tableOutput = state.outputs!["tableV3"] as TableOutput; - expect(tableOutput.ok).toBe(true); - expect(tableOutput.value).toBeDefined(); + if (!tableOutput.ok) { + throw new Error( + `tableV3 failed:\n${tableOutput.errors.map((e) => e.message ?? JSON.stringify(e)).join("\n")}`, + ); + } const pFrameDriver = ml.driverKit.pFrameDriver; @@ -28,7 +30,7 @@ blockTest( // 2-hop linked via name→group→region chain: region axis + regionName + population + linker_group_region // Total: 5 + 4 + 4 = 13 columns (at least) // Block applies filter `value > 11` → 4 rows (Alpha with value=10 excluded). - const fullHandle = tableOutput.value!.fullTableHandle as PTableHandle; + const fullHandle = tableOutput.value.fullTableHandle as PTableHandle; const fullShape = await pFrameDriver.getShape(fullHandle); expect(fullShape.rows).toBe(4); expect(fullShape.columns).toBeGreaterThan(9); // well above direct-only count @@ -59,7 +61,7 @@ blockTest( // Visible table should also resolve without "axis not present in join result" error. // columnsDisplayOptions hides "note" and marks "score" optional (hidden by default), // so visible table has fewer columns than full but still contains the remaining ones. - const visibleHandle = tableOutput.value!.visibleTableHandle as PTableHandle; + const visibleHandle = tableOutput.value.visibleTableHandle as PTableHandle; const visibleShape = await pFrameDriver.getShape(visibleHandle); expect(visibleShape.rows).toBe(4); expect(visibleShape.columns).toBeGreaterThan(3); @@ -72,3 +74,46 @@ blockTest( expect(visibleValues).toContain(JSON.stringify(["B", "A", "B", "A"])); }, ); + +blockTest( + "V3 split table fans out `count` over the `group` partition axis", + { timeout: 60000 }, + async ({ rawPrj: project, helpers, expect, ml }) => { + const blockId = await project.addBlock("Block", blockSpec); + + await project.runBlock(blockId); + const state = await helpers.awaitBlockDoneAndGetStableBlockState(blockId, 50000); + + const splitOutput = state.outputs!["tableSplitV3"] as TableOutput; + if (!splitOutput.ok) { + throw new Error( + `tableSplitV3 failed:\n${splitOutput.errors.map((e) => e.message ?? JSON.stringify(e)).join("\n")}`, + ); + } + + const pFrameDriver = ml.driverKit.pFrameDriver; + + // tableSplitV3 has no filter and no sort override — primary is `value` + // discovered via {main: value@name}; all 5 names participate. + const fullHandle = splitOutput.value.fullTableHandle as PTableHandle; + const fullShape = await pFrameDriver.getShape(fullHandle); + expect(fullShape.rows).toBe(5); + + const fullIndices = Array.from({ length: fullShape.columns }, (_, i) => i); + const fullData = await pFrameDriver.getData(fullHandle, fullIndices); + + // `count` (group, name) is partitioned by `group`. expandByPartition fans + // it out into one column per group value — G1 and G2. Each split column + // has `group` axis sliced away and its `domain.group = ""` patched + // via specOverride. The values come from count.tsv: + // G1: A=100, B=200, C=10, D=25, E=5 + // G2: A=50, B=150, C=300, D=75, E=1000 + const findInt = (expected: number[]) => + fullData.find( + (c) => c.type === "Int" && JSON.stringify([...c.data]) === JSON.stringify(expected), + ); + + expect(findInt([100, 200, 10, 25, 5])).toBeDefined(); + expect(findInt([50, 150, 300, 75, 1000])).toBeDefined(); + }, +); diff --git a/etc/blocks/table-test/ui/src/Table.vue b/etc/blocks/table-test/ui/src/Table.vue index f965191100..24bfa71691 100644 --- a/etc/blocks/table-test/ui/src/Table.vue +++ b/etc/blocks/table-test/ui/src/Table.vue @@ -4,7 +4,7 @@ import { useApp } from "./app"; const app = useApp(); -const tableSettingsV3 = usePlDataTableSettingsV2({ +const tableSettings = usePlDataTableSettingsV2({ model: () => app.model.outputs.tableV3, }); @@ -14,7 +14,7 @@ const tableSettingsV3 = usePlDataTableSettingsV2({ diff --git a/etc/blocks/table-test/ui/src/TableSplit.vue b/etc/blocks/table-test/ui/src/TableSplit.vue new file mode 100644 index 0000000000..65eaa72460 --- /dev/null +++ b/etc/blocks/table-test/ui/src/TableSplit.vue @@ -0,0 +1,21 @@ + + + diff --git a/etc/blocks/table-test/ui/src/app.ts b/etc/blocks/table-test/ui/src/app.ts index 243b4e0a56..9789428a36 100644 --- a/etc/blocks/table-test/ui/src/app.ts +++ b/etc/blocks/table-test/ui/src/app.ts @@ -1,11 +1,13 @@ import { platforma } from "@milaboratories/milaboratories.test-block-table.model"; import { defineAppV3 } from "@platforma-sdk/ui-vue"; +import TableSplit from "./TableSplit.vue"; import Table from "./Table.vue"; export const sdkPlugin = defineAppV3(platforma, () => { return { routes: { "/": () => Table, + "/split": () => TableSplit, }, }; }); diff --git a/etc/blocks/table-test/workflow/src/main.tpl.tengo b/etc/blocks/table-test/workflow/src/main.tpl.tengo index 5ed6360d69..7977211cf7 100644 --- a/etc/blocks/table-test/workflow/src/main.tpl.tengo +++ b/etc/blocks/table-test/workflow/src/main.tpl.tengo @@ -48,6 +48,21 @@ wf.body(func(args) { "G1\tR1\t1\n" + "G2\tR2\t1\n" + // --- Multi-axis column for split-by-partition demo --- + // Axes: [group, name]; partitionKeyLength=1 => partitioned by group. + // Used in tableSplitV3 output to demonstrate expandByPartition fan-out. + countTsv := "group\tname\tcount\n" + + "G1\tA\t100\n" + + "G2\tA\t50\n" + + "G1\tB\t200\n" + + "G2\tB\t150\n" + + "G1\tC\t10\n" + + "G2\tC\t300\n" + + "G1\tD\t25\n" + + "G2\tD\t75\n" + + "G1\tE\t5\n" + + "G2\tE\t1000\n" + // --- Label columns: axis value -> human-readable label --- // participate in getMatchingLabelColumns / deriveAllLabels (createPlDataTableV3.ts#L106-110) // each has single axis and column name "pl7.app/label" @@ -78,6 +93,7 @@ wf.body(func(args) { writeFile("nameLabel.tsv", nameLabelTsv). writeFile("groupLabel.tsv", groupLabelTsv). writeFile("regionLabel.tsv", regionLabelTsv). + writeFile("count.tsv", countTsv). saveFile("main.tsv"). saveFile("group.tsv"). saveFile("linker.tsv"). @@ -87,6 +103,7 @@ wf.body(func(args) { saveFile("nameLabel.tsv"). saveFile("groupLabel.tsv"). saveFile("regionLabel.tsv"). + saveFile("count.tsv"). run() mainFile := e.getFile("main.tsv") @@ -98,6 +115,7 @@ wf.body(func(args) { nameLabelFile := e.getFile("nameLabel.tsv") groupLabelFile := e.getFile("groupLabel.tsv") regionLabelFile := e.getFile("regionLabel.tsv") + countFile := e.getFile("count.tsv") nameAxisSpec := { name: "name", @@ -392,7 +410,7 @@ wf.body(func(args) { } } }], - storageFormat: "Binary", + storageFormat: "Json", partitionKeyLength: 0 } @@ -412,7 +430,7 @@ wf.body(func(args) { } } }], - storageFormat: "Binary", + storageFormat: "Json", partitionKeyLength: 0 } @@ -432,10 +450,36 @@ wf.body(func(args) { } } }], - storageFormat: "Binary", + storageFormat: "Json", partitionKeyLength: 0 } + // `count` carries (group, name) -> Int. `group` is the partition axis + // (partitionKeyLength=1) so getUniquePartitionKeys returns group values, + // which expandByPartition consumes to fan out into per-group columns. + countSpec := { + axes: [{ + column: "group", + spec: groupAxisSpec + }, { + column: "name", + spec: nameAxisSpec + }], + columns: [{ + column: "count", + id: "count", + spec: { + name: "count", + valueType: "Int", + annotations: { + "pl7.app/label": "Count" + } + } + }], + storageFormat: "Binary", + partitionKeyLength: 1 + } + importOps := { inputCache: times.second, splitDataAndSpec: true } mainPf := xsv.importFile(mainFile, "tsv", mainSpec, importOps) groupPf := xsv.importFile(groupFile, "tsv", groupSpec, importOps) @@ -446,6 +490,7 @@ wf.body(func(args) { nameLabelPf := xsv.importFile(nameLabelFile, "tsv", nameLabelSpec, importOps) groupLabelPf := xsv.importFile(groupLabelFile, "tsv", groupLabelSpec, importOps) regionLabelPf := xsv.importFile(regionLabelFile, "tsv", regionLabelSpec, importOps) + countPf := xsv.importFile(countFile, "tsv", countSpec, importOps) // Republish every column as a block export so downstream blocks // can discover the linker chain through the result pool. @@ -458,7 +503,7 @@ wf.body(func(args) { } } addAll(mainPf, groupPf, linkerPf, linkerAltPf, regionPf, linker2Pf, - nameLabelPf, groupLabelPf, regionLabelPf) + nameLabelPf, groupLabelPf, regionLabelPf, countPf) pf = pf.build() return { diff --git a/etc/blocks/ui-examples/ui/src/pages/PlAdvancedFilterPage.vue b/etc/blocks/ui-examples/ui/src/pages/PlAdvancedFilterPage.vue index 341441d48a..75bdbaef8f 100644 --- a/etc/blocks/ui-examples/ui/src/pages/PlAdvancedFilterPage.vue +++ b/etc/blocks/ui-examples/ui/src/pages/PlAdvancedFilterPage.vue @@ -17,6 +17,11 @@ import { } from "@platforma-sdk/ui-vue"; import { ref, watch } from "vue"; +/** Coerce a PlAdvancedFilterColumnId to a string key for Record lookup. */ +function columnIdKey(id: PlAdvancedFilterColumnId): string { + return typeof id === "string" ? id : JSON.stringify(id); +} + const column1Id = stringifyColumnId({ name: "1", axes: [] }) as SUniversalPColumnId; const column2Id = stringifyColumnId({ name: "2", axes: [] }) as SUniversalPColumnId; const column3Id = stringifyColumnId({ name: "3", axes: [] }) as SUniversalPColumnId; @@ -88,12 +93,11 @@ async function getSuggestOptions({ searchStr: string; axisIdx?: number; }) { + const key = columnIdKey(columnId); if (axisIdx !== undefined) { - return (uniqueValuesByAxisIdx[columnId]?.[axisIdx] || []).filter((v) => - v.label.includes(searchStr), - ); + return (uniqueValuesByAxisIdx[key]?.[axisIdx] || []).filter((v) => v.label.includes(searchStr)); } - return (uniqueValuesByColumnOrAxisId[columnId] || []).filter((v) => v.label.includes(searchStr)); + return (uniqueValuesByColumnOrAxisId[key] || []).filter((v) => v.label.includes(searchStr)); } async function getSuggestModel({ columnId, @@ -104,8 +108,9 @@ async function getSuggestModel({ searchStr: string; axisIdx?: number; }) { + const key = columnIdKey(columnId); if (axisIdx !== undefined) { - const axisValues = uniqueValuesByAxisIdx[columnId]?.[axisIdx]; + const axisValues = uniqueValuesByAxisIdx[key]?.[axisIdx]; return ( axisValues.find((v) => v.value === searchStr) || { value: searchStr, @@ -113,7 +118,7 @@ async function getSuggestModel({ } ); } - const columnValues = uniqueValuesByColumnOrAxisId[columnId]; + const columnValues = uniqueValuesByColumnOrAxisId[key]; return ( columnValues.find((v) => v.value === searchStr) || { value: searchStr, @@ -256,10 +261,10 @@ watch(
{{ option.label }}
diff --git a/etc/blocks/ui-examples/ui/src/pages/PlAnnotationPage.vue b/etc/blocks/ui-examples/ui/src/pages/PlAnnotationPage.vue index f8b8f263d5..59b38df344 100644 --- a/etc/blocks/ui-examples/ui/src/pages/PlAnnotationPage.vue +++ b/etc/blocks/ui-examples/ui/src/pages/PlAnnotationPage.vue @@ -14,6 +14,11 @@ import type { import { PlAnnotationsModal } from "@platforma-sdk/ui-vue"; import { ref, watch } from "vue"; +/** Coerce a PlAdvancedFilterColumnId to a string key for Record lookup. */ +function columnIdKey(id: PlAdvancedFilterColumnId): string { + return typeof id === "string" ? id : JSON.stringify(id); +} + const showModal = ref(true); watch(showModal, (value) => { if (!value) { @@ -141,7 +146,7 @@ async function getSuggestOptions({ searchStr: string; axisIdx?: number; }) { - return (uniqueValuesByColumnId[columnId] || []).filter((v) => + return (uniqueValuesByColumnId[columnIdKey(columnId)] || []).filter((v) => v.label.toLowerCase().includes(searchStr.toLowerCase()), ); } @@ -154,7 +159,7 @@ async function getSuggestModel({ searchStr: string; axisIdx?: number; }) { - const columnValues = uniqueValuesByColumnId[columnId]; + const columnValues = uniqueValuesByColumnId[columnIdKey(columnId)]; return ( columnValues?.find((v) => v.value === searchStr) || { value: searchStr, diff --git a/lib/model/columns-collection-driver/.gitignore b/lib/model/columns-collection-driver/.gitignore new file mode 100644 index 0000000000..404abb2212 --- /dev/null +++ b/lib/model/columns-collection-driver/.gitignore @@ -0,0 +1 @@ +coverage/ diff --git a/lib/model/columns-collection-driver/.oxfmtrc.json b/lib/model/columns-collection-driver/.oxfmtrc.json new file mode 100644 index 0000000000..7eff5e7af0 --- /dev/null +++ b/lib/model/columns-collection-driver/.oxfmtrc.json @@ -0,0 +1,4 @@ +{ + "extends": ["node_modules/@milaboratories/ts-builder/configs/oxfmt.json"], + "ignorePatterns": ["dist", "coverage", "CHANGELOG.md"] +} diff --git a/lib/model/columns-collection-driver/.oxlintrc.json b/lib/model/columns-collection-driver/.oxlintrc.json new file mode 100644 index 0000000000..b1a139038f --- /dev/null +++ b/lib/model/columns-collection-driver/.oxlintrc.json @@ -0,0 +1,3 @@ +{ + "extends": ["node_modules/@milaboratories/ts-builder/dist/configs/oxlint-node.json"] +} diff --git a/lib/model/columns-collection-driver/package.json b/lib/model/columns-collection-driver/package.json new file mode 100644 index 0000000000..b2a06ad3aa --- /dev/null +++ b/lib/model/columns-collection-driver/package.json @@ -0,0 +1,45 @@ +{ + "name": "@milaboratories/columns-collection-driver", + "version": "0.1.0", + "description": "Host-side ColumnsCollectionDriver implementation: column-collection state ownership, refcounted handles, source serialisation", + "keywords": [], + "license": "UNLICENSED", + "files": [ + "./dist/**/*", + "./src/**/*" + ], + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "scripts": { + "build": "ts-builder build --target node", + "watch": "ts-builder build --target node --watch", + "check": "ts-builder check --target node", + "formatter:check": "ts-builder formatter --check", + "linter:check": "ts-builder linter --check", + "types:check": "ts-builder type-check --target node", + "test": "vitest run --coverage", + "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz", + "fmt": "ts-builder format" + }, + "dependencies": { + "@milaboratories/helpers": "workspace:*", + "@milaboratories/pl-model-common": "workspace:*" + }, + "devDependencies": { + "@milaboratories/build-configs": "workspace:*", + "@milaboratories/ts-builder": "workspace:*", + "@milaboratories/ts-configs": "workspace:*", + "@vitest/coverage-istanbul": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/lib/model/columns-collection-driver/src/driver.ts b/lib/model/columns-collection-driver/src/driver.ts new file mode 100644 index 0000000000..261cfe0731 --- /dev/null +++ b/lib/model/columns-collection-driver/src/driver.ts @@ -0,0 +1,493 @@ +import type { + AccessorHandle, + AccessorLike, + AnchorEntry, + AxisQualification, + CollectionHandle, + ColumnEntriesProvider, + ColumnUniversalId, + ColumnsCollectionDriver, + ColumnsCollectionDriverHost, + ColumnsDiscoverOptions, + ColumnsFilterOptions, + DiscoverColumnsOptions, + DiscoverColumnsResponse, + DiscoverColumnsStepInfo, + MatchQualifications, + NativePObjectId, + PColumnSpec, + PFrameSpecDriver, + PObjectId, + PoolEntry, + SerializedColumnsSource, + SpecFrameHandle, + SpecOverrides, +} from "@milaboratories/pl-model-common"; +import { + AccessorEntriesProvider, + ResultPoolEntriesProvider, + convertColumnSelectorToMultiColumnSelector, + createGlobalPObjectId, + createColumnOverridedId, + dedupColumns, + deriveNativeId, + deriveSpecDelta, + extractPObjectId, + isPColumnSpec, + isPlRef, + isEmptySpecDelta, + matchingModeToConstraints, + reconstructSpecFromId, + stringifyColumnDiscoveredId, +} from "@milaboratories/pl-model-common"; +import { throwError } from "@milaboratories/helpers"; +import { randomUUID } from "node:crypto"; + +/** + * Single contribution to a collection's column set. Mirrors the shapes of + * {@link SerializedColumnsSource} but keeps host-resolved references — the + * driver never re-resolves an accessor handle after the initial deserialise. + */ +type SourceContribution> = + | { readonly type: "provider"; readonly provider: ColumnEntriesProvider } + | { + readonly type: "ids"; + readonly ids: ReadonlyArray; + readonly isFinal: boolean; + }; + +interface CollectionState> { + readonly contributions: ReadonlyArray>; + refs: number; +} + +/** + * Host-side `ColumnsCollectionDriver` implementation. Owns all + * collection state addressable by opaque {@link CollectionHandle} strings; + * the sandbox/UI bridge only ever sees those handles. + * + * Generic over the host's concrete accessor flavour: middle-layer passes + * `PlTreeNodeAccessor` from `@milaboratories/pl-tree`; UI-side instances + * (which never invoke `{kind:"accessor"}` / `{kind:"result_pool"}` sources) + * default to `AccessorLike`. + * + * Handle lifecycle is plain refcount: every minting method returns a fresh + * {@link PoolEntry} with `unref` and `Symbol.dispose`. + */ +export class ColumnsCollectionDriverImpl = AccessorLike> + implements ColumnsCollectionDriver, AsyncDisposable +{ + private readonly registry = new Map>(); + + create( + sources: ReadonlyArray, + host: ColumnsCollectionDriverHost, + ): PoolEntry { + const contributions = sources.map((src) => this.materialiseSource(src, host)); + return this.mint(contributions); + } + + isEmpty(handle: CollectionHandle): boolean { + const state = this.requireState(handle); + for (const c of state.contributions) { + switch (c.type) { + case "provider": + if (c.provider.getPObjectEntries().size > 0) return false; + break; + case "ids": + if (c.ids.length > 0) return false; + break; + } + } + return true; + } + + isFinal(handle: CollectionHandle): boolean { + const state = this.requireState(handle); + for (const c of state.contributions) { + switch (c.type) { + case "provider": + if (!c.provider.isFinal()) return false; + break; + case "ids": + if (!c.isFinal) return false; + break; + } + } + return true; + } + + /** + * Returns the deduplicated id list for `handle`. Dedup semantics are + * shared with sandbox-side `extractColumns` via {@link dedupColumns} — + * the same physical column reached via outputs vs. result_pool collapses + * to one (provider order decides which id is canonical). + */ + getColumns(handle: CollectionHandle, host: ColumnsCollectionDriverHost): ColumnUniversalId[] { + const state = this.requireState(handle); + const all = state.contributions.flatMap((c) => { + switch (c.type) { + case "provider": + return Array.from(c.provider.getPObjectEntries().keys()); + case "ids": + return [...c.ids]; + } + }); + return dedupColumns( + all, + (id) => id, + (id) => { + const spec = host.resolveSpec(extractPObjectId(id)); + return spec === undefined ? undefined : reconstructSpecFromId(spec, id); + }, + ); + } + + addSource( + handle: CollectionHandle, + sources: ReadonlyArray, + host: ColumnsCollectionDriverHost, + ): PoolEntry { + const state = this.requireState(handle); + const next = [ + ...state.contributions, + ...sources.map((src) => this.materialiseSource(src, host)), + ]; + return this.mint(next); + } + + discover( + handle: CollectionHandle, + options: ColumnsDiscoverOptions, + host: ColumnsCollectionDriverHost, + ): PoolEntry { + return this.runDiscovery(handle, options, host); + } + + filter( + handle: CollectionHandle, + options: ColumnsFilterOptions, + host: ColumnsCollectionDriverHost, + ): PoolEntry { + return this.runDiscovery(handle, options, host); + } + + async dispose(): Promise { + this.registry.clear(); + } + + async [Symbol.asyncDispose](): Promise { + await this.dispose(); + } + + /** Number of currently-held handles. Test-facing. */ + size(): number { + return this.registry.size; + } + + private materialiseSource( + src: SerializedColumnsSource, + host: ColumnsCollectionDriverHost, + ): SourceContribution { + switch (src.kind) { + case "ids": + return { type: "ids", ids: src.ids, isFinal: src.isFinal }; + + case "collection": + return { + type: "ids", + ids: this.getColumns(src.handle, host), + isFinal: this.isFinal(src.handle), + }; + + case "accessor": { + const accessor = host.resolveAccessor(src.accessor as AccessorHandle); + return { + type: "provider", + provider: new AccessorEntriesProvider(accessor, src.path), + }; + } + + case "result_pool": + return { + type: "provider", + provider: new ResultPoolEntriesProvider(host.getUpstreamBlockCtxes()), + }; + } + } + + private mint(contributions: ReadonlyArray>): PoolEntry { + const key = randomUUID() as CollectionHandle; + const state: CollectionState = { contributions, refs: 1 }; + this.registry.set(key, state); + + let released = false; + const unref = (): void => { + if (released) return; + released = true; + state.refs--; + if (state.refs <= 0) this.registry.delete(key); + }; + + return { + key, + resource: state as unknown as {}, + unref, + [Symbol.dispose]: unref, + }; + } + + private requireState(handle: CollectionHandle): CollectionState { + const state = this.registry.get(handle); + if (state === undefined) { + throw new Error( + `ColumnsCollectionDriverImpl: unknown CollectionHandle "${handle}" (handle expired or never minted)`, + ); + } + return state; + } + + private runDiscovery( + handle: CollectionHandle, + options: DiscoverColumnsOptions, + host: ColumnsCollectionDriverHost, + ): PoolEntry { + const sourceIsFinal = this.isFinal(handle); + const ids = this.getColumns(handle, host); + if (ids.length === 0) { + return this.mint([{ type: "ids", ids: [], isFinal: sourceIsFinal }]); + } + + const specDriver = host.getSpecDriver(); + const specMap = new Map(); + for (const id of ids) { + const leaf = extractPObjectId(id); + const spec = host.resolveSpec(leaf); + if (spec === undefined) continue; + specMap.set(id, reconstructSpecFromId(spec, id)); + } + + const anchors = options.anchors; + const hasAnchors = anchors !== undefined && Object.keys(anchors).length > 0; + const anchorsRec = hasAnchors ? resolveAnchors(anchors, specMap, specDriver) : undefined; + const anchorsList = anchorsRec ? Object.values(anchorsRec) : []; + + using specFrame = specDriver.createSpecFrame(Object.fromEntries(specMap.entries())); + + const response = specDriver.discoverColumns(specFrame.key, { + includeColumns: options.include + ? convertColumnSelectorToMultiColumnSelector(options.include) + : undefined, + excludeColumns: options.exclude + ? convertColumnSelectorToMultiColumnSelector(options.exclude) + : undefined, + constraints: matchingModeToConstraints(options.mode ?? "enrichment"), + maxHops: options.maxHops ?? (hasAnchors ? 4 : 0), + axes: anchorsList.map((anchorId) => { + const spec = + specMap.get(anchorId) ?? + throwError(`ColumnsCollectionDriverImpl: anchor "${anchorId}" lost from effective specs`); + return { axesSpec: spec.axesSpec, qualifications: [] }; + }), + }); + + const resultIds = hasAnchors + ? mapHitsWithDiscovery(response, specMap, anchorsList) + : mapHitsDirect(response, specMap); + + return this.mint([{ type: "ids", ids: resultIds, isFinal: sourceIsFinal }]); + } +} + +function mapHitsDirect( + response: DiscoverColumnsResponse, + effectiveSpecs: ReadonlyMap, +): ColumnUniversalId[] { + const known = new Set(effectiveSpecs.keys()); + const out: ColumnUniversalId[] = []; + for (const hit of response.hits) { + const id = hit.hit.columnId; + if (known.has(id)) out.push(id); + } + return out; +} + +function mapHitsWithDiscovery( + response: DiscoverColumnsResponse, + specs: ReadonlyMap, + anchors: ColumnUniversalId[], +): ColumnUniversalId[] { + const out: ColumnUniversalId[] = []; + for (const hit of response.hits) { + const originalId = hit.hit.columnId; + const originalSpec = + specs.get(originalId) ?? + throwError( + `ColumnsCollectionDriverImpl: discover hit column "${originalId}" not found in collection`, + ); + const hitDelta = deriveSpecDelta(originalSpec, hit.hit.spec); + const baseId = isEmptySpecDelta(hitDelta) ? originalId : applyOverride(originalId, hitDelta); + const path = hit.path.map((step) => buildLinkerStep(step, specs)); + + for (const variant of hit.mappingVariants) { + const quals = remapAnchorQualifications(variant.qualifications, anchors); + out.push( + path.length === 0 && quals === undefined ? baseId : applyDiscovery(baseId, path, quals), + ); + } + } + return out; +} + +function buildLinkerStep( + step: DiscoverColumnsStepInfo, + specs: ReadonlyMap, +): { type: "linker"; column: ColumnUniversalId } { + if (step.type !== "linker") { + throw new Error(`Unexpected discover-columns step type: ${step.type}`); + } + const linkerOrigId = step.linker.columnId; + const linkerSpec = + specs.get(linkerOrigId) ?? + throwError( + `ColumnsCollectionDriverImpl: linker column "${linkerOrigId}" not found in collection`, + ); + const linkerDelta = deriveSpecDelta(linkerSpec, step.linker.spec); + const linkerId = isEmptySpecDelta(linkerDelta) + ? linkerOrigId + : applyOverride(linkerOrigId, linkerDelta); + return { type: "linker", column: linkerId }; +} + +function applyDiscovery( + baseId: ColumnUniversalId, + path?: ReadonlyArray<{ type: "linker"; column: ColumnUniversalId }>, + quals?: MatchQualifications, +): ColumnUniversalId { + return stringifyColumnDiscoveredId({ + column: baseId, + path: path?.map((p) => ({ type: "linker", column: p.column })), + columnQualifications: quals?.forHit, + queriesQualifications: quals?.forQueries, + }); +} + +function applyOverride(id: ColumnUniversalId, delta: SpecOverrides): ColumnUniversalId { + return createColumnOverridedId({ source: id, specOverrides: delta }); +} + +function remapAnchorQualifications( + qualifications: { forQueries: AxisQualification[][]; forHit: AxisQualification[] }, + anchors: ColumnUniversalId[], +): undefined | MatchQualifications { + const forQueries: Record = {}; + let hasForQueries = false; + qualifications.forQueries.forEach((qs, i) => { + const anchor = anchors[i]; + if (anchor === undefined || qs.length === 0) return; + forQueries[extractPObjectId(anchor)] = qs; + hasForQueries = true; + }); + return !hasForQueries && qualifications.forHit.length === 0 + ? undefined + : { + forQueries, + forHit: qualifications.forHit, + }; +} + +function resolveAnchors( + anchors: Record, + specs: ReadonlyMap, + specDriver: PFrameSpecDriver, +): Record { + const result: Record = {}; + const resolvedIds = new Set(); + const duplicateError = (key: string) => + `Anchor "${key}": selector matched a column that was already matched by another anchor; please refine the selector to match a different column`; + + let specFrame: undefined | PoolEntry; + const discoverColumns: AnchorDiscover = (request) => { + specFrame ??= specDriver.createSpecFrame(Object.fromEntries(specs.entries())); + return specDriver.discoverColumns(specFrame.key, request); + }; + + // O(1) lookup map built lazily — only when a spec-based anchor is + // encountered. Avoids O(anchors × columns) `deriveNativeId` calls. + let byNativeId: Map | undefined; + const getByNativeId: AnchorNativeIdLookup = () => { + if (byNativeId === undefined) { + byNativeId = new Map(); + for (const [id, spec] of specs) byNativeId.set(deriveNativeId(spec), id); + } + return byNativeId; + }; + + try { + for (const [name, anchor] of Object.entries(anchors)) { + const found = matchAnchor(name, anchor, specs, discoverColumns, getByNativeId); + if (found === undefined) continue; + if (resolvedIds.has(found)) throwError(duplicateError(name)); + result[name] = found; + resolvedIds.add(found); + } + } finally { + specFrame?.unref(); + } + + if (resolvedIds.size === 0) { + throwError("At least one anchor must be resolved to a valid column"); + } + + return result; +} + +type AnchorDiscover = ( + request: Parameters[1], +) => DiscoverColumnsResponse; + +type AnchorNativeIdLookup = () => ReadonlyMap; + +function matchAnchor( + name: string, + anchor: AnchorEntry, + specs: ReadonlyMap, + discoverColumns: AnchorDiscover, + getByNativeId: AnchorNativeIdLookup, +): undefined | ColumnUniversalId { + if (isPlRef(anchor)) { + const id = createGlobalPObjectId(anchor.blockId, anchor.name); + return findById(specs, id); + } + if (typeof anchor === "string") { + const id = anchor; + return findById(specs, id); + } + if ("kind" in anchor) { + if (!isPColumnSpec(anchor)) throwError(`Anchor "${name}": invalid PColumnSpec`); + return getByNativeId().get(deriveNativeId(anchor)); + } + + // RelaxedColumnSelector + const matched = discoverColumns({ + includeColumns: convertColumnSelectorToMultiColumnSelector(anchor), + excludeColumns: undefined, + axes: [], + maxHops: 0, + constraints: matchingModeToConstraints("exact"), + }); + if (matched.hits.length === 0) return undefined; + if (matched.hits.length > 1) { + throwError( + `Anchor "${name}": selector is ambiguous and matched multiple columns; please refine the selector to match exactly one column`, + ); + } + return matched.hits[0].hit.columnId; +} + +function findById( + specs: ReadonlyMap, + needle: ColumnUniversalId, +): ColumnUniversalId | undefined { + return specs.has(needle) ? needle : undefined; +} diff --git a/lib/model/columns-collection-driver/src/index.ts b/lib/model/columns-collection-driver/src/index.ts new file mode 100644 index 0000000000..8d43184d82 --- /dev/null +++ b/lib/model/columns-collection-driver/src/index.ts @@ -0,0 +1 @@ +export { ColumnsCollectionDriverImpl } from "./driver"; diff --git a/lib/model/columns-collection-driver/tsconfig.json b/lib/model/columns-collection-driver/tsconfig.json new file mode 100644 index 0000000000..f587c33dd6 --- /dev/null +++ b/lib/model/columns-collection-driver/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@milaboratories/ts-configs/tsconfig.node.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/lib/model/columns-collection-driver/vitest.config.mts b/lib/model/columns-collection-driver/vitest.config.mts new file mode 100644 index 0000000000..b699ddc15e --- /dev/null +++ b/lib/model/columns-collection-driver/vitest.config.mts @@ -0,0 +1,10 @@ +import { createVitestConfig } from "@milaboratories/build-configs"; +import { defineConfig } from "vitest/config"; + +export default defineConfig( + createVitestConfig({ + test: { + environment: "node", + }, + }), +); diff --git a/lib/model/common/package.json b/lib/model/common/package.json index 433e80169d..6891e75990 100644 --- a/lib/model/common/package.json +++ b/lib/model/common/package.json @@ -31,6 +31,7 @@ "@milaboratories/helpers": "workspace:*", "@milaboratories/pl-error-like": "workspace:*", "canonicalize": "catalog:", + "es-toolkit": "catalog:", "zod": "catalog:" }, "devDependencies": { diff --git a/lib/model/common/src/columns/accessor_traversal.ts b/lib/model/common/src/columns/accessor_traversal.ts new file mode 100644 index 0000000000..8ad3ae5cc3 --- /dev/null +++ b/lib/model/common/src/columns/accessor_traversal.ts @@ -0,0 +1,139 @@ +import { createGlobalPObjectId, createLocalPObjectId } from "../pool"; +import { ResourceTypeName } from "../resource_types"; +import type { AccessorLike, LeafEntry, UpstreamBlockCtx } from "./types"; + +/** Resource types that hold column collections — DFS stops here and collects. */ +export const COLLECT_TYPES: ReadonlyArray = [ResourceTypeName.PFrame]; + +/** Resource types DFS descends through when looking for PFrames. */ +export const DESCEND_TYPES: ReadonlyArray = [ + ResourceTypeName.StdMap, + ResourceTypeName.StdMapSlash, +]; + +/** + * Enumerate column names backing a PFrame accessor — derived from + * `.spec` field names, without resolving the spec resources. + */ +export function listColumnNames>( + accessor: A, + prefix: string = "", +): string[] { + if (accessor.resourceType.name !== ResourceTypeName.PFrame) return []; + const out: string[] = []; + for (const field of accessor.listInputFields()) { + if (!field.endsWith(".spec")) continue; + const raw = field.slice(0, -".spec".length); + if (!raw.startsWith(prefix)) continue; + out.push(raw.slice(prefix.length)); + } + return out; +} + +/** + * One DFS hit — the accessor whose resource-type matched `collectTypes`, and + * the field-name path from the DFS root used to reach it. + */ +type DescendantHit> = { + readonly node: A; + readonly path: ReadonlyArray; +}; + +/** + * DFS over the input-field subtree of `root`. Collects nodes whose resource + * type ∈ `collectTypes`. Descends only through `descendTypes` + * (default: StdMap / StdMapSlash). Collected nodes are not descended into. + * + * Threads the field-name path from `rootPath` to each hit, so callers can + * build canonical {@link createLocalPObjectId}s without relying on the + * accessor itself to remember its path. + * + * Includes `root` itself in the walk. + */ +export function findDescendantsByType>(opts: { + root: A; + rootPath: ReadonlyArray; + collectTypes: ReadonlyArray; + descendTypes?: ReadonlyArray; +}): DescendantHit[] { + const { root, rootPath, collectTypes, descendTypes = DESCEND_TYPES } = opts; + const collectSet = new Set(collectTypes); + const descendSet = new Set(descendTypes); + const result: DescendantHit[] = []; + + const stack: DescendantHit[] = [{ node: root, path: rootPath }]; + while (stack.length > 0) { + const { node, path } = stack.pop()!; + const typeName = node.resourceType.name; + if (collectSet.has(typeName)) { + result.push({ node, path }); + continue; + } + if (!descendSet.has(typeName)) continue; + const fields = node.listInputFields(); + for (let i = fields.length - 1; i >= 0; i--) { + const child = node.traverse({ + field: fields[i], + assertFieldType: "Input", + ignoreError: true, + }); + if (child !== undefined) stack.push({ node: child, path: [...path, fields[i]] }); + } + } + return result; +} + +/** + * Walk an accessor root and return one {@link LeafEntry} per discovered column. + * Ids are {@link createLocalPObjectId}-shaped: `{resolvePath, name}`. The + * `resolvePath` is derived from `rootPath` extended by the DFS traversal. + */ +export function indexAccessorRoot>( + root: A, + rootPath: ReadonlyArray, +): LeafEntry[] { + const result: LeafEntry[] = []; + for (const { node, path } of findDescendantsByType({ + root, + rootPath, + collectTypes: COLLECT_TYPES, + descendTypes: DESCEND_TYPES, + })) { + for (const name of listColumnNames(node)) { + result.push({ + accessor: node, + name, + id: createLocalPObjectId([...path], name), + }); + } + } + return result; +} + +/** + * Walk one upstream-block ctx pair (`prodCtx` then `stagingCtx`) and return one + * {@link LeafEntry} per column. First-wins dedup by name (prod precedes staging). + * Ids are {@link createGlobalPObjectId}-shaped — `resolvePath` is not involved. + */ +export function indexPoolBlock>( + block: UpstreamBlockCtx, +): LeafEntry[] { + const accessors: A[] = []; + if (block.prodCtx) accessors.push(block.prodCtx); + if (block.stagingCtx) accessors.push(block.stagingCtx); + + const result: LeafEntry[] = []; + const seen = new Set(); + for (const accessor of accessors) { + for (const name of listColumnNames(accessor)) { + if (seen.has(name)) continue; + seen.add(name); + result.push({ + accessor, + name, + id: createGlobalPObjectId(block.blockId, name), + }); + } + } + return result; +} diff --git a/lib/model/common/src/columns/column_registry.ts b/lib/model/common/src/columns/column_registry.ts new file mode 100644 index 0000000000..c5ce197996 --- /dev/null +++ b/lib/model/common/src/columns/column_registry.ts @@ -0,0 +1,41 @@ +import type { PObjectId } from "../pool"; +import type { AccessorLike, ColumnEntriesProvider, LeafEntry } from "./types"; + +/** + * Id-index over a set of {@link ColumnEntriesProvider}s. Sole job: + * {@link PObjectId} → {@link LeafEntry}. Generic over the accessor flavour, so + * the same class backs both sandbox (`TreeNodeAccessor`) and host + * (`PlTreeNodeAccessor`) usage. + * + * Stateless beyond the provider list — every lookup goes through each + * provider's `getPObjectEntries()` (cached inside the provider). Instantiate + * directly at the call site that has the providers; there is no ambient + * singleton. + */ +export class ColumnRegistry> { + constructor(private readonly providers: ReadonlyArray>) {} + + /** + * Resolve a {@link PObjectId} to its backing {@link LeafEntry}. Returns + * `undefined` if the column is not reachable from any provider — caller + * decides whether that's `absent` or `resolving` via {@link isFinal}. + */ + resolve(id: PObjectId): LeafEntry | undefined { + return this.lookupById(id); + } + + /** Whether every indexed source has finished enumerating its columns. */ + isFinal(): boolean { + return this.providers.every((p) => p.isFinal()); + } + + private lookupById(id: PObjectId): LeafEntry | undefined { + // First-wins across providers — caller controls precedence via the + // construction order of `providers`. + for (const p of this.providers) { + const hit = p.getPObjectEntries().get(id); + if (hit !== undefined) return hit; + } + return undefined; + } +} diff --git a/sdk/model/src/columns/column_selector.ts b/lib/model/common/src/columns/column_selector.ts similarity index 97% rename from sdk/model/src/columns/column_selector.ts rename to lib/model/common/src/columns/column_selector.ts index 5fe5deabc9..9a26c02220 100644 --- a/sdk/model/src/columns/column_selector.ts +++ b/lib/model/common/src/columns/column_selector.ts @@ -4,9 +4,9 @@ import type { MultiAxisSelector, MultiColumnSelector, StringMatcher, -} from "@milaboratories/pl-model-common"; +} from "../drivers/pframe"; -export type { StringMatcher } from "@milaboratories/pl-model-common"; +export type { StringMatcher } from "../drivers/pframe"; // --- Relaxed types --- diff --git a/lib/model/common/src/columns/dedup.ts b/lib/model/common/src/columns/dedup.ts new file mode 100644 index 0000000000..1d6a58cbca --- /dev/null +++ b/lib/model/common/src/columns/dedup.ts @@ -0,0 +1,49 @@ +import type { PColumnSpec } from "../drivers/pframe/spec/spec"; +import type { NativePObjectId } from "../drivers/pframe/spec/native_id"; +import { deriveNativeId } from "../drivers/pframe/spec/native_id"; +import type { ColumnUniversalId } from "../drivers/pframe/spec/ids"; +import { isPObjectId } from "../pool"; + +/** + * Two-track dedup over an item stream keyed by `ColumnUniversalId`: + * + * - **Raw `PObjectId`s** are deduped by `deriveNativeId(spec)`. The same + * physical column reached via outputs vs. result_pool has different + * id shapes but identical nativeId — first occurrence wins so the + * provider order (outputs before pool) decides which id is canonical. + * - **Non-`PObjectId` ids** (e.g. `SUniversalPColumnUniversalId`, + * `ColumnDiscoveredId`) keep raw-id dedup — they legitimately share + * nativeId with siblings. + * + * When `getSpec` returns `undefined` for a `PObjectId`, falls back to raw-id + * dedup (so an unresolvable id still survives instead of dropping silently). + * + * Shared by sandbox-side `extractColumns` (column_providers) and host-side + * `ColumnsCollectionDriverImpl.getColumns` — both layers need identical + * dedup semantics, but operate on different concrete item types (ColumnLazy + * vs. raw ColumnUniversalId). + */ +export function dedupColumns( + items: ReadonlyArray, + getId: (item: T) => ColumnUniversalId, + getSpec: (item: T) => PColumnSpec | undefined, +): T[] { + const seenNative = new Set(); + const seenId = new Set(); + const out: T[] = []; + for (const item of items) { + const id = getId(item); + if (seenId.has(id)) continue; + if (isPObjectId(id)) { + const spec = getSpec(item); + if (spec !== undefined) { + const nativeId = deriveNativeId(spec); + if (seenNative.has(nativeId)) continue; + seenNative.add(nativeId); + } + } + seenId.add(id); + out.push(item); + } + return out; +} diff --git a/lib/model/common/src/columns/index.ts b/lib/model/common/src/columns/index.ts new file mode 100644 index 0000000000..ebd6c4d8b6 --- /dev/null +++ b/lib/model/common/src/columns/index.ts @@ -0,0 +1,6 @@ +export * from "./types"; +export * from "./accessor_traversal"; +export * from "./column_registry"; +export * from "./column_selector"; +export * from "./dedup"; +export * from "./providers"; diff --git a/lib/model/common/src/columns/providers.ts b/lib/model/common/src/columns/providers.ts new file mode 100644 index 0000000000..c68ceec95e --- /dev/null +++ b/lib/model/common/src/columns/providers.ts @@ -0,0 +1,74 @@ +import type { PObjectId } from "../pool"; +import { indexAccessorRoot, indexPoolBlock } from "./accessor_traversal"; +import type { AccessorLike, ColumnEntriesProvider, LeafEntry, UpstreamBlockCtx } from "./types"; + +/** + * Generic entries provider over a single accessor root. Walks `` once + * from the supplied `rootPath`, builds an id → {@link LeafEntry} map and + * exposes `isFinal()` via the root's `getInputsLocked()`. + * + * Used directly on the host side; sandbox extends it with `getColumns()` + * returning {@link ColumnLazy}s — see `AccessorColumnsProvider` in + * `@platforma-sdk/model`. + */ +export class AccessorEntriesProvider< + A extends AccessorLike, +> implements ColumnEntriesProvider { + protected readonly entries: ReadonlyMap>; + + constructor( + protected readonly root: A, + rootPath: ReadonlyArray, + ) { + const map = new Map>(); + for (const entry of indexAccessorRoot(root, rootPath)) { + if (!map.has(entry.id)) map.set(entry.id, entry); + } + this.entries = map; + } + + getPObjectEntries(): ReadonlyMap> { + return this.entries; + } + + isFinal(): boolean { + return this.root.getInputsLocked(); + } +} + +/** + * Generic entries provider over a list of upstream-block ctx pairs. + * + * Per-block merge: iterate `prod` then `staging`, dedupe by name with + * first-wins semantics (prod takes precedence). + * + * `isFinal()` is the AND of `getInputsLocked()` over every present ctx + * accessor and `!prodIncomplete && !stagingIncomplete` over every block. + */ +export class ResultPoolEntriesProvider< + A extends AccessorLike, +> implements ColumnEntriesProvider { + protected cachedEntries?: ReadonlyMap>; + + constructor(protected readonly blocks: ReadonlyArray>) {} + + getPObjectEntries(): ReadonlyMap> { + if (this.cachedEntries !== undefined) return this.cachedEntries; + const map = new Map>(); + for (const block of this.blocks) { + for (const entry of indexPoolBlock(block)) { + if (!map.has(entry.id)) map.set(entry.id, entry); + } + } + return (this.cachedEntries = map); + } + + isFinal(): boolean { + for (const block of this.blocks) { + if (block.prodIncomplete || block.stagingIncomplete) return false; + if (block.prodCtx && !block.prodCtx.getInputsLocked()) return false; + if (block.stagingCtx && !block.stagingCtx.getInputsLocked()) return false; + } + return true; + } +} diff --git a/lib/model/common/src/columns/types.ts b/lib/model/common/src/columns/types.ts new file mode 100644 index 0000000000..1369d7d8b6 --- /dev/null +++ b/lib/model/common/src/columns/types.ts @@ -0,0 +1,116 @@ +import type { Branded } from "../branding"; +import type { PObjectId } from "../pool"; + +/** + * Opaque sandbox/host accessor handle. + * + * Both the sandbox `TreeNodeAccessor.handle` and the host-issued accessor + * keys alias this brand — `sdk/model` re-exports the same type so the two + * sides stay structurally identical. + */ +export type AccessorHandle = Branded; + +/** + * Structural subset of {@link FieldTraversalStep} (from `@platforma-sdk/model`) + * needed by the host/sandbox column-providers traversal. + * + * Defined locally so this module does not depend on the sandbox `render` + * subtree. Both `TreeNodeAccessor.traverse` (sandbox) and + * `PlTreeNodeAccessor.traverse` (host) accept a superset of these fields, so + * structural compatibility is preserved. + */ +export interface FieldTraversalStepLike { + /** Field name */ + readonly field: string; + /** Asserted field type — used by `accessor.traverse` to validate. */ + readonly assertFieldType?: "Input" | "Output" | "Service" | "OTW" | "Dynamic" | "MTW"; + /** Don't terminate chain if current resource or field has an error associated. */ + readonly ignoreError?: true; +} + +/** + * Raw entry returned by {@link GlobalCfgRenderCtxMethods.getUpstreamBlockCtx} + * on the sandbox side, or by `collectUpstreamBlockCtx` on the host side. + * Carries handle ids only — providers wrap them into accessor instances as + * needed. + * + * Generic over the handle type so the same shape backs both: + * - sandbox (`AHandle = AccessorHandle`, a `Branded`) + * - host (`AHandle = PlTreeNodeAccessor`, the resolved accessor instance) + * + * Default `AHandle = string` since `AccessorHandle` is a brand on `string` — + * the default is safe for the sandbox case. + */ +export interface UpstreamBlockCtx { + blockId: string; + prodCtx?: AHandle; + stagingCtx?: AHandle; + /** True when the `prodCtx` ctx-holder exists but `prodUiCtx` is still rendering. */ + prodIncomplete?: boolean; + /** True when the `stagingCtx` ctx-holder exists but `stagingUiCtx` is still rendering. */ + stagingIncomplete?: boolean; +} + +/** + * Minimal accessor surface used by column-providers / column-registry + * traversal. + * + * Both sandbox `TreeNodeAccessor` and host `PlTreeNodeAccessor` satisfy this + * contract directly — `traverse(step)` is defined on both, and the other + * members already match by shape. No `resolvePath` member: when canonical + * `PObjectId`s are required (local-id construction inside the + * outputs/prerun branch), the traversal helpers thread the path explicitly. + */ +export interface AccessorLike> { + /** Resource type carried by the underlying node (only `.name` is used). */ + readonly resourceType: { readonly name: string }; + + /** + * Single-step field traversal. Returns `undefined` when the field is + * absent / unresolved (with `ignoreError`). Same shape on sandbox and host — + * sandbox `TreeNodeAccessor.traverse` is an alias for `resolveAny` and host + * `PlTreeNodeAccessor.traverse` is the canonical method. + */ + traverse(step: FieldTraversalStepLike): Self | undefined; + + /** List input-field names on this node. */ + listInputFields(): string[]; + + /** Whether the input-field collection on this node is finalized. */ + getInputsLocked(): boolean; + + /** Whether this node has a data payload attached. */ + hasData(): boolean; + + /** Decode the data payload as JSON. Returns `undefined` if no data. */ + getDataAsJson(): T | undefined; +} + +/** + * One indexed column — the canonical record produced by every traversal. + * Carries everything needed to read spec/data/status under a stable id. + * + * Generic over the accessor flavour so the same record shape works for both + * the sandbox `TreeNodeAccessor` and the host `PlTreeNodeAccessor`. + */ +export type LeafEntry> = { + /** PFrame accessor that owns `.spec` / `.data` fields. */ + accessor: A; + /** Field-name prefix inside the PFrame. */ + name: string; + /** Canonical id under which this column is reachable. */ + id: PObjectId; +}; + +/** + * Base interface for id-indexed column providers — the surface + * {@link ColumnRegistry} consumes. Generic over the concrete accessor flavour + * so it can back both sandbox (`TreeNodeAccessor`) and host + * (`PlTreeNodeAccessor` adapter) registries. + */ +export interface ColumnEntriesProvider> { + /** Map of canonical {@link PObjectId} → {@link LeafEntry} for every column reachable from this source. */ + getPObjectEntries(): ReadonlyMap>; + /** Whether enumeration of columns from this source has finalised. */ + isFinal(): boolean; +} diff --git a/lib/model/common/src/drivers/columns/columns_collection_driver.ts b/lib/model/common/src/drivers/columns/columns_collection_driver.ts new file mode 100644 index 0000000000..ea9b5a7269 --- /dev/null +++ b/lib/model/common/src/drivers/columns/columns_collection_driver.ts @@ -0,0 +1,146 @@ +import type { Branded } from "../../branding"; +import type { PoolEntry } from "../../pool_entry"; +import type { AccessorHandle, AccessorLike, UpstreamBlockCtx } from "../../columns/types"; +import type { ColumnUniversalId } from "../pframe/spec/ids"; +import type { ColumnsDiscoverOptions, ColumnsFilterOptions } from "./discover_columns_options"; +import type { PFrameSpecDriver } from "../pframe/spec_driver"; +import type { PColumnSpec } from "../pframe/spec/spec"; +import type { PObjectId } from "../../pool"; + +/** + * Opaque host-owned handle for a `ColumnsCollection` instance. Issued by + * {@link ColumnsCollectionDriver.create}, refcounted by the driver, and + * pinned to the active render ctx via the VM injector — sandbox never + * sees raw refcounting. + */ +export type CollectionHandle = Branded; + +/** + * JSON descriptor crossing the VM bridge in place of a sandbox + * `ColumnsSource`. Always plain data — no closures, no class instances. + * + * - `"collection"` – reference another driver-managed collection by handle + * (chaining, splicing). + * - `"result_pool"` – fan-out into the host's current render ctx upstream + * block ctxes. Carries no payload — the host always uses its own pool. + * - `"accessor"` – walk the host tree starting at the given accessor + * handle from a `path` prefix. + * - `"ids"` – pre-resolved id list (sandbox-materialised provider). + */ +export type SerializedColumnsSource = + | { readonly kind: "collection"; readonly handle: CollectionHandle } + | { readonly kind: "result_pool" } + | { readonly kind: "accessor"; readonly accessor: AccessorHandle; readonly path: string[] } + | { readonly kind: "ids"; readonly ids: ColumnUniversalId[]; readonly isFinal: boolean }; + +/** + * Per-call host bindings the driver needs to resolve sources whose + * shape references render-ctx state (`"accessor"`, `"result_pool"`). + * + * The VM injector inside `pl-middle-layer` supplies these on every call; + * UI-side direct callers that only build collections out of `"ids"` / + * `"collection"` sources may omit the bindings entirely. + * + * Parameterised on the concrete accessor flavour so host implementations + * keep their static types (e.g. `PlTreeNodeAccessor`) without leaking that + * dependency into `@milaboratories/pl-model-common`. + */ +export interface ColumnsCollectionDriverHost = AccessorLike> { + /** Resolve an {@link AccessorHandle} to the host's concrete accessor. */ + resolveAccessor(handle: AccessorHandle): A; + + /** Snapshot of upstream-block ctx pairs from the current render ctx. */ + getUpstreamBlockCtxes(): ReadonlyArray>; + + /** + * Per-call spec driver. The injector supplies the active render ctx's + * `PFrameSpec` service; `discover` / `filter` use it to build a spec + * frame and run a single discovery query. + */ + getSpecDriver(): PFrameSpecDriver; + + /** + * Resolve the canonical {@link PColumnSpec} for a leaf {@link PObjectId}. + * Returns `undefined` when the id is not present in the active registry + * (e.g. handed to the driver via a `{kind:"ids"}` source whose underlying + * column has since left the visible scope). Override-wrapped ids are + * unwrapped by the caller — this method only resolves the underlying leaf. + */ + resolveSpec(id: PObjectId): PColumnSpec | undefined; +} + +/** + * Sandbox / UI view of the `ColumnsCollection` driver. Same methods as + * {@link ColumnsCollectionDriver}, but the `host` parameters are dropped — + * the VM bridge / UI wrapper supplies them on every call so callers only + * pass plain data (handles + source descriptors + option objects). + * + * `getService("columnsCollection")` returns a value of this shape on both + * sandbox and UI sides. + */ +export interface ColumnsCollectionDriverModel { + /** Build a fresh collection from the supplied source descriptors. */ + create(sources: ReadonlyArray): CollectionHandle; + + /** Whether the collection currently exposes zero columns. */ + isEmpty(handle: CollectionHandle): boolean; + + /** Whether enumeration is finalised across every contributing source. */ + isFinal(handle: CollectionHandle): boolean; + + /** Canonical id list for the columns visible through this collection. */ + getColumns(handle: CollectionHandle): ColumnUniversalId[]; + + /** Append one or more sources and return a fresh collection handle. */ + addSource( + handle: CollectionHandle, + sources: ReadonlyArray, + ): CollectionHandle; + + /** Anchored/selector-driven discovery. Returns a fresh handle. */ + discover(handle: CollectionHandle, options: ColumnsDiscoverOptions): CollectionHandle; + + /** Selector-only filter (no anchor traversal). Returns a fresh handle. */ + filter(handle: CollectionHandle, options: ColumnsFilterOptions): CollectionHandle; +} + +/** + * Synchronous host-side driver for `ColumnsCollection` operations. All + * collection state lives on the host side, addressable through opaque + * {@link CollectionHandle}s. Every handle-minting method returns a + * {@link PoolEntry} so callers can wire the refcount into their own + * lifecycle; the VM bridge pins each entry to the active render ctx and + * forwards only the handle string to sandbox. + * + * Sandbox / UI callers consume {@link ColumnsCollectionDriverModel} + * instead; the bridge maps that surface onto this one by injecting + * {@link ColumnsCollectionDriverHost} bindings. + */ +export interface ColumnsCollectionDriver { + create( + sources: ReadonlyArray, + host: ColumnsCollectionDriverHost, + ): PoolEntry; + + isEmpty(handle: CollectionHandle): boolean; + isFinal(handle: CollectionHandle): boolean; + getColumns(handle: CollectionHandle, host: ColumnsCollectionDriverHost): ColumnUniversalId[]; + + addSource( + handle: CollectionHandle, + sources: ReadonlyArray, + host: ColumnsCollectionDriverHost, + ): PoolEntry; + + discover( + handle: CollectionHandle, + options: ColumnsDiscoverOptions, + host: ColumnsCollectionDriverHost, + ): PoolEntry; + + filter( + handle: CollectionHandle, + options: ColumnsFilterOptions, + host: ColumnsCollectionDriverHost, + ): PoolEntry; +} diff --git a/lib/model/common/src/drivers/columns/discover_columns_options.ts b/lib/model/common/src/drivers/columns/discover_columns_options.ts new file mode 100644 index 0000000000..d8cc40f0bc --- /dev/null +++ b/lib/model/common/src/drivers/columns/discover_columns_options.ts @@ -0,0 +1,91 @@ +import type { ColumnSelector, RelaxedColumnSelector } from "../../columns/column_selector"; +import type { PlRef } from "../../ref"; +import type { PObjectId } from "../../pool"; +import type { PColumnSpec, AxisQualification } from "../pframe/spec"; +import type { DiscoverColumnsConstraints } from "../pframe/spec_driver"; + +/** + * Axis matching behaviour applied to `discover` requests. + * + * - `enrichment` (default) — anchor axes may float over un-mapped hit axes; + * used by tooling that "extends" a query. + * - `related` — both source and hit axes may float; widest match. + * - `exact` — no floating, no qualifications; strict equality. + */ +export type MatchingMode = "enrichment" | "related" | "exact"; + +/** + * Single entry accepted by `DiscoverColumnsOptions.anchors`. All variants are + * trivially JSON-serialisable so the option carrier crosses the + * sandbox/host VM bridge unchanged. + */ +export type AnchorEntry = PlRef | PObjectId | PColumnSpec | RelaxedColumnSelector; + +/** Qualifications needed for both already-integrated anchor columns and the hit column. */ +export interface MatchQualifications { + /** Qualifications for already-integrated anchor columns */ + readonly forQueries?: Record; + /** Qualifications for the hit column. */ + readonly forHit?: AxisQualification[]; +} + +/** + * Options object accepted by sandbox `discoverColumns()` and by the host + * `ColumnsCollectionDriver.discover` / `.filter` methods. Pure JSON shape — + * no class instances, no closures. + */ +export interface DiscoverColumnsOptions { + /** Include columns matching these selectors. If omitted, includes all. */ + include?: ColumnSelector; + /** Exclude columns matching these selectors. */ + exclude?: ColumnSelector; + /** Axis matching behavior. Default: 'enrichment'. Ignored if no anchors. */ + mode?: MatchingMode; + /** Anchors enable axis-aware discovery + linker traversal. */ + // @todo: migrate to array + anchors?: Record; + /** Maximum linker hops. Default: 4 when anchors present, 0 otherwise. */ + maxHops?: number; +} + +/** + * Options accepted by `ColumnsCollection.discover` / driver `.discover`. + * Traversal scope (`mode`, `maxHops`) must be specified explicitly — the + * defaults from {@link DiscoverColumnsOptions} are intentionally surfaced as + * required choices at the discovery entrypoint. + */ +export type ColumnsDiscoverOptions = DiscoverColumnsOptions; + +/** + * Options accepted by `ColumnsCollection.filter` / driver `.filter`. Traversal + * scope is fixed by the source collection, so `mode` / `maxHops` are not part + * of the filter surface — only `include` / `exclude` / `anchors`. + */ +export type ColumnsFilterOptions = Omit; + +/** Translate a {@link MatchingMode} into the boolean-flag form the spec driver consumes. */ +export function matchingModeToConstraints(mode: MatchingMode): DiscoverColumnsConstraints { + switch (mode) { + case "enrichment": + return { + allowFloatingSourceAxes: true, + allowFloatingHitAxes: false, + allowSourceQualifications: true, + allowHitQualifications: true, + }; + case "related": + return { + allowFloatingSourceAxes: true, + allowFloatingHitAxes: true, + allowSourceQualifications: true, + allowHitQualifications: true, + }; + case "exact": + return { + allowFloatingSourceAxes: false, + allowFloatingHitAxes: false, + allowSourceQualifications: false, + allowHitQualifications: false, + }; + } +} diff --git a/lib/model/common/src/drivers/columns/index.ts b/lib/model/common/src/drivers/columns/index.ts new file mode 100644 index 0000000000..3702ecae4d --- /dev/null +++ b/lib/model/common/src/drivers/columns/index.ts @@ -0,0 +1,2 @@ +export * from "./columns_collection_driver"; +export * from "./discover_columns_options"; diff --git a/lib/model/common/src/drivers/index.ts b/lib/model/common/src/drivers/index.ts index 28da35486b..bbf233be71 100644 --- a/lib/model/common/src/drivers/index.ts +++ b/lib/model/common/src/drivers/index.ts @@ -7,4 +7,5 @@ export * from "./log"; export * from "./ls"; export * from "./pframe"; +export * from "./columns"; export * from "./ChunkedStreamReader"; diff --git a/lib/model/common/src/drivers/pframe/index.ts b/lib/model/common/src/drivers/pframe/index.ts index 498b8460ba..0546efc190 100644 --- a/lib/model/common/src/drivers/pframe/index.ts +++ b/lib/model/common/src/drivers/pframe/index.ts @@ -4,7 +4,6 @@ export * from "./filter_spec"; export * from "./data_types"; export * from "./find_columns"; export * from "./pframe"; -export * from "./spec/spec"; export * from "./table"; export * from "./table_calculate"; export * from "./table_common"; diff --git a/lib/model/common/src/drivers/pframe/query/query_common.ts b/lib/model/common/src/drivers/pframe/query/query_common.ts index 7c135a6f6d..1317a1539e 100644 --- a/lib/model/common/src/drivers/pframe/query/query_common.ts +++ b/lib/model/common/src/drivers/pframe/query/query_common.ts @@ -1157,3 +1157,27 @@ export interface QueryTransformColumns { /** Derived columns to compute (at least one). */ columns: [TransformColumnEntry, ...TransformColumnEntry[]]; } + +/** + * Spec-override query operation — client-side-only structural node. + * + * Overlays a {@link SpecOverrides} patch on top of the inner query's spec. + * Carries no topological change — it is collapsed at the host boundary + * (`resolvePColumn`) before the query reaches pframe-engine. The engine + * never sees this node. + * + * Emitted by `ColumnOverridedRecipe.getQuery()`; the only currently + * supported shape is `specOverride{ input: , override }` + * (i.e. `Overrided`). More complex projections under Overrided are + * a future engine work item. + * + * @template Q - Input query type + * @template SO - Spec override type + */ +export interface QuerySpecOverride { + type: "specOverride"; + /** Input query whose spec is to be overridden. */ + input: Q; + /** Spec override patch to overlay on the inner spec. */ + override: SO; +} diff --git a/lib/model/common/src/drivers/pframe/query/query_spec.ts b/lib/model/common/src/drivers/pframe/query/query_spec.ts index 56c74c4958..357cc8f95d 100644 --- a/lib/model/common/src/drivers/pframe/query/query_spec.ts +++ b/lib/model/common/src/drivers/pframe/query/query_spec.ts @@ -1,4 +1,5 @@ import type { PObjectId } from "../../../pool"; +import type { ColumnUniversalId } from "../spec/ids"; import type { ExprAxisRef, ExprCast, @@ -28,10 +29,12 @@ import type { QuerySliceAxes, QuerySort, QuerySparseToDenseColumn, + QuerySpecOverride, QuerySymmetricJoin, QueryTransformColumns, } from "./query_common"; import type { Domain, PColumnIdAndSpec, SingleAxisSelector } from "../spec"; +import type { SpecOverrides } from "../spec/overrided"; /** * Join entry for spec-layer queries — the base join entry extended with @@ -43,8 +46,8 @@ import type { Domain, PColumnIdAndSpec, SingleAxisSelector } from "../spec"; * qualifications: [{ axis: { name: 'sample' }, contextDomain: { ... } }] * } */ -export type SpecQueryJoinEntry = QueryJoinEntry> & { - qualifications?: { +export type SpecQueryJoinEntry = QueryJoinEntry> & { + qualifications?: readonly { /** Axis to qualify. */ axis: SingleAxisSelector; /** Additional domain constraints for this axis. */ @@ -53,46 +56,54 @@ export type SpecQueryJoinEntry = QueryJoinEntry> & { }; /** @see QueryColumn */ -export type SpecQueryColumn = QueryColumn; +export type SpecQueryColumn = QueryColumn; /** @see QueryInlineColumn */ export type SpecQueryInlineColumn = QueryInlineColumn; /** @see QuerySparseToDenseColumn */ -export type SpecQuerySparseToDenseColumn = QuerySparseToDenseColumn< +export type SpecQuerySparseToDenseColumn = QuerySparseToDenseColumn< C, SingleAxisSelector, PColumnIdAndSpec >; /** @see QuerySymmetricJoin */ -export type SpecQuerySymmetricJoin = QuerySymmetricJoin>; +export type SpecQuerySymmetricJoin = QuerySymmetricJoin< + SpecQueryJoinEntry +>; /** @see QueryOuterJoin */ -export type SpecQueryOuterJoin = QueryOuterJoin>; -/** - * Linker side of a spec-layer linker-join. - * - * At the spec layer the linker is just a column reference — integration artifacts - * (axes mapping, one-side indices) are derived during spec→data conversion. - */ -export type SpecQueryLinkerJoinLinker = { - /** Linker column reference. */ - column: C; -}; +export type SpecQueryOuterJoin = QueryOuterJoin>; /** @see QueryLinkerJoin */ -export type SpecQueryLinkerJoin = QueryLinkerJoin< - SpecQueryLinkerJoinLinker, +export type SpecQueryLinkerJoin = QueryLinkerJoin< + SpecQuery, SpecQueryJoinEntry >; /** @see QuerySliceAxes */ -export type SpecQuerySliceAxes = QuerySliceAxes, SingleAxisSelector>; +export type SpecQuerySliceAxes = QuerySliceAxes< + SpecQuery, + SingleAxisSelector +>; /** @see QuerySort */ -export type SpecQuerySort = QuerySort, SpecQueryExpression>; +export type SpecQuerySort = QuerySort, SpecQueryExpression>; /** @see QueryFilter */ -export type SpecQueryFilter = QueryFilter, SpecQueryBooleanExpression>; +export type SpecQueryFilter = QueryFilter< + SpecQuery, + SpecQueryBooleanExpression +>; /** @see QueryTransformColumns */ -export type SpecQueryTransformColumns = QueryTransformColumns< +export type SpecQueryTransformColumns = QueryTransformColumns< SpecQuery, SpecQueryExpression, PColumnIdAndSpec >; +/** + * Client-side spec-override node — collapsed at the host boundary, never + * sent to pframe-engine. + * + * @see QuerySpecOverride + */ +export type SpecQuerySpecOverride = QuerySpecOverride< + SpecQuery, + SpecOverrides +>; /** * Union of all spec layer query types. @@ -108,8 +119,9 @@ export type SpecQueryTransformColumns = QueryTransformColumns< * - Leaf nodes: column, inlineColumn, sparseToDenseColumn * - Join operations: innerJoin, fullJoin, outerJoin, linkerJoin * - Transformations: sliceAxes, sort, filter, transformColumns + * - Client-side overlays: specOverride (collapsed before reaching the engine) */ -export type SpecQuery = +export type SpecQuery = | SpecQueryColumn | SpecQueryInlineColumn | SpecQuerySparseToDenseColumn @@ -119,12 +131,13 @@ export type SpecQuery = | SpecQuerySliceAxes | SpecQuerySort | SpecQueryFilter - | SpecQueryTransformColumns; + | SpecQueryTransformColumns + | SpecQuerySpecOverride; /** @see ExprAxisRef */ export type SpecExprAxisRef = ExprAxisRef; /** @see ExprColumnRef */ -export type SpecExprColumnRef = ExprColumnRef; +export type SpecExprColumnRef = ExprColumnRef; export type SpecQueryExpression = | SpecExprColumnRef diff --git a/lib/model/common/src/drivers/pframe/query/utils.test.ts b/lib/model/common/src/drivers/pframe/query/utils.test.ts index 5ab4492dee..519d047c02 100644 --- a/lib/model/common/src/drivers/pframe/query/utils.test.ts +++ b/lib/model/common/src/drivers/pframe/query/utils.test.ts @@ -85,13 +85,13 @@ describe("traverseQuerySpec", () => { it("transforms columns inside linkerJoin", () => { const q: Q = { type: "linkerJoin", - linker: { column: "l" }, + linker: col("l"), secondary: [entry(col("a")), entry(col("b"))], }; const result = traverseQuerySpec(q, { column: (c) => c.toUpperCase() }); expect(result).toEqual({ type: "linkerJoin", - linker: { column: "L" }, + linker: { type: "column", column: "L" }, secondary: [entry({ type: "column", column: "A" }), entry({ type: "column", column: "B" })], }); }); @@ -180,7 +180,7 @@ describe("mapSpecQueryColumns", () => { primary: entry(col("a")), secondary: [entry(col("b"))], }; - const result = mapSpecQueryColumns(q, (c) => c.toUpperCase()); + const result = mapSpecQueryColumns(q, { column: (c) => c.toUpperCase() }); expect(collectSpecQueryColumns(result)).toEqual(["A", "B"]); }); }); @@ -229,7 +229,7 @@ describe("collectSpecQueryColumns", () => { it("collects linker and secondary columns from linkerJoin", () => { const q: Q = { type: "linkerJoin", - linker: { column: "l" }, + linker: col("l"), secondary: [entry(col("a")), entry(col("b"))], }; expect(collectSpecQueryColumns(q)).toEqual(["l", "a", "b"]); @@ -263,13 +263,13 @@ describe("sortSpecQuery", () => { it("sorts linkerJoin secondary entries (linker column unchanged)", () => { const q: SpecQuery = { type: "linkerJoin", - linker: { column: pid("l") }, + linker: pcol("l"), secondary: [pentry(pcol("c")), pentry(pcol("a")), pentry(pcol("b"))], }; const result = sortSpecQuery(q); expect(result).toEqual({ type: "linkerJoin", - linker: { column: "l" }, + linker: pcol("l"), secondary: [pentry(pcol("a")), pentry(pcol("b")), pentry(pcol("c"))], }); }); diff --git a/lib/model/common/src/drivers/pframe/query/utils.ts b/lib/model/common/src/drivers/pframe/query/utils.ts index 5ae9d495e0..616408af85 100644 --- a/lib/model/common/src/drivers/pframe/query/utils.ts +++ b/lib/model/common/src/drivers/pframe/query/utils.ts @@ -79,17 +79,25 @@ export function traverseQuerySpec( secondary: query.secondary.map(traverseEntry), }; break; - case "linkerJoin": + case "linkerJoin": { + // Back-compat: pre-#c7309fc8 blocks emit `linker` as `{ column }` without + // a `type` — tag it as a `column` node so recursion handles it normally. + const linker = + "type" in query.linker + ? query.linker + : ({ type: "column", column: (query.linker as { column: C1 }).column } as SpecQuery); result = { ...query, - linker: { ...query.linker, column: visitor.column(query.linker.column) }, + linker: traverseQuerySpec(linker, visitor), secondary: query.secondary.map(traverseEntry), }; break; + } case "filter": case "sort": case "sliceAxes": case "transformColumns": + case "specOverride": result = { ...query, input: traverseQuerySpec(query.input, visitor) }; break; default: @@ -102,9 +110,16 @@ export function traverseQuerySpec( /** Recursively maps all column references in a SpecQuery tree. */ export function mapSpecQueryColumns( query: SpecQuery, - cb: (c: C1) => C2, + visitor: { + /** Transform column references in leaf nodes (column, sparseToDenseColumn). */ + column: (c: C1) => C2; + /** Visit a node after its children have been traversed. */ + node?: (node: SpecQuery) => SpecQuery; + /** Visit a join entry after its inner query has been traversed. */ + joinEntry?: (entry: SpecQueryJoinEntry) => SpecQueryJoinEntry; + }, ): SpecQuery { - return traverseQuerySpec(query, { column: cb }); + return traverseQuerySpec(query, visitor); } /** Collects all column references from a SpecQuery tree. */ @@ -227,9 +242,8 @@ function cmpQuerySpec(lhs: SpecQuery, rhs: SpecQuery): number { } case "linkerJoin": { const rhsLinker = rhs as typeof lhs; - if (lhs.linker.column !== rhsLinker.linker.column) { - return lhs.linker.column < rhsLinker.linker.column ? -1 : 1; - } + const cmp = cmpQuerySpec(lhs.linker, rhsLinker.linker); + if (cmp !== 0) return cmp; if (lhs.secondary.length !== rhsLinker.secondary.length) { return lhs.secondary.length - rhsLinker.secondary.length; } @@ -245,6 +259,12 @@ function cmpQuerySpec(lhs: SpecQuery, rhs: SpecQuery): number { return cmpQuerySpec(lhs.input, (rhs as typeof lhs).input); case "filter": return cmpQuerySpec(lhs.input, (rhs as typeof lhs).input); + case "specOverride": { + const rhsSo = rhs as typeof lhs; + const cmp = cmpQuerySpec(lhs.input, rhsSo.input); + if (cmp !== 0) return cmp; + return canonicalizeJson(lhs.override).localeCompare(canonicalizeJson(rhsSo.override)); + } case "transformColumns": { const rhsTc = rhs as typeof lhs; const cmp = cmpQuerySpec(lhs.input, rhsTc.input); diff --git a/lib/model/common/src/drivers/pframe/spec/discovered_column.ts b/lib/model/common/src/drivers/pframe/spec/discovered_column.ts index f212cec23f..3082f1e388 100644 --- a/lib/model/common/src/drivers/pframe/spec/discovered_column.ts +++ b/lib/model/common/src/drivers/pframe/spec/discovered_column.ts @@ -1,35 +1,39 @@ -import { Branded, throwError } from "@milaboratories/helpers"; +import { type Branded, throwError } from "@milaboratories/helpers"; +import { type CanonicalizedJson, canonicalizeJson, parseJsonSafely } from "../../../json"; import { PObjectId } from "../../../pool"; import { AxisQualification } from "./selectors"; -import { canonicalizeJson } from "../../../json"; +import { ColumnUniversalId } from "./ids"; -export type DiscoveredPColumn = { - column: PObjectId; +export interface ColumnDiscoveredKey { + __isDiscovered: true; + column: ColumnUniversalId; path?: PathItem[]; columnQualifications?: AxisQualification[]; queriesQualifications?: Record; -}; +} -export type DiscoveredPColumnId = Branded; // CanonicalizedJson; +export type ColumnDiscoveredId = Branded< + CanonicalizedJson, + "ColumnDiscoveredId" +>; type PathItem = { type: "linker"; - column: PObjectId; + column: ColumnUniversalId; }; -export function isDiscoveredPColumn(obj: unknown): obj is DiscoveredPColumn { - return ( - typeof obj === "object" && - obj !== null && - "path" in obj && - "column" in obj && - "columnQualifications" in obj && - "queriesQualifications" in obj - ); +export function isColumnDiscoveredKey(obj: unknown): obj is ColumnDiscoveredKey { + return typeof obj === "object" && obj !== null && "__isDiscovered" in obj; } -export function distillDiscoveredPColumn(props: DiscoveredPColumn): DiscoveredPColumn { +export function isColumnDiscoveredId(str: unknown): str is ColumnDiscoveredId { + if (typeof str !== "string") return false; + return isColumnDiscoveredKey(parseJsonSafely(str)); +} + +export function distillColumnDiscoveredKey(props: ColumnDiscoveredKey): ColumnDiscoveredKey { return { + __isDiscovered: true, column: props.column, path: Array.isArray(props.path) && props.path.length > 0 ? props.path : undefined, columnQualifications: @@ -43,30 +47,41 @@ export function distillDiscoveredPColumn(props: DiscoveredPColumn): DiscoveredPC }; } -export function createDiscoveredPColumnId(props: { - column: PObjectId; +export function createColumnDiscoveredId(props: { + column: ColumnUniversalId; + path?: PathItem[]; + columnQualifications?: AxisQualification[]; + queriesQualifications?: Record; +}): ColumnDiscoveredId { + return stringifyColumnDiscoveredId(props); +} + +export function createColumnDiscoveredKey(props: { + column: ColumnUniversalId; path?: PathItem[]; columnQualifications?: AxisQualification[]; queriesQualifications?: Record; -}): DiscoveredPColumnId { - return stringifyDiscoveredPColumnId(props); +}): ColumnDiscoveredKey { + return distillColumnDiscoveredKey({ __isDiscovered: true, ...props }); } -export function parseDiscoveredPColumnId(id: DiscoveredPColumnId): DiscoveredPColumn { +export function parseColumnDiscoveredId(id: ColumnDiscoveredId): ColumnDiscoveredKey { try { const parsed = JSON.parse(id); - return isDiscoveredPColumn(parsed) + return isColumnDiscoveredKey(parsed) ? parsed : throwError("Parsed object is not a valid DiscoveredPColumn"); } catch { throw new Error( - "Invalid DiscoveredPColumnId: not a valid JSON or does not conform to DiscoveredPColumn structure", + "Invalid ColumnDiscoveredId: not a valid JSON or does not conform to DiscoveredPColumn structure", ); } } -export function stringifyDiscoveredPColumnId(id: DiscoveredPColumn) { - return canonicalizeJson( - distillDiscoveredPColumn(id), - ) as string as DiscoveredPColumnId; +export function stringifyColumnDiscoveredId( + id: Omit, +): ColumnDiscoveredId { + return canonicalizeJson( + distillColumnDiscoveredKey({ __isDiscovered: true, ...id }), + ) as ColumnDiscoveredId; } diff --git a/lib/model/common/src/drivers/pframe/spec/filtered_column.ts b/lib/model/common/src/drivers/pframe/spec/filtered_column.ts index 73ba3249dc..7e4d7552dc 100644 --- a/lib/model/common/src/drivers/pframe/spec/filtered_column.ts +++ b/lib/model/common/src/drivers/pframe/spec/filtered_column.ts @@ -1,4 +1,8 @@ +import { type Branded } from "@milaboratories/helpers"; +import { type CanonicalizedJson, canonicalizeJson, parseJsonSafely } from "../../../json"; import type { AnchoredPColumnId } from "./selectors"; +import { ColumnUniversalId } from "./ids"; +import type { PColumnSpec } from "./spec"; /** Value of an axis filter */ export type AxisFilterValue = number | string; @@ -29,13 +33,72 @@ export type FilteredPColumn = { axisFilters: AFI[]; }; +/** @deprecated */ export type FilteredPColumnId = FilteredPColumn; /** * Checks if a given value is a FilteredPColumn * @param id - The value to check * @returns True if the value is a FilteredPColumn, false otherwise + * @deprecated use {@link isColumnFilteredKey} for the key form and `source` checks for the id form */ export function isFilteredPColumn(id: unknown): id is FilteredPColumn { return typeof id === "object" && id !== null && "source" in id && "axisFilters" in id; } + +/** + * `source` is either a leaf {@link PObjectId} or a {@link ColumnDiscoveredId}. + * Filtered never nests inside Filtered (flat-merge invariant), and Filtered is + * never the outer wrapper around Overrided — Overrided is always outermost. + */ +export interface ColumnFilteredKey { + __isFiltered: true; + source: ColumnUniversalId; + axisFilters: AxisFilterByIdx[]; +} + +export type ColumnFilteredId = Branded, "ColumnFilteredId">; + +export function stringifyColumnFilteredId( + key: Omit, +): ColumnFilteredId { + return canonicalizeJson(createColumnFilteredKey(key)) as ColumnFilteredId; +} + +export function isColumnFilteredKey(obj: unknown): obj is ColumnFilteredKey { + return typeof obj === "object" && obj !== null && "__isFiltered" in obj; +} + +export function isColumnFilteredId(id: unknown): id is ColumnFilteredId { + if (typeof id !== "string") return false; + return isColumnFilteredKey(parseJsonSafely(id)); +} + +export function createColumnFilteredId(props: { + source: ColumnUniversalId; + axisFilters: AxisFilterByIdx[]; +}): ColumnFilteredId { + return stringifyColumnFilteredId(props); +} + +export function createColumnFilteredKey(props: { + source: ColumnUniversalId; + axisFilters: AxisFilterByIdx[]; +}): ColumnFilteredKey { + return { __isFiltered: true, ...props }; +} + +/** + * Drop the axes pinned by `axisFilters` from a {@link PColumnSpec}'s + * `axesSpec`. Indices are positional against `spec.axesSpec`. Returns `spec` + * unchanged when there are no filters. + * + * Single source of the Filtered-layer spec math — shared by + * `reconstructSpecFromId` (host, id-walking) and `ColumnFilteredRecipe.getSpec` + * (sandbox, recipe-graph walking). + */ +export function applyAxisFilters(spec: PColumnSpec, axisFilters: AxisFilterByIdx[]): PColumnSpec { + if (axisFilters.length === 0) return spec; + const fixed = new Set(axisFilters.map(([i]) => i)); + return { ...spec, axesSpec: spec.axesSpec.filter((_, i) => !fixed.has(i)) }; +} diff --git a/lib/model/common/src/drivers/pframe/spec/ids.ts b/lib/model/common/src/drivers/pframe/spec/ids.ts index 86b2179eec..caa34b2aeb 100644 --- a/lib/model/common/src/drivers/pframe/spec/ids.ts +++ b/lib/model/common/src/drivers/pframe/spec/ids.ts @@ -1,34 +1,159 @@ -import type { Branded } from "../../../branding"; import type { AnchoredPColumnId } from "./selectors"; -import type { FilteredPColumnId } from "./filtered_column"; -import canonicalize from "canonicalize"; -import type { PObjectId } from "../../../pool"; -import { DiscoveredPColumn } from "./discovered_column"; +import { + applyAxisFilters, + isColumnFilteredKey, + type ColumnFilteredId, + type ColumnFilteredKey, + type FilteredPColumnId, +} from "./filtered_column"; +import { + createPObjectId, + isPObjectId, + isPObjectKey, + LocalPObjectKey, + type GlobalPObjectId, + type GlobalPObjectKey, + type LocalPObjectId, + type PObjectId, +} from "../../../pool"; +import { + isColumnDiscoveredKey, + type ColumnDiscoveredId, + type ColumnDiscoveredKey, +} from "./discovered_column"; +import { throwError } from "@milaboratories/helpers"; +import { + applySpecOverrides, + isColumnOverridedKey, + type ColumnOverridedId, + type ColumnOverridedKey, +} from "./overrided"; +import { canonicalizeJson } from "../../../json"; +import { AxisSpec, PColumnSpec } from "./spec"; +import { isString } from "es-toolkit"; + +/** + * Per-axis patches keyed by positional index in the base spec's `axesSpec`. + * + * Using position rather than `name` lets us disambiguate linker-style specs + * that carry multiple axes with the same `name` differentiated by `domain` / + * `contextDomain` (e.g. a `group`, `group/primary`, `group/secondary` triple). + * + * A patch at index `>= base.axesSpec.length` appends a new axis at that slot. + */ +export type AxisPatches = Record>; /** * Universal column identifier optionally anchored and optionally filtered. + * @deprecated use {@link ColumnUniversalKey} */ -export type UniversalPColumnId = AnchoredPColumnId | FilteredPColumnId | DiscoveredPColumn; +export type UniversalPColumnId = AnchoredPColumnId | FilteredPColumnId; /** * Canonically serialized {@link UniversalPColumnId}. + * @deprecated use {@link ColumnUniversalId} */ -export type SUniversalPColumnId = Branded; +export type SUniversalPColumnId = ColumnUniversalId; +// export type SUniversalPColumnId = Branded; + +export type ColumnUniversalKey = + | LocalPObjectKey + | GlobalPObjectKey + | ColumnFilteredKey + | ColumnDiscoveredKey + | ColumnOverridedKey; + +export type ColumnUniversalId = + | LocalPObjectId + | GlobalPObjectId + | ColumnFilteredId + | ColumnDiscoveredId + | ColumnOverridedId; /** - * Canonically serializes a {@link UniversalPColumnId} to a string. - * @param id - The column identifier to serialize - * @returns The canonically serialized string + * Canonically serializes a column key to a branded string id. Accepts both + * the new {@link ColumnUniversalKey} and the deprecated {@link UniversalPColumnId} + * (anchored / old filtered object form). */ -export function stringifyColumnId(id: UniversalPColumnId): SUniversalPColumnId { - return canonicalize(id)! as SUniversalPColumnId; +export function stringifyColumnId(id: ColumnUniversalKey | UniversalPColumnId): ColumnUniversalId { + return canonicalizeJson(id) as ColumnUniversalId; +} + +/** + * Parses a canonically serialized column id back to its key form. + */ +export function parseColumnId(str: ColumnUniversalId): ColumnUniversalKey { + return JSON.parse(str) as ColumnUniversalKey; +} + +export function parseColumnIdSafety( + str: ColumnUniversalId, + fallback = undefined, +): ColumnUniversalKey | typeof fallback { + try { + return JSON.parse(str) as ColumnUniversalKey; + } catch { + return fallback; + } } /** - * Parses a canonically serialized {@link UniversalPColumnId} from a string. - * @param str - The string to parse - * @returns The parsed column identifier + * Walk a rich column id down to its terminal leaf {@link PObjectId}. */ -export function parseColumnId(str: SUniversalPColumnId): UniversalPColumnId { - return JSON.parse(str) as UniversalPColumnId; +export function extractPObjectId(id: ColumnUniversalId | ColumnUniversalKey): PObjectId { + if (isString(id)) { + if (isPObjectId(id)) return id; + + const parsed = + parseColumnIdSafety(id) ?? + throwError(`extractPObjectId: id "${id}" is not a valid canonical column id`); + return extractPObjectId(parsed); + } + + if (isPObjectKey(id)) return createPObjectId(id); + if (isColumnFilteredKey(id)) return extractPObjectId(id.source); + if (isColumnOverridedKey(id)) return extractPObjectId(id.source); + if (isColumnDiscoveredKey(id)) return extractPObjectId(id.column); + + throw new Error(`extractPObjectId: unrecognized column id structure: ${JSON.stringify(id)}`); +} + +/** + * Reconstruct the effective {@link PColumnSpec} for a rich column id by walking + * the id chain from leaf to outermost wrapper, applying each layer's spec + * transformation in the same order the corresponding recipe would. + * + * Layer semantics: + * - Leaf ({@link LocalPObjectKey} / {@link GlobalPObjectKey}): no transformation. + * - {@link ColumnDiscoveredKey}: pass-through, descends into `column`. + * - {@link ColumnFilteredKey}: drops the axes whose positional index appears in + * `axisFilters[i][0]` from the inner spec's `axesSpec` — mirrors + * `ColumnFilteredRecipe.getSpec()`. + * - {@link ColumnOverridedKey}: applies `specOverrides` via + * {@link applySpecOverrides} on top of the inner spec. + */ +export function reconstructSpecFromId( + baseSpec: PColumnSpec, + id: ColumnUniversalId | ColumnUniversalKey, +): PColumnSpec { + if (isString(id)) { + if (isPObjectId(id)) return baseSpec; + + const parsed = + parseColumnIdSafety(id) ?? + throwError(`reconstructSpecFromId: id "${id}" is not a valid canonical column id`); + return reconstructSpecFromId(baseSpec, parsed); + } + + if (isPObjectKey(id)) return baseSpec; + if (isColumnDiscoveredKey(id)) return reconstructSpecFromId(baseSpec, id.column); + if (isColumnFilteredKey(id)) { + return applyAxisFilters(reconstructSpecFromId(baseSpec, id.source), id.axisFilters); + } + if (isColumnOverridedKey(id)) { + const inner = reconstructSpecFromId(baseSpec, id.source); + return applySpecOverrides(inner, id.specOverrides); + } + + throw new Error(`reconstructSpecFromId: unrecognized column id structure: ${JSON.stringify(id)}`); } diff --git a/lib/model/common/src/drivers/pframe/spec/index.ts b/lib/model/common/src/drivers/pframe/spec/index.ts index 900c482147..2cb28f4858 100644 --- a/lib/model/common/src/drivers/pframe/spec/index.ts +++ b/lib/model/common/src/drivers/pframe/spec/index.ts @@ -5,3 +5,4 @@ export * from "./spec"; export * from "./selectors"; export * from "./native_id"; export * from "./discovered_column"; +export * from "./overrided"; diff --git a/lib/model/common/src/drivers/pframe/spec/overrided.ts b/lib/model/common/src/drivers/pframe/spec/overrided.ts new file mode 100644 index 0000000000..19dff18308 --- /dev/null +++ b/lib/model/common/src/drivers/pframe/spec/overrided.ts @@ -0,0 +1,280 @@ +import { type Branded, throwError, type Mutable, Nil } from "@milaboratories/helpers"; +import { type CanonicalizedJson, canonicalizeJson } from "../../../json"; +import { AxisPatches, type ColumnUniversalId, parseColumnIdSafety } from "./ids"; +import type { AxisSpec, PColumnSpec } from "./spec"; +import { isNil, isString } from "es-toolkit"; + +export type SpecOverrides = Pick & { + axesSpec?: AxisPatches; +}; + +/** + * `source` can reference a leaf or a Filtered/Discovered id, but never another + * Overrided id — there is no `Overrided>`. Repeated overrides + * merge at the outer wrapper via {@link mergeSpecOverrides}. + */ +export interface ColumnOverridedKey { + __isOverrided: true; + source: Exclude; + specOverrides: SpecOverrides; +} + +export type ColumnOverridedId = Branded, "ColumnOverridedId">; + +export function isColumnOverridedKey(obj: unknown): obj is ColumnOverridedKey { + return typeof obj === "object" && obj !== null && "__isOverrided" in obj; +} + +export function distillColumnOverridedKey(props: ColumnOverridedKey): ColumnOverridedKey { + return { + __isOverrided: true, + source: props.source, + specOverrides: props.specOverrides, + }; +} + +export function createColumnOverridedId(props: { + source: ColumnUniversalId; + specOverrides: SpecOverrides; +}): ColumnOverridedId { + return stringifyColumnOverridedId(createColumnOverridedKey(props)); +} + +export function createColumnOverridedKey(props: { + source: ColumnUniversalId; + specOverrides: SpecOverrides; +}): ColumnOverridedKey { + const { source, specOverrides } = props; + const unwrapped = unwrapOverrides(source); + const baseSource = unwrapped + ? unwrapped.source + : (source as Exclude); + const mergedOverrides = unwrapped + ? mergeSpecOverrides(unwrapped.specOverrides, specOverrides) + : specOverrides; + + return { + __isOverrided: true, + source: baseSource, + specOverrides: mergedOverrides, + }; +} + +export function parseColumnOverridedId(id: ColumnUniversalId): ColumnOverridedKey { + try { + const parsed = JSON.parse(id); + return isColumnOverridedKey(parsed) + ? parsed + : throwError("Parsed object is not a valid OverridedPColumn"); + } catch { + throw new Error( + "Invalid ColumnOverridedId: not a valid JSON or does not conform to OverridedPColumn structure", + ); + } +} + +export function stringifyColumnOverridedId(id: ColumnOverridedKey): ColumnOverridedId { + return canonicalizeJson(distillColumnOverridedKey(id)) as ColumnOverridedId; +} + +/** + * Peel one override-wrap layer. + * + * Invariant: `{source, specOverrides}` can only appear at the top level — + * the inner `source` is never itself an override-wrap. Anything else throws. + */ +export function unwrapOverrides(id: ColumnUniversalId): ColumnOverridedKey | undefined { + const parsed = parseColumnIdSafety(id); + if (parsed === undefined || !isColumnOverridedKey(parsed)) return undefined; + const inner = parseColumnIdSafety(parsed.source); + if (inner !== undefined && isColumnOverridedKey(inner)) { + throw new Error("nested override-wrap detected — invariant broken"); + } + return parsed; +} + +/** + * Diff two specs into a {@link SpecOverrides} patch that, when applied on top + * of `base`, reconstructs `next`. + * + * - For top-level `annotations` / `domain` / `contextDomain`: include keys + * whose value in `next` differs from `base` (or is missing in `base`). + * Keys present in `base` but missing in `next` are not emitted — overrides + * merge, they cannot delete. + * - For `axesSpec`: matched by **positional index**. Each `next[i]` is diffed + * against `base[i]`; only changed fields are emitted as a patch under key + * `i`. Indices past `base.length` carry the full new axis (append). + */ +export function deriveSpecDelta(base: PColumnSpec, next: PColumnSpec): SpecOverrides { + const annotations = recordDelta(base.annotations, next.annotations); + const domain = recordDelta(base.domain, next.domain); + const contextDomain = recordDelta(base.contextDomain, next.contextDomain); + const axesSpec = deriveAxesDelta(base.axesSpec, next.axesSpec); + return { + ...(axesSpec && { axesSpec }), + ...(annotations && { annotations }), + ...(domain && { domain }), + ...(contextDomain && { contextDomain }), + }; +} + +export function isEmptySpecDelta(delta: SpecOverrides): boolean { + return ( + (!delta.annotations || Object.keys(delta.annotations).length === 0) && + (!delta.domain || Object.keys(delta.domain).length === 0) && + (!delta.contextDomain || Object.keys(delta.contextDomain).length === 0) && + (!delta.axesSpec || Object.keys(delta.axesSpec).length === 0) + ); +} + +function deriveAxesDelta( + base: readonly AxisSpec[], + next: readonly AxisSpec[], +): AxisPatches | undefined { + const out: AxisPatches = {}; + let any = false; + for (let i = 0; i < next.length; i++) { + const a = next[i]; + const b = base[i]; + if (b === undefined) { + out[i] = a; + any = true; + continue; + } + const nameChanged = a.name !== b.name; + const typeChanged = a.type !== b.type; + const annotations = recordDelta(b.annotations, a.annotations); + const domain = recordDelta(b.domain, a.domain); + const contextDomain = recordDelta(b.contextDomain, a.contextDomain); + if (!nameChanged && !typeChanged && !annotations && !domain && !contextDomain) continue; + out[i] = { + ...(nameChanged && { name: a.name }), + ...(typeChanged && { type: a.type }), + ...(annotations && { annotations }), + ...(domain && { domain }), + ...(contextDomain && { contextDomain }), + }; + any = true; + } + return any ? out : undefined; +} + +function recordDelta( + base: Record | undefined, + next: Record | undefined, +): Record | undefined { + if (!next) return undefined; + let any = false; + const diff: Record = {}; + for (const [k, v] of Object.entries(next)) { + if (!base || base[k] !== v) { + diff[k] = v; + any = true; + } + } + return any ? diff : undefined; +} + +/** + * Apply `specOverrides` from a rich id (`SUniversalPColumnId`) on top of a + * resolved {@link PColumnSpec}. Returns `base` unchanged when the id carries + * no overrides. + */ +export function applySpecOverrides( + base: PColumnSpec, + idOrOverride: Nil | SpecOverrides | ColumnUniversalId, +): PColumnSpec { + if (isNil(idOrOverride)) { + return base; + } + + const overrides = isString(idOrOverride) + ? unwrapOverrides(idOrOverride)?.specOverrides + : idOrOverride; + + if (isNil(overrides)) { + return base; + } + + const result = { ...base }; + if (overrides.annotations) + result.annotations = mergeRecord(base.annotations, overrides.annotations); + if (overrides.domain) result.domain = mergeRecord(base.domain, overrides.domain); + if (overrides.contextDomain) + result.contextDomain = mergeRecord(base.contextDomain, overrides.contextDomain); + if (overrides.axesSpec) result.axesSpec = applyAxesPatches(base.axesSpec, overrides.axesSpec); + return result; +} + +/** + * Compose two override patches: applying `mergeSpecOverrides(prior, next)` on + * top of a base spec produces the same result as applying `prior` then `next`. + */ +export function mergeSpecOverrides(prior: SpecOverrides, next: SpecOverrides): SpecOverrides { + const result: Mutable = {}; + const axesSpec = mergeAxesPatches(prior.axesSpec, next.axesSpec); + if (axesSpec !== undefined) result.axesSpec = axesSpec; + const annotations = mergeRecord(prior.annotations, next.annotations); + if (annotations !== undefined) result.annotations = annotations; + const domain = mergeRecord(prior.domain, next.domain); + if (domain !== undefined) result.domain = domain; + const contextDomain = mergeRecord(prior.contextDomain, next.contextDomain); + if (contextDomain !== undefined) result.contextDomain = contextDomain; + return result; +} + +/** + * Apply positional `patches` onto `base.axesSpec`. Patches deep-merge + * `annotations` / `domain` / `contextDomain` and shallow-override `name` / + * `type` at their slot. Indices `>= base.length` append new axes; intermediate + * gaps (if any) are filled with the patch itself, so callers should not leave + * holes between base.length and the highest patched index. + */ +function applyAxesPatches(base: readonly AxisSpec[], patches: AxisPatches): AxisSpec[] { + const result: AxisSpec[] = base.slice(); + for (const [k, p] of Object.entries(patches)) { + const i = Number(k); + const existing = result[i]; + result[i] = (existing === undefined ? p : mergeAxisPatch(existing, p)) as AxisSpec; + } + return result; +} + +/** + * Compose two positional patch maps: `mergeAxesPatches(prior, next)` applied to + * a base spec must equal applying `prior` then `next`. Patches sharing an + * index deep-merge field-by-field. + */ +function mergeAxesPatches( + prior: AxisPatches | undefined, + next: AxisPatches | undefined, +): AxisPatches | undefined { + if (!prior || Object.keys(prior).length === 0) return next; + if (!next || Object.keys(next).length === 0) return prior; + const result: AxisPatches = { ...prior }; + for (const [k, p] of Object.entries(next)) { + const i = Number(k); + const existing = result[i]; + result[i] = existing === undefined ? p : mergeAxisPatch(existing, p); + } + return result; +} + +function mergeAxisPatch>(axis: A, patch: Partial): A { + const result: Mutable = { ...axis, ...patch }; + if (patch.annotations) result.annotations = mergeRecord(axis.annotations, patch.annotations); + if (patch.domain) result.domain = mergeRecord(axis.domain, patch.domain); + if (patch.contextDomain) { + result.contextDomain = mergeRecord(axis.contextDomain, patch.contextDomain); + } + return result; +} + +function mergeRecord( + a: Record | undefined, + b: Record | undefined, +): Record | undefined { + if (!b) return a; + if (!a) return b; + return { ...a, ...b }; +} diff --git a/lib/model/common/src/drivers/pframe/spec/selectors.ts b/lib/model/common/src/drivers/pframe/spec/selectors.ts index c094e4f8a7..12057c73a3 100644 --- a/lib/model/common/src/drivers/pframe/spec/selectors.ts +++ b/lib/model/common/src/drivers/pframe/spec/selectors.ts @@ -170,6 +170,7 @@ export interface PColumnSelector extends AnchoredPColumnSelector { /** * Strict identifier for PColumns in an anchored context * Unlike APColumnMatcher, this requires exact matches on domain and axes + * @deprecated This is part of the legacy column matching API. The new Columns API (see sdk/model/src/columns/) now handles column and axis */ export interface AnchoredPColumnId extends AnchoredPColumnSelector { /** Name is required for exact column identification */ diff --git a/lib/model/common/src/drivers/pframe/spec/spec.ts b/lib/model/common/src/drivers/pframe/spec/spec.ts index 540722ebd6..9045fba4d1 100644 --- a/lib/model/common/src/drivers/pframe/spec/spec.ts +++ b/lib/model/common/src/drivers/pframe/spec/spec.ts @@ -174,7 +174,13 @@ export const Annotation = { }, } as const; -export type AnnotationDataStatus = "computing" | "ready" | "error" | "absent"; +export type AnnotationDataStatus = + | "missing" + | "resolving" + | "absent" + | "computing" + | "ready" + | "error"; export type Annotation = Metadata & Partial<{ @@ -710,8 +716,6 @@ export interface PColumn extends PObject { readonly spec: PColumnSpec; } -export type PColumnLazy = PColumn<() => T>; - /** Columns in a PFrame also have internal identifier, this object represents * combination of specs and such id */ export interface PColumnIdAndSpec { diff --git a/lib/model/common/src/drivers/pframe/spec_driver.ts b/lib/model/common/src/drivers/pframe/spec_driver.ts index d3043743b5..bbb9fca5df 100644 --- a/lib/model/common/src/drivers/pframe/spec_driver.ts +++ b/lib/model/common/src/drivers/pframe/spec_driver.ts @@ -1,6 +1,5 @@ import type { Branded } from "@milaboratories/helpers"; import type { PoolEntry } from "../../pool_entry"; -import type { PObjectId } from "../../pool"; import type { AxisSpec, AxesSpec, @@ -15,6 +14,7 @@ import type { } from "./spec"; import type { PTableColumnId, PTableColumnSpec } from "./table_common"; import { DataQuery, SpecQuery, SpecQueryJoinEntry } from "./query"; +import { PObjectId } from "../.."; /** Matches a string value either exactly or by regex pattern */ export type StringMatcher = @@ -167,10 +167,10 @@ export interface DiscoverColumnsMappingVariant { export interface DiscoverColumnsResponseHit { /** The column that was found compatible */ hit: PColumnIdAndSpec; - /** Possible ways to integrate this column with the existing set */ - mappingVariants: DiscoverColumnsMappingVariant[]; /** Linker steps traversed to reach this hit; empty for direct matches */ path: DiscoverColumnsStepInfo[]; + /** Possible ways to integrate this column with the existing set */ + mappingVariants: DiscoverColumnsMappingVariant[]; } /** Response from discover columns */ diff --git a/lib/model/common/src/drivers/pframe/table_calculate.ts b/lib/model/common/src/drivers/pframe/table_calculate.ts index 72df47dfc9..5d2f3fa021 100644 --- a/lib/model/common/src/drivers/pframe/table_calculate.ts +++ b/lib/model/common/src/drivers/pframe/table_calculate.ts @@ -4,7 +4,7 @@ import type { PObjectId } from "../../pool"; import { assertNever } from "../../util"; import { getAxisId, type PColumn } from "./spec/spec"; import type { PColumnValues } from "./data_info"; -import type { SpecQuery } from "./query/query_spec"; +import type { SpecQuery, SpecQueryJoinEntry } from "./query/query_spec"; import { canonicalizeJson } from "../../json"; import { mapSpecQueryColumns } from "./query"; @@ -422,8 +422,18 @@ export function sortPTableDef(def: PTableDef): PTableDef { }; } -export function mapPTableDefV2(def: PTableDefV2, cb: (c: C1) => C2): PTableDefV2 { - return { query: mapSpecQueryColumns(def.query, cb) }; +export function mapPTableDefV2( + def: PTableDefV2, + visitor: { + /** Transform column references in leaf nodes (column, sparseToDenseColumn). */ + column: (c: C1) => C2; + /** Visit a node after its children have been traversed. */ + node?: (node: SpecQuery) => SpecQuery; + /** Visit a join entry after its inner query has been traversed. */ + joinEntry?: (entry: SpecQueryJoinEntry) => SpecQueryJoinEntry; + }, +): PTableDefV2 { + return { query: mapSpecQueryColumns(def.query, visitor) }; } export function mapJoinEntry(entry: JoinEntry, cb: (c: C1) => C2): JoinEntry { diff --git a/lib/model/common/src/drivers/pframe/table_common.ts b/lib/model/common/src/drivers/pframe/table_common.ts index 0b9637a7b3..7c6a12095c 100644 --- a/lib/model/common/src/drivers/pframe/table_common.ts +++ b/lib/model/common/src/drivers/pframe/table_common.ts @@ -1,5 +1,5 @@ -import type { PObjectId } from "../../pool"; import type { AxisId, AxisSpec, PColumnSpec } from "./spec/spec"; +import type { ColumnUniversalId } from "./spec/ids"; export type PTableColumnSpecAxis = { type: "axis"; @@ -9,7 +9,13 @@ export type PTableColumnSpecAxis = { export type PTableColumnSpecColumn = { type: "column"; - id: PObjectId; + /** + * Leaf column id as it appears in the SpecQuery — may be a rich + * {@link ColumnUniversalId} (Discovered / Overrided / Filtered) or a bare + * {@link PObjectId}. The host resolver strips to bare via `extractPObjectId` + * before physical lookup. + */ + id: ColumnUniversalId; spec: PColumnSpec; }; @@ -23,7 +29,8 @@ export type PTableColumnIdAxis = { export type PTableColumnIdColumn = { type: "column"; - id: PObjectId; + /** @see PTableColumnSpecColumn.id */ + id: ColumnUniversalId; }; /** Unified PTable column identifier */ diff --git a/lib/model/common/src/index.ts b/lib/model/common/src/index.ts index 5a99b236c9..9d33567ce2 100644 --- a/lib/model/common/src/index.ts +++ b/lib/model/common/src/index.ts @@ -22,3 +22,4 @@ export * from "./resource_types"; export * from "./services"; export * from "./pool_entry"; export * from "./project_id"; +export * from "./columns"; diff --git a/lib/model/common/src/pool/spec.ts b/lib/model/common/src/pool/spec.ts index 022adcb333..1ca73b0462 100644 --- a/lib/model/common/src/pool/spec.ts +++ b/lib/model/common/src/pool/spec.ts @@ -1,7 +1,9 @@ -import type { Branded } from "../branding"; -import type { JoinEntry, PColumn, PColumnLazy, PColumnSpec } from "../drivers"; +import canonicalize from "canonicalize"; +import type { JoinEntry, PColumn, PColumnSpec } from "../drivers"; import { assertNever } from "../util"; -import type { ResultPoolEntry } from "./entry"; +import { PlRef } from "../ref"; +import { CanonicalizedJson, canonicalizeJson, parseJsonSafely } from "../json"; +import { Branded } from "@milaboratories/helpers"; /** Any object exported into the result pool by the block always have spec attached to it */ export type PObjectSpec = { @@ -22,8 +24,61 @@ export type PObjectSpec = { readonly annotations?: Record; }; +export type LocalPObjectKey = { resolvePath: string[]; name: string }; +export type LocalPObjectId = Branded, "LocalPObjectId">; +export type GlobalPObjectKey = PlRef; +export type GlobalPObjectId = Branded, "GlobalPObjectId">; /** Stable PObject id */ -export type PObjectId = Branded; +export type PObjectId = LocalPObjectId | GlobalPObjectId; + +export function isPObjectId(value: unknown): value is PObjectId { + if (typeof value !== "string") return false; + return isPObjectKey(parseJsonSafely(value)); +} + +export function isPObjectKey(value: unknown): value is LocalPObjectKey | GlobalPObjectKey { + return isLocalPObjectKey(value) || isGlobalPObjectKey(value); +} + +export function createPObjectId(obj: LocalPObjectKey | GlobalPObjectKey): PObjectId { + if (isLocalPObjectKey(obj)) { + return createLocalPObjectId(obj.resolvePath, obj.name); + } + if (isGlobalPObjectKey(obj)) { + return createGlobalPObjectId(obj.blockId, obj.name); + } + throw new Error(`createPObjectId: unrecognized object key structure: ${JSON.stringify(obj)}`); +} + +export function createLocalPObjectId(resolvePath: string[], name: string): PObjectId { + return canonicalizeJson({ resolvePath, name }) as PObjectId; +} + +export function isLocalPObjectId(value: unknown): value is LocalPObjectId { + if (typeof value !== "string") return false; + return isLocalPObjectKey(parseJsonSafely(value)); +} + +export function isLocalPObjectKey(value: unknown): value is LocalPObjectKey { + if (typeof value !== "object" || value === null) return false; + const v = value as LocalPObjectKey; + return Array.isArray(v.resolvePath) && typeof v.name === "string"; +} + +export function createGlobalPObjectId(blockId: string, exportName: string): PObjectId { + return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId; +} + +export function isGlobalPObjectKey(value: unknown): value is GlobalPObjectKey { + if (typeof value !== "object" || value === null) return false; + return "__isRef" in value && "blockId" in value && "name" in value; +} + +export function isGlobalPObjectId(value: unknown): value is GlobalPObjectId { + if (typeof value !== "string") return false; + const parsed = parseJsonSafely(value); + return isGlobalPObjectKey(parsed); +} /** * Full PObject representation. @@ -41,24 +96,16 @@ export interface PObject { readonly data: Data; } -export function isPColumnSpec(spec: PObjectSpec): spec is PColumnSpec { - return spec.kind === "PColumn"; +export function isPObject(obj: unknown): obj is PObject { + return typeof obj === "object" && obj !== null && "id" in obj && "spec" in obj && "data" in obj; } -export function isPColumn(obj: PObject): obj is PColumn { - return isPColumnSpec(obj.spec); +export function isPColumn(obj: unknown): obj is PColumn { + return isPObject(obj) && isPColumnSpec(obj.spec); } -export function isPColumnSpecResult( - r: ResultPoolEntry, -): r is ResultPoolEntry { - return isPColumnSpec(r.obj); -} - -export function isPColumnResult( - r: ResultPoolEntry>, -): r is ResultPoolEntry> { - return isPColumnSpec(r.obj.spec); +export function isPColumnSpec(spec: PObjectSpec): spec is PColumnSpec { + return spec.kind === "PColumn"; } export function ensurePColumn(obj: PObject): PColumn { @@ -66,10 +113,7 @@ export function ensurePColumn(obj: PObject): PColumn { return obj; } -export function mapPObjectData( - pObj: PColumn | PColumnLazy, - cb: (d: D1) => D2, -): PColumn; +export function mapPObjectData(pObj: PColumn, cb: (d: D1) => D2): PColumn; export function mapPObjectData(pObj: PColumn, cb: (d: D1) => D2): PColumn; export function mapPObjectData( pObj: PColumn | undefined, @@ -88,7 +132,7 @@ export function mapPObjectData( ? undefined : { ...pObj, - data: cb(typeof pObj.data === "function" ? pObj.data() : pObj.data), + data: cb(pObj.data), }; } diff --git a/lib/model/common/src/services/service_declarations.ts b/lib/model/common/src/services/service_declarations.ts index bfe7e420bd..c8df5339f9 100644 --- a/lib/model/common/src/services/service_declarations.ts +++ b/lib/model/common/src/services/service_declarations.ts @@ -24,6 +24,10 @@ */ import type { DialogService } from "../dialog"; +import type { + ColumnsCollectionDriver, + ColumnsCollectionDriverModel, +} from "../drivers/columns/columns_collection_driver"; import type { PFrameDriver, PFrameModelDriver } from "../drivers/pframe/driver"; import type { PFrameSpecDriver } from "../drivers/pframe/spec_driver"; import { service } from "./service_types"; @@ -79,4 +83,31 @@ export const Services = { modelMethods: [] as const, uiMethods: ["showSaveDialog"] as const, }), + ColumnsCollection: service< + ColumnsCollectionDriverModel, + ColumnsCollectionDriverModel, + ColumnsCollectionDriver, + ColumnsCollectionDriver + >()({ + type: "wasm", + name: "columnsCollection", + modelMethods: [ + "create", + "isEmpty", + "isFinal", + "getColumns", + "addSource", + "discover", + "filter", + ] as const, + uiMethods: [ + "create", + "isEmpty", + "isFinal", + "getColumns", + "addSource", + "discover", + "filter", + ] as const, + }), }; diff --git a/lib/model/common/src/services/service_registry.ts b/lib/model/common/src/services/service_registry.ts index f291806a49..dddf2b169a 100644 --- a/lib/model/common/src/services/service_registry.ts +++ b/lib/model/common/src/services/service_registry.ts @@ -1,7 +1,7 @@ import type { ServiceTypesLike, - InferServiceModel, - InferServiceUi, + InferServiceModelHost, + InferServiceUiHost, ServiceName, ModelServiceFactoryMap, UiServiceFactoryMap, @@ -71,7 +71,9 @@ export class ModelServiceRegistry< get( id: ServiceName, ): - | ([unknown] extends [InferServiceModel] ? Record : InferServiceModel) + | ([unknown] extends [InferServiceModelHost] + ? Record + : InferServiceModelHost) | null; get(id: ServiceName): Record | null { return this.getById(id); @@ -87,7 +89,9 @@ export class UiServiceRegistry< get( id: ServiceName, - ): ([unknown] extends [InferServiceUi] ? Record : InferServiceUi) | null; + ): + | ([unknown] extends [InferServiceUiHost] ? Record : InferServiceUiHost) + | null; get(id: ServiceName): Record | null { return this.getById(id); } diff --git a/lib/model/common/src/services/service_types.ts b/lib/model/common/src/services/service_types.ts index 6a4f112c35..f213be0aff 100644 --- a/lib/model/common/src/services/service_types.ts +++ b/lib/model/common/src/services/service_types.ts @@ -7,18 +7,44 @@ export type ServiceTypesLike< Model = unknown, Ui = unknown, Kind extends ServiceType = ServiceType, + ModelHost = Model, + UiHost = Ui, > = { - readonly __types?: { model: Model; ui: Ui; kind: Kind }; + readonly __types?: { + model: Model; + ui: Ui; + kind: Kind; + modelHost: ModelHost; + uiHost: UiHost; + }; }; export type InferServiceModel = - S extends ServiceTypesLike ? M : unknown; + S extends ServiceTypesLike ? M : unknown; export type InferServiceUi = - S extends ServiceTypesLike ? U : unknown; + S extends ServiceTypesLike ? U : unknown; export type InferServiceKind = - S extends ServiceTypesLike ? K : ServiceType; + S extends ServiceTypesLike ? K : ServiceType; + +/** + * Host-side shape stored in the model service registry. Differs from + * {@link InferServiceModel} only when the registered impl carries extra + * parameters (e.g. per-call `host` bindings) that the sandbox-view does + * not expose. Defaults to {@link InferServiceModel}. + */ +export type InferServiceModelHost = + S extends ServiceTypesLike ? MH : unknown; + +/** + * Host-side shape stored in the UI service registry. Differs from + * {@link InferServiceUi} only when the registered impl carries extra + * parameters that the UI-proxy view does not expose. Defaults to + * {@link InferServiceUi}. + */ +export type InferServiceUiHost = + S extends ServiceTypesLike ? UH : unknown; export type ServiceName = Branded; @@ -39,7 +65,7 @@ export const { const modelMethodsMap = new Map(); const uiMethodsMap = new Map(); return { - service() { + service() { return < K extends ServiceType, N extends string, @@ -50,7 +76,7 @@ export const { readonly name: N; readonly modelMethods: MM; readonly uiMethods: UM; - }): Branded> => { + }): Branded> => { const { name, type, modelMethods, uiMethods } = options; if (!SERVICE_ID_PATTERN.test(name)) { throw new ServiceInvalidIdError( @@ -63,7 +89,7 @@ export const { typeMap.set(name, type); modelMethodsMap.set(name, modelMethods); uiMethodsMap.set(name, uiMethods); - return name as Branded>; + return name as Branded>; }; }, isNodeService(id: ServiceName): boolean { @@ -95,11 +121,11 @@ export type ServiceBrand = T extends Branded ? S : never; export type ModelServiceFactoryMap> = { - [K in keyof SMap]: (() => InferServiceModel>) | null; + [K in keyof SMap]: (() => InferServiceModelHost>) | null; }; export type UiServiceFactoryMap> = { - [K in keyof SMap]: (() => InferServiceUi>) | null; + [K in keyof SMap]: (() => InferServiceUiHost>) | null; }; /** Contract between any service provider and any service consumer. */ diff --git a/lib/node/pf-driver/src/driver_double.test.ts b/lib/node/pf-driver/src/driver_double.test.ts index b777717388..f86ec47f4f 100644 --- a/lib/node/pf-driver/src/driver_double.test.ts +++ b/lib/node/pf-driver/src/driver_double.test.ts @@ -168,7 +168,11 @@ test("createTableV2 support", async ({ expect }) => { { key: ["d"], val: 5 }, ]; - const column = { id: columnId, spec: columnSpec, data: inlineData }; + const column = { + id: columnId, + spec: columnSpec, + data: inlineData, + }; const columnRef: SpecQueryExpression = { type: "columnRef", value: columnId }; @@ -339,7 +343,11 @@ test("createTableV2 sorting by axis with 2 axes", async ({ expect }) => { { key: ["b", "y"], val: 6 }, ]; - const column = { id: columnId, spec: columnSpec, data: inlineData }; + const column = { + id: columnId, + spec: columnSpec, + data: inlineData, + }; const axis1Ref: SpecQueryExpression = { type: "axisRef", diff --git a/lib/node/pf-driver/src/driver_impl.ts b/lib/node/pf-driver/src/driver_impl.ts index 74ac617704..c18f14d9a5 100644 --- a/lib/node/pf-driver/src/driver_impl.ts +++ b/lib/node/pf-driver/src/driver_impl.ts @@ -234,7 +234,7 @@ export class AbstractPFrameDriver< const columns = uniqueBy(collectSpecQueryColumns(def.query), (c) => c.id); using pFrameGuard = new PoolEntryGuard(this.createPFrame(columns)); const pFrameSpec = this.pFrames.getByKey(pFrameGuard.key).pFrameSpec; - const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, (c) => c.id)); + const sortedQuery = sortSpecQuery(mapSpecQueryColumns(def.query, { column: (c) => c.id })); const { tableSpec, dataQuery } = pFrameSpec.evaluateQuery(sortedQuery); using pTableGuard = new PoolEntryGuard( diff --git a/lib/node/pl-middle-layer/package.json b/lib/node/pl-middle-layer/package.json index 50b867ee2d..094d73ee3d 100644 --- a/lib/node/pl-middle-layer/package.json +++ b/lib/node/pl-middle-layer/package.json @@ -30,6 +30,7 @@ "fmt": "ts-builder format" }, "dependencies": { + "@milaboratories/columns-collection-driver": "workspace:*", "@milaboratories/computable": "workspace:*", "@milaboratories/helpers": "workspace:*", "@milaboratories/pf-driver": "workspace:*", diff --git a/lib/node/pl-middle-layer/src/js_render/column_registry.ts b/lib/node/pl-middle-layer/src/js_render/column_registry.ts new file mode 100644 index 0000000000..533e127ea0 --- /dev/null +++ b/lib/node/pl-middle-layer/src/js_render/column_registry.ts @@ -0,0 +1,90 @@ +import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree"; +import { + AccessorEntriesProvider, + applySpecOverrides, + ColumnRegistry, + extractPObjectId, + PColumnValues, + ResultPoolEntriesProvider, + SpecOverrides, + type ColumnEntriesProvider, + type PColumn, + type PColumnSpec, + type PObjectId, + type UpstreamBlockCtx, +} from "@milaboratories/pl-model-common"; + +export interface ColumnRegistryRoots { + /** Main `outputs` accessor (if available) — usually `MainAccessorName`. */ + readonly outputs?: PlTreeNodeAccessor; + /** Prerun `staging` accessor (if available) — usually `StagingAccessorName`. */ + readonly prerun?: PlTreeNodeAccessor; + /** Upstream-block ctx accessors (as `collectUpstreamBlockCtxes` returns). */ + readonly upstreamBlockCtxes: ReadonlyArray>; +} + +/** + * Build a host-side {@link ColumnRegistry} mirroring the sandbox layout: + * `outputs` precedes `prerun` precedes `rawResultPool`, first-wins on conflict. + * + * Uses raw {@link PlTreeNodeAccessor}s directly — both `TreeNodeAccessor` + * (sandbox) and `PlTreeNodeAccessor` (host) satisfy the `AccessorLike` + * traversal surface, so no host-side adapter is required. + */ +export function buildColumnRegistry( + roots: ColumnRegistryRoots, +): ColumnRegistry { + const providers: ColumnEntriesProvider[] = []; + + if (roots.outputs !== undefined) { + providers.push(new AccessorEntriesProvider(roots.outputs, ["main"])); + } + if (roots.prerun !== undefined) { + providers.push(new AccessorEntriesProvider(roots.prerun, ["staging"])); + } + + providers.push(new ResultPoolEntriesProvider(roots.upstreamBlockCtxes)); + + return new ColumnRegistry(providers); +} + +/** + * Resolve a column id (plain {@link PObjectId}) to a materialised {@link PColumn} backed by the + * underlying {@link PlTreeNodeAccessor} for the `.data` resource. Applies any + * `specOverrides` carried by the rich id on top of the resolved spec. + * + * Throws when the id is not in the registry, when the column has no spec, or + * when its `.data` field has not yet appeared on the parent PFrame accessor. + */ +export function resolvePColumnById( + registry: ColumnRegistry, + id: PObjectId, + overrides?: SpecOverrides, +): PColumn { + const pid = extractPObjectId(id); + const leaf = registry.resolve(pid); + if (leaf === undefined) { + throw new Error(`column id ${String(pid)} not found in host column registry`); + } + const specNode = leaf.accessor.traverse({ + field: `${leaf.name}.spec`, + assertFieldType: "Input", + ignoreError: true, + }); + const spec = specNode?.getDataAsJson(); + if (spec === undefined) { + throw new Error(`column ${String(pid)} has no resolved spec`); + } + const data = + leaf.accessor.traverse({ + field: `${leaf.name}.data`, + assertFieldType: "Input", + ignoreError: true, + }) ?? []; + + return { + id: id as PObjectId, + spec: applySpecOverrides(spec, overrides), + data, + }; +} diff --git a/lib/node/pl-middle-layer/src/js_render/computable_context.ts b/lib/node/pl-middle-layer/src/js_render/computable_context.ts index 9a0c950bb4..929f041ae6 100644 --- a/lib/node/pl-middle-layer/src/js_render/computable_context.ts +++ b/lib/node/pl-middle-layer/src/js_render/computable_context.ts @@ -7,14 +7,12 @@ import type { CommonFieldTraverseOps as CommonFieldTraverseOpsFromSDK, DataInfo, FieldTraversalStep as FieldTraversalStepFromSDK, - Option, PColumn, PColumnValues, PFrameDef, PFrameHandle, PObject, PObjectSpec, - PSpecPredicate, PTableDef, PTableDefV2, PTableHandle, @@ -42,6 +40,7 @@ import type { MiddleLayerEnvironment } from "../middle_layer/middle_layer"; import type { Block } from "../model/project_model"; import { parseFinalPObjectCollection } from "../pool/p_object_collection"; import type { ResultPool } from "../pool/result_pool"; +import { collectUpstreamBlockCtx, type UpstreamBlockCtx } from "../pool/upstream_block_ctx"; import type { JsExecutionContext } from "./context"; import type { VmFunctionImplementation } from "quickjs-emscripten"; import { Scope, type QuickJSHandle } from "quickjs-emscripten"; @@ -54,10 +53,10 @@ import { ServiceMethodNotFoundError, } from "@milaboratories/pl-model-common"; import { getServiceInjectors } from "./service_injectors"; - -function bytesToBase64(data: Uint8Array | undefined): string | undefined { - return data !== undefined ? Buffer.from(data).toString("base64") : undefined; -} +import { MainAccessorName, StagingAccessorName } from "@platforma-sdk/model"; +import { buildColumnRegistry, resolvePColumnById } from "./column_registry"; +import type { ColumnRegistry, PObjectId } from "@milaboratories/pl-model-common"; +import { collapseSpecOverrideNode } from "./spec_override_collapse"; export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRenderCtxMethods< string, @@ -92,6 +91,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender public resetComputableCtx() { this.computableCtx = undefined; this.accessors.clear(); + this._resultPool = undefined; + this._upstreamBlockCtx = undefined; + this._rawUpstreamBlockCtx = undefined; + this._columnRegistry = undefined; } private get requireComputableCtx(): ComputableCtx { @@ -153,6 +156,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender return this.getAccessor(handle).getIsReadyOrError(); } + hasData(handle: string): boolean { + return this.getAccessor(handle).hasData(); + } + getIsFinal(handle: string): boolean { return this.getAccessor(handle).getIsFinal(); } @@ -350,10 +357,6 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender return this._resultPool; } - public calculateOptions(predicate: PSpecPredicate): Option[] { - return this.resultPool.calculateOptions(predicate); - } - public getDataFromResultPool(): ResultCollection> { const collection = this.resultPool.getData(); if (collection.instabilityMarker !== undefined) @@ -409,28 +412,79 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender ); } + /** + * Single source-of-truth cache of upstream-block ctx pairs as raw + * `PlTreeNodeAccessor`s. Both the sandbox-facing `getUpstreamBlockCtx()` + * (string-handle view) and the host column registry are derived from this. + */ + private _rawUpstreamBlockCtx: UpstreamBlockCtx[] | undefined = undefined; + /** + * Raw upstream-block ctx accessors for the current render. Public so the + * `ColumnsCollection` service injector (`service_injectors.ts`) can build + * `ColumnsCollectionDriverHost.getUpstreamBlockCtxes` bindings without + * round-tripping through the handle-shape view. + */ + public getRawUpstreamBlockCtx(): UpstreamBlockCtx[] { + if (this._rawUpstreamBlockCtx === undefined) { + const prjEntry = notEmpty(this.blockCtx.projectEntry, "projectEntry"); + this._rawUpstreamBlockCtx = collectUpstreamBlockCtx( + this.requireComputableCtx, + prjEntry, + this.blockCtx.blockId, + ); + } + return this._rawUpstreamBlockCtx; + } + + private _upstreamBlockCtx: UpstreamBlockCtx[] | undefined = undefined; + public getUpstreamBlockCtx(): UpstreamBlockCtx[] { + if (this._upstreamBlockCtx === undefined) { + this._upstreamBlockCtx = this.getRawUpstreamBlockCtx().map((b) => ({ + blockId: b.blockId, + prodCtx: this.wrapAccessor(b.prodCtx), + stagingCtx: this.wrapAccessor(b.stagingCtx), + prodIncomplete: b.prodIncomplete, + stagingIncomplete: b.stagingIncomplete, + })); + } + return this._upstreamBlockCtx; + } + + // + // Host-side column registry — id → LeafEntry index over outputs/prerun/raw pool. + // Mirrors the sandbox `ColumnRegistry` so `createPFrame` / `createPTable` can + // accept `PObjectId` in addition to materialised `PColumn`s. + // + private _columnRegistry: ColumnRegistry | undefined = undefined; + public getColumnRegistry(): ColumnRegistry { + if (this._columnRegistry !== undefined) return this._columnRegistry; + return (this._columnRegistry = buildColumnRegistry({ + outputs: this.accessors.get(MainAccessorName) ?? undefined, + prerun: this.accessors.get(StagingAccessorName) ?? undefined, + upstreamBlockCtxes: this.getRawUpstreamBlockCtx(), + })); + } + // // PFrames / PTables // public createPFrame( - def: PFrameDef>>, + def: PFrameDef>>, ): PFrameHandle { using guard = new PoolEntryGuard( - this.env.driverKit.pFrameDriver.createPFrame( - def.map((c) => mapPObjectData(c, (d) => this.transformInputPData(d))), - ), + this.env.driverKit.pFrameDriver.createPFrame(def.map((c) => this.resolvePColumn(c))), ); this.requireComputableCtx.addOnDestroy(guard.entry.unref); return guard.keep().key; } public createPTable( - def: PTableDef>>, + def: PTableDef>>, ): PTableHandle { using guard = new PoolEntryGuard( this.env.driverKit.pFrameDriver.createPTable( - mapPTableDef(def, (c) => mapPObjectData(c, (d) => this.transformInputPData(d))), + mapPTableDef(def, (c) => this.resolvePColumn(c)), ), ); this.requireComputableCtx.addOnDestroy(guard.entry.unref); @@ -438,17 +492,35 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender } public createPTableV2( - def: PTableDefV2>>, + def: PTableDefV2>>, ): PTableHandle { using guard = new PoolEntryGuard( this.env.driverKit.pFrameDriver.createPTableV2( - mapPTableDefV2(def, (c) => mapPObjectData(c, (d) => this.transformInputPData(d))), + mapPTableDefV2(def, { + column: (c) => this.resolvePColumn(c), + node: collapseSpecOverrideNode, + }), ), ); this.requireComputableCtx.addOnDestroy(guard.entry.unref); return guard.keep().key; } + /** + * Normalise a def element: pass-through for materialised columns (after + * mapping their `data` payload via {@link transformInputPData}), or resolve + * a bare id through the host column registry. `resolvePColumnById` extracts + * the underlying {@link PObjectId} from any id encoding + * ({@link ColumnOverridedId}, etc.). + */ + private resolvePColumn( + item: PObjectId | PColumn>, + ): PColumn> { + return typeof item === "string" + ? resolvePColumnById(this.getColumnRegistry(), item) + : mapPObjectData(item, (d) => this.transformInputPData(d)); + } + /** * Transforms input data for PFrame/PTable creation * - Converts string handles to accessors @@ -495,7 +567,13 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender // Helpers // - private getAccessor(handle: string): PlTreeNodeAccessor { + /** + * Resolve a sandbox-side `AccessorHandle` string back to its concrete + * host accessor. Used internally by every `traverse`/`getX(handle)` + * method and externally by the `ColumnsCollection` service injector + * for `SerializedColumnsSource` of kind `"accessor"`. + */ + public getAccessor(handle: string): PlTreeNodeAccessor { const accessor = this.accessors.get(handle); if (accessor === undefined) throw new Error("No such accessor"); return accessor; @@ -649,6 +727,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender return parent.exportSingleValue(this.getIsFinal(vm.getString(handle)), undefined); }); + exportCtxFunction("hasData", (handle) => { + return parent.exportSingleValue(this.hasData(vm.getString(handle)), undefined); + }); + exportCtxFunction("getError", (handle) => { return parent.exportSingleValue(this.getError(vm.getString(handle)), undefined); }); @@ -816,13 +898,6 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender return parent.exportObjectUniversal(this.getSpecsFromResultPool(), undefined); }); - exportCtxFunction("calculateOptions", (predicate) => { - return parent.exportObjectUniversal( - this.calculateOptions(parent.importObjectViaJson(predicate) as PSpecPredicate), - undefined, - ); - }); - exportCtxFunction("getSpecFromResultPoolByRef", (blockId, exportName) => { return parent.exportObjectUniversal( this.getSpecFromResultPoolByRef(vm.getString(blockId), vm.getString(exportName)), @@ -837,6 +912,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender ); }); + exportCtxFunction("getUpstreamBlockCtx", () => { + return parent.exportObjectUniversal(this.getUpstreamBlockCtx(), undefined); + }); + exportCtxFunction("createPFrame", (def) => { return parent.exportSingleValue( this.createPFrame( @@ -938,3 +1017,7 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender }); } } + +function bytesToBase64(data: Uint8Array | undefined): string | undefined { + return data !== undefined ? Buffer.from(data).toString("base64") : undefined; +} diff --git a/lib/node/pl-middle-layer/src/js_render/service_injectors.ts b/lib/node/pl-middle-layer/src/js_render/service_injectors.ts index ca54456056..a42c8b3934 100644 --- a/lib/node/pl-middle-layer/src/js_render/service_injectors.ts +++ b/lib/node/pl-middle-layer/src/js_render/service_injectors.ts @@ -1,6 +1,15 @@ import type { QuickJSHandle, VmFunctionImplementation } from "quickjs-emscripten"; -import type { InferServiceModel, ServiceBrand } from "@milaboratories/pl-model-common"; +import type { + CollectionHandle, + ColumnsCollectionDriverHost, + DiscoverColumnsOptions, + InferServiceModel, + PoolEntry, + SerializedColumnsSource, + ServiceBrand, +} from "@milaboratories/pl-model-common"; import { Services, ServiceNotRegisteredError } from "@milaboratories/pl-model-common"; +import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree"; import type { AxesId, AxesSpec, @@ -8,6 +17,7 @@ import type { PColumn, PColumnSpec, PColumnValues, + PObjectId, PTableColumnId, PTableColumnSpec, SingleAxisSelector, @@ -130,16 +140,12 @@ export function getServiceInjectors(): ServiceInjectorMap { }; }, - // Dialog has no model-side surface — workflow scripts cannot open save dialogs. - // Declared as an empty injector to satisfy the exhaustive ServiceInjectorMap. - Dialog: () => ({}) as Record, - PFrame: ({ host, vm }: ServiceInjectorContext) => ({ createPFrame: (def: QuickJSHandle) => vm.exportSingleValue( host.createPFrame( vm.importObjectViaJson(def) as PFrameDef< - PColumn> + PObjectId | PColumn> >, ), ), @@ -148,7 +154,7 @@ export function getServiceInjectors(): ServiceInjectorMap { vm.exportSingleValue( host.createPTable( vm.importObjectViaJson(def) as PTableDef< - PColumn> + PObjectId | PColumn> >, ), ), @@ -157,10 +163,102 @@ export function getServiceInjectors(): ServiceInjectorMap { vm.exportSingleValue( host.createPTableV2( vm.importObjectViaJson(def) as PTableDefV2< - PColumn> + PObjectId | PColumn> >, ), ), }), + + // Dialog has no model-side surface — workflow scripts cannot open save dialogs. + // Declared as an empty injector to satisfy the exhaustive ServiceInjectorMap. + Dialog: () => ({}) as Record, + + ColumnsCollection: ({ host, vm }: ServiceInjectorContext) => { + const driver = host.serviceRegistry.get(Services.ColumnsCollection); + if (!driver) + throw new ServiceNotRegisteredError( + `Service "${Services.ColumnsCollection}" has no factory in ModelServiceRegistry. Provide a non-null factory.`, + ); + const specDriver = host.serviceRegistry.get(Services.PFrameSpec); + if (!specDriver) + throw new ServiceNotRegisteredError( + `Service "${Services.PFrameSpec}" has no factory in ModelServiceRegistry. Provide a non-null factory.`, + ); + + const bindings: ColumnsCollectionDriverHost = { + resolveAccessor: (h) => host.getAccessor(h), + getUpstreamBlockCtxes: () => host.getRawUpstreamBlockCtx(), + getSpecDriver: () => specDriver, + resolveSpec: (id: PObjectId) => { + const leaf = host.getColumnRegistry().resolve(id); + if (leaf === undefined) return undefined; + const specNode = leaf.accessor.traverse({ + field: `${leaf.name}.spec`, + assertFieldType: "Input", + ignoreError: true, + }); + return specNode?.getDataAsJson(); + }, + }; + + const pinHandle = (entry: PoolEntry): CollectionHandle => { + using guard = new PoolEntryGuard(entry); + host.addOnDestroy(guard.entry.unref); + return guard.keep().key; + }; + + return { + create: (sources: QuickJSHandle) => + vm.exportSingleValue( + pinHandle( + driver.create(vm.importObjectViaJson(sources) as SerializedColumnsSource[], bindings), + ), + ), + + isEmpty: (handle: QuickJSHandle) => + vm.exportSingleValue(driver.isEmpty(vm.vm.getString(handle) as CollectionHandle)), + + isFinal: (handle: QuickJSHandle) => + vm.exportSingleValue(driver.isFinal(vm.vm.getString(handle) as CollectionHandle)), + + getColumns: (handle: QuickJSHandle) => + vm.exportObjectViaJson( + driver.getColumns(vm.vm.getString(handle) as CollectionHandle, bindings), + ), + + addSource: (handle: QuickJSHandle, sources: QuickJSHandle) => + vm.exportSingleValue( + pinHandle( + driver.addSource( + vm.vm.getString(handle) as CollectionHandle, + vm.importObjectViaJson(sources) as SerializedColumnsSource[], + bindings, + ), + ), + ), + + discover: (handle: QuickJSHandle, opts: QuickJSHandle) => + vm.exportSingleValue( + pinHandle( + driver.discover( + vm.vm.getString(handle) as CollectionHandle, + vm.importObjectViaJson(opts) as DiscoverColumnsOptions, + bindings, + ), + ), + ), + + filter: (handle: QuickJSHandle, opts: QuickJSHandle) => + vm.exportSingleValue( + pinHandle( + driver.filter( + vm.vm.getString(handle) as CollectionHandle, + vm.importObjectViaJson(opts) as DiscoverColumnsOptions, + bindings, + ), + ), + ), + }; + }, }; } diff --git a/lib/node/pl-middle-layer/src/js_render/spec_override_collapse.ts b/lib/node/pl-middle-layer/src/js_render/spec_override_collapse.ts new file mode 100644 index 0000000000..9ef1a88025 --- /dev/null +++ b/lib/node/pl-middle-layer/src/js_render/spec_override_collapse.ts @@ -0,0 +1,206 @@ +import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree"; +import type { DataInfo, PColumn, PColumnValues } from "@platforma-sdk/model"; +import { applySpecOverrides, matchAxis } from "@milaboratories/pl-model-common"; +import type { AxesSpec, AxisSpec, SpecOverrides, SpecQuery } from "@milaboratories/pl-model-common"; + +type Leaf = PColumn>; +type Node = SpecQuery; + +/** + * Collapse the client-side `specOverride` query node at the host boundary — + * pframe-engine never sees it. The override is pushed down through the + * subtree until it lands on `column`-leaf nodes, where it is folded into + * the leaf's spec via {@link applySpecOverrides}. Intermediate join nodes + * are left untouched — pframe-engine will recompute their result spec + * from the patched leaves. + * + * Designed to run as the `node` visitor in `mapPTableDefV2` (post-order), + * so by the time we see a `specOverride` its inner subtree is already + * mapped to {@link PColumn} leaves. + * + * Currently only `column` and `linkerJoin` are walked. Other node shapes + * under `specOverride` throw — pframe-engine support for them is still + * in progress. Extend the switch in {@link pushSpecOverrideDown} as more + * shapes become needed. + */ +export function collapseSpecOverrideNode(node: Node): Node { + if (node.type !== "specOverride") return node; + return pushSpecOverrideDown(node.input, node.override); +} + +function pushSpecOverrideDown(node: Node, overrides: SpecOverrides): Node { + switch (node.type) { + case "column": + return { + type: "column", + column: { ...node.column, spec: applySpecOverrides(node.column.spec, overrides) }, + }; + + case "linkerJoin": { + // `axesSpec` patches are positional against the linkerJoin's OUTPUT axes, + // which equal the secondary side's axes (ColumnDiscoveredRecipe emits the + // hit on the many-side of the linker). Route axesSpec only into secondary; + // applying the same positional indices to the linker would silently + // mis-target the linker's own axes. + // + // Non-axesSpec patches (domain / contextDomain / annotations) are + // column-level and not positional — keep the existing both-sides walk. + const { axesSpec: _axesSpec, ...nonAxesOverrides } = overrides; + const hasNonAxes = + nonAxesOverrides.domain !== undefined || + nonAxesOverrides.contextDomain !== undefined || + nonAxesOverrides.annotations !== undefined; + return { + ...node, + linker: hasNonAxes ? pushSpecOverrideDown(node.linker, nonAxesOverrides) : node.linker, + secondary: node.secondary.map((e) => ({ + ...e, + entry: pushSpecOverrideDown(e.entry, overrides), + })), + }; + } + + // case "outerJoin": + // return { + // ...node, + // primary: { + // ...node.primary, + // entry: pushSpecOverrideDown(node.primary.entry, overrides), + // }, + // secondary: node.secondary.map((e) => ({ + // ...e, + // entry: pushSpecOverrideDown(e.entry, overrides), + // })), + // }; + + // case "innerJoin": + // case "fullJoin": + // case "symmetricJoin": + // return { + // ...node, + // entries: node.entries.map((e) => ({ + // ...e, + // entry: pushSpecOverrideDown(e.entry, overrides), + // })), + // }; + + case "sliceAxes": { + // axesSpec patches are positional against the slice OUTPUT axes (the + // recipe-layer spec the consumer sees), but the inner input still + // carries the full pre-slice axes list. Translate outer indices to + // inner indices before pushing the override down. + // + // Non-axesSpec overrides (domain / contextDomain / annotations) are + // column-level and pass through transparently. + if (overrides.axesSpec === undefined || Object.keys(overrides.axesSpec).length === 0) { + return { ...node, input: pushSpecOverrideDown(node.input, overrides) }; + } + + const innerAxes = effectiveAxesSpec(node.input); + if (innerAxes === undefined) { + throw new Error( + `specOverride over sliceAxes: cannot translate axesSpec — ` + + `inner shape unknown for input type "${node.input.type}"`, + ); + } + + const filteredInnerIdxs = new Set(); + for (const filter of node.axisFilters) { + const idx = innerAxes.findIndex((a) => matchAxis(filter.axisSelector, a)); + if (idx < 0) { + throw new Error( + `specOverride over sliceAxes: filter selector ` + + `${JSON.stringify(filter.axisSelector)} not found in inner axes`, + ); + } + filteredInnerIdxs.add(idx); + } + + // outer index j (post-slice) ↔ inner index = j-th surviving position. + const outerToInner: number[] = []; + for (let i = 0; i < innerAxes.length; i++) { + if (!filteredInnerIdxs.has(i)) outerToInner.push(i); + } + + const translated: Record = {}; + for (const [outerKey, patch] of Object.entries(overrides.axesSpec)) { + const outerIdx = Number(outerKey); + const innerIdx = outerToInner[outerIdx]; + if (innerIdx === undefined) { + throw new Error( + `specOverride over sliceAxes: outer axesSpec index ${outerIdx} ` + + `out of range (slice output has ${outerToInner.length} axes)`, + ); + } + translated[innerIdx] = patch as AxisSpec; + } + + const translatedOverrides: SpecOverrides = { ...overrides, axesSpec: translated }; + return { ...node, input: pushSpecOverrideDown(node.input, translatedOverrides) }; + } + + // // Transparent transforms — passthrough. + // case "sort": + // case "filter": + // case "transformColumns": + // return { ...node, input: pushSpecOverrideDown(node.input, overrides) }; + + // // Synthetic leaves — patch the spec carrier they expose, or + // // throw if the node has no natural place to absorb overrides. + // case "inlineColumn": + // return { + // ...node, + // column: { ...node.column, spec: applySpecOverrides(node.column.spec, overrides) }, + // }; + + // case "sparseToDenseColumn": + // // Same idea: patch the new dense column's spec carrier. + // return { + // ...node, + // newSpec: applySpecOverrides(node.newSpec, overrides), + // }; + + default: + throw new Error( + `specOverride: inner node "${node.type}" is not supported yet — ` + + "only column / linkerJoin can carry an override " + + "(pframe-engine support for the rest is in progress)", + ); + } +} + +/** + * Effective post-traversal {@link AxesSpec} of a {@link Node} — used to + * translate positional axesSpec patches across structural nodes whose + * indexing differs from their input (currently only `sliceAxes`). + * + * Returns `undefined` when the node shape is not introspectable here. + * Callers must then refuse axesSpec translation and surface a clear error, + * rather than silently mis-targeting indices. + */ +function effectiveAxesSpec(node: Node): AxesSpec | undefined { + switch (node.type) { + case "column": + return node.column.spec.axesSpec; + + case "linkerJoin": + // Output axes equal the secondary side's axes (linker side drops out). + // All secondary entries share the same axis identity post-join — read + // the first one. + return node.secondary.length === 0 ? undefined : effectiveAxesSpec(node.secondary[0].entry); + + case "sliceAxes": { + const inner = effectiveAxesSpec(node.input); + if (inner === undefined) return undefined; + const filtered = new Set(); + for (const f of node.axisFilters) { + const i = inner.findIndex((a) => matchAxis(f.axisSelector, a)); + if (i >= 0) filtered.add(i); + } + return inner.filter((_, i) => !filtered.has(i)); + } + + default: + return undefined; + } +} diff --git a/lib/node/pl-middle-layer/src/middle_layer/block_ctx.ts b/lib/node/pl-middle-layer/src/middle_layer/block_ctx.ts index 4251a3dc5a..d088fdb935 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/block_ctx.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/block_ctx.ts @@ -22,9 +22,13 @@ export type BlockContextFull = BlockContextArgsOnly & { readonly prod: (cCtx: ComputableCtx) => PlTreeEntry | undefined; readonly staging: (cCtx: ComputableCtx) => PlTreeEntry | undefined; readonly getResultsPool: (cCtx: ComputableCtx) => ResultPool; + readonly projectEntry: PlTreeEntry; }; -export type BlockContextAny = Optional; +export type BlockContextAny = Optional< + BlockContextFull, + "prod" | "staging" | "getResultsPool" | "projectEntry" +>; export function constructBlockContextArgsOnly( projectEntry: PlTreeEntry, @@ -159,5 +163,6 @@ export function constructBlockContext( return result; }, getResultsPool: (cCtx: ComputableCtx) => ResultPool.create(cCtx, projectEntry, blockId), + projectEntry, }; } diff --git a/lib/node/pl-middle-layer/src/mutator/template/template_cache.test.ts b/lib/node/pl-middle-layer/src/mutator/template/template_cache.test.ts index fd66d6bd48..dc55f7c14b 100644 --- a/lib/node/pl-middle-layer/src/mutator/template/template_cache.test.ts +++ b/lib/node/pl-middle-layer/src/mutator/template/template_cache.test.ts @@ -39,8 +39,6 @@ function createTestCacheInTx(pl: Parameters { test("produces nodes in topological order for V2 template", () => { const data = parseTemplate(ExplicitTemplateEnterNumbers); @@ -77,8 +75,6 @@ describe("flattenTemplateTree", () => { }); }); -// ─── getOrCreateTemplateCache / dropTemplateCache ──────────────────────────── - describe("getOrCreateTemplateCache", () => { test("creates cache on first call and reuses on second", async () => { await TestHelpers.withTempRoot(async (pl) => { @@ -100,8 +96,6 @@ describe("dropTemplateCache", () => { }); }); -// ─── loadTemplateCached ────────────────────────────────────────────────────── - describe("loadTemplateCached", () => { test("cache miss then cache hit returns same SignedResourceId", async () => { await TestHelpers.withTempRoot(async (pl) => { @@ -184,8 +178,6 @@ describe("loadTemplateCached", () => { }, 15000); }); -// ─── cacheBlockPackTemplate ────────────────────────────────────────────────── - describe("cacheBlockPackTemplate", () => { test("replaces prepared template with cached reference", async () => { await TestHelpers.withTempRoot(async (pl) => { @@ -235,8 +227,6 @@ describe("cacheBlockPackTemplate", () => { }, 15000); }); -// ─── Equivalence: cached vs legacy produce identical resources ─────────────── - describe("template cache produces equivalent resources", () => { test("cached and legacy templates deduplicate to same original (V2)", async () => { await TestHelpers.withTempRoot(async (pl) => { @@ -272,8 +262,6 @@ describe("template cache produces equivalent resources", () => { }, 30000); }); -// ─── Shared library dedup ──────────────────────────────────────────────────── - describe("shared library dedup", () => { test("two different templates sharing a library reuse the same cache entry", async () => { await TestHelpers.withTempRoot(async (pl) => { @@ -303,8 +291,6 @@ describe("shared library dedup", () => { }, 15000); }); -// ─── GC ────────────────────────────────────────────────────────────────────── - describe("GC", () => { test("evicts entries when cache exceeds max entries", async () => { await TestHelpers.withTempRoot(async (pl) => { diff --git a/lib/node/pl-middle-layer/src/mutator/template/template_cache.ts b/lib/node/pl-middle-layer/src/mutator/template/template_cache.ts index bd9de75f26..ff06206f69 100644 --- a/lib/node/pl-middle-layer/src/mutator/template/template_cache.ts +++ b/lib/node/pl-middle-layer/src/mutator/template/template_cache.ts @@ -46,8 +46,6 @@ export const ACCESS_COUNT_KEY = "_accessCount"; /** @internal exported for testing */ export const ACCESS_KEY_PREFIX = "access_"; -// ─── Stats ─────────────────────────────────────────────────────────────────── - export type TemplateCacheStat = { totalMs: number; flattenMs: number; @@ -80,8 +78,6 @@ function initialStat(): TemplateCacheStat { }; } -// ─── Tree node abstraction ─────────────────────────────────────────────────── - interface CacheableNode { /** SHA-256 content hash (includes all descendant content) */ hash: string; @@ -92,8 +88,6 @@ interface CacheableNode { childHashes: string[]; } -// ─── Hash computation helpers ──────────────────────────────────────────────── - function getSourceCode(name: string, sources: Record, sourceHash: string): string { return notEmpty( sources[sourceHash], @@ -150,8 +144,6 @@ function hashSoftwareV3(sw: TemplateSoftwareDataV3): string { .digest("hex"); } -// ─── Tree flattening ───────────────────────────────────────────────────────── - function flattenV2Tree(data: TemplateData): CacheableNode[] { const nodes: CacheableNode[] = []; const seen = new Set(); @@ -404,8 +396,6 @@ export function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): Ca } } -// ─── Cache operations ──────────────────────────────────────────────────────── - /** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */ const cacheRidMap = new WeakMap(); @@ -458,8 +448,6 @@ export async function dropTemplateCache(pl: PlClient): Promise { invalidateTemplateCacheId(pl); } -// ─── GC ────────────────────────────────────────────────────────────────────── - /** * Run count-based garbage collection on the template cache. * Evicts least-recently-used entries when the cache exceeds maxEntries. @@ -505,8 +493,6 @@ export async function runGc( }); } -// ─── Batched materialization ───────────────────────────────────────────────── - /** Create a batch of cache nodes in the current transaction. */ function createBatchNodes( tx: PlTransaction, @@ -744,8 +730,6 @@ export async function loadTemplateCached( } } -// ─── Caller helper ─────────────────────────────────────────────────────────── - /** * Pre-materialize a block pack's template via cache. * Returns a copy of the spec with the template replaced by a cached reference. diff --git a/lib/node/pl-middle-layer/src/pool/data.ts b/lib/node/pl-middle-layer/src/pool/data.ts index 3fd44483d5..d983fdae05 100644 --- a/lib/node/pl-middle-layer/src/pool/data.ts +++ b/lib/node/pl-middle-layer/src/pool/data.ts @@ -1,11 +1,12 @@ import { + createGlobalPObjectId, + createLocalPObjectId, PFrameDriverError, type BinaryChunk, type ParquetChunk, type ParquetChunkMapping, type ParquetChunkMetadata, type PColumnValue, - type PlRef, type PObjectId, type PObjectSpec, } from "@platforma-sdk/model"; @@ -331,9 +332,9 @@ export function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccesso } export function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId { - return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId; + return createGlobalPObjectId(blockId, exportName); } export function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId { - return canonicalize({ resolvePath, name: outputName })! as PObjectId; + return createLocalPObjectId(resolvePath, outputName); } diff --git a/lib/node/pl-middle-layer/src/pool/result_pool.ts b/lib/node/pl-middle-layer/src/pool/result_pool.ts index f5c74661f8..062876e3ab 100644 --- a/lib/node/pl-middle-layer/src/pool/result_pool.ts +++ b/lib/node/pl-middle-layer/src/pool/result_pool.ts @@ -4,13 +4,12 @@ import type { Option, PObject, PObjectSpec, - PSpecPredicate, PlRef, ResultCollection, ResultPoolEntry, ValueOrError, } from "@platforma-sdk/model"; -import { executePSpecPredicate, mapValueInVOE } from "@platforma-sdk/model"; +import { mapValueInVOE } from "@platforma-sdk/model"; import { notEmpty } from "@milaboratories/ts-helpers"; import { outputRef } from "../model/args"; import type { Block, ProjectStructure } from "../model/project_model"; @@ -239,28 +238,6 @@ export class ResultPool { return { entries, isComplete, instabilityMarker }; } - public calculateOptions(predicate: PSpecPredicate): ExtendedOption[] { - const found: ExtendedOption[] = []; - for (const block of this.blocks.values()) { - const exportsChecked = new Set(); - const addToFound = (ctx: RawPObjectCollection) => { - for (const [exportName, result] of ctx.results) { - if (exportsChecked.has(exportName) || result.spec === undefined) continue; - exportsChecked.add(exportName); - if (executePSpecPredicate(predicate, result.spec)) - found.push({ - label: block.info.label + " / " + exportName, - ref: outputRef(block.info.id, exportName), - spec: result.spec, - }); - } - }; - if (block.staging !== undefined) addToFound(block.staging); - if (block.prod !== undefined) addToFound(block.prod); - } - return found; - } - public static create(ctx: ComputableCtx, prjEntry: PlTreeEntry, rootBlockId: string): ResultPool { const prj = ctx.accessor(prjEntry).node(); diff --git a/lib/node/pl-middle-layer/src/pool/upstream_block_ctx.ts b/lib/node/pl-middle-layer/src/pool/upstream_block_ctx.ts new file mode 100644 index 0000000000..114eee5144 --- /dev/null +++ b/lib/node/pl-middle-layer/src/pool/upstream_block_ctx.ts @@ -0,0 +1,85 @@ +import type { ComputableCtx } from "@milaboratories/computable"; +import type { PlTreeEntry, PlTreeNodeAccessor } from "@milaboratories/pl-tree"; +import { notEmpty } from "@milaboratories/ts-helpers"; +import type { UpstreamBlockCtx } from "@milaboratories/pl-model-common"; +import type { ProjectStructure } from "../model/project_model"; +import { ProjectStructureKey, projectFieldName } from "../model/project_model"; +import { allBlocks, stagingGraph } from "../model/project_model_util"; + +export type { UpstreamBlockCtx } from "@milaboratories/pl-model-common"; + +/** + * Collect upstream-block ctx accessors for the given root block. + * + * For each upstream block (per the staging graph) we try to resolve its + * `prodUiCtx` and `stagingUiCtx` resources. Only the `.ok` outcomes are + * surfaced; failures and missing fields collapse to `undefined`. + * + * SDK-side providers compose enumerate/status/data operations on top of these + * accessors using the helpers in `column_providers/accessor_traversal`. + * + * Note: this function is the only place that reads the project tree to build + * the upstream-pool view — the legacy `ResultPool` class is not used. + */ +export function collectUpstreamBlockCtx( + ctx: ComputableCtx, + prjEntry: PlTreeEntry, + rootBlockId: string, +): UpstreamBlockCtx[] { + const prj = ctx.accessor(prjEntry).node(); + const structure = notEmpty(prj.getKeyValueAsJson(ProjectStructureKey)); + const graph = stagingGraph(structure); + const targetBlocks = graph.traverseIds("upstream", rootBlockId); + + const out: UpstreamBlockCtx[] = []; + for (const blockInfo of allBlocks(structure)) { + if (!targetBlocks.has(blockInfo.id)) continue; + + const prod = resolveCtxAccessor(prj, blockInfo.id, "prod"); + const staging = resolveCtxAccessor(prj, blockInfo.id, "staging"); + + const prodIncomplete = prod.calculated && prod.accessor === undefined; + const stagingIncomplete = staging.calculated && staging.accessor === undefined; + + out.push({ + blockId: blockInfo.id, + prodCtx: prod.accessor, + stagingCtx: staging.accessor, + prodIncomplete: prodIncomplete || undefined, + stagingIncomplete: stagingIncomplete || undefined, + }); + } + return out; +} + +interface CtxProbe { + /** True if the ctx-holder field exists in the project tree (block has started rendering). */ + readonly calculated: boolean; + /** Resolved ui-ctx accessor (only when `.ok`). */ + readonly accessor: PlTreeNodeAccessor | undefined; +} + +function resolveCtxAccessor( + prj: PlTreeNodeAccessor, + blockId: string, + kind: "prod" | "staging", +): CtxProbe { + const ctxField = kind === "prod" ? "prodCtx" : "stagingCtx"; + const uiCtxField = kind === "prod" ? "prodUiCtx" : "stagingUiCtx"; + + const calculated = + prj.traverse({ + field: projectFieldName(blockId, ctxField), + ignoreError: true, + pureFieldErrorToUndefined: true, + stableIfNotFound: true, + }) !== undefined; + + const uiCtx = prj.traverseOrError({ + field: projectFieldName(blockId, uiCtxField), + stableIfNotFound: true, + }); + + if (uiCtx === undefined || !uiCtx.ok) return { calculated, accessor: undefined }; + return { calculated, accessor: uiCtx.value }; +} diff --git a/lib/node/pl-middle-layer/src/service_factories.ts b/lib/node/pl-middle-layer/src/service_factories.ts index ab2b7ecfd6..20e88281f0 100644 --- a/lib/node/pl-middle-layer/src/service_factories.ts +++ b/lib/node/pl-middle-layer/src/service_factories.ts @@ -8,6 +8,7 @@ import { ModelServiceRegistry, Services } from "@milaboratories/pl-model-common"; import { SpecDriver } from "@milaboratories/pf-spec-driver"; +import { ColumnsCollectionDriverImpl } from "@milaboratories/columns-collection-driver"; import { type MiLogger } from "@milaboratories/ts-helpers"; export type ModelServiceOptions = { @@ -19,5 +20,6 @@ export function createModelServiceRegistry(options: ModelServiceOptions) { PFrameSpec: () => new SpecDriver({ logger: options.logger }), PFrame: null, Dialog: null, + ColumnsCollection: () => new ColumnsCollectionDriverImpl(), }); } diff --git a/lib/node/pl-tree/src/accessors.ts b/lib/node/pl-tree/src/accessors.ts index 574dc1ba57..00fec715e2 100644 --- a/lib/node/pl-tree/src/accessors.ts +++ b/lib/node/pl-tree/src/accessors.ts @@ -5,17 +5,13 @@ import type { ComputableHooks, UsageGuard, } from "@milaboratories/computable"; -import type { - SignedResourceId, - ResourceType, - OptionalSignedResourceId, -} from "@milaboratories/pl-client"; import { resourceIdToString, resourceTypesEqual, resourceTypeToString, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - NullSignedResourceId, + ResourceType, + OptionalSignedResourceId, + SignedResourceId, } from "@milaboratories/pl-client"; import type { ValueAndError } from "./value_and_error"; import { mapValueAndError } from "./value_and_error"; @@ -129,7 +125,6 @@ function getResourceFromTree( : !resourceTypesEqual(ops.assertResourceType, acc.resourceType)) ) throw new Error( - // eslint-disable-next-line @typescript-eslint/no-base-to-string `wrong resource type ${resourceTypeToString(acc.resourceType)} but expected ${ops.assertResourceType}`, ); @@ -367,6 +362,11 @@ export class PlTreeNodeAccessor { return this.getResourceFromTree(rid, {}); } + public hasData(): boolean { + this.instanceData.guard(); + return this.resource.data !== undefined; + } + public getData(): Uint8Array | undefined { return this.resource.data; } diff --git a/lib/node/pl-tree/src/sync.test.ts b/lib/node/pl-tree/src/sync.test.ts index c8dcab5937..4b7576f434 100644 --- a/lib/node/pl-tree/src/sync.test.ts +++ b/lib/node/pl-tree/src/sync.test.ts @@ -284,8 +284,6 @@ test("load resources", async () => { }); }); -// ─── TraversalMode tests ────────────────────────────────────────────────────── - /** Build a minimal mock tx that records whether resourceTree / getResourceDataIfExists * was called. Both surfaces are instrumented so we can assert which path ran. */ function buildMockTx(opts: { capable: boolean }): { @@ -390,8 +388,6 @@ test("traversalMode=auto default regression: streaming on capable, BFS on incapa expect(callsNoCap.streaming).toBe(0); }); -// ─── Stop-marker handling tests ─────────────────────────────────────────────── - /** Capabilities for backends that support streaming traversal. */ const stopMarkerCaps = ["treeFilter:v2"] as const; diff --git a/lib/util/helpers/src/types/brand.ts b/lib/util/helpers/src/types/brand.ts index 5d29f43a2f..df4ef35f97 100644 --- a/lib/util/helpers/src/types/brand.ts +++ b/lib/util/helpers/src/types/brand.ts @@ -1,6 +1,20 @@ -declare const __brand: unique symbol; - -export type Branded = T & { readonly [__brand]: B }; +/** + * Phantom-property brand. The key is a plain string literal rather than a + * `unique symbol` so the resulting type stays fully nameable across packages. + * + * A `unique symbol` key forces TS, when forced to expand `Branded` into + * its structural form (e.g. inside `Record` index signatures + * during dts emit), to write out `typeof __brand` — a value-level reference + * to the symbol. If the symbol's declaring module isn't reachable from the + * compilation root, dts emit fails with TS4023 ("cannot be named"). Using a + * string key sidesteps this entirely: `{ __brand: B }` is a plain structural + * type, nameable from anywhere. + * + * Phantom keys provide compile-time discrimination only — two different brand + * tags `B1 ≠ B2` make `Branded` and `Branded` mutually + * incompatible regardless of whether the key is a symbol or a string. + */ +export type Branded = T & { readonly __brand: B }; // simple regex string, without flags or lookarounds export type RegExpString = Branded; diff --git a/migrations/2026-05-20-new-column-access-mechanism.md b/migrations/2026-05-20-new-column-access-mechanism.md new file mode 100644 index 0000000000..3bd48f9b34 --- /dev/null +++ b/migrations/2026-05-20-new-column-access-mechanism.md @@ -0,0 +1,775 @@ +# New column access mechanism + +## Why + +The block model runs in a sandbox with a hard **8 MB** input limit. The old `ColumnCollectionBuilder` API pulled every upstream `PColumnSpec` it could see into that 8 MB up-front — even specs the model never read. On projects with a wide upstream graph this routinely blew the limit, and there was nothing you could do about it short of rewriting the model. + +The whole refactor exists to fix this. The shape of the fix: + +- **Identity lives in `ColumnUniversalId`.** A column is now a canonical id string. Spec, data and dataStatus are _derived_ from the id — nothing is materialised until you ask for it. (`SUniversalPColumnId` is a legacy alias of the same type, still surfaced by `createPFrame` / `createPTable` signatures.) +- **Specs are fetched on demand, one at a time.** The host owns the upstream tree and the spec frame. The sandbox only holds opaque handles and ids. `col.getSpec()` crosses the bridge for _that_ column; uninspected columns cost ~0 bytes. +- **Columns are immutable.** A column's spec is reachable only through `getSpec()`, and the spec you get back is a fresh snapshot of what the host has — you can't write to it, and there's no "local edit" that survives anywhere. + +What this means for you, the consumer: + +- **Don't read specs you don't need.** Every `col.getSpec()` is a host round-trip. Discover columns with a selector (regex on name/domain/annotations runs host-side, for free), and only call `getSpec()` on the survivors. The old habit of "grab everything, filter in JS with `(spec) => …`" doesn't translate. +- **Don't mutate specs.** There's no field to mutate, and patching the local copy buys you nothing — nothing else can see the change, the next `getSpec()` round-trip re-fetches the canonical version, and you've burnt bridge bytes for a value you immediately overwrote. If you genuinely need a different spec (e.g. to retag annotations for a downstream PFrame), `col.withSpecs(patch)` encodes the patch into the id, so the resulting column is a real, addressable, transportable thing. +- **Prefer ids over column objects when passing things around.** `createPFrame` / `createPTable` and block args all accept `ColumnUniversalId` strings directly. The host resolves them server-side; no spec/data has to travel through the sandbox just to be handed back to the host. + +The rest of this document is the mechanical migration. Each change is an instance of one of the rules above. + +--- + +## Column access — `ColumnCollectionBuilder` → `ColumnsCollection` + +`ColumnCollectionBuilder` and its surrounding helpers are removed. The new entry point is `ColumnsCollection(sources?)` from `@platforma-sdk/model`. + +The collection itself is **host-driven** — it never ships specs into the sandbox. Discovery, filtering and addSource all mint a fresh handle on the host; only ids come back. No `dispose()`, no spec-frame lifetime to manage on the sandbox side. + +### Discovery + +```diff +- const collection = new ColumnCollectionBuilder(ctx.getService("pframeSpec")) +- .addSources(collectCtxColumnSnapshotProviders(ctx)) +- .build({ anchors: { main: spec } }); +- try { +- const variants = collection.findColumnVariants({ include, maxHops: 4 }); +- ... +- } finally { +- collection.dispose(); +- } ++ const cols = ColumnsCollection() ++ .discover({ anchors: { main: spec }, include, maxHops: 4 }) ++ .getColumns(); // ColumnRecipe[] — no specs fetched yet +``` + +`ColumnsCollection(sources?)` defaults to **current block + result pool** (`outputs` + `prerun` + upstream-block result pool) when `sources` is omitted. To scope discovery, pass an explicit array — entries can be: + +- `"current_block"` — shorthand for `outputs` + `prerun` accessors of the current block; +- `"result_pool"` — shorthand for the upstream-block result pool; +- a `TreeNodeAccessor`, a `ColumnsProvider`, another `ColumnsCollection`, or a column-like shape `{ columns, isFinal }`. The `columns` array accepts `PColumn` / `ColumnLazy` / any `ColumnRecipe` — the source only needs `.id` from each entry. + +`isFinal` says whether everything upstream has finished computing. While blocks are still running the set of visible columns keeps growing across render passes; `isFinal: true` means it won't grow anymore. The built-in sources figure this out themselves — you only set it explicitly when you pass `{ columns, isFinal }`. + +The instance itself exposes a handful of host-driven methods on top of `discover` / `filter` / `addSource`: + +```ts +collection.isEmpty(); // host-side emptiness check, no ids transferred +collection.isFinal(); // upstream done? — same flag the `{ columns, isFinal }` source carries +collection.getColumnIds(); // ColumnUniversalId[] — fast path when you only need ids +collection.getColumns(); // ColumnRecipe[] — same ids, lifted to typed recipes +collection.handle; // CollectionHandle — pass to another `ColumnsCollection(sources)` to chain +``` + +`getColumnIds()` is the fastest exit: nothing on the recipe is touched, you just get the ids back to feed into `createPFrame` / `createPTable`. `getColumns()` is a thin map over those ids. + +```ts +// only the upstream blocks +ColumnsCollection(["result_pool"]); +// only the current block's own outputs/prerun +ColumnsCollection(["current_block"]); +// current block + a specific extra subtree +ColumnsCollection(["current_block", someAccessor]); +``` + +Narrow `sources` aggressively. Every entry you add expands the host-side spec frame the collection operates against; `"current_block"` alone is much cheaper than the default triplet when you actually only care about your own outputs. + +### Selectors do filtering host-side — use them + +`(spec) => boolean` selectors are gone. `name`, `domain`, `annotations`, etc. now take regex strings (auto-wrapped as `StringMatcher` of `type: "regex"`). The filter runs against the host-side spec frame, so columns that don't match never need their spec brought into the sandbox. + +```diff +- { match: (spec) => spec.name === "value", priority: 10 } +- { match: (spec) => spec.name === "score", visibility: "optional" } ++ { match: { name: "^value$" }, priority: 10 } ++ { match: { name: "^score$" }, visibility: "optional" } +``` + +If you genuinely need a predicate the selector can't express, apply it as a post-filter — but understand that **every column reaching the predicate pays a `getSpec()` round-trip**, so push as much as possible into the selector first: + +```ts +const cols = ColumnsCollection() + .discover({ include: { name: "^value$" } }) // host-side, cheap + .getColumns() + .filter((c) => myCustomPredicate(c.getSpec())); // sandbox-side, one round-trip per survivor +``` + +### `ColumnRecipe` vs `ColumnLazy` — what you're actually holding + +Every column you get back from the new API is a **`ColumnRecipe`**: an immutable, identity-bearing _description_ of how to obtain a column — not the column data itself. The recipe interface is intentionally narrow: + +```ts +recipe.id; // PObjectId | ColumnUniversalId — addressable string, transportable +recipe.getReferencedIds(); // every PObjectId this recipe reaches (leaf + nested refs) +recipe.getSpec(); // PColumnSpec — host round-trip, memoised; always returns a value +recipe.getQuery(); // SpecQuery — the IR the host uses to derive `getSpec()` +recipe.getDataStatus(); // "present" | "absent" | "resolving" (worst across referenced ids) +recipe.withSpecs(patch); // returns a new recipe with the patch baked into the id +``` + +Note what's **not** there: there is no `getData()` on a recipe. Data is only meaningful at a leaf — `Overrided` / `Filtered` / `Discovered` recipes are descriptions whose data is fetched on the **host** when you hand the id to `createPFrame` / `createPTable`. Pulling data into the sandbox for a non-leaf recipe would defeat the whole point of the refactor. + +The id is the source of truth; the recipe is just a typed accessor over it. The id string encodes _how_ the column was produced, and each form has its own recipe class: + +| Id shape on the wire | Recipe class | What it represents | +| --------------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------- | +| bare `PObjectId` | `ColumnLazy` | a plain upstream column — spec comes from the host accessor, data is reachable as `TreeNodeAccessor` | +| `ColumnOverridedKey` | `ColumnOverridedRecipe` | "this other recipe, but with annotations/domain/axes patched" — `withSpecs()` builds these | +| `ColumnFilteredKey` | `ColumnFilteredRecipe` | "this recipe restricted to a subset of axes" — produced by `ColumnsCollection.filter` | +| `ColumnDiscoveredKey` | `ColumnDiscoveredRecipe` | the result of a `ColumnsCollection.discover` hit, carrying the join graph back to its anchor so the spec can be rebuilt | + +The `*Key` rows are the parsed JSON object shapes; the stringified-on-the-wire form for each is `Column*Id` (e.g. `ColumnOverridedId`). + +**`ColumnLazy` is the only recipe class with `getData()`.** It is the _leaf_ — `implements ColumnRecipe` — and is also the only recipe you ever construct from a `TreeNodeAccessor` / `PColumn` / `PlRef`. Everything else is a wrapper around a leaf id; you produce the wrappers by calling `discover` / `filter` / `withSpecs`, and you pass them around by `.id`. + +For most code this is all you need to know: **you hold recipes, you pass ids, you call `getSpec()` only on survivors.** The recipe-kind taxonomy only becomes visible when you `instanceof`-narrow to a leaf to call `getData()`, and even then most call sites are better off using the helpers in `@platforma-sdk/model` (`collectLinkerColumns`, `hitQualifications`, …) than walking the recipe tree by hand. + +### Reading and constructing columns + +`id` is a field; spec / data status / query are getter methods. Each round-trips the host once and memoises per instance: + +```ts +col.id; // recipe id +col.getReferencedIds(); // leaf-set this recipe depends on +col.getSpec(); // PColumnSpec — bridge round-trip on first call +col.getDataStatus(); // ColumnFieldStatus: "present" | "absent" | "resolving" + +// Only on ColumnLazy (the leaf) — narrow first. +// `ColumnLazy` (the value) is the dispatcher function with statics attached, +// so `instanceof ColumnLazy` does not narrow. Always use the `isColumnLazy` +// guard — do **not** import the internal `ColumnLazyImpl` symbol to +// `instanceof`-narrow either; that's an implementation detail. +if (isColumnLazy(col)) { + col.getData(); // PColumnDataUniversal | undefined — bridge round-trip +} +``` + +Iterating a 5k-column collection to call `getSpec()` on every entry will fetch 5k specs. If you only need ids, pass `col.id` straight through to whatever needs it — `createPFrame` / `createPTable` accept ids directly, so most pipelines never call `getSpec()` at all. + +### Type guards — `isColumn` / `isColumnRecipe` / `isColumnLazy` / `isLeafColumn` + +Four guards are exported alongside the factories so consumers don't need to import the internal `*Impl` symbols just to narrow: + +```ts +isColumn(value); // value is Column (alias of isColumnRecipe) +isColumnRecipe(value); // value is ColumnRecipe (any recipe — leaf or wrapper) +isColumnLazy(value); // value is ColumnLazy (bare leaf only — the one with getData()) +isLeafColumn(recipe); // boolean (leaf-form: chain contains no Discovered) +``` + +Use `isColumnLazy` instead of `instanceof ColumnLazy`: `ColumnLazy` (the value) is the dispatcher function with statics, not the class, so `instanceof` does not narrow there. + +**`isColumnLazy` vs `isLeafColumn` — pick by what you do with the result.** + +- `isColumnLazy(c)` — strict, type-narrowing predicate. Returns `true` only for the **bare leaf** (`ColumnLazy`). Use it when you need access to `getData()` directly, or when the type forces you to a bare leaf — e.g. `PColumnIdAndSpec.columnId` and `PColumn.id` are `PObjectId`, which only bare leaves carry; wrappers expose `ColumnUniversalId`. +- `isLeafColumn(recipe)` — broader, boolean (not a type guard). Returns `true` for any recipe whose wrapper chain contains no `ColumnDiscoveredRecipe` — so `Overrided` / `Filtered` over a bare leaf still count. Use it for the **primary vs linker-joined** classification at the `createPlDataTableV3` boundary: anything that has no `Discovered` in its chain is a valid primary, projections over a plain leaf included. The SDK's own `createPlDataTableV3` uses `isLeafColumn` for this split — mirror it in your block code. + +Counterpart for reading data without a strict `isColumnLazy` narrow: `getLeafColumnData(recipe)` walks the wrapper chain down to the bottom leaf and returns its data (throws when the chain reaches a `Discovered`). + +There is a single top-level entry point, `Column(source)`. It picks the right factory by source shape and returns a `ColumnRecipe`: + +```ts +Column(source); // id string | PlRef | PColumn | LeafEntry | ColumnLazy +``` + +Routing rules: + +- **String id** (`PObjectId` or `ColumnUniversalId`) → `ColumnRecipe(id)`. The id shape decides which recipe class comes back (see the table above) — you don't pick, the id does. +- **Object source** (`PlRef` / `LeafEntry` / `PColumn` / `ColumnLazy`) → `ColumnLazy(source)`. These shapes only ever map to the bare `ColumnLazy` case. + +`Column(source)` returns `undefined` when the source can't be resolved (e.g. an id whose accessor isn't reachable from the current ctx, or a `PlRef` with no matching entry). Always check before chaining. + +When the intent at the call site is unambiguous you can call the dispatchers directly: + +```ts +ColumnRecipe(id); // string id → routed by id shape +ColumnLazy(source); // id | PlRef | PColumn | LeafEntry | ColumnLazy +``` + +Or the explicit factories when the source shape would otherwise be ambiguous: + +```ts +ColumnLazy.fromId(id); // PObjectId +ColumnLazy.fromPlRef(ref); // PlRef +ColumnLazy.fromColumn(pColumn); // already-materialised PColumn +ColumnLazy.fromAccessor(leafEntry); // direct accessor binding +``` + +### Two flavours of "not yet" — `resolving` vs `absent` + +Two distinct status types live next to recipes: + +- **`ColumnFieldStatus`** = `"present" | "resolving" | "absent"` — the worst-case across every PObjectId the recipe references. Returned by `recipe.getDataStatus()`. This is the only thing you usually look at when iterating recipes. +- **`ColumnResolutionStatus`** = `"present" | "resolving" | "absent"` — same value space, different meaning: it folds spec readiness + leaf-registry readiness, so you can ask "can this recipe be _constructed_ at all in the active ctx" without actually building it. + +Use the static `getStatus` helpers when you need the answer without paying for the recipe: + +```ts +ColumnRecipe.getStatus(id); // any id shape — dispatches by parsed key +ColumnLazy.getStatus(source); // PObjectId / PlRef / LeafEntry / PColumn / ColumnLazy +ColumnLazy.getStatusById(id); +ColumnLazy.getStatusByPlRef(ref); +ColumnLazy.getStatusByAccessor(entry); +``` + +A recipe factory returns `undefined` for `resolving` (try again on the next render pass — more accessor inputs may still arrive), and throws **`ColumnAbsentError`** for `absent` (every relevant accessor is `inputsLocked`; the column will not appear in this ctx). Catch `ColumnAbsentError` at boundaries — resolver / filter-and-sort wiring / data-table assembly — when partial absence should be surfaced rather than silently producing an empty result. Inside a tight loop over already-known ids, prefer `getStatus(...)` once up-front over a try/catch around the factory. + +### Columns are immutable — don't poke at specs + +Previously columns were plain objects: callers spread/rewrote `col.spec` in place. That "worked" only locally — the mutation didn't survive a bridge crossing, and the original spec had already cost you bytes to fetch. Doing it in a loop over upstream columns was a frequent way to blow the 8 MB limit for nothing. + +No recipe has a spec field. The spec is derived from the id, lazily, on first `getSpec()`. To produce a column with a different spec, use `withSpecs(patch)`: the patch becomes part of the **id** (the recipe class for the result is `ColumnOverridedRecipe`, but you don't need to care — you still hold a `ColumnRecipe`), so the new column is a real, addressable thing — the override travels with the id, no extra bridge traffic needed to re-describe it elsewhere. + +```diff +- // ad-hoc mutation — paid for the original spec, change was invisible to everyone else +- col.spec = { ...col.spec, annotations: { ...col.spec.annotations, x: "y" } }; +- const tagged = { ...col, spec: { ...col.spec, annotations: { ...col.spec.annotations, x: "y" } } }; ++ // new column, new id; original `col` is unchanged, no spec read required ++ const tagged = col.withSpecs({ annotations: { x: "y" } }); ++ tagged.id !== col.id; +``` + +`withSpecs` accepts a `SpecOverrides` — `Pick & { axesSpec?: AxisPatches }`. `axesSpec` is an `AxisPatches` map (positional patches keyed by axis index, **not** an `AxisSpec[]`). Repeated calls flatten into a single override on the id, so `col.withSpecs(a).withSpecs(b)` is equal-by-id to `col.withSpecs(merge(a, b))`. + +If you find yourself reaching for `getSpec()` just to feed it back into `withSpecs`, you're paying for a round-trip you don't need — express the change as a patch. + +--- + +## `createPFrame` / `createPTable` accept ids directly + +`PFrameDef` / `PTableDefV2` now accept `ColumnUniversalId` and `PObjectId` strings mixed with `PColumn` objects. (The runtime signature of `createPFrame` still names the legacy alias `SUniversalPColumnId` — same type, no migration needed.) The host resolves ids server-side, so the sandbox doesn't have to fetch spec+data just to hand them back: + +```diff +- ctx.createPFrame(cols.map(c => ({ id: c.id, spec: c.getSpec(), data: c.getData() }))); ++ ctx.createPFrame(cols.map(c => c.id)); // strings — resolved on host, zero spec reads +``` + +When you already have a `ColumnsCollection`, skip the intermediate recipes entirely: + +```ts +ctx.createPFrame(collection.getColumnIds()); +``` + +This is also the canonical way to feed **wrapped recipes** (`ColumnDiscoveredRecipe` / `ColumnFilteredRecipe` / `ColumnOverridedRecipe`) into a PFrame. They have no `getData()` — but `recipe.id` is a `ColumnUniversalId` that `createPFrame` accepts directly, and the host materialises it server-side. So instead of trying to narrow a recipe list to leaves before building a PFrame, pass `.id`: + +```ts +ctx.createPFrame(recipes.map((c) => c.id)); // works for any recipe — leaf or wrapped +``` + +(Code that still needs a `PColumn<...>[]`, e.g. `createPFrameForGraphs`, does not have this id-form yet — see the "ColumnLazy → PColumn adapter" section below for the materialisation pattern that only works for leaves.) + +You can still pass `PColumn` objects with live `TreeNodeAccessor` / `DataInfo` data when the column was assembled in the sandbox (e.g. via `expandByPartition`); the helper that converts those (`transformPColumnData`) was renamed and exported as `finalizePColumnData`. Prefer the id form whenever the column came from the host in the first place. + +--- + +## `ResultPool` / `RenderCtxBase` — all column access goes through `ColumnsCollection` + +The whole `ResultPool` class is `@deprecated`. Every column-discovery / column-fetch entry point on it (and the convenience wrappers on `RenderCtxBase`) pulled specs eagerly — exactly the pattern this refactor exists to kill. Use `ColumnsCollection` + `ColumnLazy` for **everything**. + +| Old | New | +| -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ctx.resultPool.getData()` / `getDataFromResultPool()` | `ColumnsCollection(["result_pool"]).getColumns()` — iterate the resulting `ColumnRecipe[]` and pass `.id`s into `createPFrame` / `createPTable`; never iterate `getData()` over the whole pool | +| `ctx.resultPool.getDataWithErrors()` / `getDataWithErrorsFromResultPool()` | same; per-column status comes from `recipe.getDataStatus()` ("present" / "absent" / "resolving"). Errors live on the host — surface them at the consumer (PFrame/PTable) instead of the result-pool boundary | +| `ctx.resultPool.getSpecs()` / `getSpecsFromResultPool()` | `ColumnsCollection(["result_pool"]).getColumns()` and `getSpec()` only on the survivors of a discover/filter — never iterate `getSpec()` over the whole pool | +| `ctx.resultPool.getDataByRef(ref)` | `ColumnLazy.fromPlRef(ref)?.getData()` — `getData()` lives on the leaf `ColumnLazy`, not on the generic `ColumnRecipe` | +| `ctx.resultPool.getPColumnByRef(ref)` | `ColumnLazy.fromPlRef(ref)` (or the unified `Column(ref)`, which for `PlRef` likewise yields a `ColumnLazy`) | +| `ctx.resultPool.getSpecByRef(ref)` / `getPColumnSpecByRef(ref)` | `Column(ref)?.getSpec()` (or `ColumnLazy.fromPlRef(ref)?.getSpec()`) — `getSpec()` is on every recipe, so either form types out | +| `ctx.resultPool.selectColumns(predicate)` | `ColumnsCollection(["result_pool"]).discover({ include }).getColumns()` — push the predicate into `include` so filtering runs host-side; fall back to a JS post-`.filter((c) => …(c.getSpec()))` only if you must | +| `ctx.resultPool.findDataWithCompatibleSpec(spec)` | `ColumnsCollection(["result_pool"]).discover({ anchors: { main: spec }, mode: "default" }).getColumns()` | +| `ctx.resultPool.getOptions(predicate, labelOps)` | **Stay on `ctx.resultPool.getOptions(...)`** — it preserves the `Option[]` = `{ ref: PlRef, label }` wire shape and `refsWithEnrichments`. (The `@deprecated` JSDoc on `ResultPool` lists `ctx.getOptions` as the migration target, but the method has **not** been promoted to `RenderCtxBase` yet — `ctx.getOptions` is not callable.) Switch to `ColumnsCollection(["result_pool"]).discover({ include }).getColumns()` + `deriveLabels` **only when** the consumer wants column **ids** (e.g. for `createPFrame`/`createPTable`), not PlRefs | +| `ctx.resultPool.getAnchoredPColumns(anchors, selectors, opts)` | `ColumnsCollection(["result_pool"]).discover({ anchors, include: selectors, … }).getColumnIds()` — each id is already a `ColumnUniversalId`, pass directly into `createPFrame` / `createPTable` | +| `ctx.resultPool.getCanonicalOptions(anchors, selectors, opts)` | same discover call; map `(col) => ({ value: col.id, label: deriveLabel(col) })` | +| `ctx.resultPool.resolveAnchorCtx({ key: PlRef })` | per-entry: `isPlRef(v) ? Column(v)?.getSpec() : v`. `ColumnsCollection.discover` accepts the resulting `{ key: PColumnSpec }` directly | +| `ctx.resultPool.findLabels(axis)` / `findLabelsForColumnAxis(spec, axisIdx)` / `ctx.findLabels(…)` | discover label columns via `ColumnsCollection` (`{ name: "^pl7.app/label$", axes: [{ name: axis.name }] }`) and read JSON data off the resulting `ColumnLazy` | +| `ctx.getBlockLabel(blockId)` | `@deprecated` on `RenderCtxBase` — still callable but slated to return dummy values; do not write new code against it | + +Deprecated aliases that just forward to the non-`*FromResultPool` versions (`getDataFromResultPool`, `getDataWithErrorsFromResultPool`, `getSpecsFromResultPool`) follow the same row as their canonical name. `ctx.resultPool` itself is `@deprecated` — the `*ByRef` reads have already moved to `ctx.*ByRef` on `RenderCtxBase` (see the migration map in the JSDoc above `class ResultPool` in `sdk/model/src/render/api.ts`), but the recommended migration is straight to `Column(ref)` / `ColumnsCollection`, which doesn't touch `ctx.resultPool` at all. + +Always pass `["result_pool"]` (or the narrowest source set that works) — the default ctx-wide triplet pulls the upstream tree **and** the current block. Use `["current_block"]` if you only want your own outputs/prerun. + +--- + +## Recipe utilities (`@platforma-sdk/model`) + +A few helpers in `sdk/model/src/columns/utils.ts` cover the recipe-walk cases you'd otherwise hand-roll: + +- **`collectLinkerIds(recipe): PObjectId[]`** — every non-hit `PObjectId` referenced by `recipe.getQuery()`, deduped in traversal order. Pure query walk, no registry access. +- **`collectLinkerColumns(recipe, opts?): ColumnLazy[]`** — same set, resolved against the ambient ctx as leaf `ColumnLazy` instances. Throws if any linker fails to resolve — direct replacement for the legacy `resolveLinkers`. +- **`hitQualifications(recipe): readonly AxisQualification[]`** — hit-side axis qualifications, pulled out of the inner `ColumnDiscoveredRecipe` (if any). Returns `[]` for plain leaves / overrided-over-leaf chains. +- **`queriesQualifications(recipe): Readonly>`** — per-primary-column qualifications, applied to outer primary anchors at the consumer boundary. + +Prefer these over walking the recipe class hierarchy yourself — they encapsulate the wrapper-peeling invariants (`Overrided` / `Filtered` over at most one `Discovered`) so they keep working as new wrappers are added. + +--- + +## Removed types — mechanical renames + +These names are gone from `@platforma-sdk/model`. Replace one-to-one before chasing semantic differences: + +| Old | New | Notes | +| --------------------------------------------------- | --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ColumnCollectionBuilder` | `ColumnsCollection` (function) | Not a class — call as `ColumnsCollection(sources?)`. No `.build()` / `.dispose()`. | +| `AnchoredColumnCollection` | `ColumnsCollection` (type) | Same name, function _and_ instance type. | +| `AnchoredFindColumnsOptions` | `ColumnsDiscoverOptions` / `ColumnsFilterOptions` | `mode` / `maxHops` only on `discover`; `filter` takes a strict subset. | +| `ColumnMatch` | `ColumnRecipe` | **Shape changes** — see "ColumnMatch → ColumnRecipe" below. There is no `.column` wrapper field anymore. | +| `collection.findColumns({...})` | `collection.discover({...}).getColumns()` | `findColumns` is gone. `discover` returns a new `ColumnsCollection`; call `getColumns()` / `getColumnIds()` to materialise. | +| `collection.findColumnVariants({...})` | `collection.discover({...}).getColumns()` | Same — single discovery method, no `variants` vs `columns` split. | +| `ArrayColumnProvider` | `ArrayColumnsProvider` (renamed) **or** `{ columns, isFinal }` source | Prefer the inline `{ columns, isFinal: true }` shape inside `sources` — no wrapper class needed. | +| `namePattern: "..."` | `name: "..."` (inside a selector) | Selectors accept plain regex strings as `name`. The old `namePattern` key is gone. | +| `ctx.getService("pframeSpec")` | _not needed_ | `ColumnsCollection` resolves its own driver via `getService("columnsCollection")`. | +| `ColumnSource` (legacy meaning: source for builder) | `ColumnsSource` (note the **`s`**) | The legacy `ColumnSource` is now the input type of `Column(source)` (singular, id/PlRef/PColumn). Source arrays for `sources` use `ColumnsSource[]`. | + +--- + +## `ColumnMatch` → `ColumnRecipe` + +The biggest mechanical hazard. `ColumnMatch` had a nested `.column` field carrying a materialised `PColumn`; `ColumnRecipe` is flat, identity-only, and field access is replaced by method calls. + +| Old | New | Cost | +| --------------------------- | --------------------------- | ------------------------------------------------------------------------------------- | +| `m.column.id` | `c.id` | field, free | +| `m.column.spec` | `c.getSpec()` | **host round-trip on first call**, memoised after. Do not iterate this in a hot loop. | +| `m.column.spec.name` | `c.getSpec().name` | host round-trip | +| `m.column.spec.axesSpec` | `c.getSpec().axesSpec` | host round-trip | +| `m.column.spec.annotations` | `c.getSpec().annotations` | host round-trip | +| `m.column.data` | _only on `ColumnLazy`_ | narrow first: `if (isColumnLazy(c)) c.getData()` | +| `m.column.data.get()` | `c.getData()` (no `.get()`) | the `.get()` indirection is gone — `getData()` is the read. | + +The rule is the same as for any other recipe: **filter host-side via `include` / `exclude` whenever possible, call `getSpec()` only on survivors**. Whenever you see a long `.filter(m => …m.column.spec…)` chain in legacy code, the first migration step is "what of this is expressible as a selector?". + +--- + +## Predicates → `collection.filter()` cookbook + +The minimal filter surface on `ColumnsCollection` covers almost every JS-side predicate the legacy code used. Anything that runs through `collection.filter({ include, exclude })` stays on the host and never pays the spec round-trip. + +The selector schema is `{ name, type, domain, contextDomain, annotations, axes: [{ name, type, domain, contextDomain, annotations }], partialAxesMatch }`. Plain string values become `{ type: "regex", value }` matchers automatically. + +| Legacy JS predicate | Host-side equivalent | +| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `m.column.spec.name === 'pl7.app/label'` | `filter({ include: { name: '^pl7\\.app/label$' } })` | +| `m.column.spec.name !== 'pl7.app/label'` | `filter({ exclude: { name: '^pl7\\.app/label$' } })` | +| `m.column.spec.valueType !== 'String'` | `filter({ exclude: { type: 'String' } })` | +| `(spec.valueType as string) !== 'File'` | `filter({ exclude: { type: 'File' } })` | +| `spec.annotations?.['pl7.app/isLinkerColumn'] === 'true'` | `filter({ include: { annotations: { 'pl7.app/isLinkerColumn': 'true' } } })` | +| `spec.annotations?.['pl7.app/isAnchor'] !== 'true'` | `filter({ exclude: { annotations: { 'pl7.app/isAnchor': 'true' } } })` | +| `spec.axesSpec.some(a => a.name === sampleAxisName)` | `filter({ include: { axes: [{ name: `^${escape(sampleAxisName)}$` }], partialAxesMatch: true } })` | +| `!spec.axesSpec.some(a => a.name === sampleAxisName)` | `filter({ exclude: { axes: [{ name: `^${escape(sampleAxisName)}$` }], partialAxesMatch: true } })` | +| `spec.axesSpec.length === 1 && spec.axesSpec[0].name === 'pl7.app/vdj/clonotypeKey'` | `filter({ include: { axes: [{ name: '^pl7\\.app/vdj/clonotypeKey$' }] } })` (omit `partialAxesMatch` to require exact axis-set length) | +| `spec.domain?.['pl7.app/alphabet'] === 'aminoacid'` | `filter({ include: { domain: { 'pl7.app/alphabet': 'aminoacid' } } })` | +| `spec.annotations?.[Annotation.Trace]?.includes('antibody-tcr-lead-selection')` | `filter({ exclude: { annotations: { 'pl7.app/trace': '.*antibody-tcr-lead-selection.*' } } })` (regex value — selectors match values as regex) | + +Multiple `include` clauses combine as **OR** (pass arrays of `RelaxedColumnSelector`); multiple keys inside one selector combine as **AND**. `exclude` is the same shape and removes hits. + +If a predicate still doesn't fit (cross-column logic, JSON-parsing the annotations payload, computing a derived value), keep it as a JS post-filter on `getColumns()`, but minimise the survivor set with `filter()` first: + +```ts +const matches = ColumnsCollection(["result_pool"]) + .discover({ anchors: { main: anchorSpec }, mode: "enrichment", maxHops: 2 }) + .filter({ + exclude: [ + { annotations: { "pl7.app/isLinkerColumn": "true" } }, + { annotations: { "pl7.app/sequence/isAnnotation": "true" } }, + ], + }) + .filter({ exclude: { axes: [{ name: `^${escape(sampleAxisName)}$` }], partialAxesMatch: true } }) + .getColumns() + // post-filter only what selectors can't express + .filter((c) => parseTrace(c.getSpec().annotations?.[Annotation.Trace]).type !== "lead-selection"); +``` + +--- + +## `createPlDataTableV3` — full example + +The new `createPlDataTableV3` takes a declarative `columns: { sources, anchors, selector }` block and does discovery itself host-side. You no longer need to gather `ColumnSource[]` via `selectColumns` + `ArrayColumnProvider` and feed them in. + +Legacy shape (the `table` output of `antibody-tcr-lead-selection`): + +```ts +// Legacy +const resultPoolColumns = ctx.resultPool.selectColumns( + (spec) => + spec.valueType !== "File" && + !(spec.annotations?.["pl7.app/isLinkerColumn"] === "true" && spec.axesSpec.length > 2) && + !spec.annotations?.[Annotation.Trace]?.includes("antibody-tcr-lead-selection"), +); +const sources: ColumnSource[] = [ + new ArrayColumnProvider(resultPoolColumns), + new ArrayColumnProvider(sampledRows), +]; +if (assemblingKabatAccessor) + sources.push(new ArrayColumnProvider(assemblingKabatAccessor.getPColumns())); + +return createPlDataTableV3(ctx, { + columns: { + sources, + anchors: { main: leadSelectionCol.spec }, + selector: { mode: "enrichment" }, + }, + // ... +}); +``` + +New shape: + +There are two shapes of `createPlDataTableOptionsV3.columns`: + +```ts +// Form A — declarative; the helper does discovery for you. +createPlDataTableV3(ctx, { + columns: { + sources: [...], // ColumnsSource[] — NO string shorthand here + anchors: { main: leadSelectionCol.spec }, + selector: { mode: 'enrichment', exclude: [...] }, + }, + ... +}); + +// Form B — you do discovery yourself and hand recipes in directly. +createPlDataTableV3(ctx, { + primaryColumns: [leadSelection], // ColumnRecipe[] + columns: secondaryRecipes, // ColumnRecipe[] + ... +}); +``` + +**Form A pitfalls:** + +- `sources` is typed `ColumnsSource[]` — the `"result_pool"` / `"current_block"` shorthand strings are **not** accepted there (only by `ColumnsCollection(sources?)` itself). Pass `TreeNodeAccessor`s or a `ColumnsCollection` instance explicitly. +- `displayOptions.ordering[].match` and `visibility[].match` are typed **`ColumnSelector`**, not `(spec) => boolean` lambdas. Anything you can't express as a selector (axis cardinality `axesSpec.length === N`, closures over runtime args, runtime-built Sets of canonical specs) has no place in display-options anymore. + +**When to pick Form B:** whenever the legacy code's display-rules depend on lambda predicates that can't be re-expressed as selectors. In Form B you do the discovery up-front (`ColumnsCollection.discover().filter().getColumns()`), filter aggressively in JS for anything not selector-expressible, and hand the helper the recipe list it should render. The columns the table sees are already the right ones — there's nothing to hide via `visibility`. + +```ts +// Form B example — discovery + JS post-filter for non-expressible parts, +// then split into primary (anchor leaf) vs secondary (everything else). +const cols = ColumnsCollection(['result_pool']) + .discover({ anchors: { main: leadSelectionCol.spec }, mode: 'enrichment' }) + .filter({ exclude: [ + { type: 'File' }, + { annotations: { 'pl7.app/isLinkerColumn': 'true' } }, + { annotations: { 'pl7.app/trace': '.*antibody-tcr-lead-selection.*' } }, + ] }) + .getColumns() + // Non-selector-expressible: axis-count, runtime-Set membership, etc. + .filter((c) => !(isLinkerColumn(c) && c.getSpec().axesSpec.length > 2)); + +const [primary, secondary] = partitionByLeaf(cols); // bare ColumnLazy vs wrapped recipes + +return createPlDataTableV3(ctx, { + primaryColumns: primary, + columns: secondary, + ... +}); +``` + +**Always push as much filtering as possible into `collection.filter`/`discover` selectors before falling back to `.getColumns().filter(...)` — every survivor of the post-filter pays one `getSpec()` round-trip.** + +### NOT-predicates inside `displayOptions.match` + +`ColumnSelector` has no nested negation operator, and `displayOptions.ordering[]` / `visibility[]` accept only one selector per rule (no sibling `exclude` like `discover`/`filter` have). Common workarounds: + +- **`match: {}` + first-match-wins ordering.** The catch-all selector matches everything; place earlier rules that explicitly handle the "exception" set. Linker columns hidden, everything else optional: + + ```ts + visibility: [ + { match: { annotations: { "pl7.app/isLinkerColumn": "true" } }, visibility: "hidden" }, + { match: {}, visibility: "optional" }, // catch-all — only reached when the rule above didn't match + ]; + ``` + + This is the safe, declarative path. Use it for any NOT-predicate that hinges on an **optional annotation** — the WASM matcher requires a selector's annotation key to be present in the spec, so a regex like `^(?!true$).*$` won't match columns that don't carry the annotation at all. + +- **Synthetic-annotation tag via `withSpecs`.** When the "exception" set is computed from runtime state and can't be expressed as a single selector, tag the recipes upstream: + + ```ts + const TAG = "__myblock/highlighted"; + const tagged = recipes.map((c) => + specialIds.has(c.id) ? c.withSpecs({ annotations: { [TAG]: "true" } }) : c, + ); + // ... + visibility: [ + { match: { annotations: { [TAG]: "true" } }, visibility: "default" }, + { match: {}, visibility: "optional" }, + ]; + ``` + + Trade-off: each tagged recipe gets wrapped as `ColumnOverridedRecipe`, so its `id` changes — downstream resolvers must accept `ColumnUniversalId`. Use only when the `{ match: {} }` ordering approach can't express the rule. + +- **Don't bet on regex negation** (`^(?!X$).*$`) for selector values until the WASM matcher's regex flavour and absent-key semantics are confirmed. The TS-side `MatcherMap` type is identical to the legacy form, but the active matcher is in `pframes-rs` — JS-side regex-negation behaviour may not transfer. + +--- + +## `resultPool.getOptions(selectors, { refsWithEnrichments: true })` + +There are two paths to drop `ctx.resultPool.getOptions(...)`, and the choice depends on whether you can change the **wire shape** of the option-list: + +### Preferred: `deriveColumnOptions` (new helper, `ColumnUniversalId`-valued) + +`@platforma-sdk/model` exports `deriveColumnOptions(source, labelOptions?): ColumnOption[]` where `ColumnOption = { id: ColumnUniversalId; label: string }`. It accepts a pre-filtered `ColumnsCollection` or a raw `ColumnsSource[]` (provider, accessor, `"result_pool"` / `"current_block"` shorthand) and runs `deriveDistinctLabels` internally. + +```ts +import { ColumnsCollection, deriveColumnOptions } from "@platforma-sdk/model"; + +const options = deriveColumnOptions( + ColumnsCollection(["result_pool"]).discover({ + include: { axes: [{ name: "pl7.app/sampleId" }], annotations: { "pl7.app/isAnchor": "true" } }, + mode: "enrichment", + }), +); +// options: [{ id: ColumnUniversalId, label: string }, ...] +``` + +**Use this whenever you can.** It runs filtering host-side, returns ids that `createPFrame` / `createPTable` accept directly, and removes the deprecated `ctx.resultPool` surface. + +**Wire-shape implication.** The old `Option[]` shape was `{ ref: PlRef, label }`. The new shape is `{ id: ColumnUniversalId, label }`. Migrating a `getOptions` consumer means: + +- `BlockArgs` field type: `PlRef` → `ColumnUniversalId`. +- Sandbox-side reads that fed the `PlRef` into `Column(ref)` continue to work — `Column(source)` accepts both `PlRef` and `ColumnUniversalId` / `PObjectId`. No further change needed in model code. +- **Workflow (tengo) and UI consumers must be updated separately.** Workflow helpers built around `bundleBuilder.addAnchor(name, plRef)` and `columns.getSpec(plRef)` accept the `PlRef` shape; the `ColumnUniversalId` shape is a string id, not a `PlRef`, so the workflow side needs to switch to whatever accepts the id-form (out of scope for this guide). Same for UI: `option.ref === model.data.X` comparisons become `option.id === model.data.X`, with `plRefsEqual` replaced by string equality. + +If the workflow/UI updates are out of scope for the current change, **stay on `ctx.resultPool.getOptions(...)`** until you can land them together. + +### Persisted block data — migrate `PlRef` → `ColumnUniversalId` + +Changing the `BlockArgs` / `BlockData` field type from `PlRef` to `ColumnUniversalId` is not just a TypeScript rename — every block that has ever opened a project under the old shape has a stored `inputAnchor: PlRef` value persisted under the previous data-model version. Without a migration step it stays a `PlRef` object on disk, the model loads it under the new `ColumnUniversalId` type, and downstream code (`Column(id)`, `createPFrame`, etc.) silently sees an object where it expects a string. + +The two traps that catch this: + +1. **The `DataModelBuilder` chain typechecks even when no step touches the field.** A migration like `.migrate("vN+1", (prev) => ({ ...prev, ... }))` where `BlockData.inputAnchor` is now `ColumnUniversalId` will pass — but `prev.inputAnchor` is whatever was on disk, and the spread carries the `PlRef` object straight through. The type system can't catch this because the *previous* version was also annotated as `BlockData` (or `Omit`), which got dragged into the new type when `BlockData` itself changed. + +2. **Reusing the current `BlockData` for historical versions hides the conversion.** Anchor each migration's input/output type to the **shape persisted at that version**, not to a relative of the current `BlockData`. Otherwise the chain reads as if every step already handles the new field, and the only step where the actual `PlRef → ColumnUniversalId` conversion belongs is invisible. + +**Pattern.** Freeze each historical version as its own type with `inputAnchor: PlRef`, and add one new migration step whose output is the current `BlockData` and whose body actually converts the value: + +```ts +import { + createGlobalPObjectId, + DataModelBuilder, + type ColumnUniversalId, + type PlRef, +} from "@platforma-sdk/model"; + +// Stored shape at v1 — what's actually on disk under "Ver_2026_04_07". +type StoredV1 = { + inputAnchor?: PlRef; + // …other fields exactly as they were at v1… +}; + +// Stored shape at v2 — same as v1 with whatever v2's migration added. +type StoredV2 = Omit & { + sampleTableState: PlDataTableStateV2; +}; + +// For a result-pool leaf, the canonical id is `createGlobalPObjectId(blockId, name)` +// of the ref. Wrappers (Filtered/Overrided/Discovered) have no PlRef form — +// they only exist as ids, so they never need this conversion. +function plRefToUniversalId(ref: PlRef | undefined): ColumnUniversalId | undefined { + return ref ? createGlobalPObjectId(ref.blockId, ref.name) : undefined; +} + +export const blockDataModel = new DataModelBuilder() + .from("Ver_2026_04_07") + .upgradeLegacy(({ args, uiState }) => ({ + inputAnchor: args.inputAnchor, // still PlRef — upgradeLegacy outputs the v1 shape + // … + })) + .migrate("Ver_2026_04_14", (prev) => ({ + ...prev, + sampleTableState: prev.sampleTableState ?? createPlDataTableStateV2(), + })) + // New step — the only place where PlRef becomes ColumnUniversalId. + .migrate("Ver_2026_05_28", (prev) => ({ + ...prev, + inputAnchor: plRefToUniversalId(prev.inputAnchor), + })) + .init(() => ({ /* current shape */ })); +``` + +Three things to check at review time: + +- **`upgradeLegacy` returns the v1 shape, not the current `BlockData`.** Doing the `plRefToUniversalId` conversion inside `upgradeLegacy` looks tempting but means stored-v1 data (which never went through `upgradeLegacy`) still arrives as `PlRef`. Keep `upgradeLegacy` aligned with `from<...>` and put the conversion in a dedicated migration step that every code path traverses. +- **Each `migrate(...)` is typed against the next version's stored shape, not against `BlockData`.** Reusing `BlockData` (or `Omit`) for historical versions is what hid this bug in the first place — when `BlockData.inputAnchor` changed type, every historical alias quietly changed with it. +- **`init()` is the current shape.** If the new migration step's output equals `BlockData`, `init()` naturally types as `BlockData` and any drift gets caught. + +For consumers whose stored field used a value other than a bare result-pool leaf (a `PObjectId` already, a wrapper-shaped ref produced by `withEnrichments`, etc.), `plRefToUniversalId` is the wrong helper — those should round-trip through the corresponding factory. But for the common case of "options came from `ctx.resultPool.getOptions` and were saved as `PlRef`", `createGlobalPObjectId(ref.blockId, ref.name)` is the canonical translation. + +### Fallback: `ctx.resultPool.getOptions(...)` (preserves `PlRef` wire shape) + +```ts +ctx.resultPool.getOptions(selectors, { refsWithEnrichments: true }); +``` + +The `@deprecated` JSDoc on `ResultPool` lists `ctx.getOptions` on `RenderCtxBase` as the migration target, but the method has not yet been promoted there. Until it lands — or until you migrate the consumer to `ColumnUniversalId` — `ctx.resultPool.getOptions(...)` remains the supported entry point and preserves the `PlRef` enrichment-rewrite (`withEnrichments`) plus `label` derivation options. + +### Selector shape — `PColumnSelector` (legacy) vs `RelaxedColumnSelector` (new) + +`ctx.resultPool.getOptions(...)` still accepts the **legacy** `PColumnSelector`, while `ColumnsCollection.discover(...)` / `.filter(...)` take the **new** `RelaxedColumnSelector`. The two look superficially similar but differ in one important place — `axes`: + +| Field | Legacy `PColumnSelector` (getOptions) | New `RelaxedColumnSelector` (discover/filter) | +| -------------- | ------------------------------------------ | ---------------------------------------------------------------------------------------- | +| `name` | `string` | `string \| StringMatcher \| (string \| StringMatcher)[]` | +| `domain` | `Record` | `Record` | +| `annotations` | `Record` | `Record` | +| `axes` | `AxisSelector[]` with `{ name?, idx?, … }` | `RelaxedAxisSelector[]` with `{ name?, type?, domain?, … }` | +| anchor binding | `axes: [{ anchor: 'main', idx: 1 }]` | **not supported** — extract `anchorSpec.axesSpec[i].name` and pass as `axes: [{ name }]` | +| `namePattern` | `namePattern: '^pl7\\.app/foo$'` | **gone** — use `name: '^pl7\\.app/foo$'` (auto-wrapped as regex matcher) | + +Don't reuse a legacy selector literal with `ColumnsCollection`. Translate by: + +- dropping `namePattern` → fold into `name`, +- replacing `{ anchor, idx }` axis bindings with `axes: [{ name: anchorSpec.axesSpec[idx].name }]` derived from the anchor's resolved spec, +- relaxing typed `Record` values into the same plain strings (they're auto-wrapped as `{ type: "regex", value }`). + +**Prefer `{ type: 'exact', value }` over `^…$` regex for exact-name match.** Anywhere you'd write `name: '^pl7\\.app/foo$'` to mean "this exact name", write `name: [{ type: 'exact', value: 'pl7.app/foo' }]` instead. Same for `axes: [{ name: [{ type: 'exact', value: anchorSpec.axesSpec[1].name }] }]` when binding to the resolved anchor axis name. Reasons: + +- No double-escaping of `.` / `/` / other regex metachars in column-namespace strings — the value goes through literally. +- No `escapeRegex(...)` helper needed when the axis name is read off `anchorSpec` at runtime; the value is wrapped in the matcher object as-is. +- Reads as intent: "exact match on this string", not "regex that happens to be anchored and have its dots escaped". + +Keep the regex form only when you actually need pattern semantics (prefix, suffix, alternation). Example: + +```ts +// regex form — needed when intent is a pattern +{ + name: "^(pl7\\.app/vdj/sampleCount|pl7\\.app/sampleCount)$"; +} + +// exact form — preferred when intent is a single literal name +{ + name: [{ type: "exact", value: "pl7.app/vdj/sampleCount" }]; +} +``` + +Switch to `ColumnsCollection(["result_pool"]).discover({...}).getColumns()` only when the consumer wants column **ids** (e.g. for `createPFrame` / `createPTable`), not refs — `ColumnRecipe` exposes `.id`, not a reconstructible `PlRef`, and for linker-reachable hits (`ColumnDiscoveredRecipe`) there is no underlying single ref at all. + +--- + +## `accessor.getPColumns()` / `getIsFinal()` → `ColumnsCollection([accessor])` + +`TreeNodeAccessor.getPColumns()` is `@deprecated` and now returns `ColumnLazy[]` (the transitive break: every `.spec` / `.data` read on its result needs `.getSpec()` / `.getData()`). The replacement is **wrap the accessor as a source of a `ColumnsCollection`** — once that's done, every read goes through the collection's host-side surface: + +```diff +- const sampledRowsAccessor = ctx.outputs?.resolve({ field: 'sampledRows', ... }); +- const sampledRows = sampledRowsAccessor?.getPColumns(); +- const sampledRowsAreFinal = sampledRowsAccessor?.getIsFinal() ?? false; +- const leadSelectionCol = sampledRows?.find((c) => c.spec.name === 'pl7.app/lead-selection'); ++ const sampledRowsAccessor = ctx.outputs?.resolve({ field: 'sampledRows', ... }); ++ if (!sampledRowsAccessor) return undefined; ++ const sampledRowsCollection = ColumnsCollection([sampledRowsAccessor]); ++ const sampledRowsAreFinal = sampledRowsCollection.isFinal(); ++ const leadSelectionCol = sampledRowsCollection ++ .filter({ include: { name: '^pl7\\.app/lead-selection$' } }) ++ .getColumns()[0]; +``` + +Two things to notice: + +1. **Don't iterate `getPColumns()` to do name lookups.** Each named lookup → one `collection.filter({ include: { name: '^...$' } })`. The host resolves the filter and returns only the surviving recipe, so a five-named-lookup loop is five host-side queries returning ~1 id each — not five JS scans over an N-element array paying `getSpec()` per element. + +2. **`isFinal()` lives on the collection.** Don't reach back into the accessor for `getIsFinal()` once you've built the collection — they're equivalent for a single-accessor source, and the collection is the supported API going forward. + +When the same accessor is reused as a discovery **source** (e.g. `sampledRows` is both queried by name above _and_ fed into `createPlDataTableV3` as part of `sources`), pass the accessor itself or the collection's `handle` into the next `ColumnsCollection` — no need to fetch `getPColumns()` once and pass the array. + +--- + +## Helpers that still need `PColumn[]` — `ColumnLazy` → `PColumn` adapter + +A handful of SDK helpers (most notably `createPFrameForGraphs`, and any consumer typed against `PColumn[]`) still take materialised `PColumn[]` only — they do not yet accept `ColumnRecipe[]` / ids. The bridge is straightforward when every recipe in the set is a **bare leaf** (`ColumnLazy`): + +```ts +const leaves = recipes.filter(isColumnLazy); +const pCols: PColumn[] = leaves.map((c) => ({ + id: c.id, + spec: c.getSpec(), + data: c.getData()!, +})); +return createPFrameForGraphs(ctx, pCols); +``` + +Use strict `isColumnLazy` here, **not** the broader `isLeafColumn`. The `PColumn.id` slot is typed `PObjectId`, which only bare `ColumnLazy` carries — wrappers (`Overrided` / `Filtered`-over-leaf) expose `ColumnUniversalId`. Same reasoning applies to `PColumnIdAndSpec.columnId`. `isColumnLazy` is both a type guard and the canonical narrow — pass it straight to `Array.prototype.filter` (no manual predicate, no `ColumnLazyImpl` import). + +Notes: + +- This **only works for `ColumnLazy`**. `ColumnOverridedRecipe` / `ColumnDiscoveredRecipe` / `ColumnFilteredRecipe` have no `getData()` — there is no "materialise this recipe to a PColumn" path in the sandbox. If your discovery emits any wrapped recipes and you need to feed them into a PColumn-only helper, the helper needs to grow id-form support (or you avoid it for that source). +- `PColumnIdAndSpec.columnId: PObjectId` — same constraint by the same logic. Only `ColumnLazy.id` (bare `PObjectId`) goes there; never a `ColumnUniversalId` from a wrapper recipe. +- The `{ anchor: 'main', idx: 1 }` axis-binding from legacy `getAnchoredPColumns` has no `RelaxedAxisSelector` form. Workaround: read the anchor's `axesSpec[i].name` and pass it as `axes: [{ name }]` in the new selector — the axis-name match is enough for the common case. + +### Splitting columns by partition — use `expandByPartition` + +The legacy "snapshot to N synthetic PColumns" pattern (read `data` accessor, fan out, wrap each item with a fresh id, re-wrap via `ColumnLazyImpl.fromColumn`) does not fit the new recipe model. Two traps: + +1. **You cannot fabricate ids by string concatenation.** `${col.id}#${value} as PObjectId` looks fine, but the result is not canonical JSON — `JSON.parse` rejects the trailing suffix and any consumer walking the id graph (`discoverLabelColumns` → `collectLinkerIds` → `extractPObjectId` inside `createPlDataTableV3`) throws with `id "..." is not a valid canonical column id`. The cast silences the type-checker, not the runtime invariant. + +2. **You cannot embed a foreign canonical id inside `LocalPObjectId.resolvePath` either.** `createLocalPObjectId([col.id, ...], value)` parses and passes `extractPObjectId`, but it abuses the semantics of `LocalPObjectKey` (a path inside a _block's own_ local tree) by stuffing a `GlobalPObjectId` JSON in as a path element. Nested canonical ids are not how this layer composes. + +**Use the SDK helper `expandByPartition`** instead. Internally it pairs `ColumnFilteredRecipe.wrap` (pins specific axes to specific values, generates a `sliceAxes` query node) with `ColumnOverridedRecipe.wrap` (overlays domain + trace annotations). The result per split is a canonical `ColumnOverridedId(source: ColumnFilteredId, specOverrides)` — distinct, parseable, linked to the source for linker discovery — and the PFrame engine does the data slicing, so you never materialise filtered `data` in the sandbox. + +For human-readable axis-value labels in the trace annotation, pair it with `deriveAxisValuesLabels()` — the modern replacement for the legacy `ctx.resultPool.findLabels(axisId)`. It reads all label columns in scope and returns a `(axisId) => Record` resolver. + +```ts +import { + ColumnsCollection, + createPlDataTableV3, + deriveAxisValuesLabels, + expandByPartition, + isColumnLazy, + isLeafColumn, +} from "@platforma-sdk/model"; + +.outputWithStatus("tableSplit", (ctx) => { + const valueAnchor = { name: "value", axes: [{ name: "name" }] }; + + // Primary: only leaf-form hits. `discover({mode: "exact"})` with anchors + // can also surface multi-axis Discovered variants (e.g. `count [group, name]` + // reached via a linker); those belong in secondary, not in the join's primary + // side. Mirrors what `discoverTableColumns` does internally for the + // selector-form path. `isLeafColumn` accepts bare `ColumnLazy` plus + // `Overrided` / `Filtered` over a leaf — anything whose chain reaches a + // `Discovered` is rejected. + const primary = ColumnsCollection() + .discover({ anchors: { main: valueAnchor }, mode: "exact" }) + .getColumns() + .filter(isLeafColumn); + if (primary.length === 0) return undefined; + + // Inputs for the split must be unwrapped bare leaves — `expandByPartition` + // reads `getData()` on each, which only `ColumnLazy` exposes directly. Use + // strict `isColumnLazy` here, not `isLeafColumn`. + const countLeaves = ColumnsCollection() + .filter({ include: { name: [{ type: "exact", value: "count" }] } }) + .getColumns() + .filter(isColumnLazy); + + const splitRecipes = expandByPartition(countLeaves, [{ idx: 0 }], { + axisValuesLabels: deriveAxisValuesLabels(), + }); + if (splitRecipes === undefined) return undefined; + + const primaryIds = new Set(primary.map((c) => c.id)); + const secondary = ColumnsCollection() + .discover({ anchors: { main: valueAnchor }, mode: "enrichment", maxHops: 4 }) + .getColumns() + .filter((c) => !primaryIds.has(c.id)); + + return createPlDataTableV3(ctx, { + tableState: ctx.data.tableState, + primaryColumns: primary, + columns: [...secondary, ...splitRecipes], + }); +}); +``` + +What you gain over hand-rolling Filtered/Overrided yourself: + +- **No data materialisation in the sandbox** — partition inspection reads `getUniquePartitionKeys` once; the actual slicing moves into the engine via the `sliceAxes` node generated by `ColumnFilteredRecipe.getQuery()`. +- **No synthetic ids** — each split's `ColumnOverridedId` is canonical and uniquely keyed by the override patch. +- **Spec correctness for free** — `ColumnFilteredRecipe.getSpec()` removes the pinned axis from `axesSpec` automatically; `domain[axisName]` and the `pl7.app/trace` entry (used by `deriveDistinctLabels` for human-readable disambiguation) land in the outer `ColumnOverridedRecipe`. +- **Linker discovery still works** — the recipe's `getQuery()` references the inner via the rebrand-leaf-id mechanism, so `collectLinkerIds` resolves correctly. + +Two architectural invariants worth remembering when reading or extending this path: + +- **`ColumnOverridedRecipe` is always the outermost wrap.** `ColumnFilteredRecipe.withSpecs(overrides)` yields `Overrided>` automatically. Do not try to construct `Filtered>` — the SDK has no public path to that layering and the invariant is enforced inside `unwrapOverrides`. + +- **`primaryColumns` must be leaf-form only.** `discover` with anchors returns a mix of bare `ColumnLazy` (direct anchor hits), `Overrided` / `Filtered` over a leaf (projections — still leaf-form), and `ColumnDiscoveredRecipe` (multi-hop hits via linker chains). The selector-form of `createPlDataTableV3` (via `discoverTableColumns`) splits these into `primary` / `secondary` for you using `isLeafColumn`; the `primaryColumns` form trusts you to do the same. If a multi-axis Discovered slips into `primaryColumns`, `discoverLabelColumns` flat-maps its extra axes into the include set and pulls in label columns on those axes — which then appear in the engine join as disjoint-axes tables and crash with `axes sets are disjoint`. The fix is `.filter(isLeafColumn)` on the discover result — **not** `isColumnLazy`, which is stricter and drops valid `Filtered` / `Overrided`-over-leaf primaries. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82e533bd3e..043f7f0f3e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1790,6 +1790,34 @@ importers: specifier: 'catalog:' version: 4.1.3(@types/node@24.5.2)(@vitest/coverage-istanbul@4.1.3)(happy-dom@15.11.7)(jsdom@25.0.1)(vite@8.0.6(@types/node@24.5.2)(esbuild@0.27.3)(jiti@2.6.1)(sass@1.83.4)(tsx@4.19.1)(yaml@2.8.0)) + lib/model/columns-collection-driver: + dependencies: + '@milaboratories/helpers': + specifier: workspace:* + version: link:../../util/helpers + '@milaboratories/pl-model-common': + specifier: workspace:* + version: link:../common + devDependencies: + '@milaboratories/build-configs': + specifier: workspace:* + version: link:../../../tools/build-configs + '@milaboratories/ts-builder': + specifier: workspace:* + version: link:../../../tools/ts-builder + '@milaboratories/ts-configs': + specifier: workspace:* + version: link:../../../tools/ts-configs + '@vitest/coverage-istanbul': + specifier: 'catalog:' + version: 4.1.3(vitest@4.1.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.3(@types/node@24.5.2)(@vitest/coverage-istanbul@4.1.3)(happy-dom@15.11.7)(jsdom@25.0.1)(vite@8.0.6(@types/node@24.5.2)(esbuild@0.27.3)(jiti@2.6.1)(sass@1.83.4)(tsx@4.19.1)(yaml@2.8.0)) + lib/model/common: dependencies: '@milaboratories/helpers': @@ -1801,6 +1829,9 @@ importers: canonicalize: specifier: 'catalog:' version: 2.1.0 + es-toolkit: + specifier: 'catalog:' + version: 1.39.10 zod: specifier: 'catalog:' version: 3.25.76 @@ -2454,6 +2485,9 @@ importers: lib/node/pl-middle-layer: dependencies: + '@milaboratories/columns-collection-driver': + specifier: workspace:* + version: link:../../model/columns-collection-driver '@milaboratories/computable': specifier: workspace:* version: link:../computable @@ -2957,6 +2991,9 @@ importers: fast-json-patch: specifier: 'catalog:' version: 3.1.1 + lru-cache: + specifier: 'catalog:' + version: 11.2.2 utility-types: specifier: 'catalog:' version: 3.11.0 @@ -2967,6 +3004,9 @@ importers: '@milaboratories/build-configs': specifier: workspace:* version: link:../../tools/build-configs + '@milaboratories/columns-collection-driver': + specifier: workspace:* + version: link:../../lib/model/columns-collection-driver '@milaboratories/pf-driver': specifier: workspace:* version: link:../../lib/node/pf-driver @@ -3031,6 +3071,9 @@ importers: sdk/ui-vue: dependencies: + '@milaboratories/columns-collection-driver': + specifier: workspace:* + version: link:../../lib/model/columns-collection-driver '@milaboratories/pf-spec-driver': specifier: workspace:* version: link:../../lib/model/pf-spec-driver diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6f59f6c262..8745276832 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -23,6 +23,7 @@ packages: - lib/node/computable - lib/node/pf-driver - lib/model/pf-spec-driver + - lib/model/columns-collection-driver - lib/node/pl-client - lib/node/pl-healthcheck - lib/node/pl-tree diff --git a/sdk/model/package.json b/sdk/model/package.json index 10df2972b2..3b6485c786 100644 --- a/sdk/model/package.json +++ b/sdk/model/package.json @@ -44,11 +44,13 @@ "canonicalize": "catalog:", "es-toolkit": "catalog:", "fast-json-patch": "catalog:", + "lru-cache": "catalog:", "utility-types": "catalog:", "zod": "catalog:" }, "devDependencies": { "@milaboratories/build-configs": "workspace:*", + "@milaboratories/columns-collection-driver": "workspace:*", "@milaboratories/pf-driver": "workspace:*", "@milaboratories/pf-spec-driver": "workspace:*", "@milaboratories/ts-builder": "workspace:*", diff --git a/sdk/model/src/columns/__test_helpers__/collection_driver.ts b/sdk/model/src/columns/__test_helpers__/collection_driver.ts new file mode 100644 index 0000000000..a088d6b48d --- /dev/null +++ b/sdk/model/src/columns/__test_helpers__/collection_driver.ts @@ -0,0 +1,197 @@ +/** + * Test-only wiring for `ColumnsCollectionDriverModel`. Wraps a real + * `ColumnsCollectionDriverImpl` (so the discover / filter algebra is exercised + * end-to-end) with stub bindings backed by an in-memory `(id → spec)` map. + * + * Accessor-based source kinds (`"accessor"`, `"result_pool"`) throw — they + * have no meaning outside a render ctx. Tests should stick to `"ids"` / + * `"collection"` sources. + * + * Calling {@link TestCollectionDriverHandle.installAmbientCtx} additionally + * installs a minimal `globalThis.cfgRenderCtx` AND plants a stub + * `ColumnEntriesProvider` into the ctx-providers cache so registered specs + * are reachable via `ColumnLazy.fromId` (and therefore via + * `ColumnsCollection.getColumns()`). + */ +import type { + ColumnEntriesProvider, + ColumnsCollectionDriverHost, + ColumnsCollectionDriverModel, + LeafEntry, + PColumnSpec, + PObjectId, + ServiceName, +} from "@milaboratories/pl-model-common"; +import { extractPObjectId, Services } from "@milaboratories/pl-model-common"; +import { ColumnsCollectionDriverImpl } from "@milaboratories/columns-collection-driver"; +import { SpecDriver } from "@milaboratories/pf-spec-driver"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import { TreeNodeAccessor } from "../../render/accessor"; +import { _ctxProvidersCache } from "../column_providers"; +import type { ColumnsProvider } from "../column_providers"; +import { ColumnLazy, ColumnLazyImpl } from "../column_lazy"; + +export interface TestCollectionDriverHandle { + readonly driver: ColumnsCollectionDriverModel; + /** Register `(id → spec)` pairs the driver will see via `resolveSpec`. */ + register(columns: ReadonlyArray<{ readonly id: PObjectId; readonly spec: PColumnSpec }>): void; + /** + * Install a minimal `globalThis.cfgRenderCtx` so `getService("columnsCollection")` + * resolves to this driver and registered columns are reachable via + * `ColumnLazy.fromId`. Call inside `beforeEach` if the code under test uses + * the ambient ctx instead of an explicit `driver` option. + */ + installAmbientCtx(): void; + /** Remove the ambient ctx installed by {@link installAmbientCtx}. */ + uninstallAmbientCtx(): void; + /** Dispose the underlying SpecDriver. */ + dispose(): Promise; +} + +const COLUMNS_COLLECTION_METHODS: ReadonlyArray = [ + "create", + "isEmpty", + "isFinal", + "getColumns", + "addSource", + "discover", + "filter", +]; + +const PFRAME_SPEC_METHODS: ReadonlyArray = [ + "createSpecFrame", + "listColumns", + "discoverColumns", + "deleteColumn", + "evaluateQuery", + "buildQuery", + "expandAxes", + "collapseAxes", + "findAxis", + "findTableColumn", +]; + +export function createTestCollectionDriver(): TestCollectionDriverHandle { + const specMap = new Map(); + const specDriver = new SpecDriver(); + const impl = new ColumnsCollectionDriverImpl(); + + const bindings: ColumnsCollectionDriverHost = { + resolveAccessor: () => { + throw new Error("test collection driver: no accessor support"); + }, + getUpstreamBlockCtxes: () => [], + getSpecDriver: () => specDriver, + resolveSpec: (id) => specMap.get(extractPObjectId(id)), + }; + + const driver: ColumnsCollectionDriverModel = { + create: (sources) => impl.create(sources, bindings).key, + isEmpty: (h) => impl.isEmpty(h), + isFinal: (h) => impl.isFinal(h), + getColumns: (h) => impl.getColumns(h, bindings), + addSource: (h, srcs) => impl.addSource(h, srcs, bindings).key, + discover: (h, o) => impl.discover(h, o, bindings).key, + filter: (h, o) => impl.filter(h, o, bindings).key, + }; + + let installedCtx: GlobalCfgRenderCtx | undefined; + + return { + driver, + register(columns) { + for (const c of columns) specMap.set(c.id, c.spec); + if (installedCtx !== undefined) { + _ctxProvidersCache.set(installedCtx, [buildStubProvider(specMap)]); + } + }, + installAmbientCtx() { + const ctx = { + getAccessorHandleByName: () => undefined, + getUpstreamBlockCtx: () => [], + getServiceNames: () => [Services.ColumnsCollection, Services.PFrameSpec] as ServiceName[], + getServiceMethods: (id: ServiceName) => { + if ((id as unknown) === Services.ColumnsCollection) + return COLUMNS_COLLECTION_METHODS.slice(); + if ((id as unknown) === Services.PFrameSpec) return PFRAME_SPEC_METHODS.slice(); + return []; + }, + callServiceMethod: (id: ServiceName, method: string, ...args: unknown[]) => { + if ((id as unknown) === Services.ColumnsCollection) { + const fn = (driver as unknown as Record unknown>)[method]; + return fn(...args); + } + if ((id as unknown) === Services.PFrameSpec) { + const fn = (specDriver as unknown as Record unknown>)[ + method + ]; + return fn.apply(specDriver, args); + } + throw new Error(`test ctx: service "${id}" not stubbed`); + }, + } as unknown as GlobalCfgRenderCtx; + installedCtx = ctx; + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + _ctxProvidersCache.set(ctx, [buildStubProvider(specMap)]); + }, + uninstallAmbientCtx() { + installedCtx = undefined; + delete (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx; + }, + async dispose() { + await specDriver.dispose(); + }, + }; +} + +/** + * Construct a {@link ColumnEntriesProvider} + {@link ColumnsProvider} backed + * by an `(id → spec)` map. Each entry carries a fake {@link TreeNodeAccessor} + * whose only contract is the surface `readSpec` / `readData` / + * `readDataStatus` (in `p_column_lazy.ts`) actually call: + * - `traverse({field: `${name}.spec`, ignoreError: true})` → + * stub-accessor whose `getDataAsJson()` returns the spec. + * - `traverse({field: `${name}.data`})` → `undefined` (data is not + * materialised in tests). + * - `listInputFields()` → `[]` (data absent). + * - `getInputsLocked()` → `true` (no pending resolution). + */ +function buildStubProvider( + specMap: ReadonlyMap, +): ColumnEntriesProvider & ColumnsProvider { + const entries = new Map>(); + for (const [id, spec] of specMap) { + entries.set(id, { accessor: stubAccessorFor(id, spec), name: id, id }); + } + const columns: ColumnLazy[] = []; + for (const [id, spec] of specMap) { + columns.push(ColumnLazyImpl.fromColumn({ id, spec, data: undefined as never })); + } + return { + getPObjectEntries: () => entries, + isFinal: () => true, + getColumns: () => columns, + }; +} + +function stubAccessorFor(id: PObjectId, spec: PColumnSpec): TreeNodeAccessor { + const specField = `${id}.spec`; + const specHolder = { + getDataAsJson: () => spec as unknown as T, + hasData: () => true, + }; + const stub = { + handle: id, + resolvePath: [], + traverse: (...steps: unknown[]) => { + if (steps.length === 0) return stub; + const first = steps[0] as { field?: string } | string; + const field = typeof first === "string" ? first : first?.field; + return field === specField ? specHolder : undefined; + }, + getDataAsJson: () => spec as unknown as T, + listInputFields: () => [], + getInputsLocked: () => true, + }; + return stub as unknown as TreeNodeAccessor; +} diff --git a/sdk/model/src/columns/__test_helpers__/stub_registry.ts b/sdk/model/src/columns/__test_helpers__/stub_registry.ts new file mode 100644 index 0000000000..4c0a8afb9c --- /dev/null +++ b/sdk/model/src/columns/__test_helpers__/stub_registry.ts @@ -0,0 +1,134 @@ +/** + * Test-only: plant a stub provider into `_ctxProvidersCache` so that + * `ColumnRegistry.resolve(id)` returns a `LeafEntry` whose spec accessor + * reports `hasData() === true` and `getDataAsJson()` returns the supplied + * spec. Sufficient for code that goes through + * `getCtxProviders` / `readLeafSpecAccessor` (e.g. `ColumnLazy.fromId`, + * `ColumnDiscoveredRecipe.fromKey`). + * + * Two leaf shapes: + * - `PColumnSpec` (shorthand) — spec present, accessor locked, `hasData` + * true. Discriminated at runtime by the presence of a `kind` field. + * - {@link StubLeafConfig} — fine-grained control: omit `spec` to simulate + * "no spec field reachable from this leaf"; set `accessorLocked: false` + * to simulate a still-resolving leaf accessor; set `specHasData: false` + * to simulate "spec resource exists but bytes not yet written". + * + * Registry-wide `isFinal` is also configurable — pass `{ isFinal: false }` + * to simulate a still-enumerating registry where missing ids should report + * `resolving` rather than `absent`. + */ +import type { + ColumnEntriesProvider, + LeafEntry, + PColumnSpec, + PObjectId, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import type { TreeNodeAccessor } from "../../render/accessor"; +import { _ctxProvidersCache } from "../column_providers"; +import type { ColumnsProvider } from "../column_providers"; +import { ColumnLazy, ColumnLazyImpl } from "../column_lazy"; + +/** Fine-grained per-leaf stub config. Defaults match the shorthand form. */ +export type StubLeafConfig = { + /** Spec returned by `${id}.spec` traverse. Omit to simulate a missing field. */ + spec?: PColumnSpec; + /** Drives `leaf.accessor.getInputsLocked()`. Default `true`. */ + accessorLocked?: boolean; + /** Drives `spec.hasData()` (ignored when `spec` is omitted). Default `true`. */ + specHasData?: boolean; +}; + +/** Either the shorthand `PColumnSpec` or a {@link StubLeafConfig}. */ +export type StubLeafInput = PColumnSpec | StubLeafConfig; + +export function installStubRegistry( + ctx: GlobalCfgRenderCtx, + leaves: ReadonlyMap | Record, + { isFinal = true }: { isFinal?: boolean } = {}, +): void { + const map = + leaves instanceof Map + ? leaves + : new Map(Object.entries(leaves) as [PObjectId, StubLeafInput][]); + _ctxProvidersCache.set(ctx, [buildStubProvider(map, isFinal)]); +} + +function buildStubProvider( + leaves: ReadonlyMap, + isFinal: boolean, +): ColumnEntriesProvider & ColumnsProvider { + const { entries, columns } = [...leaves].reduce( + (acc, [id, raw]) => { + const cfg = normalizeLeafInput(raw); + acc.entries.set(id, { accessor: stubAccessorFor(id, cfg), name: id, id }); + if (cfg.spec !== undefined && cfg.specHasData) { + acc.columns.push( + ColumnLazyImpl.fromColumn({ id, spec: cfg.spec, data: undefined as never }), + ); + } + return acc; + }, + { + entries: new Map>(), + columns: [] as ColumnLazy[], + }, + ); + return { + getPObjectEntries: () => entries, + isFinal: () => isFinal, + getColumns: () => columns, + }; +} + +type NormalizedLeafConfig = { + spec?: PColumnSpec; + accessorLocked: boolean; + specHasData: boolean; +}; + +function normalizeLeafInput(raw: StubLeafInput): NormalizedLeafConfig { + if (isPColumnSpec(raw)) { + return { spec: raw, accessorLocked: true, specHasData: true }; + } + return { + spec: raw.spec, + accessorLocked: raw.accessorLocked ?? true, + specHasData: raw.specHasData ?? true, + }; +} + +function isPColumnSpec(v: StubLeafInput): v is PColumnSpec { + return typeof v === "object" && v !== null && "kind" in v; +} + +let stubAccessorSeq = 0; + +function stubAccessorFor(id: PObjectId, cfg: NormalizedLeafConfig): TreeNodeAccessor { + const specField = `${id}.spec`; + const specHolder = + cfg.spec === undefined + ? undefined + : { + getDataAsJson: () => cfg.spec as unknown as T, + hasData: () => cfg.specHasData, + }; + // Per-installation unique handle. The module-level `readSpecAccessor` LRU + // cache keys by `${accessor.handle}:${name}`, and reinstalling the stub + // for the same id should produce a fresh cache lookup (otherwise tests + // that vary leaf state for the same id see stale spec accessors). + const handle = `${id}#${++stubAccessorSeq}`; + const stub = { + handle, + resolvePath: [], + traverse: (step: { field?: string } | string) => { + const field = typeof step === "string" ? step : step?.field; + return field === specField ? specHolder : undefined; + }, + getDataAsJson: () => cfg.spec as unknown as T, + listInputFields: () => (cfg.spec !== undefined ? [specField] : []), + getInputsLocked: () => cfg.accessorLocked, + }; + return stub as unknown as TreeNodeAccessor; +} diff --git a/sdk/model/src/columns/column.ts b/sdk/model/src/columns/column.ts new file mode 100644 index 0000000000..32994768d2 --- /dev/null +++ b/sdk/model/src/columns/column.ts @@ -0,0 +1,46 @@ +import type { + ColumnUniversalId, + LeafEntry, + PColumn, + PlRef, + PObjectId, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../render/internal"; +import type { TreeNodeAccessor } from "../render"; +import { ColumnLazy, ColumnLazyImpl, type ColumnLazyData } from "./column_lazy"; +import { ColumnRecipe, ColumnRecipeId, isColumnRecipe } from "./column_recipes"; + +export type ColumnSource = + | PObjectId + | ColumnUniversalId + | PlRef + | LeafEntry + | PColumn + | ColumnLazyImpl; + +/** + * Unified entry point — routes between the two top-level dispatchers: + * - string id (`PObjectId` / `ColumnUniversalId`) → {@link ColumnRecipe} + * - object source (`PlRef` / `LeafEntry` / `PColumn` / `ColumnLazy`) → {@link ColumnLazy} + * + * `ColumnLazy` is itself a `ColumnRecipe`, so the return type is + * the common `ColumnRecipe`. + */ +export function Column( + source: ColumnSource, + opts?: { ctx?: GlobalCfgRenderCtx }, +): undefined | ColumnRecipe { + if (typeof source === "string") return ColumnRecipe(source, opts); + return ColumnLazy(source, opts); +} + +export type Column = ColumnRecipe; + +/** + * Type-guard for the `Column` type — currently aliased to `ColumnRecipe`, so + * this is equivalent to {@link isColumnRecipe}. Kept as a separate export to + * mirror the `Column` / `ColumnRecipe` / `ColumnLazy` factory triple. + */ +export function isColumn(value: unknown): value is Column { + return isColumnRecipe(value); +} diff --git a/sdk/model/src/columns/column_collection_builder.test.ts b/sdk/model/src/columns/column_collection_builder.test.ts deleted file mode 100644 index fb28a5c701..0000000000 --- a/sdk/model/src/columns/column_collection_builder.test.ts +++ /dev/null @@ -1,484 +0,0 @@ -import type { AxisSpec, PColumnSpec, PObjectId } from "@milaboratories/pl-model-common"; -import { SpecDriver } from "@milaboratories/pf-spec-driver"; - -import { afterEach, describe, expect, test } from "vitest"; -import type { ColumnSnapshotProvider } from "./column_snapshot_provider"; -import type { ColumnSnapshot } from "./column_snapshot"; -import { ColumnCollectionBuilder } from "./column_collection_builder"; - -const drivers: SpecDriver[] = []; - -function createSpecFrameCtx() { - const driver = new SpecDriver(); - drivers.push(driver); - return driver; -} - -afterEach(async () => { - for (const driver of drivers) await driver.dispose(); - drivers.length = 0; -}); - -// --- Helpers --- - -/** Default axis used when none specified — WASM requires at least one axis. */ -const DEFAULT_AXIS: AxisSpec = { name: "id", type: "String" } as AxisSpec; - -function createSpec( - name: string, - options?: { domain?: Record; axesSpec?: AxisSpec[] }, -): PColumnSpec { - return { - kind: "PColumn", - name, - valueType: "Int", - axesSpec: options?.axesSpec ?? [DEFAULT_AXIS], - annotations: {}, - ...(options?.domain !== undefined ? { domain: options.domain } : {}), - } as PColumnSpec; -} - -function createSnapshot( - id: string, - spec: PColumnSpec, - dataStatus: "ready" | "computing" | "absent" = "ready", -): ColumnSnapshot { - return { - id: id as PObjectId, - spec, - dataStatus, - data: - dataStatus === "absent" - ? undefined - : { get: () => (dataStatus === "ready" ? ({} as never) : undefined) }, - }; -} - -function createProvider( - snapshots: ColumnSnapshot[], - complete = true, -): ColumnSnapshotProvider { - return { - getAllColumns() { - return snapshots; - }, - isColumnListComplete() { - return complete; - }, - }; -} - -function sampleAxis(name: string, type = "String"): AxisSpec { - return { name, type } as AxisSpec; -} - -// --- Tests --- - -describe("ColumnCollectionBuilder", () => { - test("empty builder returns empty collection", () => { - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - const collection = builder.build(); - expect(collection).toBeDefined(); - expect(collection!.findColumns()).toEqual([]); - }); - - test("addSource returns this for chaining", () => { - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - const result = builder.addSource([]); - expect(result).toBe(builder); - }); - - test("build returns undefined when providers are incomplete", () => { - const snap = createSnapshot("id1", createSpec("col1")); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([snap], false)); - expect(builder.build()).toBeUndefined(); - }); - - test("build with allowPartialColumnList returns collection even when incomplete", () => { - const snap = createSnapshot("id1", createSpec("col1")); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([snap], false)); - - const collection = builder.build({ allowPartialColumnList: true }); - expect(collection).toBeDefined(); - expect(collection.findColumns()).toHaveLength(1); - }); - - test("allowPartialColumnList with complete providers returns collection", () => { - const snap = createSnapshot("id1", createSpec("col1")); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([snap], true)); - - const collection = builder.build({ allowPartialColumnList: true }); - expect(collection).toBeDefined(); - expect(collection.findColumns()).toHaveLength(1); - }); -}); - -describe("ColumnCollection lookup by id", () => { - test("findColumns includes snapshot with existing id", () => { - const spec = createSpec("col1"); - const snap = createSnapshot("id1", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap]); - - const collection = builder.build()!; - const found = collection.findColumns().find((c) => c.id === ("id1" as PObjectId)); - expect(found).toBeDefined(); - expect(found!.spec.name).toBe("col1"); - }); - - test("findColumns does not include missing id", () => { - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([createSnapshot("id1", createSpec("col1"))]); - - const collection = builder.build()!; - const found = collection.findColumns().find((c) => c.id === ("missing" as PObjectId)); - expect(found).toBeUndefined(); - }); -}); - -describe("ColumnCollection.findColumns", () => { - test("no options returns all columns", () => { - const s1 = createSnapshot("id1", createSpec("col1")); - const s2 = createSnapshot("id2", createSpec("col2")); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([s1, s2]); - - const collection = builder.build()!; - expect(collection.findColumns()).toHaveLength(2); - }); - - test("exclude filters out matching columns", () => { - const s1 = createSnapshot("id1", createSpec("col1")); - const s2 = createSnapshot("id2", createSpec("col2")); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([s1, s2]); - - const collection = builder.build()!; - const results = collection.findColumns({ exclude: { name: "col1" } }); - expect(results).toHaveLength(1); - expect(results[0].spec.name).toBe("col2"); - }); - - test("include filters by exact name among disjoint-axis columns", () => { - const vdjAxis = sampleAxis("pl7.app/vdj/clonotypeKey"); - const itemAxis = sampleAxis("item"); - - const vdjColumns = Array.from({ length: 10 }, (_, i) => - createSnapshot(`vdj${i}`, createSpec(`pl7.app/vdj/col${i}`, { axesSpec: [vdjAxis] })), - ); - const mockScore = createSnapshot("mock", createSpec("mock_score", { axesSpec: [itemAxis] })); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([...vdjColumns, mockScore]); - - const collection = builder.build()!; - const results = collection.findColumns({ - include: [{ name: [{ type: "exact", value: "mock_score" }] }], - }); - - expect(results).toHaveLength(1); - expect(results[0].spec.name).toBe("mock_score"); - }); -}); - -describe("dedup by native ID", () => { - test("first source wins when specs have same native ID", () => { - const spec = createSpec("col1"); - const snap1 = createSnapshot("id-first", spec); - const snap2 = createSnapshot("id-second", spec); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap1]); - builder.addSource([snap2]); - - const collection = builder.build()!; - const all = collection.findColumns(); - expect(all).toHaveLength(1); - expect(all[0].id).toBe("id-first"); - }); - - test("different specs are not deduped", () => { - const snap1 = createSnapshot("id1", createSpec("col1")); - const snap2 = createSnapshot("id2", createSpec("col2")); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap1]); - builder.addSource([snap2]); - - const collection = builder.build()!; - expect(collection.findColumns()).toHaveLength(2); - }); -}); - -describe("data status handling", () => { - test("ready column data is accessible", () => { - const data = { someKey: "someValue" }; - const snap: ColumnSnapshot = { - id: "id1" as PObjectId, - spec: createSpec("col1"), - dataStatus: "ready", - data: { get: () => data as never }, - }; - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap]); - const collection = builder.build()!; - - const found = collection.findColumns().find((c) => c.id === ("id1" as PObjectId))!; - expect(found.dataStatus).toBe("ready"); - expect(found.data).toBeDefined(); - expect(found.data!.get()).toBe(data); - }); - - test("computing column data returns undefined", () => { - const snap = createSnapshot("id1", createSpec("col1"), "computing"); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap]); - const collection = builder.build()!; - - const found = collection.findColumns().find((c) => c.id === ("id1" as PObjectId))!; - expect(found.dataStatus).toBe("computing"); - expect(found.data).toBeDefined(); - - const result = found.data!.get(); - expect(result).toBeUndefined(); - }); - - test("absent column has no data accessor", () => { - const snap = createSnapshot("id1", createSpec("col1"), "absent"); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap]); - const collection = builder.build()!; - - const found = collection.findColumns().find((c) => c.id === ("id1" as PObjectId))!; - expect(found.dataStatus).toBe("absent"); - expect(found.data).toBeUndefined(); - }); -}); - -describe("multiple providers", () => { - test("columns from multiple sources are merged", () => { - const s1 = createSnapshot("id1", createSpec("col1")); - const s2 = createSnapshot("id2", createSpec("col2")); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([s1]); - builder.addSource([s2]); - - const collection = builder.build()!; - expect(collection.findColumns()).toHaveLength(2); - }); - - test("build returns undefined if any provider is incomplete", () => { - const s1 = createSnapshot("id1", createSpec("col1")); - const s2 = createSnapshot("id2", createSpec("col2")); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([s1], true)); - builder.addSource(createProvider([s2], false)); - - expect(builder.build()).toBeUndefined(); - }); - - test("allowPartialColumnList returns collection when any provider incomplete", () => { - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([createSnapshot("id1", createSpec("col1"))], true)); - builder.addSource(createProvider([createSnapshot("id2", createSpec("col2"))], false)); - - const collection = builder.build({ allowPartialColumnList: true }); - expect(collection).toBeDefined(); - expect(collection.findColumns()).toHaveLength(2); - }); -}); - -describe("AnchoredColumnCollection", () => { - // The anchor spec must also exist as a source column — the new implementation - // resolves PColumnSpec anchors by matching native ID in the collected columns. - const anchorSpec = createSpec("anchor-col", { - axesSpec: [sampleAxis("sample"), sampleAxis("gene")], - }); - const anchorSnap = createSnapshot("anchor-snap-id", anchorSpec); - - test("build with PColumnSpec anchor returns anchored collection", () => { - const s1 = createSnapshot("id1", createSpec("col1", { axesSpec: [sampleAxis("sample")] })); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - // anchorSnap must be in sources so resolveAnchorMap can find it by native ID - builder.addSource([s1, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } }); - expect(collection).toBeDefined(); - }); - - test("build with PObjectId anchor resolves from sources", () => { - const spec = createSpec("col1", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("anchor-id", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap]); - - const collection = builder.build({ anchors: { main: "anchor-id" as PObjectId } }); - expect(collection).toBeDefined(); - }); - - test("build with unknown PObjectId anchor throws", () => { - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([createSnapshot("id1", createSpec("col1"))]); - - expect(() => builder.build({ anchors: { main: "missing" as PObjectId } })).toThrow( - /not found in sources/, - ); - }); - - test("findColumns surfaces column by original PObjectId", () => { - const spec = createSpec("col1", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("id1", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const matches = collection.findColumns(); - const found = matches.find((m) => m.column.id === ("id1" as PObjectId)); - expect(found).toBeDefined(); - expect(found!.column.spec.name).toBe("col1"); - }); - - test("findColumns does not include unknown id", () => { - const snap = createSnapshot("id1", createSpec("col1", { axesSpec: [sampleAxis("sample")] })); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - const found = collection - .findColumns() - .find((m) => m.column.id === ("not-a-real-id" as PObjectId)); - expect(found).toBeUndefined(); - }); - - test("getAnchors returns resolved anchor map", () => { - const spec = createSpec("col1", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("id1", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - const anchors = collection.getAnchors(); - expect(anchors.size).toBe(1); - expect(anchors.get("main")).toBeDefined(); - expect(anchors.get("main")!.spec.name).toBe("anchor-col"); - }); - - test("findColumns returns ColumnMatch with column id and variants", () => { - const spec = createSpec("col1", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("id1", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - // anchorSnap itself also appears in findColumns results (axes ⊆ anchor axes) - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - const matches = collection.findColumns(); - - // col1 + anchor-col are both discovered - expect(matches.length).toBeGreaterThanOrEqual(1); - const col1Match = matches.find((m) => m.column.spec.name === "col1")!; - expect(col1Match).toBeDefined(); - expect(col1Match.column.id).toBe("id1"); - expect(col1Match.variants).toBeDefined(); - }); - - test("variants carry forQueries keyed by anchor PObjectId", () => { - const spec = createSpec("col1", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("id1", spec); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - const matches = collection.findColumns(); - const col1Match = matches.find((m) => m.column.spec.name === "col1")!; - - expect(col1Match.variants.length).toBeGreaterThan(0); - for (const v of col1Match.variants) { - expect(v.qualifications.forQueries).toBeUndefined(); - expect(v.qualifications.forHit).toBeUndefined(); - } - }); - - test("anchors sharing same axes group produce forQueries entries pointing to same array", () => { - // Two anchors with identical axesSpec share an axes-group bucket in the reverse index. - const sharedAxes = [sampleAxis("sample"), sampleAxis("gene")]; - const anchorA = createSpec("anchor-a", { axesSpec: sharedAxes }); - const anchorB = createSpec("anchor-b", { axesSpec: sharedAxes }); - const anchorASnap = createSnapshot("anchor-a-id", anchorA); - const anchorBSnap = createSnapshot("anchor-b-id", anchorB); - - const colSpec = createSpec("c1", { axesSpec: [sampleAxis("sample")] }); - const colSnap = createSnapshot("c1-id", colSpec); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([colSnap, anchorASnap, anchorBSnap]); - - const collection = builder.build({ anchors: { a: anchorA, b: anchorB } })!; - const matches = collection.findColumns(); - const c1 = matches.find((m) => m.column.spec.name === "c1")!; - expect(c1).toBeDefined(); - - for (const v of c1.variants) { - const forQueries = v.qualifications.forQueries; - if (forQueries && anchorASnap.id in forQueries && anchorBSnap.id in forQueries) { - // Shared axes group → equal qualifications for both anchor keys. - expect(forQueries[anchorASnap.id]).toStrictEqual(forQueries[anchorBSnap.id]); - } - } - }); - - test("findColumns exclude filters out matching columns", () => { - const snap1 = createSnapshot("id1", createSpec("col1", { axesSpec: [sampleAxis("sample")] })); - const snap2 = createSnapshot("id2", createSpec("col2", { axesSpec: [sampleAxis("sample")] })); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap1, snap2, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - const results = collection.findColumns({ exclude: { name: "col1" } }); - expect(results.every((r) => r.column.spec.name !== "col1")).toBe(true); - expect(results.some((r) => r.column.spec.name === "col2")).toBe(true); - }); - - test("allowPartialColumnList with anchors returns collection when incomplete", () => { - const snap = createSnapshot("id1", createSpec("col1", { axesSpec: [sampleAxis("sample")] })); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([snap, anchorSnap], false)); - - const collection = builder.build({ - anchors: { main: anchorSpec }, - allowPartialColumnList: true, - }); - expect(collection).toBeDefined(); - }); - - test("build returns undefined with anchors when incomplete and no allowPartial", () => { - const snap = createSnapshot("id1", createSpec("col1", { axesSpec: [sampleAxis("sample")] })); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource(createProvider([snap], false)); - - const result = builder.build({ anchors: { main: anchorSpec } }); - expect(result).toBeUndefined(); - }); - - test("data status is preserved through anchored snapshots", () => { - const spec = createSpec("computing-col", { axesSpec: [sampleAxis("sample")] }); - const snap = createSnapshot("id1", spec, "computing"); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([snap, anchorSnap]); - - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const found = collection.findColumns().find((m) => m.column.id === ("id1" as PObjectId))!; - expect(found.column.dataStatus).toBe("computing"); - expect(found.column.data!.get()).toBeUndefined(); - }); -}); diff --git a/sdk/model/src/columns/column_collection_builder.ts b/sdk/model/src/columns/column_collection_builder.ts deleted file mode 100644 index 11ab4aa0ee..0000000000 --- a/sdk/model/src/columns/column_collection_builder.ts +++ /dev/null @@ -1,485 +0,0 @@ -import type { - AxisQualification, - DiscoverColumnsConstraints, - DiscoverColumnsRequest, - DiscoverColumnsResponse, - MultiColumnSelector, - NativePObjectId, - PColumnSpec, - PObjectId, -} from "@milaboratories/pl-model-common"; -import { deriveNativeId, isPColumnSpec } from "@milaboratories/pl-model-common"; -import type { ColumnSelector, RelaxedColumnSelector } from "./column_selector"; -import { convertColumnSelectorToMultiColumnSelector } from "./column_selector"; -import { TreeNodeAccessor } from "../render/accessor"; -import type { ColumnSnapshot } from "./column_snapshot"; -import type { ColumnSnapshotProvider, ColumnSource } from "./column_snapshot_provider"; -import { ArrayColumnProvider, toColumnSnapshotProvider } from "./column_snapshot_provider"; - -import type { PFrameSpecDriver, PoolEntry, SpecFrameHandle } from "@milaboratories/pl-model-common"; -import { throwError } from "@milaboratories/helpers"; -import { getService } from "../services"; - -/** Options for plain collection findColumns. */ -export interface FindColumnsOptions { - /** Include columns matching these selectors. If omitted, includes all columns. */ - include?: ColumnSelector; - /** Exclude columns matching these selectors. */ - exclude?: ColumnSelector; -} - -/** Plain collection — no axis context, selector-based filtering only. */ -export interface ColumnCollection extends Disposable { - /** Release the underlying spec frame WASM resource. */ - dispose(): void; - - /** Find columns matching selectors. Returns flat list of snapshots. - * No axis compatibility matching, no linker traversal. - * Never returns undefined — the "not ready" state was absorbed by the builder. */ - findColumns(options?: FindColumnsOptions): ColumnSnapshot[]; -} - -/** Axis-aware column collection with anchored identity derivation. */ -export interface AnchoredColumnCollection extends Disposable { - /** Release the underlying spec frame WASM resource. */ - dispose(): void; - - /** List of anchors used for discovery, with their resolved specs. */ - getAnchors(): Map>; - - /** Axis-aware column discovery. */ - findColumns(options?: AnchoredFindColumnsOptions): ColumnMatch[]; - - /** Variant discovery with detailed mapping info for each hit. */ - findColumnVariants(options?: AnchoredFindColumnsOptions): ColumnVariant[]; -} - -/** Controls axis matching behavior for anchored discovery. */ -export type MatchingMode = "enrichment" | "related" | "exact"; - -/** Options for anchored collection findColumns. */ -export interface AnchoredFindColumnsOptions extends FindColumnsOptions { - /** Controls axis matching behavior. Default: 'enrichment'. */ - mode?: MatchingMode; - /** Maximum linker hops for cross-domain discovery (0 = direct only, default: 4). */ - maxHops?: number; -} - -/** Result of anchored discovery — column snapshot + routing info. */ -export interface ColumnMatch { - /** Column snapshot with anchored SUniversalPColumnId. */ - readonly column: ColumnSnapshot; - /** Match variants — different ways (paths/qualifications) to reach this column. */ - readonly variants: MatchVariant[]; -} - -export interface ColumnVariant { - /** Column snapshot with anchored SUniversalPColumnId. */ - readonly column: ColumnSnapshot; - /** Linker steps traversed to reach this hit; empty for direct matches. */ - readonly path?: { linker: ColumnSnapshot }[]; - /** Full qualifications needed for integration. */ - readonly qualifications?: MatchQualifications; -} - -/** A single mapping variant describing how a hit column can be integrated. */ -export interface MatchVariant { - /** Full qualifications needed for integration. */ - readonly qualifications: MatchQualifications; - /** Linker steps traversed to reach this hit; empty for direct matches. */ - readonly path: { linker: ColumnSnapshot }[]; -} - -/** Qualifications needed for both already-integrated anchor columns and the hit column. */ -export interface MatchQualifications { - /** Qualifications for already-integrated anchor columns */ - readonly forQueries?: Record; - /** Qualifications for the hit column. */ - readonly forHit?: AxisQualification[]; -} - -export interface BuildOptions { - allowPartialColumnList?: true; -} - -export type AnchorEntry = PObjectId | PColumnSpec | RelaxedColumnSelector; - -export interface AnchoredBuildOptions extends BuildOptions { - anchors: Record; -} - -/** - * Mutable builder that accumulates column sources, then produces - * a ColumnCollection (plain) or AnchoredColumnCollection (with anchors). - * - * Each output lambda creates its own builder — a constraint of the - * computable framework where each output tracks its own dependencies. - */ -export class ColumnCollectionBuilder { - private readonly providers: ColumnSnapshotProvider[] = []; - - constructor(private readonly specDriver: PFrameSpecDriver = getService("pframeSpec")) {} - - /** - * Register a column source. Sources added first take precedence for dedup. - * Does NOT accept undefined — if a source isn't available yet, - * the caller should return undefined from the output lambda. - */ - addSource(source: ColumnSource | TreeNodeAccessor): this { - if (source instanceof TreeNodeAccessor) { - const columns = source.getPColumns(); - if (columns) this.providers.push(new ArrayColumnProvider(columns)); - } else { - this.providers.push(toColumnSnapshotProvider(source)); - } - return this; - } - - addSources(sources: (ColumnSource | TreeNodeAccessor)[]): this { - for (const source of sources) { - this.addSource(source); - } - return this; - } - - /** Plain collection — selector-based filtering, PObjectId namespace. */ - build(): undefined | ColumnCollection; - build(options: { - allowPartialColumnList: true; - }): ColumnCollection & { readonly columnListComplete: boolean }; - /** Anchored collection — axis-aware discovery, SUniversalPColumnId namespace. */ - build( - options: AnchoredBuildOptions & { allowPartialColumnList: true }, - ): AnchoredColumnCollection & { readonly columnListComplete: boolean }; - build(options: AnchoredBuildOptions): undefined | AnchoredColumnCollection; - build( - options?: BuildOptions | AnchoredBuildOptions, - ): - | undefined - | ColumnCollection - | AnchoredColumnCollection - | (ColumnCollection & { readonly columnListComplete: boolean }) - | (AnchoredColumnCollection & { readonly columnListComplete: boolean }) { - const allowPartial = options?.allowPartialColumnList === true; - - // Check column list completeness - const allComplete = this.providers.every((p) => p.isColumnListComplete()); - if (!allComplete && !allowPartial) return undefined; - - // Collect all columns, dedup by native ID (first source wins) - const columns = collectColumns(this.providers); - const hasAnchors = options !== undefined && "anchors" in options; - - if (hasAnchors) { - return new AnchoredColumnCollectionImpl(this.specDriver, { - anchors: options.anchors, - columns, - }); - } else { - return new ColumnCollectionImpl(this.specDriver, { - columns, - }); - } - } -} - -interface ColumnCollectionImplOptions { - readonly columns: ColumnSnapshot[]; -} - -class ColumnCollectionImpl implements ColumnCollection, Disposable { - private readonly columns: Map>; - private readonly specFrameEntry: PoolEntry; - - constructor( - private readonly specDriver: PFrameSpecDriver, - options: ColumnCollectionImplOptions, - ) { - this.columns = new Map(options.columns.map((col) => [col.id, col])); - this.specFrameEntry = this.specDriver.createSpecFrame( - Object.fromEntries(options.columns.map((col) => [col.id, col.spec])), - ); - } - - dispose(): void { - this.specFrameEntry.unref(); - } - - [Symbol.dispose](): void { - this.dispose(); - } - - findColumns(options?: FindColumnsOptions): ColumnSnapshot[] { - const includeColumns = options?.include ? toMultiColumnSelectors(options.include) : undefined; - const excludeColumns = options?.exclude ? toMultiColumnSelectors(options.exclude) : undefined; - - const response = this.specDriver.discoverColumns(this.specFrameEntry.key, { - includeColumns, - excludeColumns, - axes: [], - maxHops: 0, - constraints: matchingModeToConstraints("enrichment"), - }); - - // Map hits back to snapshots - const results = response.hits - .map((hit) => this.columns.get(hit.hit.columnId as PObjectId)) - .filter((col): col is ColumnSnapshot => col !== undefined); - - return results; - } -} - -interface AnchoredColumnCollectionImplOptions extends ColumnCollectionImplOptions { - readonly anchors: Record; -} - -class AnchoredColumnCollectionImpl implements AnchoredColumnCollection, Disposable { - private readonly anchorsMap: Map>; - private readonly columnsMap: Map>; - private readonly specFrameEntry: PoolEntry; - - constructor( - private readonly specDriver: PFrameSpecDriver, - options: AnchoredColumnCollectionImplOptions, - ) { - // Create spec frame from all collected columns - this.specFrameEntry = this.specDriver.createSpecFrame( - Object.fromEntries(options.columns.map((col) => [col.id, col.spec])), - ); - this.columnsMap = new Map(options.columns.map((col) => [col.id, col])); - this.anchorsMap = resolveAnchorMap( - options.anchors, - options.columns, - this.specDriver.discoverColumns.bind(this.specDriver, this.specFrameEntry.key), - ); - } - - dispose(): void { - this.specFrameEntry.unref(); - } - - [Symbol.dispose](): void { - this.dispose(); - } - - getAnchors(): Map> { - return this.anchorsMap; - } - - findColumns(options?: AnchoredFindColumnsOptions): ColumnMatch[] { - const mode = options?.mode ?? "enrichment"; - const constraints = matchingModeToConstraints(mode); - const includeColumns = options?.include ? toMultiColumnSelectors(options.include) : undefined; - const excludeColumns = options?.exclude ? toMultiColumnSelectors(options.exclude) : undefined; - const anchors = Array.from(this.anchorsMap.values()); - const response = this.specDriver.discoverColumns(this.specFrameEntry.key, { - includeColumns, - excludeColumns, - constraints, - maxHops: options?.maxHops ?? 4, - axes: anchors.map((anchor) => ({ - axesSpec: anchor.spec.axesSpec, - qualifications: [], - })), - }); - - const byColumn = response.hits.reduce>((acc, hit) => { - const origId = hit.hit.columnId as PObjectId; - const col = - this.columnsMap.get(origId) ?? - throwError(`Column with id ${origId} not found in collection`); - - const path = hit.path.map((step) => { - if (step.type !== "linker") { - throw new Error(`Unexpected discover-columns step type: ${step.type}`); - } - - return { - linker: - this.columnsMap.get(step.linker.columnId) ?? - throwError(`Linker column with id ${step.linker.columnId} not found in collection`), - }; - }); - const variants: MatchVariant[] = hit.mappingVariants.map((v) => ({ - path, - qualifications: remapFromIdxToId(v.qualifications, anchors), - })); - const existing = acc.get(origId); - return acc.set( - origId, - existing === undefined - ? { column: col, variants } - : { ...existing, variants: [...existing.variants, ...variants] }, - ); - }, new Map()); - - return Array.from(byColumn.values()); - } - - findColumnVariants(options?: AnchoredFindColumnsOptions): ColumnVariant[] { - const matches = this.findColumns(options); - return matches.flatMap((match) => - match.variants.map((variant) => ({ - column: match.column, - path: variant.path, - qualifications: variant.qualifications, - })), - ); - } -} - -/** - * Collect all columns from all providers, dedup by NativePObjectId. - * First source wins. - */ -function collectColumns(providers: ColumnSnapshotProvider[]): ColumnSnapshot[] { - const seen = new Set(); - const result: ColumnSnapshot[] = []; - - for (const provider of providers) { - const columns = provider.getAllColumns(); - for (const col of columns) { - const nativeId = deriveNativeId(col.spec); - if (seen.has(nativeId)) continue; - seen.add(nativeId); - result.push(col); - } - } - - return result; -} - -/** Normalize ColumnSelector (relaxed, single or array) to MultiColumnSelector[]. */ -function toMultiColumnSelectors(input: ColumnSelector): MultiColumnSelector[] { - return convertColumnSelectorToMultiColumnSelector(input); -} - -/** - * Resolve each anchor entry to a ColumnSnapshot from the collected columns. - * - PObjectId (string): looked up by id in the collected columns - * - PColumnSpec: matched by deriveNativeId against collected columns - * - RelaxedColumnSelector: resolved via discoverColumns in "exact" mode; - * must match exactly one column - * Throws on unresolved, ambiguous, or duplicated matches. Requires at least one - * anchor to resolve. - */ -function resolveAnchorMap( - anchors: Record, - columns: ColumnSnapshot[], - discoverColumns: (request: DiscoverColumnsRequest) => DiscoverColumnsResponse, -): Map> { - const result = new Map>(); - const resovedIds = new Set(); - const getDuplicateError = (key: string) => - `Anchor "${key}": selector matched a column that was already matched by another anchor; please refine the selector to match a different column`; - - // O(1) lookup maps built lazily — only when an anchor of the matching kind is - // encountered. Avoids O(anchors × columns) `deriveNativeId` calls, which were - // tripping the QuickJS interrupt deadline on tables with many anchors. - let byId: Map> | undefined; - let byNativeId: Map> | undefined; - - for (const [name, anchor] of Object.entries(anchors)) { - if (typeof anchor === "string") { - if (byId === undefined) byId = new Map(columns.map((col) => [col.id, col])); - const found = - byId.get(anchor) ?? - throwError(`Anchor "${name}": column with id "${anchor}" not found in sources`); - if (resovedIds.has(found.id)) { - throwError(getDuplicateError(name)); - } - result.set(name, found); - resovedIds.add(found.id); - } else if ("kind" in anchor) { - if (!isPColumnSpec(anchor)) throwError(`Anchor "${name}": invalid PColumnSpec`); - if (byNativeId === undefined) { - byNativeId = new Map(columns.map((col) => [deriveNativeId(col.spec), col])); - } - const found = - byNativeId.get(deriveNativeId(anchor)) ?? - throwError(`Anchor "${name}": no column matching spec found in sources`); - if (resovedIds.has(found.id)) { - throwError(getDuplicateError(name)); - } - result.set(name, found); - resovedIds.add(found.id); - } else { - const matched = discoverColumns({ - includeColumns: toMultiColumnSelectors(anchor), - excludeColumns: undefined, - axes: [], - maxHops: 0, - constraints: matchingModeToConstraints("related"), - }); - - if (matched.hits.length === 0) { - throwError(`Anchor "${name}": no columns matched selector`); - } - if (matched.hits.length > 1) { - throwError( - `Anchor "${name}": selector is ambiguous and matched multiple columns; please refine the selector to match exactly one column`, - ); - } - if (resovedIds.has(matched.hits[0].hit.columnId as PObjectId)) { - throwError(getDuplicateError(name)); - } - - const id = matched.hits[0].hit.columnId as PObjectId; - const snap = - columns.find((col) => col.id === id) ?? - throwError(`Anchor "${name}": matched column with id "${id}" not found in sources`); - result.set(name, snap); - resovedIds.add(snap.id); - } - } - - if (resovedIds.size === 0) { - throwError("At least one anchor must be resolved to a valid column"); - } - - return result; -} - -function remapFromIdxToId( - qualifications: { - forQueries: AxisQualification[][]; - forHit: AxisQualification[]; - }, - anchors: ColumnSnapshot[], -): MatchQualifications { - const forQueries = qualifications.forQueries.reduce( - (acc, qs, i) => (anchors[i] && qs.length > 0 ? acc.set(anchors[i].id, qs) : acc), - new Map(), - ); - return { - forQueries: forQueries.size > 0 ? Object.fromEntries(forQueries) : undefined, - forHit: qualifications.forHit.length > 0 ? qualifications.forHit : undefined, - }; -} - -function matchingModeToConstraints(mode: MatchingMode): DiscoverColumnsConstraints { - switch (mode) { - case "enrichment": - return { - allowFloatingSourceAxes: true, - allowFloatingHitAxes: false, - allowSourceQualifications: true, - allowHitQualifications: true, - }; - case "related": - return { - allowFloatingSourceAxes: true, - allowFloatingHitAxes: true, - allowSourceQualifications: true, - allowHitQualifications: true, - }; - case "exact": - return { - allowFloatingSourceAxes: false, - allowFloatingHitAxes: false, - allowSourceQualifications: false, - allowHitQualifications: false, - }; - } -} diff --git a/sdk/model/src/columns/column_lazy.test.ts b/sdk/model/src/columns/column_lazy.test.ts new file mode 100644 index 0000000000..5354691b4d --- /dev/null +++ b/sdk/model/src/columns/column_lazy.test.ts @@ -0,0 +1,201 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + createLocalPObjectId, + parseColumnOverridedId, + SpecOverrides, + type PColumnSpec, + type PObjectId, +} from "@milaboratories/pl-model-common"; +import { ColumnAbsentError, ColumnLazy, ColumnLazyImpl } from "./column_lazy"; +import { _ctxProvidersCache } from "./column_providers"; +import { installStubRegistry } from "./__test_helpers__/stub_registry"; + +// --- Ambient ctx mock ------------------------------------------------------ +// ColumnLazy constructor pulls `getCfgRenderCtx()` from globalThis. A minimal +// mock with empty index is enough — these tests don't exercise resolve paths. + +const mockCtx = { + getAccessorHandleByName: () => undefined, + getUpstreamBlockCtx: () => [], +} as unknown as never; + +beforeEach(() => { + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = mockCtx; +}); + +afterEach(() => { + delete (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx; +}); + +// --- Helpers ---------------------------------------------------------------- + +const baseLeaf = createLocalPObjectId(["main", "out"], "col-1"); + +const stubSpec = { kind: "PColumn", name: "stub" } as unknown as PColumnSpec; + +function lazyOf(id: PObjectId, spec: PColumnSpec = stubSpec) { + return ColumnLazyImpl.fromColumn({ id, spec, data: undefined }); +} + +// --- clone flattening ------------------------------------------------------- + +describe("ColumnLazy.clone", () => { + test("two successive clones produce flat wrap (no nesting)", () => { + const c0 = lazyOf(baseLeaf); + const c1 = c0.withSpecs({ + annotations: { a: "1" }, + domain: {}, + contextDomain: {}, + axesSpec: {}, + }); + const c2 = c1.withSpecs({ + annotations: { b: "2" }, + domain: {}, + contextDomain: {}, + axesSpec: {}, + }); + + const parsed = parseColumnOverridedId(c2.id); + // source must be the canonical leaf — not a nested wrap + expect(parsed.source).toBe(baseLeaf); + expect(parsed.specOverrides.annotations).toEqual({ a: "1", b: "2" }); + }); + + test("idempotent: clone with same overrides twice → same id", () => { + const c0 = lazyOf(baseLeaf); + const patch: SpecOverrides = { + annotations: { x: "1" }, + domain: {}, + contextDomain: {}, + axesSpec: {}, + }; + const a = c0.withSpecs(patch); + const b = c0.withSpecs(patch); + expect(a.id).toBe(b.id); + }); +}); + +// --- spec() override application ------------------------------------------- + +describe("ColumnLazy.spec — applySpecOverrides", () => { + test("base spec with no overrides returns readers.spec verbatim", () => { + const c = lazyOf(baseLeaf); + expect(c.getSpec()).toBe(stubSpec); + }); + + test("fromId throws ColumnAbsentError when ctx has no handles", () => { + // Empty registry — `isFinal()` returns true vacuously, so the leaf is + // provably absent (no provider will ever supply it). + expect(() => ColumnLazyImpl.fromId(baseLeaf)).toThrow(ColumnAbsentError); + }); + + test("appends axes at indices past base.axesSpec.length", () => { + const base = { + kind: "PColumn", + name: "col", + valueType: "String", + axesSpec: [{ name: "group", type: "String" }], + annotations: {}, + } as unknown as PColumnSpec; + const c = lazyOf(baseLeaf, base).withSpecs({ + annotations: {}, + domain: {}, + contextDomain: {}, + axesSpec: { 1: { name: "name", type: "String" } }, + }); + expect(c.getSpec().axesSpec).toEqual([ + { name: "group", type: "String" }, + { name: "name", type: "String" }, + ]); + }); + + test("patch on existing axis merges annotations/domain (does not replace whole object)", () => { + const base = { + kind: "PColumn", + name: "col", + valueType: "String", + axesSpec: [{ name: "group", type: "String", annotations: { k1: "v1" } }], + annotations: {}, + } as unknown as PColumnSpec; + const c = lazyOf(baseLeaf, base).withSpecs({ + annotations: {}, + domain: {}, + contextDomain: {}, + axesSpec: { 0: { annotations: { k2: "v2" } } }, + }); + expect(c.getSpec().axesSpec[0].annotations).toEqual({ k1: "v1", k2: "v2" }); + }); +}); + +// --- resolution status / absent throw -------------------------------------- + +describe("ColumnLazy resolution: fromId / fromAccessor / getStatus", () => { + test("fromId returns undefined when registry is unfinalized and id not found", () => { + // `isFinal: false` → resolver can't tell yet → resolving → undefined. + installStubRegistry(mockCtx, {}, { isFinal: false }); + expect(ColumnLazyImpl.fromId(baseLeaf)).toBeUndefined(); + }); + + test("fromId throws ColumnAbsentError when leaf has no spec field on locked accessor", () => { + // Leaf entry present, no spec field, accessor locked → absent forever. + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: true } }); + expect(() => ColumnLazyImpl.fromId(baseLeaf)).toThrow(ColumnAbsentError); + }); + + test("fromId returns undefined when leaf has no spec field on unlocked accessor", () => { + // Leaf entry present, no spec field, accessor unlocked → spec may still arrive. + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: false } }); + expect(ColumnLazyImpl.fromId(baseLeaf)).toBeUndefined(); + }); + + test("fromId returns undefined when spec accessor exists but hasData is false", () => { + // Spec resource present, bytes not yet written — transient resolving. + installStubRegistry(mockCtx, { [baseLeaf]: { spec: stubSpec, specHasData: false } }); + expect(ColumnLazyImpl.fromId(baseLeaf)).toBeUndefined(); + }); + + test("fromAccessor throws ColumnAbsentError when accessor is locked without spec", () => { + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: true } }); + const entry = pluckLeafEntry(baseLeaf); + expect(() => ColumnLazyImpl.fromAccessor(entry)).toThrow(ColumnAbsentError); + }); + + test("fromAccessor returns undefined when accessor is unlocked without spec", () => { + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: false } }); + const entry = pluckLeafEntry(baseLeaf); + expect(ColumnLazyImpl.fromAccessor(entry)).toBeUndefined(); + }); + + test("getStatusById returns 'absent' / 'resolving' / 'present' across configurations", () => { + // present — spec with data on a locked, populated leaf. + installStubRegistry(mockCtx, { [baseLeaf]: stubSpec }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("present"); + + // resolving — same leaf id, spec bytes not yet written. + installStubRegistry(mockCtx, { [baseLeaf]: { spec: stubSpec, specHasData: false } }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("resolving"); + + // resolving — leaf entry exists but spec field missing on an unlocked accessor. + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: false } }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("resolving"); + + // absent — leaf entry exists with no spec field on a locked accessor. + installStubRegistry(mockCtx, { [baseLeaf]: { accessorLocked: true } }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("absent"); + + // absent — id not in any provider and registry is final. + installStubRegistry(mockCtx, {}, { isFinal: true }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("absent"); + + // resolving — id not in any provider but registry still enumerating. + installStubRegistry(mockCtx, {}, { isFinal: false }); + expect(ColumnLazy.getStatusById(baseLeaf)).toBe("resolving"); + }); +}); + +/** Pull the `LeafEntry` planted by the stub registry for a given id. */ +function pluckLeafEntry(id: PObjectId) { + const entry = _ctxProvidersCache.get(mockCtx)?.[0]?.getPObjectEntries().get(id); + if (entry === undefined) throw new Error(`pluckLeafEntry: no entry for ${id}`); + return entry; +} diff --git a/sdk/model/src/columns/column_lazy.ts b/sdk/model/src/columns/column_lazy.ts new file mode 100644 index 0000000000..5bb2f24416 --- /dev/null +++ b/sdk/model/src/columns/column_lazy.ts @@ -0,0 +1,311 @@ +import { + ColumnRegistry, + createGlobalPObjectId, + isPColumn, + isPlRef, + PlRef, + PObjectId, + type ColumnUniversalId, + type LeafEntry, + type PColumn, + type PColumnSpec, + type SpecOverrides, + type SpecQuery, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx, PColumnDataUniversal } from "../render/internal"; +import { getCtxProviders } from "./column_providers"; +import { isNil } from "es-toolkit"; +import { LRUCache } from "lru-cache"; +import { TreeNodeAccessor } from "../render"; +import type { + ColumnFieldStatus, + ColumnRecipe, + ColumnResolutionStatus, +} from "./column_recipes/types"; +import { ColumnOverridedRecipe } from "./column_recipes/column_overrided_recipe"; + +export type ColumnLazyId = PObjectId; +export type ColumnLazyData = undefined | PColumnDataUniversal; +export type { ColumnFieldStatus, ColumnResolutionStatus } from "./column_recipes/types"; + +/** + * Thrown by leaf-recipe factories when the requested column is provably + * absent in the active render ctx — i.e. every relevant accessor reports + * `inputsLocked` and the column did not appear. Distinct from the + * `undefined` return that the factories still use for the "resolving" case + * (the column may still appear later). + * + * Catch at the boundary (resolver / filter+sort wiring / data-table assembly) + * when partial absence should be surfaced to the user rather than silently + * producing an empty result. + */ +export class ColumnAbsentError extends Error { + constructor(public readonly id: ColumnUniversalId) { + super(`Column is absent in the active render ctx and will not appear: ${id}`); + this.name = "ColumnAbsentError"; + } +} + +/** + * ColumnLazy is the leaf-recipe building block: a {@link ColumnRecipe} whose + * `id` is a bare {@link PObjectId} and whose readers are bound to a single + * tree-accessor leaf. Layered encodings (Overrided / Discovered / Filtered) + * are reified through their dedicated recipe classes and reference leaf + * columns by id. + */ +export class ColumnLazyImpl implements ColumnRecipe { + private specCache?: { readonly value: PColumnSpec }; + private dataCache?: { readonly value: ColumnLazyData }; + private dataStatusCache?: { readonly value: ColumnFieldStatus }; + + private constructor( + public readonly id: PObjectId, + private readonly options: { + getSpec: () => PColumnSpec; + getData: () => ColumnLazyData; + getDataStatus: () => ColumnFieldStatus; + }, + ) {} + + /** A leaf-recipe references exactly one column — its own id. */ + getReferencedIds(): PObjectId[] { + return [this.id]; + } + + getSpec(): PColumnSpec { + if (this.specCache === undefined) { + this.specCache = { value: this.options.getSpec() }; + } + return this.specCache.value; + } + + /** Leaf-shaped — points straight at `this.id`. */ + getQuery(): SpecQuery { + return { type: "column", column: this.id } as SpecQuery; + } + + /** Leaf-only: not on the recipe interface — only the leaf can read data directly. */ + getData(): ColumnLazyData { + if (this.dataCache === undefined) this.dataCache = { value: this.options.getData() }; + return this.dataCache.value; + } + + getDataStatus(): ColumnFieldStatus { + if (this.dataStatusCache === undefined) { + this.dataStatusCache = { value: this.options.getDataStatus() }; + } + return this.dataStatusCache.value; + } + + /** + * Overlay overrides → produces a {@link ColumnOverridedRecipe} wrapping + * this leaf. ColumnLazy itself stays bare (id remains a plain PObjectId); + * layering lives entirely in the recipe wrappers. + */ + withSpecs(overrides: SpecOverrides): ColumnRecipe { + return ColumnOverridedRecipe.wrap(this, overrides); + } + + /** + * Resolve via the ambient {@link ColumnRegistry}. Spec is resolved eagerly: + * returns `undefined` if the leaf isn't reachable yet (resolving). Throws + * {@link ColumnAbsentError} when every relevant accessor is `inputsLocked` + * and the column did not appear — the column will not exist in this ctx. + * Data and dataStatus stay lazy. + */ + static fromId(id: PObjectId, { ctx }: { ctx?: GlobalCfgRenderCtx } = {}): undefined | ColumnLazy { + const registry = new ColumnRegistry(getCtxProviders({ ctx })); + const leaf = registry.resolve(id); + if (isNil(leaf)) { + if (registry.isFinal()) throw new ColumnAbsentError(id); + return undefined; + } + const spec = readSpecAccessor(leaf); + if (isNil(spec)) { + if (leaf.accessor.getInputsLocked()) throw new ColumnAbsentError(id); + return undefined; + } + if (!spec.hasData()) return undefined; + return new ColumnLazyImpl(id, { + getSpec: () => spec.getDataAsJson(), + getData: () => readDataAccessor(leaf), + getDataStatus: () => readDataStatus(leaf), + }); + } + + /** {@link PlRef} wrapper over {@link fromId}. */ + static fromPlRef(ref: PlRef): undefined | ColumnLazy { + return ColumnLazyImpl.fromId(createGlobalPObjectId(ref.blockId, ref.name)); + } + + /** + * Bind directly to an accessor-backed {@link LeafEntry} — no registry. + * Throws {@link ColumnAbsentError} if the leaf has no spec field and its + * accessor is `inputsLocked`. Returns `undefined` while still resolving. + */ + static fromAccessor(entry: LeafEntry): undefined | ColumnLazy { + const spec = readSpecAccessor(entry); + if (isNil(spec)) { + if (entry.accessor.getInputsLocked()) throw new ColumnAbsentError(entry.id); + return undefined; + } + if (!spec.hasData()) return undefined; + return new ColumnLazyImpl(entry.id, { + getSpec: () => spec.getDataAsJson(), + getData: () => readDataAccessor(entry), + getDataStatus: () => readDataStatus(entry), + }); + } + + /** + * Wrap a materialised {@link PColumn}. If the input is already a + * {@link ColumnLazy} it is returned as-is. + */ + static fromColumn(column: PColumn | ColumnLazy): ColumnLazy { + if (column instanceof ColumnLazyImpl) return column; + return new ColumnLazyImpl(column.id, { + getSpec: () => column.spec, + getData: () => column.data, + getDataStatus: () => "present", + }); + } + + /** + * Distinguishes `present` / `resolving` / `absent` for a {@link PObjectId} + * in the active render ctx. Falls back to the registry's `isFinal()` + * when the id has no entry — only then we can say `absent` instead of + * `resolving`. + */ + static getStatusById( + id: PObjectId, + { ctx }: { ctx?: GlobalCfgRenderCtx } = {}, + ): ColumnResolutionStatus { + const registry = new ColumnRegistry(getCtxProviders({ ctx })); + const leaf = registry.resolve(id); + if (isNil(leaf)) return registry.isFinal() ? "absent" : "resolving"; + return getLeafEntryStatus(leaf); + } + + /** {@link PlRef} wrapper over {@link getStatusById}. */ + static getStatusByPlRef( + ref: PlRef, + opts: { ctx?: GlobalCfgRenderCtx } = {}, + ): ColumnResolutionStatus { + return ColumnLazyImpl.getStatusById(createGlobalPObjectId(ref.blockId, ref.name), opts); + } + + /** No registry — reads straight off the entry's accessor. */ + static getStatusByAccessor(entry: LeafEntry): ColumnResolutionStatus { + return getLeafEntryStatus(entry); + } +} + +/** + * Public type alias — `ColumnLazy` (the type) refers to the underlying + * {@link ColumnLazyImpl} class instance. `ColumnLazy` (the value) is the + * callable below with the static factories attached. + */ +export type ColumnLazy = ColumnLazyImpl; + +/** + * Unified dispatcher — picks the right `ColumnLazyImpl.fromX` by source + * shape. For ambiguous inputs callers can still use the explicit factories + * (also attached as properties: `ColumnLazy.fromId`, `.fromPlRef`, + * `.fromAccessor`, `.fromColumn`). + */ +function ColumnLazyDispatch( + source: PObjectId | PlRef | LeafEntry | PColumn | ColumnLazy, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): undefined | ColumnLazy { + if (typeof source === "string") return ColumnLazyImpl.fromId(source, opts); + if (source instanceof ColumnLazyImpl) return source; + if ("accessor" in source) return ColumnLazyImpl.fromAccessor(source); + if (isPlRef(source)) return ColumnLazyImpl.fromPlRef(source); + if (isPColumn(source)) return ColumnLazyImpl.fromColumn(source); + throw new Error("ColumnLazy: unknown source shape"); +} + +/** + * Polymorphic counterpart to {@link ColumnLazyDispatch}: returns the + * {@link ColumnResolutionStatus} for any factory input without constructing + * the recipe. For already-materialised sources ({@link PColumn} value, + * existing {@link ColumnLazy}) status is `present` by construction. + */ +function ColumnLazyGetStatus( + source: PObjectId | PlRef | LeafEntry | PColumn | ColumnLazy, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): ColumnResolutionStatus { + if (typeof source === "string") return ColumnLazyImpl.getStatusById(source, opts); + if (source instanceof ColumnLazyImpl) return "present"; + if ("accessor" in source) return ColumnLazyImpl.getStatusByAccessor(source); + if (isPlRef(source)) return ColumnLazyImpl.getStatusByPlRef(source, opts); + if (isPColumn(source)) return "present"; + throw new Error("ColumnLazy.getStatus: unknown source shape"); +} + +export const ColumnLazy = Object.assign(ColumnLazyDispatch, { + fromId: ColumnLazyImpl.fromId, + fromPlRef: ColumnLazyImpl.fromPlRef, + fromAccessor: ColumnLazyImpl.fromAccessor, + fromColumn: ColumnLazyImpl.fromColumn, + getStatus: ColumnLazyGetStatus, + getStatusById: ColumnLazyImpl.getStatusById, + getStatusByPlRef: ColumnLazyImpl.getStatusByPlRef, + getStatusByAccessor: ColumnLazyImpl.getStatusByAccessor, +}); + +/** + * Type-guard narrowing to the leaf recipe. Prefer this over + * `instanceof ColumnLazyImpl` — `ColumnLazy` (the value) is the dispatcher + * with statics, not the class, so `instanceof` requires the impl symbol. + */ +export function isColumnLazy(value: unknown): value is ColumnLazy { + return value instanceof ColumnLazyImpl; +} + +const readSpecAccessor = memoizeByEntry( + ({ accessor, name }: LeafEntry): undefined | TreeNodeAccessor => + accessor.traverse({ field: `${name}.spec`, assertFieldType: "Input", ignoreError: true }), +); + +/** + * Per-entry counterpart to {@link readDataStatus}: tells whether the leaf's + * **spec** can be read in this ctx, and — for the negative cases — + * distinguishes `resolving` from `absent` via `getInputsLocked()`. + * + * - spec field not yet on the entry's accessor + inputs locked → `absent` + * - spec field not yet on the entry's accessor + still resolving → `resolving` + * - spec resource present but bytes not yet written → `resolving` + * (transient — the spec resource is connected, just unfilled) + * - spec resource present and `hasData()` → `present` + */ +function getLeafEntryStatus(entry: LeafEntry): ColumnResolutionStatus { + const spec = readSpecAccessor(entry); + if (isNil(spec)) return entry.accessor.getInputsLocked() ? "absent" : "resolving"; + if (!spec.hasData()) return "resolving"; + return "present"; +} + +const readDataAccessor = memoizeByEntry( + ({ accessor, name }: LeafEntry): undefined | TreeNodeAccessor => + accessor.traverse({ field: `${name}.data`, assertFieldType: "Input", ignoreError: true }), +); + +const readDataStatus = memoizeByEntry( + ({ accessor, name }: LeafEntry): ColumnFieldStatus => { + if (accessor.listInputFields().includes(`${name}.data`)) return "present"; + return accessor.getInputsLocked() ? "absent" : "resolving"; + }, +); + +function memoizeByEntry( + fn: (entry: LeafEntry) => R, +): (entry: LeafEntry) => R { + const cache = new LRUCache({ max: 1000 }); + return (entry) => { + const key = `${entry.accessor.handle}:${entry.name}`; + let hit = cache.get(key); + if (!hit) cache.set(key, (hit = { value: fn(entry) })); + return hit.value; + }; +} diff --git a/sdk/model/src/columns/column_providers/index.ts b/sdk/model/src/columns/column_providers/index.ts new file mode 100644 index 0000000000..ddb11ab4c9 --- /dev/null +++ b/sdk/model/src/columns/column_providers/index.ts @@ -0,0 +1,78 @@ +import { PColumn, type ColumnEntriesProvider } from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx, PColumnDataUniversal } from "../../render/internal"; +import { getCfgRenderCtx } from "../../internal"; +import { MainAccessorName, StagingAccessorName } from "../../render/internal"; +import { TreeNodeAccessor } from "../../render/accessor"; +import { ColumnLazy } from "../column_lazy"; +import type { ColumnsSource } from "./types"; +import { ArrayColumnsProvider, ColumnsProvider } from "./providers"; + +export * from "./types"; +export * from "./providers"; + +/** + * Build the default set of ColumnsProviders for the ambient render ctx: + * - `AccessorColumnsProvider` over `outputs` (if present) + * - `AccessorColumnsProvider` over `prerun` (if present) + * - `ResultPoolColumnsProvider` over `rawResultPool` + * + * Pulls handles directly from the ambient `cfgRenderCtx`. Returns `[]` when + * called outside a render context. + * + * Result is memoised per-ctx (WeakMap → array). Repeated calls within the + * same render cycle return the same provider triplet — inner providers are + * also LRU-memoised by their content keys, so even a cache miss here doesn't + * re-walk the trees. + */ +export const _ctxProvidersCache = new WeakMap< + GlobalCfgRenderCtx, + (ColumnEntriesProvider & ColumnsProvider)[] +>(); + +export function getCtxProviders(deps?: { + ctx?: GlobalCfgRenderCtx; +}): (ColumnEntriesProvider & ColumnsProvider)[] { + const ctx = deps?.ctx ?? getCfgRenderCtx(); + + const cached = _ctxProvidersCache.get(ctx); + if (cached !== undefined) return cached; + + const providers: (ColumnEntriesProvider & ColumnsProvider)[] = []; + + const outputs = ctx.getAccessorHandleByName(MainAccessorName); + if (outputs !== undefined) { + providers.push(ColumnsProvider(new TreeNodeAccessor(outputs, [MainAccessorName]))); + } + + const prerun = ctx.getAccessorHandleByName(StagingAccessorName); + if (prerun !== undefined) { + providers.push(ColumnsProvider(new TreeNodeAccessor(prerun, [StagingAccessorName]))); + } + + providers.push(ColumnsProvider(ctx.getUpstreamBlockCtx())); + + _ctxProvidersCache.set(ctx, providers); + return providers; +} + +export function isColumnProvider(source: unknown): source is ColumnsProvider { + if (typeof source !== "object" || source === null) return false; + const p = source as ColumnsProvider; + return typeof p.getColumns === "function" && typeof p.isFinal === "function"; +} + +export function toColumnProvider(source: ColumnsSource): ColumnsProvider { + if (isColumnArray(source)) return new ArrayColumnsProvider(source.columns, source.isFinal); + if (isColumnProvider(source)) return source; + if (source instanceof TreeNodeAccessor) return ColumnsProvider(source); + throw new Error("Unknown ColumnsSource type"); +} + +function isColumnArray(source: unknown): source is { + readonly columns: ReadonlyArray | ColumnLazy>; + readonly isFinal: boolean; +} { + if (typeof source !== "object" || source === null) return false; + const s = source as { columns?: unknown; isFinal?: unknown }; + return Array.isArray(s.columns) && typeof s.isFinal === "boolean"; +} diff --git a/sdk/model/src/columns/column_providers/providers.ts b/sdk/model/src/columns/column_providers/providers.ts new file mode 100644 index 0000000000..28d17050aa --- /dev/null +++ b/sdk/model/src/columns/column_providers/providers.ts @@ -0,0 +1,193 @@ +import { + AccessorEntriesProvider, + ResultPoolEntriesProvider, + type PColumn, + type UpstreamBlockCtx, +} from "@milaboratories/pl-model-common"; +import { AccessorHandle, PColumnDataUniversal, TreeNodeAccessor } from "../../render"; +import { ColumnLazy, ColumnLazyImpl } from "../column_lazy"; +import { getCfgRenderCtx } from "../../internal"; +import { isNil } from "es-toolkit"; +import { LRUCache } from "lru-cache"; + +/** + * Data source for sandbox column-list builders. Has nothing to say about + * id-resolution — `ColumnRegistry` (in `@milaboratories/pl-model-common`) + * consumes the separate `ColumnEntriesProvider` interface for that. + * + * Same name as the factory function below — they merge into one TS + * declaration: `ColumnsProvider` is callable AND the interface type. + */ +export interface ColumnsProvider { + /** Returns all currently known PColumn columns as lazy views. */ + getColumns(): ColumnLazy[]; + /** Whether the provider has finished enumerating all its columns. */ + isFinal(): boolean; +} + +/** + * Unified memoised factory dispatching to the right provider flavour by the + * source shape: + * - `TreeNodeAccessor` → {@link AccessorColumnsProvider} + * (indexed by `root.handle`, walks the accessor tree once per root). + * - `UpstreamBlockCtx[]` (or omitted — pulled from the + * ambient ctx) → {@link ResultPoolColumnsProvider} + * (indexed by a content-hash of the block list). + * + * Both branches return cached instances within their LRU window, so repeated + * calls with the same source skip re-walking / re-hashing. + */ +export function ColumnsProvider( + source?: TreeNodeAccessor | ReadonlyArray>, +) { + if (isNil(source) || Array.isArray(source)) { + return ResultPoolColumnsProvider(source); + } else if (source instanceof TreeNodeAccessor) { + return AccessorColumnsProvider(source); + } else { + throw new Error("Unknown columns provider source"); + } +} + +/** + * Memoised constructor — same `root.handle` returns the same + * {@link AccessorColumnsProviderImpl}, so `indexAccessorRoot` runs once per + * accessor root (within the LRU window). + */ +const _accessorProviderCache = new LRUCache({ + max: 64, +}); +export function AccessorColumnsProvider(root: TreeNodeAccessor): AccessorColumnsProvider { + let provider = _accessorProviderCache.get(root.handle); + if (provider === undefined) { + provider = new AccessorColumnsProviderImpl(root); + _accessorProviderCache.set(root.handle, provider); + } + return provider; +} + +/** + * Memoised constructor — same `rawPool` content returns the same + * {@link ResultPoolColumnsProviderImpl}, so `indexPoolBlock` runs once per + * pool snapshot (within the LRU window). + */ +const _resultPoolProviderCache = new LRUCache({ max: 4 }); +export function ResultPoolColumnsProvider( + rawPool: ReadonlyArray< + UpstreamBlockCtx + > = getCfgRenderCtx().getUpstreamBlockCtx(), +): ResultPoolColumnsProvider { + const key = hashRawPool(rawPool); + let provider = _resultPoolProviderCache.get(key); + if (provider === undefined) { + provider = new ResultPoolColumnsProviderImpl(rawPool); + _resultPoolProviderCache.set(key, provider); + } + return provider; +} + +/** + * Simple provider wrapping an array of PColumns. Sandbox-only — + * implements {@link ColumnsProvider} (lazy-view enumeration). It is *not* an + * id-index source — `ColumnRegistry` does not consume it. + */ +export class ArrayColumnsProvider implements ColumnsProvider { + private readonly columns: ColumnLazy[]; + + constructor( + columns: ReadonlyArray | ColumnLazy>, + private readonly _isFinal: boolean, + ) { + this.columns = columns.map((c) => ColumnLazyImpl.fromColumn(c)); + } + + getColumns(): ColumnLazy[] { + return this.columns; + } + + isFinal(): boolean { + return this._isFinal; + } +} + +/** + * Sandbox-specific provider over a {@link TreeNodeAccessor} root. Uses the + * accessor's own `resolvePath` as the canonical root path for id construction, + * and adds `getColumns()` returning {@link ColumnLazy}s on top of the generic + * entries index. + */ +export class AccessorColumnsProviderImpl + extends AccessorEntriesProvider + implements ColumnsProvider +{ + private cachedColumns?: ColumnLazy[]; + + constructor(root: TreeNodeAccessor) { + super(root, root.resolvePath); + } + + getColumns(): ColumnLazy[] { + if (this.cachedColumns !== undefined) return this.cachedColumns; + return (this.cachedColumns = Array.from(this.entries.values()) + .map((e) => ColumnLazyImpl.fromAccessor(e)) + .filter((v): v is ColumnLazy => !isNil(v))); + } +} + +/** Public type alias — value of the same name is the memoised factory above. */ +export type AccessorColumnsProvider = AccessorColumnsProviderImpl; + +/** + * Sandbox-specific result-pool provider. Accepts the raw handle-shaped blocks + * returned by {@link GlobalCfgRenderCtxMethods.getUpstreamBlockCtx} and + * wraps them into {@link TreeNodeAccessor}s, then adds `getColumns()`. + */ +export class ResultPoolColumnsProviderImpl + extends ResultPoolEntriesProvider + implements ColumnsProvider +{ + private cachedColumns?: ColumnLazy[]; + + constructor( + rawPool: ReadonlyArray< + UpstreamBlockCtx + > = getCfgRenderCtx().getUpstreamBlockCtx(), + ) { + const blocks = rawPool.map((b) => ({ + blockId: b.blockId, + prodCtx: + b.prodCtx !== undefined + ? new TreeNodeAccessor(b.prodCtx, [b.blockId, "prodCtx"]) + : undefined, + stagingCtx: + b.stagingCtx !== undefined + ? new TreeNodeAccessor(b.stagingCtx, [b.blockId, "stagingCtx"]) + : undefined, + prodIncomplete: b.prodIncomplete, + stagingIncomplete: b.stagingIncomplete, + })); + + super(blocks); + } + + getColumns(): ColumnLazy[] { + if (this.cachedColumns !== undefined) return this.cachedColumns; + return (this.cachedColumns = Array.from(this.getPObjectEntries().values()) + .map((e) => ColumnLazyImpl.fromAccessor(e)) + .filter((v): v is ColumnLazy => !isNil(v))); + } +} + +/** Public type alias — value of the same name is the memoised factory above. */ +export type ResultPoolColumnsProvider = ResultPoolColumnsProviderImpl; + +function hashRawPool(rawPool: ReadonlyArray>): string { + return rawPool + .map( + (b) => + `${b.blockId}\x1f${b.prodCtx ?? ""}\x1f${b.stagingCtx ?? ""}\x1f${ + b.prodIncomplete ? 1 : 0 + }\x1f${b.stagingIncomplete ? 1 : 0}`, + ) + .join("\x1e"); +} diff --git a/sdk/model/src/columns/column_providers/types.ts b/sdk/model/src/columns/column_providers/types.ts new file mode 100644 index 0000000000..fa0e546b53 --- /dev/null +++ b/sdk/model/src/columns/column_providers/types.ts @@ -0,0 +1,24 @@ +import type { PColumn } from "@milaboratories/pl-model-common"; +import { TreeNodeAccessor } from "../../render"; +import type { PColumnDataUniversal } from "../../render/internal"; +import type { ColumnLazy } from "../column_lazy"; +import type { ColumnRecipe } from "../column_recipes"; +import type { ColumnsProvider } from "./providers"; + +/** + * Union of types that can serve as column sources for helpers and builders. + * Includes TreeNodeAccessor, ColumnsProvider, and arrays of columns. + * + * The array form accepts plain {@link PColumn}s (materialized snapshots), + * {@link ColumnLazy} leaves, or any {@link ColumnRecipe} — builders only + * need each entry's `id` for serialization. + */ +export type ColumnsSource = + | ColumnsProvider + | TreeNodeAccessor + | { + readonly columns: ReadonlyArray< + PColumn | ColumnLazy | ColumnRecipe + >; + readonly isFinal: boolean; + }; diff --git a/sdk/model/src/columns/column_recipes/column_discovered_recipe.ts b/sdk/model/src/columns/column_recipes/column_discovered_recipe.ts new file mode 100644 index 0000000000..3095893530 --- /dev/null +++ b/sdk/model/src/columns/column_recipes/column_discovered_recipe.ts @@ -0,0 +1,229 @@ +import { + distillColumnDiscoveredKey, + extractPObjectId, + stringifyColumnDiscoveredId, + type AxisQualification, + type ColumnDiscoveredId, + type ColumnDiscoveredKey, + type ColumnUniversalId, + type PColumnSpec, + type PObjectId, + type SpecOverrides, + type SpecQuery, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import type { ColumnFieldStatus, ColumnResolutionStatus } from "./types"; +import { ColumnRecipe } from "./index"; +import { ColumnOverridedRecipe } from "./column_overrided_recipe"; +import { Column } from "../column"; +import { rebrandLeafId } from "./leaf_rebrand"; +import { throwError } from "@milaboratories/helpers"; + +/** + * Recipe for columns identified by a {@link ColumnDiscoveredKey}: a base + * column + a `path` of linkers + qualifications. + * + * Specifics: + * - References more than one column (`column` + each `path[i].column`); + * {@link getDataStatus} aggregates across the whole set. + * - {@link getQuery} assembles the {@link SpecQuery} locally + * ({@link buildSpecQuery}) — the hit is inlined as its own recipe's query + * tree, linkers fold inside-out into nested `linkerJoin` nodes. + * - {@link getSpec} runs `createSpecFrame` + `evaluateQuery` over that query + * to project the final {@link PColumnSpec} after the linker chain. + */ +export class ColumnDiscoveredRecipe implements ColumnRecipe { + /** + * Validate the key + resolve every referenced column via + * {@link ColumnLazy.fromId} (which itself checks the leaf accessor reports + * `hasData()`). Returns `undefined` if the key is malformed or any + * referenced column is not yet spec-resolvable. + * + * Spec JSON is deserialized lazily on first {@link getSpec}/{@link getQuery}. + */ + static fromKey( + key: ColumnDiscoveredKey, + opts: { ctx?: GlobalCfgRenderCtx } = {}, + ): undefined | ColumnDiscoveredRecipe { + const distilled = distillColumnDiscoveredKey(key); + const columns: Record = {}; + for (const uniId of referencedUniIdsOf(distilled)) { + const recipe = Column(uniId, opts); + if (recipe === undefined) return undefined; + columns[uniId] = recipe; + } + return new ColumnDiscoveredRecipe(distilled, columns); + } + + /** + * Static counterpart to {@link fromKey}: returns the resolution status of + * a {@link ColumnDiscoveredKey} without constructing the recipe. Worst-wins + * across every referenced {@link ColumnUniversalId} (hit + path linkers), + * each resolved via the top-level {@link ColumnRecipe.getStatus} dispatcher. + */ + static getStatusByKey( + key: ColumnDiscoveredKey, + opts: { ctx?: GlobalCfgRenderCtx } = {}, + ): ColumnResolutionStatus { + const distilled = distillColumnDiscoveredKey(key); + let worst: ColumnResolutionStatus = "present"; + for (const uniId of referencedUniIdsOf(distilled)) { + const s = ColumnRecipe.getStatus(uniId, opts); + if (s === "absent") return "absent"; + if (s === "resolving") worst = "resolving"; + } + return worst; + } + + readonly id: ColumnDiscoveredId; + private specCache?: { readonly value: PColumnSpec }; + private queryCache?: { readonly value: SpecQuery }; + private dataStatusCache?: { readonly value: ColumnFieldStatus }; + + private constructor( + private readonly key: ColumnDiscoveredKey, + private readonly columns: Record, + ) { + this.id = stringifyColumnDiscoveredId(this.key); + } + + /** + * Universal id of the hit column (excluding path/qualifications on top). + * May itself be a wrapped id (Overrided / Filtered / nested Discovered) — + * preserves projections applied to the underlying column. + */ + getHitId(): ColumnUniversalId { + return this.key.column; + } + + /** + * Axis qualifications attached to the hit column itself — applied to the + * outer-join entry that wraps this column at the consumer boundary. + */ + getColumnQualifications(): readonly AxisQualification[] { + return this.key.columnQualifications ?? []; + } + + /** + * Per-primary-column qualifications applied to the primary anchors on + * this group's side. Keyed by the external primary column id. + */ + getQueriesQualifications(): Readonly> { + return this.key.queriesQualifications ?? {}; + } + + /** + * Leaf {@link PObjectId}s the recipe physically depends on: hit column + + * each linker in `path`, with any projection wrappers unwrapped down to + * their underlying storage column. Used to track real-data dependencies + * (e.g. {@link getDataStatus}), not projected identities. + */ + getReferencedIds(): PObjectId[] { + return [...new Set([...referencedUniIdsOf(this.key)].map((uniId) => extractPObjectId(uniId)))]; + } + + /** + * Final spec of a discovered hit is just the hit column's own spec: the + * linker chain only enables co-indexing, it does not remap the hit's + * `axesSpec`, and axis qualifications live on the outer join entry (see + * {@link buildSpecQuery}), not in the spec. So pass through to the hit + * recipe's `getSpec()` — no engine round-trip needed. This mirrors the + * Discovered branch of `reconstructSpecFromId` on the host side. + */ + getSpec(): PColumnSpec { + if (this.specCache === undefined) { + const hit = + this.columns[this.key.column] ?? + throwError(`ColumnDiscoveredRecipe.getSpec: missing column ${this.key.column}`); + this.specCache = { value: hit.getSpec() }; + } + return this.specCache.value; + } + + /** IR for building the final spec — see {@link buildSpecQuery}. */ + getQuery(): SpecQuery { + if (this.queryCache === undefined) { + this.queryCache = { value: this.buildSpecQuery() }; + } + return this.queryCache.value; + } + + getDataStatus(): ColumnFieldStatus { + if (this.dataStatusCache === undefined) { + this.dataStatusCache = { + value: Object.values(this.columns).reduce((worst, lazy) => { + const s = lazy.getDataStatus(); + if (s === "absent" || worst === "absent") return "absent"; + if (s === "resolving") return "resolving"; + return worst; + }, "present"), + }; + } + return this.dataStatusCache.value; + } + + /** + * Discovered + overrides → Overrided. Delegates to the + * flat-merge factory of ColumnOverridedRecipe; subsequent `withSpecs` + * calls merge at the Overrided level (one wrapper, no nesting). + */ + withSpecs(overrides: SpecOverrides): ColumnRecipe { + return ColumnOverridedRecipe.wrap(this, overrides); + } + + /** + * Assembles the SpecQuery for this discovered hit by folding `key.path` + * around the hit column. + * + * The hit (innermost) is built by inlining `columns[key.column].getQuery()` + * — recursing into the hit's own recipe — so wrapping layers on the hit + * (Overrided / Filtered / nested Discovered) appear in the produced tree. + * The previous `pframeSpec.buildQuery` path was a flat + * {@link PObjectId}-only call and could not express that. + * + * Linkers stay as plain column references on the linker side of + * `linkerJoin` (the spec node has no slot for a query-side linker); the + * linker's recipe id is used directly. The frame registered in + * {@link getSpec} carries each recipe's final spec under that same id, so + * the engine resolves linker axes from the post-projection spec. + * + * Path ordering matches {@link ColumnDiscoveredKey.path}: `path[0]` is + * outermost, `path[N-1]` is closest to the hit. We fold from the inside + * out — innermost wrap first — so the result lines up with that convention. + * + * `columnQualifications` / `queriesQualifications` are not emitted here — + * qualifications live on a {@link SpecQueryJoinEntry} wrapper (lost on + * unwrapping to {@link SpecQuery}). External consumers apply them at the + * outer join via {@link getColumnQualifications} / {@link getQueriesQualifications}. + */ + private buildSpecQuery(): SpecQuery { + // Preserve the inner hit recipe's full query tree — it may carry + // wrappers (specOverride / sliceAxes / nested linkerJoin) that the + // host needs to apply correctly. We only rebrand the terminal column + // leaf to this.id so two variants of the same physical hit reached via + // different linker chains stay distinct columns in the engine output. + const hitRecipe = + this.columns[this.key.column] ?? + throwError(`ColumnDiscoveredRecipe.buildSpecQuery: missing column ${this.key.column}`); + let query: SpecQuery = rebrandLeafId(hitRecipe.getQuery(), hitRecipe.id, this.id); + const path = this.key.path ?? []; + + for (let i = path.length - 1; i >= 0; i--) { + const linker = + this.columns[path[i].column] ?? + throwError( + `ColumnDiscoveredRecipe.buildSpecQuery: missing linker column ${path[i].column}`, + ); + query = { + type: "linkerJoin", + linker: linker.getQuery(), + secondary: [{ entry: query }], + }; + } + return query; + } +} + +function referencedUniIdsOf(key: ColumnDiscoveredKey): Set { + return new Set([key.column, ...(key.path ?? []).map((item) => item.column)]); +} diff --git a/sdk/model/src/columns/column_recipes/column_filtered_recipe.ts b/sdk/model/src/columns/column_recipes/column_filtered_recipe.ts new file mode 100644 index 0000000000..e9e45c1045 --- /dev/null +++ b/sdk/model/src/columns/column_recipes/column_filtered_recipe.ts @@ -0,0 +1,181 @@ +import { + applyAxisFilters, + stringifyColumnFilteredId, + type AxisFilterByIdx, + type AxisSpec, + type ColumnFilteredId, + type ColumnFilteredKey, + type PColumnSpec, + type PObjectId, + type SingleAxisSelector, + type SpecOverrides, + type SpecQuery, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import type { ColumnFieldStatus, ColumnResolutionStatus } from "./types"; +import { ColumnRecipe } from "./index"; +import { ColumnOverridedRecipe } from "./column_overrided_recipe"; +import { rebrandLeafId } from "./leaf_rebrand"; + +/** + * Recipe wrapper that fixes the values of selected axes of an inner recipe + * (axis slicing). The id is `FilteredPColumnId { source: inner.id, + * axisFilters }`. + * + * Flat-merge invariant: `inner` is never another {@link ColumnFilteredRecipe}. + * Repeated {@link wrap} concatenates `axisFilters` at the current level. + * + * Layering: `withSpecs` on a Filtered recipe yields + * `Overrided>` — Overrided is always the outermost layer. + * The reverse layering (`Filtered>`) is not reachable via + * the public interface. + */ +export class ColumnFilteredRecipe implements ColumnRecipe { + /** + * Wrap any recipe with axis filters. If `inner` is already a + * {@link ColumnFilteredRecipe}, concatenates `axisFilters` and returns a + * new Filtered sharing the original `inner`. + * + * Validates every axis index against `inner.getSpec().axesSpec` at + * construction — throws on an out-of-range index. After {@link wrap}, the + * recipe is guaranteed to be spec-complete. + * + * Order semantics: new filters are appended after existing ones, preserving + * their relative order. AND composition is otherwise commutative; order + * only affects canonicalize/equality. + */ + static wrap(col: ColumnRecipe, axisFilters: AxisFilterByIdx[]): ColumnFilteredRecipe { + const combined = + col instanceof ColumnFilteredRecipe ? [...col.axisFilters, ...axisFilters] : axisFilters; + const base = col instanceof ColumnFilteredRecipe ? col.inner : col; + + if (combined.length === 0) { + throw new Error("ColumnFilteredRecipe.wrap: at least one axis filter must be provided"); + } + + const innerSpec = base.getSpec(); + const fixed = new Set(); + for (const [idx] of combined) { + if (innerSpec.axesSpec[idx] === undefined) { + throw new Error( + `ColumnFilteredRecipe.wrap: axis index ${idx} out of range (inner has ${innerSpec.axesSpec.length} axes)`, + ); + } + fixed.add(idx); + } + if (fixed.size >= innerSpec.axesSpec.length) { + throw new Error( + `ColumnFilteredRecipe.wrap: filters fix all ${innerSpec.axesSpec.length} axes — at least one axis must remain`, + ); + } + + return new ColumnFilteredRecipe(base, combined); + } + + /** + * Static counterpart to {@link wrap}: returns the resolution status of a + * {@link ColumnFilteredKey} without constructing the recipe. Axis filters + * do not add references — status collapses to the source's status. + */ + static getStatusByKey( + key: ColumnFilteredKey, + opts: { ctx?: GlobalCfgRenderCtx } = {}, + ): ColumnResolutionStatus { + if (typeof key.source !== "string") { + throw new Error( + "ColumnFilteredRecipe.getStatusByKey: filtered column with non-string source", + ); + } + return ColumnRecipe.getStatus(key.source, opts); + } + + readonly id: ColumnFilteredId; + private specCache?: { readonly value: PColumnSpec }; + private queryCache?: { readonly value: SpecQuery }; + private dataStatusCache?: { readonly value: ColumnFieldStatus }; + + private constructor( + private readonly inner: ColumnRecipe, + private readonly axisFilters: AxisFilterByIdx[], + ) { + this.id = stringifyColumnFilteredId({ + source: inner.id, + axisFilters, + }); + } + + /** Wrapped recipe — exposed so external walkers can descend. */ + getInner(): ColumnRecipe { + return this.inner; + } + + /** Axis filters do not introduce references to other columns. */ + getReferencedIds(): PObjectId[] { + return this.inner.getReferencedIds(); + } + + getDataStatus(): ColumnFieldStatus { + if (this.dataStatusCache === undefined) { + this.dataStatusCache = { value: this.inner.getDataStatus() }; + } + return this.dataStatusCache.value; + } + + /** + * Final spec — the inner spec with fixed axes removed. Indices in + * `axisFilters` are positional against `inner.getSpec().axesSpec` and were + * validated at {@link wrap} time. + */ + getSpec(): PColumnSpec { + if (this.specCache === undefined) { + this.specCache = { value: applyAxisFilters(this.inner.getSpec(), this.axisFilters) }; + } + return this.specCache.value; + } + + /** + * Wraps the inner query in `SpecQuerySliceAxes`. Index→selector resolution + * goes through the inner spec — guaranteed available since {@link wrap} + * validated all indices at construction. + */ + getQuery(): SpecQuery { + if (this.queryCache === undefined) { + const innerQuery = this.inner.getQuery(); + if (this.axisFilters.length === 0) { + // Pass-through (no filters): rebrand the inner leaf to this.id so + // wrappers in the chain don't reintroduce the inner's id at the + // leaf — keeps the resolver/leaf-id contract uniform. + this.queryCache = { value: rebrandLeafId(innerQuery, this.inner.id, this.id) }; + } else { + const innerSpec = this.inner.getSpec(); + this.queryCache = { + value: { + type: "sliceAxes", + input: rebrandLeafId(innerQuery, this.inner.id, this.id), + axisFilters: this.axisFilters.map(([idx, value]) => ({ + axisSelector: toAxisSelector(innerSpec.axesSpec[idx]), + constant: value, + })), + }, + }; + } + } + return this.queryCache.value; + } + + /** + * Filtered + overrides → Overrided. Overrided is always the + * outermost layer. + */ + withSpecs(overrides: SpecOverrides): ColumnRecipe { + return ColumnOverridedRecipe.wrap(this, overrides); + } +} + +function toAxisSelector(axis: AxisSpec): SingleAxisSelector { + const selector: SingleAxisSelector = { name: axis.name, type: axis.type }; + if (axis.domain !== undefined && Object.keys(axis.domain).length > 0) { + selector.domain = axis.domain; + } + return selector; +} diff --git a/sdk/model/src/columns/column_recipes/column_overrided_recipe.ts b/sdk/model/src/columns/column_recipes/column_overrided_recipe.ts new file mode 100644 index 0000000000..cb94002101 --- /dev/null +++ b/sdk/model/src/columns/column_recipes/column_overrided_recipe.ts @@ -0,0 +1,146 @@ +import { + applySpecOverrides, + createColumnOverridedId, + mergeSpecOverrides, + type ColumnOverridedId, + type ColumnOverridedKey, + type ColumnUniversalId, + type PColumnSpec, + type PObjectId, + type SpecOverrides, + type SpecQuery, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import type { ColumnFieldStatus, ColumnResolutionStatus } from "./types"; +import { ColumnRecipe } from "./index"; +import { Column } from "../column"; +import { rebrandLeafId } from "./leaf_rebrand"; + +/** + * A recipe whose id is not an Overrided wrap — valid `source` for Overrided. + * Mirrors the spec-level invariant on {@link ColumnOverridedKey.source}. + */ +type NonOverridedColumn = Column>; + +/** + * Recipe wrapper that overlays {@link SpecOverrides} on top of any other + * recipe. The id is `ColumnOverridedKey { source: inner.id, specOverrides }`. + * + * Flat-merge invariant: `inner` is never another {@link ColumnOverridedRecipe}. + * Repeated overrides go through {@link wrap} / {@link withSpecs}, which merge + * `specOverrides` at the current level and keep wrapper depth constant. + * + * This invariant matches the one enforced inside `unwrapOverrides` from + * pl-model-common: it explicitly throws on a nested override-wrap. + */ +export class ColumnOverridedRecipe implements Column { + /** + * Wrap any recipe with overrides. If `inner` is already a + * {@link ColumnOverridedRecipe}, merges `overrides` into the existing ones + * and returns a new Overrided sharing the original `inner`. No + * `Overrided>` is produced. + */ + static wrap(col: Column, overrides: SpecOverrides): ColumnOverridedRecipe { + if (col instanceof ColumnOverridedRecipe) { + return new ColumnOverridedRecipe(col.inner, mergeSpecOverrides(col.overrides, overrides)); + } + // `inner` is not an Overrided wrap (the `instanceof` branch above handles + // that). Narrow the id union here so the constructor parameter stays + // honest with `ColumnOverridedKey.source`. + return new ColumnOverridedRecipe(col as NonOverridedColumn, overrides); + } + + /** + * Static counterpart to {@link wrap}: returns the resolution status of a + * {@link ColumnOverridedKey} without constructing the recipe. Overrides + * do not add references — status collapses to the source's status. + */ + static getStatusByKey( + key: ColumnOverridedKey, + opts: { ctx?: GlobalCfgRenderCtx } = {}, + ): ColumnResolutionStatus { + return ColumnRecipe.getStatus(key.source, opts); + } + + readonly id: ColumnOverridedId; + private specCache?: { readonly value: PColumnSpec }; + private queryCache?: { readonly value: SpecQuery }; + private dataStatusCache?: { readonly value: ColumnFieldStatus }; + + private constructor( + private readonly inner: NonOverridedColumn, + private readonly overrides: SpecOverrides, + ) { + this.id = createColumnOverridedId({ + source: inner.id, + specOverrides: overrides, + }); + } + + /** Wrapped recipe — exposed so external walkers can descend. */ + getInner(): Column { + return this.inner; + } + + /** + * `inner` owns all references; overrides are a pure spec patch and do not + * introduce new column refs. + */ + getReferencedIds(): PObjectId[] { + return this.inner.getReferencedIds(); + } + + getDataStatus(): ColumnFieldStatus { + if (this.dataStatusCache === undefined) { + this.dataStatusCache = { value: this.inner.getDataStatus() }; + } + return this.dataStatusCache.value; + } + + /** + * Take the inner spec and overlay our overrides. `applySpecOverrides` + * extracts the overrides from `this.id` itself. + */ + getSpec(): PColumnSpec { + if (this.specCache === undefined) { + this.specCache = { value: applySpecOverrides(this.inner.getSpec(), this.id) }; + } + return this.specCache.value; + } + + /** + * Emit a client-side `specOverride` node wrapping the inner query. + * + * This is a structural-but-collapsible node: it carries the override + * patch through the query tree so consumers see the full recipe shape, + * but it is collapsed at the host boundary (`resolvePColumn`) before + * the query reaches pframe-engine. The engine never sees this node. + * + * Overrides do not change query topology — they are a pure spec patch. + */ + getQuery(): SpecQuery { + if (this.queryCache === undefined) { + // Rebrand the inner column leaf to this.id (rich Overrided id) so + // filter/sort references emitted with this recipe's id match the + // leaf the engine sees. The specOverride node still wraps the leaf + // and is collapsed at the host boundary into a spec patch on the + // resolved PColumn. + this.queryCache = { + value: { + type: "specOverride", + input: rebrandLeafId(this.inner.getQuery(), this.inner.id, this.id), + override: this.overrides, + }, + }; + } + return this.queryCache.value; + } + + /** + * Flat merge: new overrides merge with existing ones; `inner` is unchanged. + * The result is an Overrided of the same depth. + */ + withSpecs(overrides: SpecOverrides): Column { + return ColumnOverridedRecipe.wrap(this, overrides); + } +} diff --git a/sdk/model/src/columns/column_recipes/column_recipes.test.ts b/sdk/model/src/columns/column_recipes/column_recipes.test.ts new file mode 100644 index 0000000000..0d95d7dfbd --- /dev/null +++ b/sdk/model/src/columns/column_recipes/column_recipes.test.ts @@ -0,0 +1,659 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + createColumnDiscoveredKey, + createColumnFilteredKey, + createColumnOverridedId, + createColumnOverridedKey, + createGlobalPObjectId, + createLocalPObjectId, + isFilteredPColumn, + isColumnOverridedKey, + isColumnDiscoveredKey, + parseColumnIdSafety, + parseColumnOverridedId, + stringifyColumnDiscoveredId, + stringifyColumnFilteredId, + type ColumnDiscoveredKey, + type PColumnSpec, + type PObjectId, + type SpecOverrides, + type SpecQuery, + type ColumnUniversalId, +} from "@milaboratories/pl-model-common"; +import { ColumnDiscoveredRecipe } from "./column_discovered_recipe"; +import { ColumnOverridedRecipe } from "./column_overrided_recipe"; +import { ColumnFilteredRecipe } from "./column_filtered_recipe"; +import { ColumnRecipe } from "./index"; +import { ColumnAbsentError } from "../column_lazy"; +import type { ColumnFieldStatus } from "./types"; +import { installStubRegistry } from "../__test_helpers__/stub_registry"; +import { throwError } from "@milaboratories/helpers"; + +// ── ctx mock ──────────────────────────────────────────────────────────────── +// Recipe constructors pull `getCfgRenderCtx()` from globalThis. A minimal +// mock with a no-op service dispatch is enough so that +// `new ColumnDiscoveredRecipe` does not throw in the constructor +// (getService("pframeSpec")). + +function makeCtx(opts: { serviceMethods?: Record unknown> } = {}) { + const methods = opts.serviceMethods ?? {}; + return { + getAccessorHandleByName: () => undefined, + getUpstreamBlockCtx: () => [], + getServiceNames: () => ["pframeSpec"], + getServiceMethods: () => Object.keys(methods), + callServiceMethod: (_id: string, method: string, ...args: unknown[]) => + methods[method]?.(...args), + } as unknown as never; +} + +beforeEach(() => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; +}); + +afterEach(() => { + delete (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx; +}); + +/** Install a fresh cfgRenderCtx (optionally pframeSpec-aware) plus a stub + * registry resolving the given (id → spec) map. */ +function setupCtx( + specs: Record, + opts: Parameters[0] = {}, +): void { + const ctx = makeCtx(opts); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, specs); +} + +// ── fixtures ──────────────────────────────────────────────────────────────── + +const baseLeaf = createLocalPObjectId(["main", "out"], "col-1"); +const linkerId = createLocalPObjectId(["main"], "L1"); + +const stubSpec = ( + axes: { name: string; type: string; domain?: Record }[] = [], +): PColumnSpec => + ({ + kind: "PColumn", + name: "stub", + valueType: "String", + axesSpec: axes, + annotations: {}, + }) as unknown as PColumnSpec; + +const overrides = (patch: Partial): SpecOverrides => ({ + annotations: patch.annotations ?? {}, + domain: patch.domain ?? {}, + contextDomain: patch.contextDomain ?? {}, + axesSpec: patch.axesSpec ?? {}, +}); + +/** + * Minimal `ColumnRecipe` stub — lets us test Overrided/Filtered without + * ctx mocking and without touching the pframeSpec service. + */ +class StubRecipe implements ColumnRecipe { + constructor( + readonly id: ColumnUniversalId, + private readonly state: { + spec?: PColumnSpec; + query?: SpecQuery; + refs?: PObjectId[]; + status?: ColumnFieldStatus; + } = {}, + ) {} + + getReferencedIds(): PObjectId[] { + return this.state.refs ?? [this.id as unknown as PObjectId]; + } + getDataStatus(): ColumnFieldStatus { + return this.state.status ?? "present"; + } + getSpec(): PColumnSpec { + if (this.state.spec === undefined) throw new Error("StubRecipe: no stub spec"); + return this.state.spec; + } + getQuery(): SpecQuery { + if (!this.state.query) throw new Error("StubRecipe: no stub query"); + return this.state.query; + } + withSpecs(o: SpecOverrides): ColumnRecipe { + return ColumnOverridedRecipe.wrap(this, o); + } +} + +const leafQuery: SpecQuery = { type: "column", column: baseLeaf } as SpecQuery; + +const stubInner = (state: ConstructorParameters[1] = {}) => + new StubRecipe(baseLeaf as unknown as ColumnUniversalId, state); + +// ════════════════════════════════════════════════════════════════════════════ +// ColumnOverridedRecipe +// ════════════════════════════════════════════════════════════════════════════ + +describe("ColumnOverridedRecipe", () => { + test("wrap(plain inner): id is Overrided{source: inner.id, specOverrides}", () => { + const inner = stubInner(); + const ov = overrides({ annotations: { a: "1" } }); + const wrapped = ColumnOverridedRecipe.wrap(inner, ov); + const parsed = parseColumnOverridedId(wrapped.id); + expect(parsed.__isOverrided).toBe(true); + expect(parsed.source).toBe(inner.id); + expect(parsed.specOverrides.annotations).toEqual({ a: "1" }); + }); + + test("wrap on already-Overrided → flat merge (single layer, merged overrides)", () => { + const inner = stubInner(); + const a = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { x: "1" } })); + const b = ColumnOverridedRecipe.wrap(a, overrides({ annotations: { y: "2" } })); + + const parsed = parseColumnOverridedId(b.id); + // source is the leaf, not a nested Overrided + expect(parsed.source).toBe(inner.id); + expect(parsed.specOverrides.annotations).toEqual({ x: "1", y: "2" }); + + // and source itself is not an Overrided wrap + const innerParsed = parseColumnIdSafety(parsed.source); + expect(innerParsed === undefined || !isColumnOverridedKey(innerParsed)).toBe(true); + }); + + test("withSpecs(o1).withSpecs(o2) has same id as withSpecs(merge(o1,o2))", () => { + const inner = stubInner(); + const a = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1" } })).withSpecs( + overrides({ annotations: { b: "2" } }), + ); + const b = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1", b: "2" } })); + expect(a.id).toBe(b.id); + }); + + test("getSpec applies overrides to inner.getSpec()", () => { + const inner = stubInner({ spec: stubSpec() }); + const wrapped = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1" } })); + expect(wrapped.getSpec().annotations).toEqual({ a: "1" }); + }); + + test("getSpec propagates throws from inner.getSpec()", () => { + // Inner without a stubbed spec throws — Overrided forwards the error. + const inner = stubInner({ spec: undefined }); + expect(() => + ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1" } })).getSpec(), + ).toThrow(/no stub spec/); + }); + + test("getReferencedIds / getDataStatus forward to inner", () => { + const refs = [baseLeaf, linkerId]; + const inner = stubInner({ + query: leafQuery, + refs, + status: "resolving", + }); + const wrapped = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1" } })); + expect(wrapped.getReferencedIds()).toEqual(refs); + expect(wrapped.getDataStatus()).toBe("resolving"); + }); + + test("getQuery wraps inner query in a specOverride node carrying the overrides", () => { + const inner = stubInner({ query: leafQuery }); + const ov = overrides({ annotations: { a: "1" } }); + const wrapped = ColumnOverridedRecipe.wrap(inner, ov); + const query = wrapped.getQuery(); + expect(query.type).toBe("specOverride"); + if (query.type === "specOverride") { + // Inner leaf gets rebranded to the Overrided id so every variant of the + // same physical column produces a distinct leaf for the engine. + expect(query.input).toEqual({ type: "column", column: wrapped.id }); + expect(query.override).toBe(ov); + } + }); + + test("getQuery is cached: repeated calls return the same node", () => { + const inner = stubInner({ query: leafQuery }); + const wrapped = ColumnOverridedRecipe.wrap(inner, overrides({ annotations: { a: "1" } })); + expect(wrapped.getQuery()).toBe(wrapped.getQuery()); + }); +}); + +// ════════════════════════════════════════════════════════════════════════════ +// ColumnFilteredRecipe +// ════════════════════════════════════════════════════════════════════════════ + +describe("ColumnFilteredRecipe", () => { + test("wrap(plain inner): id is FilteredPColumnId{source, axisFilters}", () => { + // Two axes so the filter on axis 0 leaves axis 1 — wrap rejects filters + // that fix every axis (an empty axis set has no useful column). + const inner = stubInner({ + spec: stubSpec([ + { name: "a", type: "String" }, + { name: "b", type: "String" }, + ]), + }); + const filtered = ColumnFilteredRecipe.wrap(inner, [[0, "v1"]]); + const parsed = parseColumnIdSafety(filtered.id); + expect(parsed && isFilteredPColumn(parsed)).toBe(true); + if (parsed && isFilteredPColumn(parsed)) { + expect(parsed.source).toBe(inner.id); + expect(parsed.axisFilters).toEqual([[0, "v1"]]); + } + }); + + test("wrap on already-Filtered → flat concat axisFilters", () => { + // Three axes so two stacked filters (axes 0, 1) leave axis 2. + const inner = stubInner({ + spec: stubSpec([ + { name: "a", type: "String" }, + { name: "b", type: "String" }, + { name: "c", type: "String" }, + ]), + }); + const a = ColumnFilteredRecipe.wrap(inner, [[0, "v1"]]); + const b = ColumnFilteredRecipe.wrap(a, [[1, "v2"]]); + const parsed = parseColumnIdSafety(b.id); + expect(parsed && isFilteredPColumn(parsed)).toBe(true); + if (parsed && isFilteredPColumn(parsed)) { + expect(parsed.source).toBe(inner.id); + expect(parsed.axisFilters).toEqual([ + [0, "v1"], + [1, "v2"], + ]); + } + }); + + test("getSpec projects axes — drops by indices from axisFilters", () => { + const inner = stubInner({ + spec: stubSpec([ + { name: "sample", type: "String" }, + { name: "gene", type: "String" }, + { name: "rep", type: "Int" }, + ]), + }); + const filtered = ColumnFilteredRecipe.wrap(inner, [ + [0, "S1"], + [2, 3], + ]); + expect(filtered.getSpec().axesSpec).toEqual([{ name: "gene", type: "String" }]); + }); + + test("wrap throws when no axis filter is provided", () => { + // The no-op `wrap(inner, [])` form is rejected at construction: a + // Filtered without filters has no projection over the inner and would + // collapse to identity in the wrapper chain. + const inner = stubInner({ + spec: stubSpec([{ name: "a", type: "String" }]), + query: leafQuery, + }); + expect(() => ColumnFilteredRecipe.wrap(inner, [])).toThrow( + /at least one axis filter must be provided/, + ); + }); + + test("wrap throws when filters fix every axis", () => { + // A filter set that fixes every axis would leave a 0-axis projection — + // not a useful column. Construction-time validation rejects it. + const inner = stubInner({ + spec: stubSpec([{ name: "a", type: "String" }]), + query: leafQuery, + }); + expect(() => ColumnFilteredRecipe.wrap(inner, [[0, "v"]])).toThrow( + /at least one axis must remain/, + ); + }); + + test("getQuery builds SpecQuerySliceAxes with correct selectors derived from indices", () => { + const inner = stubInner({ + spec: stubSpec([ + { name: "sample", type: "String", domain: { kind: "patient" } }, + { name: "gene", type: "String" }, + ]), + query: leafQuery, + }); + const filtered = ColumnFilteredRecipe.wrap(inner, [[0, "S1"]]); + const q = filtered.getQuery() as { type: string; input: SpecQuery; axisFilters: unknown[] }; + expect(q.type).toBe("sliceAxes"); + // Inner leaf gets rebranded to the Filtered id so the engine sees a + // distinct column for each axis-slice variant of the same physical hit. + expect(q.input).toEqual({ type: "column", column: filtered.id }); + expect(q.axisFilters).toEqual([ + { + axisSelector: { name: "sample", type: "String", domain: { kind: "patient" } }, + constant: "S1", + }, + ]); + }); + + test("wrap throws on out-of-range axis index (validated at construction)", () => { + const inner = stubInner({ + spec: stubSpec([{ name: "a", type: "String" }]), + query: leafQuery, + }); + expect(() => ColumnFilteredRecipe.wrap(inner, [[5, "v"]])).toThrow(/out of range/); + }); + + test("withSpecs → Overrided> (Overrided on the outside)", () => { + const inner = stubInner({ + spec: stubSpec([ + { name: "a", type: "String" }, + { name: "b", type: "String" }, + ]), + }); + const filtered = ColumnFilteredRecipe.wrap(inner, [[0, "v"]]); + const layered = filtered.withSpecs(overrides({ annotations: { a: "1" } })); + + const parsedOuter = parseColumnIdSafety(layered.id); + expect(parsedOuter && isColumnOverridedKey(parsedOuter)).toBe(true); + if (parsedOuter && isColumnOverridedKey(parsedOuter)) { + const parsedInner = parseColumnIdSafety(parsedOuter.source); + expect(parsedInner && isFilteredPColumn(parsedInner)).toBe(true); + } + }); + + test("getReferencedIds / getDataStatus forward to inner", () => { + const refs = [baseLeaf, linkerId]; + const inner = stubInner({ + refs, + status: "absent", + spec: stubSpec([ + { name: "a", type: "String" }, + { name: "b", type: "String" }, + ]), + }); + const filtered = ColumnFilteredRecipe.wrap(inner, [[0, "v"]]); + expect(filtered.getReferencedIds()).toEqual(refs); + expect(filtered.getDataStatus()).toBe("absent"); + }); +}); + +// ════════════════════════════════════════════════════════════════════════════ +// ColumnDiscoveredRecipe (id / refs / transformations / getSpec pass-through). +// getQuery is covered separately; getSpec no longer needs the engine. +// ════════════════════════════════════════════════════════════════════════════ + +describe("ColumnDiscoveredRecipe", () => { + const trivialKey: ColumnDiscoveredKey = createColumnDiscoveredKey({ column: baseLeaf }); + + test("fromKey throws on an empty column id (unrecognized id variant)", () => { + // Empty string isn't a valid `ColumnUniversalId` — `parseColumnIdSafety` + // returns undefined and the dispatcher falls through to the "unsupported + // variant" throw rather than silently returning undefined. + expect(() => + ColumnDiscoveredRecipe.fromKey(createColumnDiscoveredKey({ column: "" as PObjectId })), + ).toThrow(/Unsupported ColumnRecipe id variant/); + }); + + test("fromKey propagates ColumnAbsentError from a referenced absent column", () => { + // Default ctx mock has no providers → empty registry → leaf id is provably + // absent. The leaf-level throw inside `Column(uniId, opts)` propagates + // unchanged through `fromKey`. + expect(() => ColumnDiscoveredRecipe.fromKey(trivialKey)).toThrow(ColumnAbsentError); + }); + + test("id is the canonical stringify(ColumnDiscoveredKey)", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + const r = + ColumnDiscoveredRecipe.fromKey(trivialKey) ?? throwError("fromKey returned undefined"); + const parsed = parseColumnIdSafety(r.id); + expect(parsed && isColumnDiscoveredKey(parsed)).toBe(true); + if (parsed && isColumnDiscoveredKey(parsed)) { + expect(parsed.column).toBe(baseLeaf); + } + }); + + test("getHitId returns key.column", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + const r = + ColumnDiscoveredRecipe.fromKey(trivialKey) ?? throwError("fromKey returned undefined"); + expect(r.getHitId()).toBe(baseLeaf); + }); + + test("getReferencedIds: hit + path linkers, deduplicated", () => { + const linker2 = createGlobalPObjectId("blk", "L2"); + setupCtx({ [baseLeaf]: stubSpec(), [linkerId]: stubSpec(), [linker2]: stubSpec() }); + const r = + ColumnDiscoveredRecipe.fromKey( + createColumnDiscoveredKey({ + column: baseLeaf, + path: [ + { type: "linker", column: linkerId }, + { type: "linker", column: linker2 }, + { type: "linker", column: linkerId }, // duplicate + ], + }), + ) ?? throwError("fromKey returned undefined"); + const refs = r.getReferencedIds().sort(); + expect(refs).toEqual([baseLeaf, linkerId, linker2].sort()); + }); + + test("getSpec is the hit column's own spec, unaffected by the linker path", () => { + const linker2 = createGlobalPObjectId("blk", "L2"); + const hitSpec = stubSpec([ + { name: "group", type: "String" }, + { name: "name", type: "String" }, + ]); + setupCtx({ [baseLeaf]: hitSpec, [linkerId]: stubSpec(), [linker2]: stubSpec() }); + const r = + ColumnDiscoveredRecipe.fromKey( + createColumnDiscoveredKey({ + column: baseLeaf, + path: [ + { type: "linker", column: linkerId }, + { type: "linker", column: linker2 }, + ], + }), + ) ?? throwError("fromKey returned undefined"); + // Pass-through: the linker chain enables co-indexing but does not remap + // the hit's axesSpec, so getSpec === the hit recipe's spec. + expect(r.getSpec().axesSpec).toEqual(hitSpec.axesSpec); + }); + + test("withSpecs → Overrided, flat-merge on repeat", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + const r = + ColumnDiscoveredRecipe.fromKey(trivialKey) ?? throwError("fromKey returned undefined"); + const a = r.withSpecs(overrides({ annotations: { x: "1" } })); + const b = a.withSpecs(overrides({ annotations: { y: "2" } })); + + const parsed = parseColumnOverridedId(b.id); + expect(parsed.specOverrides.annotations).toEqual({ x: "1", y: "2" }); + + // source must be Discovered, not Overrided + const inner = parseColumnIdSafety(parsed.source); + expect(inner && isColumnDiscoveredKey(inner)).toBe(true); + }); +}); + +// ════════════════════════════════════════════════════════════════════════════ +// Static resolution status (getStatusByKey + ColumnRecipe.getStatus dispatcher) +// — verifies the new contract: status answered before construction; worst-wins +// aggregation; absent throw propagates through recursive id resolution. +// ════════════════════════════════════════════════════════════════════════════ + +describe("ColumnDiscoveredRecipe.getStatusByKey (worst-wins across references)", () => { + const hit = createLocalPObjectId(["main", "out"], "hit"); + const linker = createLocalPObjectId(["main"], "linker"); + + test("present when hit and every linker are present", () => { + setupCtx({ [hit]: stubSpec(), [linker]: stubSpec() }); + expect( + ColumnDiscoveredRecipe.getStatusByKey( + createColumnDiscoveredKey({ + column: hit, + path: [{ type: "linker", column: linker }], + }), + ), + ).toBe("present"); + }); + + test("resolving when any reference is still resolving (hit present, linker resolving)", () => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, { + [hit]: stubSpec(), + [linker]: { accessorLocked: false }, // resolving + }); + expect( + ColumnDiscoveredRecipe.getStatusByKey( + createColumnDiscoveredKey({ + column: hit, + path: [{ type: "linker", column: linker }], + }), + ), + ).toBe("resolving"); + }); + + test("absent when any reference is provably absent (hit present, linker absent)", () => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, { + [hit]: stubSpec(), + [linker]: { accessorLocked: true }, // absent + }); + expect( + ColumnDiscoveredRecipe.getStatusByKey( + createColumnDiscoveredKey({ + column: hit, + path: [{ type: "linker", column: linker }], + }), + ), + ).toBe("absent"); + }); +}); + +describe("Wrapper getStatusByKey delegates to source", () => { + test("ColumnFilteredRecipe.getStatusByKey returns the source's status", () => { + setupCtx({ [baseLeaf]: stubSpec([{ name: "axis", type: "String" }]) }); + expect( + ColumnFilteredRecipe.getStatusByKey( + createColumnFilteredKey({ source: baseLeaf, axisFilters: [] }), + ), + ).toBe("present"); + + // Now make the source absent — wrapper status follows. + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, { [baseLeaf]: { accessorLocked: true } }); + expect( + ColumnFilteredRecipe.getStatusByKey( + createColumnFilteredKey({ source: baseLeaf, axisFilters: [] }), + ), + ).toBe("absent"); + }); + + test("ColumnOverridedRecipe.getStatusByKey returns the source's status", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + expect( + ColumnOverridedRecipe.getStatusByKey( + createColumnOverridedKey({ source: baseLeaf, specOverrides: overrides({}) }), + ), + ).toBe("present"); + + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, {}, { isFinal: false }); + expect( + ColumnOverridedRecipe.getStatusByKey( + createColumnOverridedKey({ source: baseLeaf, specOverrides: overrides({}) }), + ), + ).toBe("resolving"); + }); +}); + +describe("ColumnRecipe.getStatus (top-level dispatcher)", () => { + test("routes bare PObjectId to ColumnLazyImpl.getStatusById", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + expect(ColumnRecipe.getStatus(baseLeaf)).toBe("present"); + }); + + test("routes ColumnDiscoveredKey id to ColumnDiscoveredRecipe.getStatusByKey", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + const id = stringifyColumnDiscoveredId({ column: baseLeaf }); + expect(ColumnRecipe.getStatus(id)).toBe("present"); + }); + + test("routes ColumnFilteredKey id to ColumnFilteredRecipe.getStatusByKey", () => { + setupCtx({ [baseLeaf]: stubSpec([{ name: "axis", type: "String" }]) }); + const id = stringifyColumnFilteredId({ + source: baseLeaf, + axisFilters: [], + }); + expect(ColumnRecipe.getStatus(id)).toBe("present"); + }); + + test("routes ColumnOverridedKey id to ColumnOverridedRecipe.getStatusByKey", () => { + setupCtx({ [baseLeaf]: stubSpec() }); + const id = createColumnOverridedId({ + source: baseLeaf, + specOverrides: overrides({}), + }); + expect(ColumnRecipe.getStatus(id)).toBe("present"); + }); +}); + +describe("ColumnRecipe(id) throw propagation on absent leaves", () => { + test("filtered id over an absent source throws ColumnAbsentError", () => { + // Empty registry with isFinal=true → leaf is provably absent → throw + // propagates through the Filtered dispatch branch. + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, {}, { isFinal: true }); + const id = stringifyColumnFilteredId({ + source: baseLeaf, + axisFilters: [], + }); + expect(() => ColumnRecipe(id)).toThrow(ColumnAbsentError); + }); + + test("overrided id over an absent source throws ColumnAbsentError", () => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, {}, { isFinal: true }); + const id = createColumnOverridedId({ + source: baseLeaf, + specOverrides: overrides({}), + }); + expect(() => ColumnRecipe(id)).toThrow(ColumnAbsentError); + }); + + test("discovered id over an absent hit throws ColumnAbsentError", () => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, {}, { isFinal: true }); + const id = stringifyColumnDiscoveredId({ column: baseLeaf }); + expect(() => ColumnRecipe(id)).toThrow(ColumnAbsentError); + }); +}); + +describe("ColumnDiscoveredRecipe.getQuery (local assembly)", () => { + test("trivial (no path) → leaf column query with the Discovered id", () => { + // `buildSpecQuery` no longer delegates to `pframeSpec.buildQuery`; it + // assembles the SpecQuery locally and rebrands the hit leaf to `this.id` + // so sibling variants of the same physical hit stay distinguishable. + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, { [baseLeaf]: stubSpec() }); + + const r = + ColumnDiscoveredRecipe.fromKey(createColumnDiscoveredKey({ column: baseLeaf })) ?? + throwError("fromKey returned undefined"); + expect(r.getQuery()).toEqual({ type: "column", column: r.id }); + }); + + test("with linker path → linkerJoin nesting, hit leaf rebranded to the Discovered id", () => { + const ctx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = ctx; + installStubRegistry(ctx, { [baseLeaf]: stubSpec(), [linkerId]: stubSpec() }); + + const r = + ColumnDiscoveredRecipe.fromKey( + createColumnDiscoveredKey({ + column: baseLeaf, + path: [{ type: "linker", column: linkerId }], + }), + ) ?? throwError("fromKey returned undefined"); + expect(r.getQuery()).toEqual({ + type: "linkerJoin", + linker: { type: "column", column: linkerId }, + secondary: [{ entry: { type: "column", column: r.id } }], + }); + }); +}); diff --git a/sdk/model/src/columns/column_recipes/index.ts b/sdk/model/src/columns/column_recipes/index.ts new file mode 100644 index 0000000000..08309b0141 --- /dev/null +++ b/sdk/model/src/columns/column_recipes/index.ts @@ -0,0 +1,127 @@ +import { + isColumnDiscoveredKey, + isColumnOverridedKey, + parseColumnIdSafety, + type ColumnUniversalId, + type PObjectId, + isPObjectKey, + isColumnFilteredKey, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../../render/internal"; +import { ColumnLazyImpl } from "../column_lazy"; +import { ColumnDiscoveredRecipe } from "./column_discovered_recipe"; +import { ColumnFilteredRecipe } from "./column_filtered_recipe"; +import { ColumnOverridedRecipe } from "./column_overrided_recipe"; +import type { + ColumnRecipe as ColumnRecipeBase, + ColumnRecipeId, + ColumnResolutionStatus, +} from "./types"; + +export type { ColumnFieldStatus, ColumnRecipeId, ColumnResolutionStatus, SpecQuery } from "./types"; +export { ColumnAbsentError } from "../column_lazy"; +export { ColumnDiscoveredRecipe } from "./column_discovered_recipe"; +export { ColumnFilteredRecipe } from "./column_filtered_recipe"; +export { ColumnOverridedRecipe } from "./column_overrided_recipe"; + +// Re-declare the interface locally so the name can merge with the dispatcher +// value declared below: a single name `ColumnRecipe` resolves to the +// interface in type position and to the factory function (`Object.assign`d +// with `.getStatus`) in value position. +// (Type-only re-exports do not participate in TS declaration merging.) +export interface ColumnRecipe< + ID extends ColumnRecipeId = ColumnRecipeId, +> extends ColumnRecipeBase {} + +/** + * Build a `ColumnRecipe` from a stringified id. Dispatches to the matching + * concrete recipe variant based on the id's encoding and recurses on + * wrappers' inner `source`: + * + * - bare {@link PObjectId} (non-JSON) → `ColumnLazy.fromId` + * - `ColumnDiscoveredKey` → {@link ColumnDiscoveredRecipe} + * - `ColumnOverridedKey { source, specOverrides }` → recurse on `source`, + * then {@link ColumnOverridedRecipe.wrap} + * - `ColumnFilteredKey { source, axisFilters }` → recurse on `source`, + * then {@link ColumnFilteredRecipe.wrap} + * + * Returns `undefined` for unsupported variants (e.g. AnchoredPColumnId, + * which needs an anchor map to resolve) or when a nested source itself + * cannot be built. + */ +function ColumnRecipeBuild( + id: ColumnUniversalId, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): undefined | ColumnRecipe { + const parsed = parseColumnIdSafety(id); + + if (isPObjectKey(parsed)) { + return ColumnLazyImpl.fromId(id as PObjectId, opts); + } + + if (isColumnDiscoveredKey(parsed)) { + return ColumnDiscoveredRecipe.fromKey(parsed, opts); + } + + if (isColumnOverridedKey(parsed)) { + const inner = ColumnRecipe(parsed.source, opts); + if (inner === undefined) return undefined; + return ColumnOverridedRecipe.wrap(inner, parsed.specOverrides); + } + + if (isColumnFilteredKey(parsed)) { + // ColumnFilteredKey.source is a stringified ColumnUniversalId (leaf or + // any nested wrapper). Legacy `FilteredPColumnId` with an AnchoredPColumnId + // (object) source is not supported here — guard against it explicitly. + if (typeof parsed.source !== "string") { + throw new Error( + `Unsupported ColumnRecipe id variant: filtered column with non-string source (${JSON.stringify(parsed.source)})`, + ); + } + const inner = ColumnRecipe(parsed.source, opts); + if (inner === undefined) return undefined; + return ColumnFilteredRecipe.wrap(inner, parsed.axisFilters); + } + + throw new Error(`Unsupported ColumnRecipe id variant: ${id}`); +} + +/** + * Resolution status counterpart to {@link ColumnRecipeBuild}. Dispatches on + * the parsed id shape to the matching `getStatusBy*` static — never + * constructs the recipe. Use this at boundaries (resolver, filter/sort + * targets) to throw early on `absent` instead of silently producing an empty + * recipe via {@link ColumnRecipeBuild}. + */ +function ColumnRecipeGetStatus( + id: ColumnUniversalId, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): ColumnResolutionStatus { + const parsed = parseColumnIdSafety(id); + if (isPObjectKey(parsed)) return ColumnLazyImpl.getStatusById(id as PObjectId, opts); + if (isColumnDiscoveredKey(parsed)) return ColumnDiscoveredRecipe.getStatusByKey(parsed, opts); + if (isColumnOverridedKey(parsed)) return ColumnOverridedRecipe.getStatusByKey(parsed, opts); + if (isColumnFilteredKey(parsed)) return ColumnFilteredRecipe.getStatusByKey(parsed, opts); + throw new Error(`Unsupported ColumnRecipe id variant: ${id}`); +} + +export const ColumnRecipe: typeof ColumnRecipeBuild & { + readonly getStatus: typeof ColumnRecipeGetStatus; +} = Object.assign(ColumnRecipeBuild, { + getStatus: ColumnRecipeGetStatus, +}); + +/** + * Type-guard for any recipe class — leaf {@link ColumnLazyImpl} or any of the + * wrapper recipes ({@link ColumnDiscoveredRecipe}, {@link ColumnFilteredRecipe}, + * {@link ColumnOverridedRecipe}). Use when a value may be a raw id, a `PColumn`, + * or a recipe and you want to branch on the recipe case. + */ +export function isColumnRecipe(value: unknown): value is ColumnRecipe { + return ( + value instanceof ColumnLazyImpl || + value instanceof ColumnDiscoveredRecipe || + value instanceof ColumnFilteredRecipe || + value instanceof ColumnOverridedRecipe + ); +} diff --git a/sdk/model/src/columns/column_recipes/leaf_rebrand.ts b/sdk/model/src/columns/column_recipes/leaf_rebrand.ts new file mode 100644 index 0000000000..fe0c78f88d --- /dev/null +++ b/sdk/model/src/columns/column_recipes/leaf_rebrand.ts @@ -0,0 +1,28 @@ +import { + mapSpecQueryColumns, + type ColumnUniversalId, + type SpecQuery, +} from "@milaboratories/pl-model-common"; + +/** + * Replace every column leaf whose id equals `fromId` with `toId`, leaving + * other column refs (linkers, sub-anchors) intact. Wrapper recipes + * (Overrided / Filtered) and Discovered use this to lift their own id onto + * the hit leaf, so per-variant uniqueness propagates to the engine and + * resolver-emitted ids match leaves in the emitted SpecQuery. + * + * Generic over all SpecQuery node shapes — including `linkerJoin`, which + * occurs when a wrapper sits over a {@link ColumnDiscoveredRecipe}: only + * the deepest hit leaf carries `fromId`, every linker leaf has a different + * id and is left untouched. + */ +export function rebrandLeafId( + node: SpecQuery, + fromId: ColumnUniversalId, + toId: ColumnUniversalId, +): SpecQuery { + if (fromId === toId) return node; + return mapSpecQueryColumns(node, { + column: (id) => (id === fromId ? toId : id), + }); +} diff --git a/sdk/model/src/columns/column_recipes/types.ts b/sdk/model/src/columns/column_recipes/types.ts new file mode 100644 index 0000000000..bd82dc9d80 --- /dev/null +++ b/sdk/model/src/columns/column_recipes/types.ts @@ -0,0 +1,113 @@ +import type { + ColumnUniversalId, + PColumnSpec, + PObjectId, + SpecOverrides, + SpecQuery, +} from "@milaboratories/pl-model-common"; + +export type { SpecQuery }; + +/** + * Stable identifier of a recipe: either the canonical {@link PObjectId} of a + * leaf or a {@link ColumnUniversalId} encoding `Overrided / Discovered / + * Filtered` layers on top of it. + * + * Recipes are equivalent as value-objects iff their `id`s match (structurally, + * after canonicalization). + */ +export type ColumnRecipeId = ColumnUniversalId; + +/** + * Aggregate data status of a recipe: the "worst" status across every + * PObjectId the recipe references (the leaf itself plus any refs inside + * overrides/discovery). + * + * "All or nothing" semantics: spec/query built from a recipe are meaningful + * only when every referenced column is `present`. + */ +export type ColumnFieldStatus = "resolving" | "absent" | "present"; + +/** + * Resolution status of a recipe — whether the recipe is meaningful in the + * active render ctx. Aggregates spec readiness (the recipe can be + * constructed/used) and data readiness (the underlying leaves can be read). + * + * - `present`: spec + data readable end-to-end. + * - `resolving`: more accessor inputs may still arrive — recheck later. + * - `absent`: every relevant accessor is `inputsLocked`; nothing more will + * appear. Used at boundaries to throw early instead of silently + * producing an empty/broken recipe. + * + * Distinct from {@link ColumnFieldStatus}: that one is the leaf's data field + * status only. {@link ColumnResolutionStatus} also folds in spec/registry + * readiness, so the recipe interface exposes both. + */ +export type ColumnResolutionStatus = "present" | "resolving" | "absent"; + +/** + * Base contract of a column recipe — an immutable description of HOW to + * obtain a column, not the column itself. + * + * A recipe is a value-object. The only transformation exposed via the + * interface is {@link withSpecs}: it overlays overrides on the spec. + * `withDiscovery` / `withAxisFilters` are intentionally absent — discovered + * and filtered recipes are built only via their own factory constructors + * from a leaf recipe; nothing further can be stacked on top of Discovered + * or Filtered. + * + * Invariant: `withSpecs` never nests. A recipe is either bare or wrapped + * in exactly one Overrided layer. Repeated `withSpecs` merges the new + * overrides into the existing ones, keeping the same depth. + * + * A recipe intentionally exposes no `getSpecOverrides / getDiscovery / + * getAxisFilters` getters: those encoding pieces are details of the id. + * Consumers that need to decompose call utilities from `pl-model-common` + * over `recipe.id`. + */ +export interface ColumnRecipe { + /** Canonical recipe identifier. */ + readonly id: ID; + + /** PObjectIds referenced by this recipe (leaf + any nested refs). */ + getReferencedIds(): PObjectId[]; + + /** + * Final spec — built via `pframespec` from id + base spec of the source. + * No partial JS logic: the result matches what the engine returns when + * executing {@link getQuery}. + * + * Always returns a value: the recipe factories validate all referenced + * specs at construction time and return `undefined` from the factory if + * anything is missing. A constructed recipe is by contract spec-complete. + */ + getSpec(): PColumnSpec; + + /** + * IR describing how to obtain the column — a {@link SpecQuery} from + * `pl-model-common`. The recipe builds the description; execution belongs + * to the layer above (pframespec/engine). + * + * The return is parameterized with default `C = PObjectId`. Concrete + * recipes may narrow the parameter (e.g. `SpecQuery`) if + * they need to thread nested id variants. + */ + getQuery(): SpecQuery; + + /** + * Aggregate status across ALL PObjectIds this recipe reaches: own leaf + + * refs from overrides + the parent chain of discovery. + * + * Rule: `absent ▸ resolving ▸ present` ("worst wins"). + */ + getDataStatus(): ColumnFieldStatus; + + /** + * Apply overrides on the spec. Always returns a new recipe. + * + * Flat-merge invariant: if `self` is already Overrided, the implementation + * merges `overrides` with the existing ones and returns the same depth + * (one Overrided wrapper). No `Overrided>`. + */ + withSpecs(overrides: SpecOverrides): ColumnRecipe; +} diff --git a/sdk/model/src/columns/column_selector.test.ts b/sdk/model/src/columns/column_selector.test.ts deleted file mode 100644 index 4ceb5398f8..0000000000 --- a/sdk/model/src/columns/column_selector.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { convertColumnSelectorToMultiColumnSelector } from "./column_selector"; -import type { RegExpString } from "@milaboratories/helpers"; - -// --- convertColumnSelectorToMultiColumnSelector --- - -describe("convertColumnSelectorToMultiColumnSelector", () => { - test("wraps single selector in array", () => { - const result = convertColumnSelectorToMultiColumnSelector({ name: "foo" }); - expect(result).toHaveLength(1); - expect(result[0].name).toEqual([{ type: "regex", value: "foo" }]); - }); - - test("passes through array of selectors", () => { - const result = convertColumnSelectorToMultiColumnSelector([{ name: "a" }, { name: "b" }]); - expect(result).toHaveLength(2); - }); - - test("normalizes plain string name to RegexMatcher[]", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ name: "foo" }); - expect(sel.name).toEqual([{ type: "regex", value: "foo" }]); - }); - - test("normalizes array of mixed strings and matchers", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ - name: ["foo", { type: "regex", value: "bar.*" as RegExpString }], - }); - expect(sel.name).toEqual([ - { type: "regex", value: "foo" }, - { type: "regex", value: "bar.*" }, - ]); - }); - - test("normalizes single StringMatcher object", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ - name: { type: "exact", value: "foo" }, - }); - expect(sel.name).toEqual([{ type: "exact", value: "foo" }]); - }); - - test("normalizes single ValueType to array", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ type: "Int" }); - expect(sel.type).toEqual(["Int"]); - }); - - test("passes through ValueType array", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ type: ["Int", "Long"] }); - expect(sel.type).toEqual(["Int", "Long"]); - }); - - test("normalizes record with plain string values", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ - domain: { "pl7.app/chain": "IGHeavy" }, - }); - expect(sel.domain).toEqual({ - "pl7.app/chain": [{ type: "regex", value: "IGHeavy" }], - }); - }); - - test("normalizes record with mixed array values", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ - annotations: { label: ["a", { type: "regex", value: "b.*" as RegExpString }] }, - }); - expect(sel.annotations).toEqual({ - label: [ - { type: "regex", value: "a" }, - { type: "regex", value: "b.*" }, - ], - }); - }); - - test("normalizes axes", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ - axes: [{ name: "axisName", type: "String" }], - }); - expect(sel.axes).toEqual([{ name: [{ type: "regex", value: "axisName" }], type: ["String"] }]); - }); - - test("preserves partialAxesMatch", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ partialAxesMatch: false }); - expect(sel.partialAxesMatch).toBe(false); - }); - - test("omits undefined fields", () => { - const [sel] = convertColumnSelectorToMultiColumnSelector({ name: "foo" }); - expect(sel.type).toBeUndefined(); - expect(sel.domain).toBeUndefined(); - expect(sel.axes).toBeUndefined(); - }); -}); diff --git a/sdk/model/src/columns/column_snapshot.ts b/sdk/model/src/columns/column_snapshot.ts deleted file mode 100644 index 4e52e37a56..0000000000 --- a/sdk/model/src/columns/column_snapshot.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type { PColumnSpec, PObjectId, SUniversalPColumnId } from "@milaboratories/pl-model-common"; -import type { PColumnDataUniversal } from "../render/internal"; - -// --- ColumnSnapshot --- - -/** Data status of a column snapshot. */ -export type ColumnDataStatus = "ready" | "computing" | "absent"; - -/** - * Immutable snapshot of a column: spec, data status, and lazy data accessor. - * - * - `dataStatus` is readable without marking the render context unstable. - * - `data` holds an active object when data exists (ready or computing), - * or `undefined` when data is permanently absent. - */ -export interface ColumnSnapshot< - Id extends PObjectId | SUniversalPColumnId, - Data = PColumnDataUniversal, -> { - readonly id: Id; - readonly spec: PColumnSpec; - readonly dataStatus: ColumnDataStatus; - - /** - * Lazy data accessor. - * - `'ready'`: `data.get()` returns column data, context stays stable. - * - `'computing'`: `data.get()` returns `undefined`, marks context unstable. - * - `'absent'`: `data` is `undefined` — no active object, no instability. - */ - readonly data: ColumnData | undefined; -} - -// --- ColumnData --- - -/** - * Active object wrapping lazy column data access. - * Accessing data on a computing column marks the render context unstable. - */ -export interface ColumnData { - get(): Data | undefined; -} - -/** Creates a ColumnData active object for a ready column. */ -export function createReadyColumnData(getData: () => PColumnDataUniversal | undefined): ColumnData { - return { get: getData }; -} - -// --- Snapshot construction helpers --- - -/** Creates a ColumnSnapshot from parts. */ -export function createColumnSnapshot( - id: Id, - spec: PColumnSpec, - data: undefined | ColumnData, - dataStatus: ColumnDataStatus, -): ColumnSnapshot { - return { id, spec, data, dataStatus }; -} diff --git a/sdk/model/src/columns/column_snapshot_provider.ts b/sdk/model/src/columns/column_snapshot_provider.ts deleted file mode 100644 index c06a1ed11b..0000000000 --- a/sdk/model/src/columns/column_snapshot_provider.ts +++ /dev/null @@ -1,194 +0,0 @@ -import type { PObjectId } from "@milaboratories/pl-model-common"; -import { isDataInfo, PColumn, visitDataInfo } from "@milaboratories/pl-model-common"; -import { TreeNodeAccessor } from "../render/accessor"; -import type { PColumnDataUniversal } from "../render/internal"; -import type { ColumnDataStatus, ColumnSnapshot } from "./column_snapshot"; - -/** - * Data source interface for column enumeration. - * - * Knows nothing about the render framework, stability tracking, labels, - * anchoring, or splitting. All that complexity lives in the collection layer. - */ -export interface ColumnSnapshotProvider { - /** Returns all currently known columns. */ - getAllColumns(): ColumnSnapshot[]; - - /** Whether the provider has finished enumerating all its columns. - * Calling this may mark the render context unstable — it touches - * the reactive tree to check field resolution state. */ - isColumnListComplete(): boolean; -} - -/** - * Union of types that can serve as column sources for helpers and builders. - * Does NOT include TreeNodeAccessor — call `.toColumnSource()` on it first. - */ -export type ColumnSource = - | ColumnSnapshotProvider - | ColumnSnapshot[] - | PColumn[]; - -/** - * Simple provider wrapping an array of PColumns. - * Always complete, data status always 'ready'. - */ -export class ArrayColumnProvider implements ColumnSnapshotProvider { - private readonly columns: ColumnSnapshot[]; - - constructor(columns: PColumn[]) { - this.columns = columns.map((col) => ({ - id: col.id, - spec: col.spec, - data: { get: () => col.data }, - dataStatus: this.getStatus(col.data), - })); - } - - getAllColumns(): ColumnSnapshot[] { - return this.columns; - } - - isColumnListComplete(): boolean { - return true; - } - - protected getStatus( - d: undefined | PColumnDataUniversal | (() => undefined | PColumnDataUniversal), - ): ColumnDataStatus { - if (d == null) { - return "absent"; - } - if (typeof d === "function") { - return this.getStatus(d()); - } - if (d instanceof TreeNodeAccessor) { - if (d.getIsReadyOrError()) return "ready"; - if (d.getIsFinal()) return "absent"; - return "computing"; - } - if (isDataInfo(d)) { - let ready = true; - let final = true; - visitDataInfo(d, (v) => { - ready &&= v.getIsReadyOrError(); - final &&= v.getIsFinal(); - }); - if (ready) return "ready"; - if (final) return "absent"; - return "computing"; - } - return "ready"; - } -} - -/** - * Provider wrapping an array of ColumnSnapshots. - * Always complete. Data status taken from each snapshot. - */ -export class SnapshotColumnProvider implements ColumnSnapshotProvider { - constructor(private readonly snapshots: ColumnSnapshot[]) {} - - getAllColumns(): ColumnSnapshot[] { - return this.snapshots; - } - - isColumnListComplete(): boolean { - return true; - } -} - -export interface OutputColumnProviderOpts { - /** When true and the accessor is final, columns with no ready data get status 'absent'. */ - allowPermanentAbsence?: boolean; -} - -/** - * Provider wrapping a TreeNodeAccessor (output/prerun resolve result). - * Detects data status from accessor readiness state. - */ -export class OutputColumnProvider implements ColumnSnapshotProvider { - constructor( - private readonly accessor: TreeNodeAccessor, - private readonly opts?: OutputColumnProviderOpts, - ) {} - - getAllColumns(): ColumnSnapshot[] { - return this.getColumns(); - } - - isColumnListComplete(): boolean { - return this.accessor.getInputsLocked(); - } - - private getColumns(): ColumnSnapshot[] { - const pColumns = this.accessor.getPColumns(); - if (pColumns === undefined) return []; - - const isFinal = this.accessor.getIsFinal(); - const allowAbsence = this.opts?.allowPermanentAbsence === true; - - return pColumns.map((col) => { - const dataAccessor = col.data; - const isReady = dataAccessor.getIsReadyOrError(); - - let dataStatus: ColumnDataStatus; - if (isReady) { - dataStatus = "ready"; - } else if (allowAbsence && isFinal) { - dataStatus = "absent"; - } else { - dataStatus = "computing"; - } - - return { - id: col.id, - spec: col.spec, - dataStatus, - data: { get: () => (isReady ? dataAccessor : undefined) }, - }; - }); - } -} - -/** Checks if a value is a ColumnSnapshotProvider (duck-typing). */ -export function isColumnSnapshotProvider(source: unknown): source is ColumnSnapshotProvider { - return ( - typeof source === "object" && - source !== null && - "getAllColumns" in source && - "isColumnListComplete" in source && - typeof (source as ColumnSnapshotProvider).getAllColumns === "function" && - typeof (source as ColumnSnapshotProvider).isColumnListComplete === "function" - ); -} - -/** Checks if a value looks like a PColumn (has id, spec, data). */ -function isPColumnArray(source: unknown): source is PColumn[] { - if (!Array.isArray(source)) return false; - if (source.length === 0) return true; // empty array — treat as PColumn[] - const first = source[0]; - return "id" in first && "spec" in first && "data" in first && !("dataStatus" in first); -} - -/** Checks if a value looks like a ColumnSnapshot array. */ -function isColumnSnapshotArray(source: unknown): source is ColumnSnapshot[] { - if (!Array.isArray(source)) return false; - if (source.length === 0) return true; // empty array — treat as snapshots - const first = source[0]; - return "id" in first && "spec" in first && "dataStatus" in first; -} - -/** - * Normalize any ColumnSource into a ColumnSnapshotProvider. - * - ColumnSnapshotProvider → returned as-is - * - ColumnSnapshot[] → wrapped in SnapshotColumnProvider - * - PColumn[] → wrapped in ArrayColumnProvider - */ -export function toColumnSnapshotProvider(source: ColumnSource): ColumnSnapshotProvider { - if (isColumnSnapshotProvider(source)) return source; - if (isColumnSnapshotArray(source)) return new SnapshotColumnProvider(source); - if (isPColumnArray(source)) return new ArrayColumnProvider(source); - // Should not reach here given the type union, but be safe - throw new Error("Unknown ColumnSource type"); -} diff --git a/sdk/model/src/columns/columns_collection.test.ts b/sdk/model/src/columns/columns_collection.test.ts new file mode 100644 index 0000000000..eebce4db1b --- /dev/null +++ b/sdk/model/src/columns/columns_collection.test.ts @@ -0,0 +1,164 @@ +import type { AxisSpec, PColumnSpec, PObjectId } from "@milaboratories/pl-model-common"; +import { createGlobalPObjectId } from "@milaboratories/pl-model-common"; + +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { ColumnsCollection } from "./columns_collection"; +import { + createTestCollectionDriver, + type TestCollectionDriverHandle, +} from "./__test_helpers__/collection_driver"; + +let handle: TestCollectionDriverHandle; + +beforeEach(() => { + handle = createTestCollectionDriver(); +}); + +afterEach(async () => { + await handle.dispose(); +}); + +const DEFAULT_AXIS: AxisSpec = { name: "id", type: "String" } as AxisSpec; + +function gid(name: string): PObjectId { + return createGlobalPObjectId("test-block", name); +} + +function spec(name: string, axesSpec: AxisSpec[] = [DEFAULT_AXIS]): PColumnSpec { + return { kind: "PColumn", name, valueType: "Int", axesSpec, annotations: {} } as PColumnSpec; +} + +function makeCollection( + cols: ReadonlyArray<{ id: PObjectId; spec: PColumnSpec }>, + options?: { isFinal?: boolean }, +): ColumnsCollection { + handle.register(cols); + return ColumnsCollection( + [ + { + columns: cols.map((c) => ({ id: c.id, spec: c.spec, data: undefined })) as never, + isFinal: options?.isFinal ?? true, + }, + ], + { driver: handle.driver }, + ); +} + +describe("ColumnsCollection constructor", () => { + test("accepts column array source", () => { + const c = { id: gid("c1"), spec: spec("col1") }; + expect(makeCollection([c]).getColumnIds()).toEqual([c.id]); + }); + + test("merges multiple sources", () => { + const c1 = { id: gid("c1"), spec: spec("col1") }; + const c2 = { id: gid("c2"), spec: spec("col2") }; + handle.register([c1, c2]); + const ids = ColumnsCollection( + [ + { columns: [{ id: c1.id, spec: c1.spec, data: undefined }] as never, isFinal: true }, + { columns: [{ id: c2.id, spec: c2.spec, data: undefined }] as never, isFinal: true }, + ], + { driver: handle.driver }, + ).getColumnIds(); + expect(ids).toEqual([c1.id, c2.id]); + }); + + test("empty sources array yields empty collection", () => { + const coll = ColumnsCollection([], { driver: handle.driver }); + expect(coll.getColumnIds()).toEqual([]); + }); +}); + +describe("ColumnsCollection.isEmpty", () => { + test("true when no columns", () => { + expect(ColumnsCollection([], { driver: handle.driver }).isEmpty()).toBe(true); + }); + + test("false when at least one column", () => { + const c = { id: gid("c1"), spec: spec("col1") }; + expect(makeCollection([c]).isEmpty()).toBe(false); + }); +}); + +describe("ColumnsCollection.isFinal", () => { + test("true when all sources final", () => { + const c = { id: gid("c1"), spec: spec("col1") }; + expect(makeCollection([c], { isFinal: true }).isFinal()).toBe(true); + }); + + test("false when any source incomplete", () => { + const c = { id: gid("c1"), spec: spec("col1") }; + expect(makeCollection([c], { isFinal: false }).isFinal()).toBe(false); + }); + + test("true with no sources (vacuous)", () => { + expect(ColumnsCollection([], { driver: handle.driver }).isFinal()).toBe(true); + }); +}); + +describe("ColumnsCollection.getColumnIds", () => { + test("deduplicates by id across sources (first wins)", () => { + const id = gid("c1"); + const c1 = { id, spec: spec("col1") }; + const c2 = { id, spec: spec("col1-dup") }; + handle.register([c1]); + const ids = ColumnsCollection( + [ + { columns: [{ id: c1.id, spec: c1.spec, data: undefined }] as never, isFinal: true }, + { columns: [{ id: c2.id, spec: c2.spec, data: undefined }] as never, isFinal: true }, + ], + { driver: handle.driver }, + ).getColumnIds(); + expect(ids).toEqual([id]); + }); +}); + +describe("ColumnsCollection.addSource", () => { + test("returns new instance, original unchanged", () => { + const c1 = { id: gid("c1"), spec: spec("col1") }; + const c2 = { id: gid("c2"), spec: spec("col2") }; + handle.register([c1, c2]); + const base = makeCollection([c1]); + const next = base.addSource({ + columns: [{ id: c2.id, spec: c2.spec, data: undefined } as never], + isFinal: true, + }); + + expect(next).not.toBe(base); + expect(base.getColumnIds()).toEqual([c1.id]); + expect(next.getColumnIds()).toEqual([c1.id, c2.id]); + }); + + test("isFinal reflects appended source", () => { + const c1 = { id: gid("c1"), spec: spec("col1") }; + const c2 = { id: gid("c2"), spec: spec("col2") }; + handle.register([c1, c2]); + const base = makeCollection([c1], { isFinal: true }); + expect(base.isFinal()).toBe(true); + + const next = base.addSource({ + columns: [{ id: c2.id, spec: c2.spec, data: undefined } as never], + isFinal: false, + }); + expect(next.isFinal()).toBe(false); + }); +}); + +describe("ColumnsCollection.filter", () => { + test("returns collection with filtered ids", () => { + const c1 = { id: gid("c1"), spec: spec("col1") }; + const c2 = { id: gid("c2"), spec: spec("col2") }; + const ids = makeCollection([c1, c2]) + .filter({ exclude: { name: "col1" } }) + .getColumnIds(); + expect(ids).toEqual([c2.id]); + }); + + test("no filters returns all ids", () => { + const c1 = { id: gid("c1"), spec: spec("col1") }; + const c2 = { id: gid("c2"), spec: spec("col2") }; + const ids = makeCollection([c1, c2]).filter({}).getColumnIds(); + expect(ids).toEqual([c1.id, c2.id]); + }); +}); diff --git a/sdk/model/src/columns/columns_collection.ts b/sdk/model/src/columns/columns_collection.ts new file mode 100644 index 0000000000..c17ff9cda5 --- /dev/null +++ b/sdk/model/src/columns/columns_collection.ts @@ -0,0 +1,170 @@ +import type { + CollectionHandle, + ColumnUniversalId, + ColumnsCollectionDriverModel, + ColumnsDiscoverOptions, + ColumnsFilterOptions, + SerializedColumnsSource, +} from "@milaboratories/pl-model-common"; +import type { GlobalCfgRenderCtx } from "../render/internal"; +import { MainAccessorName, StagingAccessorName } from "../render/internal"; +import type { ColumnsSource } from "./column_providers"; +import { isColumnProvider } from "./column_providers"; +import { TreeNodeAccessor } from "../render/accessor"; +import { getService } from "../services/get_services"; +import { getCfgRenderCtx } from "../internal"; +import { ColumnRecipe } from "./column_recipes"; +import { isNil } from "es-toolkit"; + +export interface ColumnsCollectionDeps { + /** Render ctx used both for service resolution and for default ctx sources. */ + readonly ctx?: GlobalCfgRenderCtx; + /** Override the resolved `columnsCollection` service — primarily for tests. */ + readonly driver?: ColumnsCollectionDriverModel; +} + +/** + * Shorthand literals accepted inside `ColumnsCollection`'s `sources` + * array, expanded against the active render ctx: + * + * - `"result_pool"` – fan-out into the host's upstream-block result pool. + * - `"current_block"` – main outputs + prerun (staging) accessors of the + * current block, when present. + */ +export type ColumnsSourceShorthand = "result_pool" | "current_block"; + +/** + * Build a {@link ColumnsCollection} from sandbox-side source descriptors. + * Resolves the `columnsCollection` driver (either from `deps.driver` or via + * `getService("columnsCollection")`) and asks the host to mint a fresh + * handle covering the supplied sources. Pass `sources === undefined` to + * fall back to the active render ctx (main outputs + prerun + result pool). + * + * `sources` accepts {@link ColumnsSourceShorthand} string literals + * (`"result_pool"`, `"current_block"`) for the common ctx-derived sources. + */ +export function ColumnsCollection( + sources?: (ColumnsCollection | ColumnsSource | ColumnsSourceShorthand)[], + deps?: ColumnsCollectionDeps, +): ColumnsCollection { + const driver = deps?.driver ?? getService("columnsCollection", { ctx: deps?.ctx }); + const serialized: SerializedColumnsSource[] = sources + ? sources.flatMap((s) => toSerializedSources(s, deps?.ctx)) + : defaultCtxSources(deps?.ctx); + return new ColumnsCollectionImpl(driver.create(serialized), driver); +} + +export function isColumnsCollection(value: unknown): value is ColumnsCollection { + return value instanceof ColumnsCollectionImpl; +} + +/** + * Sandbox proxy over the host-side `ColumnsCollection` driver. The driver + * owns all column-set state; this object only carries an opaque + * {@link CollectionHandle} plus a reference to the driver service. + * + * Construct via {@link ColumnsCollection} — the bare constructor is a + * trivial `(handle, driver)` pair so that `addSource` / `discover` / `filter` + * can rebind a freshly-minted handle without going through source + * serialisation again. + * + * Every method returning a `ColumnsCollection` mints a fresh handle on the + * host — the host VM bridge pins it to the active render ctx, so the sandbox + * never has to manage refcounts. + */ +export class ColumnsCollectionImpl { + constructor( + public readonly handle: CollectionHandle, + private readonly driver: ColumnsCollectionDriverModel, + ) {} + + isEmpty(): boolean { + return this.driver.isEmpty(this.handle); + } + + isFinal(): boolean { + return this.driver.isFinal(this.handle); + } + + getColumnIds(): ColumnUniversalId[] { + return this.driver.getColumns(this.handle); + } + + getColumns(): ColumnRecipe[] { + return this.getColumnIds() + .map((id) => ColumnRecipe(id)) + .filter((c): c is ColumnRecipe => !isNil(c)); + } + + addSource(source: ColumnsSource | ColumnsCollection): ColumnsCollection { + return new ColumnsCollectionImpl( + this.driver.addSource(this.handle, toSerializedSources(source)), + this.driver, + ); + } + + discover(options: ColumnsDiscoverOptions): ColumnsCollection { + return new ColumnsCollectionImpl(this.driver.discover(this.handle, options), this.driver); + } + + filter(options: ColumnsFilterOptions): ColumnsCollection { + return new ColumnsCollectionImpl(this.driver.filter(this.handle, options), this.driver); + } +} + +/** Public type alias — value of the same name is the factory function above. */ +export type ColumnsCollection = ColumnsCollectionImpl; + +function toSerializedSources( + source: ColumnsCollection | ColumnsSource | ColumnsSourceShorthand, + ctx?: GlobalCfgRenderCtx, +): SerializedColumnsSource[] { + if (source === "result_pool") { + return [{ kind: "result_pool" }]; + } + if (source === "current_block") { + return currentBlockSources(ctx); + } + if (source instanceof ColumnsCollectionImpl) { + return [{ kind: "collection", handle: source.handle }]; + } + if (source instanceof TreeNodeAccessor) { + return [{ kind: "accessor", accessor: source.handle, path: source.resolvePath }]; + } + if (isColumnArray(source)) { + return [{ kind: "ids", ids: source.columns.map((c) => c.id), isFinal: source.isFinal }]; + } + if (isColumnProvider(source)) { + return [{ kind: "ids", ids: source.getColumns().map((c) => c.id), isFinal: source.isFinal() }]; + } + throw new Error("ColumnsCollection: unrecognised ColumnsSource shape"); +} + +function isColumnArray(source: unknown): source is { + columns: ReadonlyArray<{ id: ColumnUniversalId; spec?: unknown; data?: unknown }>; + isFinal: boolean; +} { + if (typeof source !== "object" || source === null) return false; + const s = source as { columns?: unknown; isFinal?: unknown }; + return Array.isArray(s.columns) && typeof s.isFinal === "boolean"; +} + +function currentBlockSources(ctx?: GlobalCfgRenderCtx): SerializedColumnsSource[] { + const renderCtx = ctx ?? getCfgRenderCtx(); + const sources: SerializedColumnsSource[] = []; + + const outputs = renderCtx.getAccessorHandleByName(MainAccessorName); + if (outputs !== undefined) { + sources.push({ kind: "accessor", accessor: outputs, path: [MainAccessorName] }); + } + const prerun = renderCtx.getAccessorHandleByName(StagingAccessorName); + if (prerun !== undefined) { + sources.push({ kind: "accessor", accessor: prerun, path: [StagingAccessorName] }); + } + + return sources; +} + +function defaultCtxSources(ctx?: GlobalCfgRenderCtx): SerializedColumnsSource[] { + return [...currentBlockSources(ctx), { kind: "result_pool" }]; +} diff --git a/sdk/model/src/columns/ctx_column_sources.ts b/sdk/model/src/columns/ctx_column_sources.ts deleted file mode 100644 index 17f3771a47..0000000000 --- a/sdk/model/src/columns/ctx_column_sources.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { PColumnSpec, PObjectId } from "@milaboratories/pl-model-common"; -import { TreeNodeAccessor } from "../render/accessor"; -import type { RenderCtxBase, ResultPool } from "../render"; -import type { ColumnSnapshot } from "./column_snapshot"; -import type { ColumnDataStatus } from "./column_snapshot"; -import type { ColumnSnapshotProvider } from "./column_snapshot_provider"; -import { OutputColumnProvider } from "./column_snapshot_provider"; -import { ResourceTypeName } from "@milaboratories/pl-model-common"; -import type { ValueOf } from "@milaboratories/helpers"; - -/** - * Collect ColumnSnapshotProviders from `outputs`, `prerun`, and - * `resultPool` in that order. Dedup keeps the first occurrence per - * `NativePObjectId`, so a block re-publishing its own columns keeps - * the `outputs`-rooted canonical id instead of the result-pool variant. - */ -export function collectCtxColumnSnapshotProviders(ctx: RenderCtxBase): ColumnSnapshotProvider[] { - const providers: ColumnSnapshotProvider[] = []; - - const outputs = ctx.outputs; - if (outputs) { - providers.push(...collectPFrameProviders(outputs)); - } - - const prerun = ctx.prerun; - if (prerun) { - providers.push(...collectPFrameProviders(prerun)); - } - - providers.push(new ResultPoolColumnSnapshotProvider(ctx.resultPool)); - - return providers; -} - -/** - * Adapter wrapping ResultPool into the new ColumnSnapshotProvider interface. - * - * - `isColumnListComplete()` always returns true — the result pool - * is a stable snapshot within a single render cycle. - * - Data status is derived from the underlying TreeNodeAccessor: - * ready (getIsReadyOrError), computing, or absent (no data resource). - */ -export class ResultPoolColumnSnapshotProvider implements ColumnSnapshotProvider { - constructor(private readonly pool: ResultPool) {} - - getAllColumns(): ColumnSnapshot[] { - const pColumns = this.pool.selectColumns(() => true); - return pColumns.map((col) => toSnapshot(col.id, col.spec, col.data)); - } - - isColumnListComplete(): boolean { - return true; - } -} - -function toSnapshot( - id: PObjectId, - spec: PColumnSpec, - accessor: TreeNodeAccessor | undefined, -): ColumnSnapshot { - if (accessor === undefined) { - return { id, spec, dataStatus: "absent" as ColumnDataStatus, data: undefined }; - } - const isReady = accessor.getIsReadyOrError(); - return { - id, - spec, - dataStatus: (isReady ? "ready" : "computing") as ColumnDataStatus, - data: { get: () => (isReady ? accessor : undefined) }, - }; -} - -/** - * Recursively walk the output tree starting from `accessor`. - * - If a node's resourceType is PFrame → wrap it as OutputColumnProvider. - * - If a node's resourceType is StdMap/std/map → recurse into its output fields. - * - Otherwise → skip (leaf of unknown type). - */ -function collectPFrameProviders(accessor: TreeNodeAccessor): ColumnSnapshotProvider[] { - const out: ColumnSnapshotProvider[] = []; - walkTree(accessor, out); - return out; -} - -function walkTree(node: TreeNodeAccessor, out: ColumnSnapshotProvider[]): void { - const typeName = node.resourceType.name as ValueOf; - - if (typeName === ResourceTypeName.PFrame) { - out.push(new OutputColumnProvider(node)); - return; - } - - if (typeName === ResourceTypeName.StdMap || typeName === ResourceTypeName.StdMapSlash) { - for (const field of node.listInputFields()) { - const child = node.resolve(field); - if (child) walkTree(child, out); - } - } -} diff --git a/sdk/model/src/columns/derive_axis_values_labels.ts b/sdk/model/src/columns/derive_axis_values_labels.ts new file mode 100644 index 0000000000..d1e4d85bde --- /dev/null +++ b/sdk/model/src/columns/derive_axis_values_labels.ts @@ -0,0 +1,98 @@ +import type { AxisId } from "@milaboratories/pl-model-common"; +import { canonicalizeAxisId, getAxisId, PColumnName } from "@milaboratories/pl-model-common"; +import type { ColumnsSource } from "./column_providers"; +import { ColumnsCollection, isColumnsCollection } from "./columns_collection"; +import type { GlobalCfgRenderCtx } from "../render/internal"; +import { TreeNodeAccessor } from "../render"; +import { isColumnLazy } from "./column_lazy"; + +const RT_JSON = "PColumnData/Json"; +const RT_JSON_PARTITIONED = "PColumnData/JsonPartitioned"; + +/** + * Build an `axisValuesLabels` callback by discovering `pl7.app/label` columns + * from the given source, eagerly materialising their value→label maps, and + * keying them by canonical axis id. + * + * Modern replacement for `ctx.resultPool.findLabels` — uses the new + * column-access mechanism (filtered {@link ColumnsCollection}) instead of + * walking the raw result pool. + * + * Pair with {@link expandByPartition} (or any consumer expecting the + * `(axisId) => Record` shape). + * + * Skips: + * - non-leaf recipes (only direct `ColumnLazy` data is read); + * - label columns whose `axesSpec.length !== 1`; + * - label columns whose data resource type isn't `PColumnData/Json` / + * `PColumnData/JsonPartitioned`. + */ +export function deriveAxisValuesLabels( + source?: ColumnsCollection | (ColumnsCollection | ColumnsSource)[], + opts?: { ctx?: GlobalCfgRenderCtx }, +): (axisId: AxisId) => Record | undefined { + const collection = + source === undefined + ? ColumnsCollection(undefined, opts) + : isColumnsCollection(source) + ? source + : ColumnsCollection(source, opts); + + const labelCols = collection + .filter({ include: { name: [{ type: "exact", value: PColumnName.Label }] } }) + .getColumns(); + + const byAxis = labelCols.reduce>>((map, col) => { + if (!isColumnLazy(col)) return map; + const spec = col.getSpec(); + if (spec.axesSpec.length !== 1) return map; + + const data = col.getData(); + if (!(data instanceof TreeNodeAccessor)) return map; + + const labelMap = readLabelMap(data); + if (!labelMap) return map; + + map.set(canonicalizeAxisId(getAxisId(spec.axesSpec[0])), labelMap); + return map; + }, new Map()); + + return (axisId) => byAxis.get(canonicalizeAxisId(axisId)); +} + +function readLabelMap(acc: TreeNodeAccessor): undefined | Record { + const rt = acc.resourceType.name; + + if (rt === RT_JSON) { + const json = acc.getDataAsJson<{ data?: Record }>(); + return json?.data ? parseLabelKeys(json.data) : undefined; + } + + if (rt === RT_JSON_PARTITIONED) { + return acc.listInputFields().reduce>((merged, partKey) => { + const part = acc.resolve({ + field: partKey, + assertFieldType: "Input", + ignoreError: true, + }); + if (!part) return merged; + const json = part.getDataAsJson<{ data?: Record }>(); + if (!json?.data) return merged; + return Object.assign(merged, parseLabelKeys(json.data)); + }, {}); + } + + return undefined; +} + +function parseLabelKeys(raw: Record): Record { + return Object.entries(raw).reduce>((acc, [k, v]) => { + try { + const parsed = JSON.parse(k); + acc[Array.isArray(parsed) ? parsed[0] : parsed] = v; + } catch { + acc[k] = v; + } + return acc; + }, {}); +} diff --git a/sdk/model/src/columns/expand_by_partition.test.ts b/sdk/model/src/columns/expand_by_partition.test.ts index ea0859e862..058f20f0dc 100644 --- a/sdk/model/src/columns/expand_by_partition.test.ts +++ b/sdk/model/src/columns/expand_by_partition.test.ts @@ -1,13 +1,20 @@ -import type { - AxisSpec, - JsonPartitionedDataInfoEntries, - PColumnSpec, - PObjectId, +import { + Annotation, + canonicalizeAxisId, + getAxisId, + isColumnFilteredKey, + isColumnOverridedKey, + parseColumnIdSafety, + parseColumnOverridedId, + type AxisSpec, + type JsonPartitionedDataInfoEntries, + type PColumnSpec, + type PObjectId, } from "@milaboratories/pl-model-common"; -import { canonicalizeAxisId, getAxisId } from "@milaboratories/pl-model-common"; import { describe, expect, test } from "vitest"; import type { PColumnDataUniversal } from "../render/internal"; -import type { ColumnSnapshot } from "./column_snapshot"; +import type { ColumnLazy } from "./column_lazy"; +import { ColumnLazyImpl } from "./column_lazy"; import { expandByPartition } from "./expand_by_partition"; // --- Helpers --- @@ -26,74 +33,62 @@ function createSpec(name: string, axes: AxisSpec[]): PColumnSpec { } as PColumnSpec; } -/** Create a ready snapshot whose data.get() returns a JsonPartitioned DataInfoEntries. */ -function createReadySnapshot( +/** Build a synthetic ColumnLazy whose data is a JsonPartitioned DataInfoEntries. */ +function createReadyLazy( id: string, - columnSpec: PColumnSpec, + spec: PColumnSpec, partitionKeyLength: number, parts: { key: (string | number)[]; value: unknown }[], -): ColumnSnapshot { +): ColumnLazy { const dataEntries: JsonPartitionedDataInfoEntries = { type: "JsonPartitioned", partitionKeyLength, parts, }; - return { - id: id as PObjectId, - spec: columnSpec, - dataStatus: "ready", - // convertOrParsePColumnData checks isDataInfoEntries first (duck-type), - // so this works at runtime despite the PColumnDataUniversal type - data: { get: () => dataEntries as unknown as PColumnDataUniversal }, - }; -} - -function createComputingSnapshot(id: string, columnSpec: PColumnSpec): ColumnSnapshot { - return { + // `data` is typed `PColumnDataUniversal` but `getUniquePartitionKeys` + // duck-types DataInfoEntries via `isDataInfoEntries`, so the JsonPartitioned + // payload works at runtime. + return ColumnLazyImpl.fromColumn({ id: id as PObjectId, - spec: columnSpec, - dataStatus: "computing", - data: { - get: () => undefined, - }, - }; + spec, + data: dataEntries as unknown as PColumnDataUniversal, + }); } -function createAbsentSnapshot(id: string, columnSpec: PColumnSpec): ColumnSnapshot { - return { +function createComputingLazy(id: string, spec: PColumnSpec): ColumnLazy { + return ColumnLazyImpl.fromColumn({ id: id as PObjectId, - spec: columnSpec, - dataStatus: "absent", + spec, data: undefined, - }; + }); } interface Trace { type: string; label: string; - importance: number; + importance?: number; } -function extractTrace(snapshot: ColumnSnapshot): Trace[] { - const raw = snapshot.spec.annotations?.["pl7.app/trace"]; +function extractTrace(recipe: { getSpec(): PColumnSpec }): Trace[] { + const raw = recipe.getSpec().annotations?.[Annotation.Trace]; return raw ? (JSON.parse(raw) as Trace[]) : []; } // --- Tests --- describe("expandByPartition", () => { - test("no split axes returns snapshots as-is", () => { - const s = createReadySnapshot("col1", createSpec("c", [createAxis("a")]), 1, [ + test("no split axes returns inputs unchanged", () => { + const s = createReadyLazy("col1", createSpec("c", [createAxis("a")]), 1, [ { key: ["x"], value: {} }, ]); const result = expandByPartition([s], []); - expect(result.complete).toBe(true); - expect(result.items).toHaveLength(1); - expect(result.items[0]).toBe(s); // same reference + expect(result).toBeDefined(); + expect(result).toHaveLength(1); + expect(result![0]).toBe(s); // same reference }); - test("single axis split produces K snapshots", () => { - const s = createReadySnapshot( + test("single axis split produces K recipes", () => { + const s = createReadyLazy( "col1", createSpec("c", [createAxis("sample"), createAxis("gene")]), 2, @@ -105,20 +100,20 @@ describe("expandByPartition", () => { ); const result = expandByPartition([s], [{ idx: 0 }]); - - expect(result.complete).toBe(true); - // unique values on axis 0: s1, s2 → 2 snapshots - expect(result.items).toHaveLength(2); - - // split axis removed from axesSpec - for (const snap of result.items) { - expect(snap.spec.axesSpec).toHaveLength(1); - expect(snap.spec.axesSpec[0].name).toBe("gene"); + expect(result).toBeDefined(); + // unique values on axis 0: s1, s2 → 2 recipes + expect(result).toHaveLength(2); + + // split axis removed from axesSpec on each recipe + for (const recipe of result!) { + const spec = recipe.getSpec(); + expect(spec.axesSpec).toHaveLength(1); + expect(spec.axesSpec[0].name).toBe("gene"); } }); - test("multi-axis split produces K1 x K2 snapshots", () => { - const s = createReadySnapshot( + test("multi-axis split produces K1 x K2 recipes", () => { + const s = createReadyLazy( "col1", createSpec("c", [createAxis("a"), createAxis("b"), createAxis("value")]), 2, @@ -130,20 +125,20 @@ describe("expandByPartition", () => { ); const result = expandByPartition([s], [{ idx: 0 }, { idx: 1 }]); - - expect(result.complete).toBe(true); + expect(result).toBeDefined(); // a: a1, a2 (2) × b: b1, b2 (2) = 4 - expect(result.items).toHaveLength(4); + expect(result).toHaveLength(4); // both split axes removed, only "value" remains - for (const snap of result.items) { - expect(snap.spec.axesSpec).toHaveLength(1); - expect(snap.spec.axesSpec[0].name).toBe("value"); + for (const recipe of result!) { + const spec = recipe.getSpec(); + expect(spec.axesSpec).toHaveLength(1); + expect(spec.axesSpec[0].name).toBe("value"); } }); - test("trace annotations include split info", () => { - const s = createReadySnapshot( + test("trace annotations include split info appended to base trace", () => { + const s = createReadyLazy( "col1", createSpec("c", [createAxis("sample"), createAxis("gene")]), 1, @@ -154,31 +149,53 @@ describe("expandByPartition", () => { ); const result = expandByPartition([s], [{ idx: 0 }]); - expect(result.complete).toBe(true); - expect(result.items).toHaveLength(2); + expect(result).toBeDefined(); + expect(result).toHaveLength(2); - const trace0 = extractTrace(result.items[0]); + const expectedType = `split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`; + + const trace0 = extractTrace(result![0]); expect(trace0).toEqual([ { - type: `split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`, + type: expectedType, label: "s1", importance: 1_000_000, }, ]); - const trace1 = extractTrace(result.items[1]); + const trace1 = extractTrace(result![1]); expect(trace1).toEqual([ { - type: `split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`, + type: expectedType, label: "s2", importance: 1_000_000, }, ]); }); - test("axisLabels option resolves labels", () => { + test("existing trace is preserved and split entry is appended", () => { + const baseTrace = [{ type: "ingest", label: "Input" }]; + const spec: PColumnSpec = { + ...createSpec("c", [createAxis("sample"), createAxis("gene")]), + annotations: { + [Annotation.Trace]: JSON.stringify(baseTrace), + }, + }; + const s = createReadyLazy("col1", spec, 1, [{ key: ["s1"], value: {} }]); + + const result = expandByPartition([s], [{ idx: 0 }]); + expect(result).toBeDefined(); + expect(result).toHaveLength(1); + + const trace0 = extractTrace(result![0]); + expect(trace0[0]).toEqual({ type: "ingest", label: "Input" }); + expect(trace0[1].type).toBe(`split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`); + expect(trace0[1].label).toBe("s1"); + }); + + test("axisValuesLabels option resolves labels in trace entries", () => { const sampleAxis = createAxis("sample"); - const s = createReadySnapshot("col1", createSpec("c", [sampleAxis, createAxis("gene")]), 1, [ + const s = createReadyLazy("col1", createSpec("c", [sampleAxis, createAxis("gene")]), 1, [ { key: ["s1"], value: {} }, { key: ["s2"], value: {} }, ]); @@ -189,32 +206,22 @@ describe("expandByPartition", () => { }; const result = expandByPartition([s], [{ idx: 0 }], { - axisLabels: () => labels, + axisValuesLabels: () => labels, }); - expect(result.complete).toBe(true); - const trace0 = extractTrace(result.items[0]); - expect(trace0[0].label).toBe("Sample One"); - const trace1 = extractTrace(result.items[1]); - expect(trace1[0].label).toBe("Sample Two"); - }); - - test("computing snapshot returns incomplete", () => { - const s = createComputingSnapshot("col1", createSpec("c", [createAxis("a")])); - const result = expandByPartition([s], [{ idx: 0 }]); - expect(result.complete).toBe(false); - expect(result.items).toHaveLength(0); + expect(result).toBeDefined(); + expect(extractTrace(result![0])[0].label).toBe("Sample One"); + expect(extractTrace(result![1])[0].label).toBe("Sample Two"); }); - test("absent snapshot returns incomplete", () => { - const s = createAbsentSnapshot("col1", createSpec("c", [createAxis("a")])); + test("computing input (data undefined) returns undefined", () => { + const s = createComputingLazy("col1", createSpec("c", [createAxis("a")])); const result = expandByPartition([s], [{ idx: 0 }]); - expect(result.complete).toBe(false); - expect(result.items).toHaveLength(0); + expect(result).toBeUndefined(); }); - test("empty unique keys for an axis produces no snapshots for that column", () => { - const s = createReadySnapshot( + test("empty unique keys for an axis produces no recipes for that column", () => { + const s = createReadyLazy( "col1", createSpec("c", [createAxis("a"), createAxis("b")]), 2, @@ -222,35 +229,64 @@ describe("expandByPartition", () => { ); const result = expandByPartition([s], [{ idx: 0 }]); - expect(result.complete).toBe(true); - expect(result.items).toHaveLength(0); + expect(result).toBeDefined(); + expect(result).toHaveLength(0); }); - test("filtered data is accessible on expanded snapshots", () => { - const s = createReadySnapshot( + test("recipe ids are canonical Overrided>", () => { + const s = createReadyLazy( "col1", createSpec("c", [createAxis("sample"), createAxis("gene")]), 1, [ - { key: ["s1"], value: { payload: "data-s1" } }, - { key: ["s2"], value: { payload: "data-s2" } }, + { key: ["s1"], value: {} }, + { key: ["s2"], value: {} }, ], ); const result = expandByPartition([s], [{ idx: 0 }]); - expect(result.complete).toBe(true); - expect(result.items).toHaveLength(2); - - // Each expanded snapshot should have accessible data - for (const snap of result.items) { - expect(snap.data).toBeDefined(); - const data = snap.data!.get(); - expect(data).toBeDefined(); + expect(result).toBeDefined(); + expect(result).toHaveLength(2); + + for (const recipe of result!) { + // Outer wrap is Overrided + const overridedKey = parseColumnOverridedId(recipe.id); + expect(isColumnOverridedKey(overridedKey)).toBe(true); + // Inner is a stringified Filtered id + expect(typeof overridedKey.source).toBe("string"); + const innerParsed = parseColumnIdSafety(overridedKey.source); + expect(isColumnFilteredKey(innerParsed)).toBe(true); } + + // Each recipe's id is distinct + const ids = new Set(result!.map((r) => r.id)); + expect(ids.size).toBe(2); + }); + + test("domain override carries axis name → value", () => { + const s = createReadyLazy( + "col1", + createSpec("c", [createAxis("sample"), createAxis("gene")]), + 1, + [ + { key: ["s1"], value: {} }, + { key: ["s2"], value: {} }, + ], + ); + + const result = expandByPartition([s], [{ idx: 0 }]); + expect(result).toBeDefined(); + expect(result).toHaveLength(2); + + const overrided0 = parseColumnOverridedId(result![0].id); + expect(overrided0.specOverrides.domain).toEqual({ sample: "s1" }); + + const overrided1 = parseColumnOverridedId(result![1].id); + expect(overrided1.specOverrides.domain).toEqual({ sample: "s2" }); }); - test("multiple input snapshots are all expanded", () => { - const s1 = createReadySnapshot( + test("multiple input columns are all expanded", () => { + const s1 = createReadyLazy( "col1", createSpec("c1", [createAxis("sample"), createAxis("gene")]), 1, @@ -259,7 +295,7 @@ describe("expandByPartition", () => { { key: ["s2"], value: {} }, ], ); - const s2 = createReadySnapshot( + const s2 = createReadyLazy( "col2", createSpec("c2", [createAxis("sample"), createAxis("gene")]), 1, @@ -271,13 +307,13 @@ describe("expandByPartition", () => { ); const result = expandByPartition([s1, s2], [{ idx: 0 }]); - expect(result.complete).toBe(true); + expect(result).toBeDefined(); // s1: 2 unique + s2: 3 unique = 5 total - expect(result.items).toHaveLength(5); + expect(result).toHaveLength(5); }); test("throws when split axis exceeds partition key length", () => { - const s = createReadySnapshot( + const s = createReadyLazy( "col1", createSpec("c", [createAxis("a"), createAxis("b"), createAxis("c")]), 1, // only 1 partition key diff --git a/sdk/model/src/columns/expand_by_partition.ts b/sdk/model/src/columns/expand_by_partition.ts index ce38096796..5e47c094ab 100644 --- a/sdk/model/src/columns/expand_by_partition.ts +++ b/sdk/model/src/columns/expand_by_partition.ts @@ -2,20 +2,22 @@ import type { AxisFilterByIdx, AxisFilterValue, AxisId, - PartitionedDataInfoEntries, - PObjectId, + TraceEntry, } from "@milaboratories/pl-model-common"; import { + Annotation, canonicalizeAxisId, - entriesToDataInfo, getAxisId, - isPartitionedDataInfoEntries, + isDataInfoEntries, + readAnnotation, } from "@milaboratories/pl-model-common"; -import type { TreeNodeAccessor } from "../render/accessor"; -import { filterDataInfoEntries } from "../render/util/axis_filtering"; -import { convertOrParsePColumnData, getUniquePartitionKeys } from "../render/util/pcolumn_data"; -import type { ColumnSnapshot } from "./column_snapshot"; -import { createReadyColumnData } from "./column_snapshot"; +import { TreeNodeAccessor } from "../render/accessor"; +import { getUniquePartitionKeys } from "../render/util/pcolumn_data"; +import type { ColumnRecipe } from "./column_recipes"; +import { ColumnFilteredRecipe } from "./column_recipes/column_filtered_recipe"; +import { ColumnOverridedRecipe } from "./column_recipes/column_overrided_recipe"; +import { Column } from "./column"; +import { getLeafColumnData } from "./utils"; // --- Types --- @@ -26,128 +28,114 @@ export interface SplitAxis { export interface ExpandByPartitionOpts { /** Resolve axis values to human-readable labels. */ - axisLabels?: (axisId: AxisId) => undefined | Record; -} - -export interface ExpandByPartitionResult { - /** Expanded snapshots (one per key combination per original snapshot). */ - readonly items: ColumnSnapshot[]; - /** False if any column's data was not ready for splitting. */ - readonly complete: boolean; + axisValuesLabels?: (axisId: AxisId) => undefined | Record; } // --- Implementation --- +const MAX_KEY_COMBINATIONS = 10_000; + /** - * Expand snapshots by splitting along partition axes. + * Expand each input column along the requested partition axes into one + * {@link ColumnRecipe} per Cartesian combination of unique partition values. + * + * Each split is produced as `ColumnOverridedRecipe.wrap(ColumnFilteredRecipe.wrap(inner, axisFilters), { domain, annotations })`: + * - `ColumnFilteredRecipe` pins all split axes at once (one wrap call with + * every `[idx, value]` pair) and removes them from `axesSpec`. The + * engine performs the data slicing via the `sliceAxes` query node. + * - `ColumnOverridedRecipe` overlays a `domain[axisName] = String(value)` + * entry per split axis and appends a `split:` trace + * entry per split axis to the existing `pl7.app/trace` annotation. * - * For each snapshot, reads partition data, enumerates unique keys on the - * split axes, and produces one output snapshot per key combination — - * with the split axes removed from `axesSpec` and a `pl7.app/trace` - * annotation recording the split origin. + * The recipe id is a canonical + * `ColumnOverridedId(source: ColumnFilteredId(source: inner.id, axisFilters), specOverrides)` + * — distinct per split combination, parseable by `extractPObjectId`, and + * traversable by recipe walkers. * - * Returns `{ items: [], complete: false }` when any snapshot's data - * is not ready (status !== 'ready' or partition data unavailable). + * Returns `undefined` when any input's `getData()` is not a + * {@link TreeNodeAccessor} — partition inspection is not yet possible. */ export function expandByPartition( - snapshots: ColumnSnapshot[], + inputs: Column[], splitAxes: SplitAxis[], opts?: ExpandByPartitionOpts, -): ExpandByPartitionResult { +): ColumnRecipe[] | undefined { if (splitAxes.length === 0) { - return { items: snapshots, complete: true }; + return [...inputs]; } const splitAxisIdxs = splitAxes.map((a) => a.idx).sort((a, b) => a - b); - const result: ColumnSnapshot[] = []; + const maxSplitIdx = splitAxisIdxs[splitAxisIdxs.length - 1]; - for (const snapshot of snapshots) { - if (snapshot.dataStatus !== "ready" || snapshot.data === undefined) { - return { items: [], complete: false }; - } + const result: ColumnRecipe[] = []; - const rawData = snapshot.data.get(); - const dataEntries = convertOrParsePColumnData(rawData as TreeNodeAccessor | undefined); - - if (dataEntries === undefined) { - return { items: [], complete: false }; - } - - if (!isPartitionedDataInfoEntries(dataEntries)) { - throw new Error( - `Splitting requires Partitioned DataInfoEntries, but got ${dataEntries.type} for column ${String(snapshot.id)}`, - ); + for (const inner of inputs) { + const data = getLeafColumnData(inner); + // Partition inspection requires either a live tree-accessor or already + // parsed DataInfoEntries. Anything else means the input is not yet ready. + if (!(data instanceof TreeNodeAccessor) && !isDataInfoEntries(data)) { + return undefined; } - const uniqueKeys = getUniquePartitionKeys(dataEntries); + const uniqueKeys = getUniquePartitionKeys(data); + if (uniqueKeys === undefined) return undefined; - const maxSplitIdx = splitAxisIdxs[splitAxisIdxs.length - 1]; - if (maxSplitIdx >= dataEntries.partitionKeyLength) { + if (maxSplitIdx >= uniqueKeys.length) { throw new Error( - `Not enough partition keys (${dataEntries.partitionKeyLength}) for requested split axes (max index ${maxSplitIdx}) in column ${snapshot.spec.name}`, + `Not enough partition keys (${uniqueKeys.length}) for requested split axes (max index ${maxSplitIdx}) in column ${inner.getSpec().name}`, ); } - // Resolve labels for each split axis - const axesLabels: (undefined | Record)[] = splitAxisIdxs.map((idx) => - opts?.axisLabels?.(getAxisId(snapshot.spec.axesSpec[idx])), - ); + const spec = inner.getSpec(); + const axesSpec = spec.axesSpec; + const splitAxisSpecs = splitAxisIdxs.map((idx) => axesSpec[idx]); + const splitAxisIds = splitAxisSpecs.map((axisSpec) => getAxisId(axisSpec)); + const axesLabels = splitAxisIds.map((axisId) => opts?.axisValuesLabels?.(axisId)); + + const existingTraceRaw = readAnnotation(spec, Annotation.Trace); + const baseTrace: TraceEntry[] = existingTraceRaw + ? ((JSON.parse(existingTraceRaw) as TraceEntry[]) ?? []) + : []; - // Generate all key combinations across split axes const keyCombinations = generateKeyCombinations(uniqueKeys, splitAxisIdxs); if (keyCombinations.length === 0) continue; - // Build adjusted axesSpec (remove split axes in reverse order) - const newAxesSpec = [...snapshot.spec.axesSpec]; - for (let i = splitAxisIdxs.length - 1; i >= 0; i--) { - newAxesSpec.splice(splitAxisIdxs[i], 1); - } - for (const keyCombo of keyCombinations) { const axisFilters: AxisFilterByIdx[] = keyCombo.map( (value, sAxisIdx): AxisFilterByIdx => [splitAxisIdxs[sAxisIdx], value as AxisFilterValue], ); - const traceEntries = keyCombo.map((value, sAxisIdx) => { - const axisIdx = splitAxisIdxs[sAxisIdx]; - const axisId = getAxisId(snapshot.spec.axesSpec[axisIdx]); + const domain: Record = {}; + const traceEntries: TraceEntry[] = []; + for (let sAxisIdx = 0; sAxisIdx < keyCombo.length; sAxisIdx++) { + const value = keyCombo[sAxisIdx]; + const axisSpec = splitAxisSpecs[sAxisIdx]; + const axisId = splitAxisIds[sAxisIdx]; const labelMap = axesLabels[sAxisIdx]; const label = labelMap?.[value] ?? String(value); - return { + domain[axisSpec.name] = String(value); + traceEntries.push({ type: `split:${canonicalizeAxisId(axisId)}`, label, importance: 1_000_000, - }; - }); - - const filteredData = filterDataInfoEntries( - dataEntries as PartitionedDataInfoEntries, - axisFilters, - ); + }); + } - const adjustedSpec = { - ...snapshot.spec, - axesSpec: newAxesSpec, + const filtered = ColumnFilteredRecipe.wrap(inner, axisFilters); + const overrided = ColumnOverridedRecipe.wrap(filtered, { + domain, annotations: { - ...snapshot.spec.annotations, - "pl7.app/trace": JSON.stringify(traceEntries), + [Annotation.Trace]: JSON.stringify([...baseTrace, ...traceEntries]), }, - }; - - result.push({ - id: snapshot.id, - spec: adjustedSpec, - dataStatus: "ready", - data: createReadyColumnData(() => entriesToDataInfo(filteredData)), }); + + result.push(overrided); } } - return { items: result, complete: true }; + return result; } -const MAX_KEY_COMBINATIONS = 10_000; - function generateKeyCombinations( uniqueKeys: (string | number)[][], splitAxisIdxs: number[], diff --git a/sdk/model/src/columns/index.ts b/sdk/model/src/columns/index.ts index 3a56371b19..d563dccfdf 100644 --- a/sdk/model/src/columns/index.ts +++ b/sdk/model/src/columns/index.ts @@ -1,6 +1,8 @@ -export * from "./column_snapshot"; -export * from "./column_snapshot_provider"; -export * from "./column_selector"; -export * from "./column_collection_builder"; -export * from "./ctx_column_sources"; +export * from "./column_providers"; export * from "./expand_by_partition"; +export * from "./derive_axis_values_labels"; +export * from "./columns_collection"; +export * from "./column_lazy"; +export * from "./column_recipes"; +export * from "./column"; +export * from "./utils"; diff --git a/sdk/model/src/columns/utils.test.ts b/sdk/model/src/columns/utils.test.ts new file mode 100644 index 0000000000..bf574b5111 --- /dev/null +++ b/sdk/model/src/columns/utils.test.ts @@ -0,0 +1,249 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + createColumnDiscoveredKey, + createGlobalPObjectId, + createLocalPObjectId, + stringifyColumnDiscoveredId, + type AxisQualification, + type PColumnSpec, + type PObjectId, + type SpecOverrides, + type SpecQuery, + type ColumnUniversalId, +} from "@milaboratories/pl-model-common"; +import { collectLinkerIds, hitQualifications, queriesQualifications } from "./utils"; +import { ColumnDiscoveredRecipe } from "./column_recipes/column_discovered_recipe"; +import { ColumnFilteredRecipe } from "./column_recipes/column_filtered_recipe"; +import { ColumnOverridedRecipe } from "./column_recipes/column_overrided_recipe"; +import type { ColumnFieldStatus, ColumnRecipe } from "./column_recipes/types"; +import { installStubRegistry } from "./__test_helpers__/stub_registry"; + +// ── ctx mock ──────────────────────────────────────────────────────────────── +// ColumnDiscoveredRecipe pulls `getCfgRenderCtx()` for `pframeSpec`. Minimal +// no-op stub keeps the constructor happy. + +function makeCtx() { + // Minimal pframeSpec mock — `ColumnFilteredRecipe.wrap` validates axis + // indices against `inner.getSpec()`, which for a real `ColumnDiscoveredRecipe` + // runs `createSpecFrame` + `evaluateQuery` through this service. The mock + // echoes the registered specs as `tableSpec` entries so `getSpec()` returns + // the spec planted by the discovered fixture. + const services: Record unknown> = { + // Wrap whatever specs map the caller passes; we don't inspect its shape. + createSpecFrame: (...args: unknown[]) => ({ key: { specs: args[0] }, unref: () => {} }), + // Echo the registered specs as `tableSpec` entries — `getSpec` looks up + // the entry by id, so we need the shape we planted in `createSpecFrame`. + evaluateQuery: (...args: unknown[]) => { + const frameKey = args[0] as { specs: Record }; + return { + tableSpec: Object.entries(frameKey.specs).map(([id, spec]) => ({ + type: "column" as const, + id, + spec, + })), + }; + }, + }; + return { + getAccessorHandleByName: () => undefined, + getUpstreamBlockCtx: () => [], + getServiceNames: () => ["pframeSpec"], + getServiceMethods: () => Object.keys(services), + callServiceMethod: (_id: string, method: string, ...args: unknown[]) => + services[method]?.(...args), + } as unknown as never; +} + +let activeCtx: ReturnType; +beforeEach(() => { + activeCtx = makeCtx(); + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = activeCtx; +}); + +afterEach(() => { + delete (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx; +}); + +// ── fixtures ──────────────────────────────────────────────────────────────── + +const hit = createLocalPObjectId(["main", "out"], "hit"); +const linker1 = createLocalPObjectId(["main"], "L1"); +const linker2 = createLocalPObjectId(["main"], "L2"); +const otherCol = createGlobalPObjectId("block-x", "extra"); + +const colNode = (c: PObjectId): SpecQuery => ({ type: "column", column: c }); + +const axisQual = (name: string): AxisQualification => ({ axis: { name }, contextDomain: {} }); + +const overrides = (patch: Partial = {}): SpecOverrides => ({ + annotations: patch.annotations ?? {}, + domain: patch.domain ?? {}, + contextDomain: patch.contextDomain ?? {}, + axesSpec: patch.axesSpec ?? {}, +}); + +class StubRecipe implements ColumnRecipe { + constructor( + readonly id: ColumnUniversalId, + private readonly query: SpecQuery, + ) {} + getReferencedIds(): PObjectId[] { + return []; + } + getDataStatus(): ColumnFieldStatus { + return "present"; + } + getSpec(): PColumnSpec { + return { kind: "PColumn", valueType: "String", name: "stub", axesSpec: [] }; + } + getQuery(): SpecQuery { + return this.query; + } + withSpecs(o: SpecOverrides): ColumnRecipe { + return ColumnOverridedRecipe.wrap(this, o); + } +} + +const recipe = (id: PObjectId, query: SpecQuery): ColumnRecipe => new StubRecipe(id, query); + +/** + * Build a real ColumnDiscoveredRecipe from a manually-crafted key. Plants a + * stub registry that resolves the hit column so {@link + * ColumnDiscoveredRecipe.fromKey} succeeds without a real ctx. + */ +function discovered(opts: { + column: PObjectId; + columnQualifications?: AxisQualification[]; + queriesQualifications?: Record; +}): ColumnDiscoveredRecipe { + const distilled = createColumnDiscoveredKey({ + column: opts.column, + columnQualifications: opts.columnQualifications, + queriesQualifications: opts.queriesQualifications, + }); + // Spec with two axes — leaves room for `ColumnFilteredRecipe.wrap` callers + // in dependent tests (need ≥1 filter and ≥1 remaining axis). + const stubSpec: PColumnSpec = { + kind: "PColumn", + name: "stub", + valueType: "String", + axesSpec: [ + { name: "donor", type: "String" }, + { name: "sample", type: "String" }, + ], + annotations: {}, + }; + installStubRegistry(activeCtx, { [opts.column]: stubSpec }); + const rec = ColumnDiscoveredRecipe.fromKey(distilled); + if (rec === undefined) throw new Error("discovered fixture: fromKey returned undefined"); + expect(rec.id).toBe(stringifyColumnDiscoveredId(distilled)); + return rec; +} + +// ════════════════════════════════════════════════════════════════════════════ +// collectLinkerIds +// ════════════════════════════════════════════════════════════════════════════ + +describe("collectLinkerIds", () => { + test("leaf recipe → empty", () => { + expect(collectLinkerIds(recipe(hit, colNode(hit)))).toEqual([]); + }); + + test("linkerJoin chain → linker ids in outer-to-inner order, hit excluded", () => { + const q: SpecQuery = { + type: "linkerJoin", + linker: colNode(linker1), + secondary: [ + { + entry: { + type: "linkerJoin", + linker: colNode(linker2), + secondary: [{ entry: colNode(hit) }], + }, + }, + ], + }; + expect(collectLinkerIds(recipe(hit, q))).toEqual([linker1, linker2]); + }); + + test("dedupes repeated references", () => { + const q: SpecQuery = { + type: "linkerJoin", + linker: colNode(linker1), + secondary: [ + { + entry: { + type: "linkerJoin", + linker: colNode(linker1), + secondary: [{ entry: colNode(hit) }], + }, + }, + ], + }; + expect(collectLinkerIds(recipe(hit, q))).toEqual([linker1]); + }); + + test("ignores the hit even when it appears multiple times", () => { + const q: SpecQuery = { + type: "innerJoin", + entries: [{ entry: colNode(hit) }, { entry: colNode(hit) }, { entry: colNode(otherCol) }], + }; + expect(collectLinkerIds(recipe(hit, q))).toEqual([otherCol]); + }); +}); + +// ════════════════════════════════════════════════════════════════════════════ +// hitQualifications / queriesQualifications — recipe-structure walk +// ════════════════════════════════════════════════════════════════════════════ + +describe("hitQualifications", () => { + test("leaf recipe → empty list", () => { + expect(hitQualifications(recipe(hit, colNode(hit)))).toEqual([]); + }); + + test("reads ColumnDiscoveredRecipe.columnQualifications", () => { + const d = discovered({ column: hit, columnQualifications: [axisQual("sample")] }); + expect(hitQualifications(d)).toEqual([axisQual("sample")]); + }); + + test("descends through Overrided wrapper", () => { + const d = discovered({ column: hit, columnQualifications: [axisQual("donor")] }); + const wrapped = ColumnOverridedRecipe.wrap(d, overrides({ annotations: { x: "1" } })); + expect(hitQualifications(wrapped)).toEqual([axisQual("donor")]); + }); + + test("descends through Filtered wrapper", () => { + // Filter axis 0 — axis 1 remains, satisfying the wrap invariant. + const d = discovered({ column: hit, columnQualifications: [axisQual("donor")] }); + const wrapped = ColumnFilteredRecipe.wrap(d, [[0, "v"]]); + expect(hitQualifications(wrapped)).toEqual([axisQual("donor")]); + }); + + test("descends through stacked Overrided>", () => { + const d = discovered({ column: hit, columnQualifications: [axisQual("donor")] }); + const stacked = ColumnOverridedRecipe.wrap( + ColumnFilteredRecipe.wrap(d, [[0, "v"]]), + overrides({ annotations: { x: "1" } }), + ); + expect(hitQualifications(stacked)).toEqual([axisQual("donor")]); + }); +}); + +describe("queriesQualifications", () => { + test("leaf recipe → empty record", () => { + expect(queriesQualifications(recipe(hit, colNode(hit)))).toEqual({}); + }); + + test("reads ColumnDiscoveredRecipe.queriesQualifications", () => { + const quals = { [otherCol]: [axisQual("primary-side")] }; + const d = discovered({ column: hit, queriesQualifications: quals }); + expect(queriesQualifications(d)).toEqual(quals); + }); + + test("descends through wrappers", () => { + const quals = { [otherCol]: [axisQual("primary-side")] }; + const d = discovered({ column: hit, queriesQualifications: quals }); + const wrapped = ColumnOverridedRecipe.wrap(d, overrides()); + expect(queriesQualifications(wrapped)).toEqual(quals); + }); +}); diff --git a/sdk/model/src/columns/utils.ts b/sdk/model/src/columns/utils.ts new file mode 100644 index 0000000000..2588a5697d --- /dev/null +++ b/sdk/model/src/columns/utils.ts @@ -0,0 +1,226 @@ +import { + collectSpecQueryColumns, + extractPObjectId, + isDataInfo, + visitDataInfo, + type AxisQualification, + type ColumnUniversalId, + type PObjectId, +} from "@milaboratories/pl-model-common"; +import { throwError } from "@milaboratories/helpers"; +import type { GlobalCfgRenderCtx } from "../render/internal"; +import { TreeNodeAccessor } from "../render"; +import { deriveDistinctLabels, type DeriveLabelsOptions } from "../labels/derive_distinct_labels"; +import { ColumnLazy, ColumnLazyImpl, type ColumnLazyData } from "./column_lazy"; +import { ColumnDiscoveredRecipe } from "./column_recipes/column_discovered_recipe"; +import { ColumnFilteredRecipe } from "./column_recipes/column_filtered_recipe"; +import { ColumnOverridedRecipe } from "./column_recipes/column_overrided_recipe"; +import type { ColumnRecipe } from "./column_recipes/types"; +import { ColumnsCollection, isColumnsCollection } from "./columns_collection"; +import type { ColumnsSource } from "./column_providers/types"; + +/** + * PObjectIds of every non-hit column referenced by `recipe.getQuery()`, + * deduped in traversal order. The hit column is + * `extractPObjectId(recipe.id)`; anything else in the query is a linker or + * other engine-consumed column. + * + * Pure query walk — no registry access. Use {@link collectLinkerColumns} for + * the resolved {@link ColumnLazy} variant. + */ +export function collectLinkerIds(recipe: ColumnRecipe): PObjectId[] { + const hit = extractPObjectId(recipe.id); + const seen = new Set(); + const out: PObjectId[] = []; + for (const id of collectSpecQueryColumns(recipe.getQuery())) { + // Leaf ids in SpecQuery may be rich (e.g. ColumnDiscoveredId) — drop to + // bare PObjectId before comparing/registering so we hand the ambient + // ColumnRegistry physical ids it can resolve. + const bare = extractPObjectId(id); + if (bare === hit) continue; + if (seen.has(bare)) continue; + seen.add(bare); + out.push(bare); + } + return out; +} + +/** + * {@link collectLinkerIds} resolved against the ambient context as + * {@link ColumnLazy} instances. Throws if any id fails to resolve — + * matches the contract of the legacy `resolveLinkers` it replaces. + */ +export function collectLinkerColumns( + recipe: ColumnRecipe, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): ColumnLazy[] { + return collectLinkerIds(recipe).map( + (id) => + ColumnLazyImpl.fromId(id, opts) ?? + throwError(`materializeLinkers: linker ${id} not resolvable`), + ); +} + +/** + * Hit-side axis qualifications for the recipe — the ones that should land on + * the outer-join entry wrapping this column at the consumer boundary. + * + * Descends {@link ColumnOverridedRecipe} / {@link ColumnFilteredRecipe} via + * `getInner()` until it finds a {@link ColumnDiscoveredRecipe}; otherwise + * returns `[]`. Pure recipe walk — no query-tree introspection. + */ +export function hitQualifications(recipe: ColumnRecipe): readonly AxisQualification[] { + const discovered = findDiscovered(recipe); + return discovered?.getColumnQualifications() ?? []; +} + +/** + * Per-primary-column axis qualifications for the recipe — applied to the + * outer primary anchors on this recipe's group side. Keyed by the external + * primary column id (NOT by columns inside this recipe's own query). + * + * Same walk strategy as {@link hitQualifications}. + */ +export function queriesQualifications( + recipe: ColumnRecipe, +): Readonly> { + const discovered = findDiscovered(recipe); + return discovered?.getQueriesQualifications() ?? {}; +} + +/** + * Whether the recipe's underlying data resources actually carry bytes — + * strictly stronger than `getDataStatus() === "present"`, which only tells + * that the `.data` field is wired onto the accessor. + * + * Walks every physical leaf the recipe depends on via + * {@link ColumnRecipe.getReferencedIds} (resolves each id back to a + * {@link ColumnLazy} through the ambient ctx) and ANDs `hasData()` across + * all of them. Inline {@link PColumnValues} payloads count as present. + * + * Returns `false` if any leaf cannot be re-resolved in `opts.ctx` (treat as + * "not yet ready") rather than throwing — this is a UI-facing predicate, not + * a contract assertion. + */ +export function hasColumnData( + recipe: ColumnRecipe, + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): boolean { + for (const id of recipe.getReferencedIds()) { + const lazy = ColumnLazyImpl.fromId(id, opts); + if (lazy === undefined) return false; + if (lazy.getDataStatus() !== "present") return false; + if (!isLazyDataPresent(lazy.getData())) return false; + } + return true; +} + +function isLazyDataPresent(data: ColumnLazyData): boolean { + if (data === undefined) return false; + if (Array.isArray(data)) return true; + if (data instanceof TreeNodeAccessor) return data.hasData(); + if (isDataInfo(data)) { + let ok = true; + visitDataInfo(data, (blob) => { + ok &&= blob.hasData(); + }); + return ok; + } + return false; +} + +/** + * Walk wrapper layers (Overrided, Filtered) until a {@link + * ColumnDiscoveredRecipe} is found. Returns `undefined` if the recipe chain + * has no Discovered layer (bare leaves and Overrided-over-leaf cases). + * + * Invariant from the wrapper classes themselves: there is at most one + * Discovered layer in any recipe chain — Discovered is constructed only via + * its own factory and is never re-wrapped by Discovered. + */ +function findDiscovered(recipe: ColumnRecipe): undefined | ColumnDiscoveredRecipe { + let current: ColumnRecipe = recipe; + while (true) { + if (current instanceof ColumnDiscoveredRecipe) return current; + if (current instanceof ColumnOverridedRecipe || current instanceof ColumnFilteredRecipe) { + current = current.getInner(); + continue; + } + return undefined; + } +} + +/** + * A recipe is a "leaf" (directly co-indexed on the anchor, reached without a + * linker chain) iff its wrapper chain contains no {@link ColumnDiscoveredRecipe}. + * + * `Filtered` / `Overrided` over a bare leaf stay leaves; anything wrapping a + * `Discovered` (e.g. `Overrided(Discovered(...))`) is linked. Use this — not an + * `instanceof ColumnLazyImpl` check — to split direct vs. linker-joined columns, + * since projections over a plain leaf are still direct. + */ +export function isLeafColumn(recipe: ColumnRecipe): boolean { + return findDiscovered(recipe) === undefined; +} + +/** + * Data of the bare leaf a "leaf" recipe bottoms out at — reads the leaf-only + * {@link ColumnLazy.getData} after walking the wrapper chain down via + * {@link extractLeafColumn}. Returns `undefined` when the recipe is not a leaf (its + * chain reaches a {@link ColumnDiscoveredRecipe}); the symmetric counterpart + * of {@link isLeafColumn} returning `false`. + */ +export function getLeafColumnData(recipe: ColumnRecipe): ColumnLazyData { + if (!isLeafColumn(recipe)) { + throw new Error(`getLeafColumnData: recipe ${recipe.id} is not a leaf column`); + } + return extractLeafColumn(recipe)?.getData(); +} + +/** + * Walk wrapper layers (Overrided, Filtered) down to the bare {@link ColumnLazy} + * leaf the chain bottoms out at. Returns `undefined` if the chain reaches a + * {@link ColumnDiscoveredRecipe} instead — i.e. the recipe is not a leaf + * ({@link isLeafColumn} would return `false`). Mirror of {@link findDiscovered}. + */ +function extractLeafColumn(recipe: ColumnRecipe): undefined | ColumnLazy { + let current: ColumnRecipe = recipe; + while (true) { + if (current instanceof ColumnLazyImpl) return current; + if (current instanceof ColumnOverridedRecipe || current instanceof ColumnFilteredRecipe) { + current = current.getInner(); + continue; + } + if (current instanceof ColumnDiscoveredRecipe) { + throw new Error(`extractLeafColumn: recipe ${recipe.id} is not a leaf column`); + } + throw new Error(`extractLeafColumn: unrecognized recipe layer for ${recipe.id}`); + } +} + +/** Drop-down option built over a {@link ColumnsCollection} — universal-id valued. */ +export type ColumnOption = { + readonly id: ColumnUniversalId; + readonly label: string; +}; + +/** + * Enumerates every column already in `source` and renders distinct labels + * from each recipe's spec. Filtering is the caller's job — pass a + * {@link ColumnsCollection} that was already narrowed via `.filter(...)` / + * `.discover(...)`, or hand over a raw {@link ColumnsSource} (provider, + * accessor, column array, or `"result_pool"` / `"current_block"` shorthand) + * to be wrapped on the fly. + */ +export function deriveColumnOptions( + source: ColumnsCollection | ColumnsSource[], + labelOptions: DeriveLabelsOptions = {}, +): ColumnOption[] { + const collection = isColumnsCollection(source) ? source : ColumnsCollection(source); + const recipes = collection.getColumns(); + const labels = deriveDistinctLabels( + recipes.map((r) => r.getSpec()), + labelOptions, + ); + return recipes.map((r, i) => ({ id: r.id, label: labels[i] })); +} diff --git a/sdk/model/src/components/PFrameForGraphs.ts b/sdk/model/src/components/PFrameForGraphs.ts index 42dc421f18..306f3a54a9 100644 --- a/sdk/model/src/components/PFrameForGraphs.ts +++ b/sdk/model/src/components/PFrameForGraphs.ts @@ -27,8 +27,10 @@ export function isHiddenFromUIColumn(column: PColumnSpec): boolean { */ export function createPFrameForGraphs( ctx: RenderCtxBase, - blockColumns?: PColumn[], + blockColumns?: PColumn[], ): PFrameHandle | undefined { + if (blockColumns?.some((v) => v.data === undefined)) return undefined; + const suitableSpec = (spec: PColumnSpec) => !isHiddenFromUIColumn(spec) && !isHiddenFromGraphColumn(spec); // if current block doesn't produce own columns then use all columns from result pool diff --git a/sdk/model/src/components/PlAnnotations/filters_ui.test.ts b/sdk/model/src/components/PlAnnotations/filters_ui.test.ts index 8b06afbf45..8db6fadd7a 100644 --- a/sdk/model/src/components/PlAnnotations/filters_ui.test.ts +++ b/sdk/model/src/components/PlAnnotations/filters_ui.test.ts @@ -1,5 +1,4 @@ // @DEPRECATED - use sdk/model/src/filters + sdk/model/src/annotations -import type { SUniversalPColumnId } from "@milaboratories/pl-model-common"; import { describe, expect, it, test } from "vitest"; import type { AnnotationFilter, @@ -37,10 +36,10 @@ describe("compileAnnotationScript", () => { filter: { type: "and", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + { type: "isNA", column: "colA" }, { type: "patternEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "abc", }, ], @@ -57,10 +56,10 @@ describe("compileAnnotationScript", () => { filter: { type: "and", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + { type: "isNA", column: "colA" }, { type: "pattern", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", predicate: { type: "equals", value: "abc" }, }, ], @@ -77,11 +76,11 @@ describe("compileFilter", () => { it('should compile "or" filter', () => { const uiFilter: FilterUi = { type: "or", - filters: [{ type: "isNA", column: "colA" as unknown as SUniversalPColumnId }], + filters: [{ type: "isNA", column: "colA" }], }; const expectedFilter: AnnotationFilter = { type: "or", - filters: [{ type: "isNA", column: "colA" as unknown as SUniversalPColumnId }], + filters: [{ type: "isNA", column: "colA" }], }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); }); @@ -89,11 +88,11 @@ describe("compileFilter", () => { it('should compile "and" filter', () => { const uiFilter: FilterUi = { type: "and", - filters: [{ type: "isNA", column: "colA" as unknown as SUniversalPColumnId }], + filters: [{ type: "isNA", column: "colA" }], }; const expectedFilter: AnnotationFilter = { type: "and", - filters: [{ type: "isNA", column: "colA" as unknown as SUniversalPColumnId }], + filters: [{ type: "isNA", column: "colA" }], }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); }); @@ -102,13 +101,13 @@ describe("compileFilter", () => { const uiFilter: FilterUi = { type: "and", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + { type: "isNA", column: "colA" }, { type: "or", filters: [ { type: "patternEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "test", }, ], @@ -118,13 +117,13 @@ describe("compileFilter", () => { const expectedFilter: AnnotationFilter = { type: "and", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + { type: "isNA", column: "colA" }, { type: "or", filters: [ { type: "pattern", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", predicate: { type: "equals", value: "test" }, }, ], @@ -137,20 +136,20 @@ describe("compileFilter", () => { it('should compile "not" filter', () => { const uiFilter: FilterUi = { type: "not", - filter: { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + filter: { type: "isNA", column: "colA" }, }; const expectedFilter: AnnotationFilter = { type: "not", - filter: { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + filter: { type: "isNA", column: "colA" }, }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); }); it('should compile "isNA" filter', () => { - const uiFilter: FilterUi = { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }; + const uiFilter: FilterUi = { type: "isNA", column: "colA" }; const expectedFilter: AnnotationFilter = { type: "isNA", - column: "colA" as unknown as SUniversalPColumnId, + column: "colA", }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); }); @@ -158,9 +157,9 @@ describe("compileFilter", () => { it('should compile "isNotNA" filter', () => { const uiFilter: FilterUi = { type: "isNotNA", - column: "colA" as unknown as SUniversalPColumnId, + column: "colA", }; - const expectedIsNA: IsNA = { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }; + const expectedIsNA: IsNA = { type: "isNA", column: "colA" }; const expectedFilter: NotFilter = { type: "not", filter: expectedIsNA }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); }); @@ -168,12 +167,12 @@ describe("compileFilter", () => { it('should compile "patternEquals" filter', () => { const uiFilter: FilterUi = { type: "patternEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "abc", }; const expectedFilter: AnnotationFilter = { type: "pattern", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", predicate: { type: "equals", value: "abc" }, }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); @@ -182,12 +181,12 @@ describe("compileFilter", () => { it('should compile "patternNotEquals" filter', () => { const uiFilter: FilterUi = { type: "patternNotEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "abc", }; const expectedPatternFilter: PatternFilter = { type: "pattern", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", predicate: { type: "equals", value: "abc" }, }; const expectedFilter: NotFilter = { type: "not", filter: expectedPatternFilter }; @@ -197,12 +196,12 @@ describe("compileFilter", () => { it('should compile "patternContainSubsequence" filter', () => { const uiFilter: FilterUi = { type: "patternContainSubsequence", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", value: "sub", }; const expectedFilter: AnnotationFilter = { type: "pattern", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", predicate: { type: "containSubsequence", value: "sub" }, }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); @@ -211,12 +210,12 @@ describe("compileFilter", () => { it('should compile "patternNotContainSubsequence" filter', () => { const uiFilter: FilterUi = { type: "patternNotContainSubsequence", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", value: "sub", }; const expectedPatternFilter: PatternFilter = { type: "pattern", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", predicate: { type: "containSubsequence", value: "sub" }, }; const expectedFilter: NotFilter = { type: "not", filter: expectedPatternFilter }; @@ -226,12 +225,12 @@ describe("compileFilter", () => { it('should compile "topN" filter (Top 5)', () => { const uiFilter: FilterUi = { type: "topN", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", n: 5, }; const expectedRank: ValueRank = { transformer: "rank", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", descending: true, }; const expectedFilter: NumericalComparisonFilter = { @@ -246,12 +245,12 @@ describe("compileFilter", () => { it('should compile "topN" filter (Bottom 3)', () => { const uiFilter: FilterUi = { type: "bottomN", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", n: 3, }; const expectedRank: ValueRank = { transformer: "rank", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", }; const expectedFilter: NumericalComparisonFilter = { type: "numericalComparison", @@ -265,12 +264,12 @@ describe("compileFilter", () => { it('should compile "lessThan" filter', () => { const uiFilter: FilterUi = { type: "lessThan", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", x: 10, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - lhs: "colNum" as unknown as SUniversalPColumnId, + lhs: "colNum", rhs: 10, }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); @@ -279,12 +278,12 @@ describe("compileFilter", () => { it('should compile "greaterThan" filter', () => { const uiFilter: FilterUi = { type: "greaterThan", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", x: 5, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - rhs: "colNum" as unknown as SUniversalPColumnId, + rhs: "colNum", lhs: 5, }; expect(compileFilter(uiFilter)).toEqual(expectedFilter); @@ -293,12 +292,12 @@ describe("compileFilter", () => { it('should compile "lessThanOrEqual" filter', () => { const uiFilter: FilterUi = { type: "lessThanOrEqual", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", x: 20, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - lhs: "colNum" as unknown as SUniversalPColumnId, + lhs: "colNum", rhs: 20, allowEqual: true, }; @@ -308,12 +307,12 @@ describe("compileFilter", () => { it('should compile "greaterThanOrEqual" filter', () => { const uiFilter: FilterUi = { type: "greaterThanOrEqual", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", x: 0, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - rhs: "colNum" as unknown as SUniversalPColumnId, + rhs: "colNum", lhs: 0, allowEqual: true, }; @@ -323,14 +322,14 @@ describe("compileFilter", () => { it('should compile "lessThanColumn" filter', () => { const uiFilter: FilterUi = { type: "lessThanColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 5, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - lhs: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + lhs: "colNum1", + rhs: "colNum2", minDiff: 5, allowEqual: undefined, }; @@ -340,14 +339,14 @@ describe("compileFilter", () => { it('should compile "lessThanColumnOrEqual" filter', () => { const uiFilter: FilterUi = { type: "lessThanColumnOrEqual", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 6, }; const expectedFilter: AnnotationFilter = { type: "numericalComparison", - lhs: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + lhs: "colNum1", + rhs: "colNum2", minDiff: 6, allowEqual: true, }; diff --git a/sdk/model/src/components/PlDataTable/columnResolver.test.ts b/sdk/model/src/components/PlDataTable/columnResolver.test.ts new file mode 100644 index 0000000000..613535e807 --- /dev/null +++ b/sdk/model/src/components/PlDataTable/columnResolver.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { + createLocalPObjectId, + createColumnOverridedId, + type AxisId, + type AxisSpec, + type PColumnSpec, + type PObjectId, + type PTableColumnId, + type SUniversalPColumnId, +} from "@milaboratories/pl-model-common"; +import { ColumnLazyImpl } from "../../columns"; +import type { ColumnRecipe } from "../../columns"; +import { createColumnResolver } from "./columnResolver"; + +function wrap(id: PObjectId | SUniversalPColumnId, tag: string): SUniversalPColumnId { + return createColumnOverridedId({ + source: id, + specOverrides: { annotations: { tag }, domain: {}, contextDomain: {}, axesSpec: {} }, + }); +} + +// ColumnLazy constructor pulls `getCfgRenderCtx()` from globalThis on some +// paths — keep a minimal mock the same way p_column_lazy.test.ts does. +const mockCtx = { + getAccessorHandleByName: () => undefined, + getUpstreamBlockCtx: () => [], +} as unknown as never; + +beforeEach(() => { + (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx = mockCtx; +}); +afterEach(() => { + delete (globalThis as { cfgRenderCtx?: unknown }).cfgRenderCtx; +}); + +function lazyOf(id: PObjectId, axes: AxisSpec[] = []) { + const spec = { kind: "PColumn", name: "stub", axesSpec: axes } as unknown as PColumnSpec; + return ColumnLazyImpl.fromColumn({ + id, + spec, + data: undefined, + }); +} + +function axisSpec(name: string): AxisSpec { + return { type: "String", name } as unknown as AxisSpec; +} + +describe("createColumnResolver", () => { + test("exact lazy id match returns the same id (column)", () => { + const id = createLocalPObjectId(["main", "out"], "col-1"); + const r = createColumnResolver([lazyOf(id)]); + const ref: PTableColumnId = { type: "column", id }; + expect(r(ref)).toEqual({ type: "column", id }); + }); + + test("bare PObjectId resolves to the recipe's full id", () => { + const pid = createLocalPObjectId(["main", "out"], "col-1"); + const wrapped = wrap(pid, "A"); + const fake: ColumnRecipe = { + id: wrapped, + getReferencedIds: () => [pid], + getSpec: () => ({ kind: "PColumn", name: "stub", axesSpec: [] }) as unknown as PColumnSpec, + getQuery: () => ({ type: "column", column: pid }) as never, + getDataStatus: () => "present", + withSpecs: () => fake, + }; + + const r = createColumnResolver([fake]); + const out = r({ type: "column", id: pid }); + expect(out).toEqual({ type: "column", id: wrapped }); + }); + + test("axis hit passes through unchanged", () => { + const ax = axisSpec("sampleId"); + const id = createLocalPObjectId(["main", "out"], "col-1"); + const r = createColumnResolver([lazyOf(id, [ax])]); + const ref: PTableColumnId = { + type: "axis", + id: { type: "String", name: "sampleId" } as AxisId, + }; + expect(r(ref)).toEqual(ref); + }); + + test("axis miss → undefined", () => { + const id = createLocalPObjectId(["main", "out"], "col-1"); + const r = createColumnResolver([lazyOf(id, [axisSpec("sampleId")])]); + const ref: PTableColumnId = { type: "axis", id: { type: "String", name: "missing" } as AxisId }; + expect(r(ref)).toBeUndefined(); + }); + + test("column miss → undefined", () => { + const id = createLocalPObjectId(["main", "out"], "col-1"); + const other = createLocalPObjectId(["main", "out"], "col-2"); + const r = createColumnResolver([lazyOf(id)]); + expect(r({ type: "column", id: other })).toBeUndefined(); + }); + + test("collision warns once and keeps first", () => { + const pid = createLocalPObjectId(["main", "out"], "col-1"); + const wrappedA = wrap(pid, "A"); + const wrappedB = wrap(pid, "B"); + const makeFake = (wrapped: SUniversalPColumnId): ColumnRecipe => { + const r: ColumnRecipe = { + id: wrapped, + getReferencedIds: () => [pid], + getSpec: () => ({ kind: "PColumn", name: "stub", axesSpec: [] }) as unknown as PColumnSpec, + getQuery: () => ({ type: "column", column: pid }) as never, + getDataStatus: () => "present", + withSpecs: () => r, + }; + return r; + }; + + const warn = vi.fn(); + const r = createColumnResolver([makeFake(wrappedA), makeFake(wrappedB)], { warn }); + expect(warn).toHaveBeenCalledTimes(1); + const out = r({ type: "column", id: pid }); + expect(out).toEqual({ type: "column", id: wrappedA }); + }); +}); diff --git a/sdk/model/src/components/PlDataTable/columnResolver.ts b/sdk/model/src/components/PlDataTable/columnResolver.ts new file mode 100644 index 0000000000..210ec1fb92 --- /dev/null +++ b/sdk/model/src/components/PlDataTable/columnResolver.ts @@ -0,0 +1,59 @@ +import type { + AxisId, + CanonicalizedJson, + ColumnUniversalId, + PObjectId, + PTableColumnId, + PTableColumnIdColumn, +} from "@milaboratories/pl-model-common"; +import { canonicalizeJson, extractPObjectId, getAxisId } from "@milaboratories/pl-model-common"; +import type { ColumnRecipe } from "../../columns"; + +/** + * Resolves a user-supplied PTableColumnId against the discovered set of + * columns and axes. For column refs, accepts either the full recipe id (rich + * {@link ColumnUniversalId}) or the bare {@link PObjectId} and returns a ref + * carrying the recipe's full id — the same id that appears as the leaf in the + * emitted SpecQuery, so engine-side dedup is consistent with the resolver's + * view. Axis refs are checked against the union of axes spanned by the + * columns and pass through unchanged on hit. + * Returns `undefined` when the reference cannot be matched. + */ +export type ColumnResolver = (ref: PTableColumnId) => PTableColumnId | undefined; + +export function createColumnResolver( + columns: ColumnRecipe[], + deps?: { warn?: (msg: string) => void }, +): ColumnResolver { + const axisSet = new Set>(); + for (const c of columns) { + for (const ax of c.getSpec().axesSpec) { + axisSet.add(canonicalizeJson(getAxisId(ax))); + } + } + + const byFullId = new Map(); + const byPObjectId = new Map(); + for (const c of columns) { + byFullId.set(c.id, c); + const pid = extractPObjectId(c.id); + const existing = byPObjectId.get(pid); + if (existing === undefined) { + byPObjectId.set(pid, c); + } else if (existing.id !== c.id) { + deps?.warn?.( + `Ambiguous PObjectId ${pid}: recipe ids ${existing.id} and ${c.id} both match — keeping first.`, + ); + } + } + + return (ref: PTableColumnId): PTableColumnId | undefined => { + if (ref.type === "axis") { + const key = canonicalizeJson(ref.id); + return axisSet.has(key) ? ref : undefined; + } + const hit = byFullId.get(ref.id) ?? byPObjectId.get(extractPObjectId(ref.id)); + if (hit === undefined) return undefined; + return { type: "column", id: hit.id } satisfies PTableColumnIdColumn; + }; +} diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV2.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV2.ts index d27a393967..6a120fe958 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV2.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV2.ts @@ -1,25 +1,25 @@ import type { + ColumnUniversalId, PColumn, PColumnIdAndSpec, PTableSorting, PTableDefV2, - DataInfo, - PColumnValues, } from "@milaboratories/pl-model-common"; import { getColumnIdAndSpec } from "@milaboratories/pl-model-common"; -import type { PColumnDataUniversal, TreeNodeAccessor } from "../../../render"; +import type { PColumnDataUniversal } from "../../../render"; import { isFunction } from "es-toolkit"; -import type { PlDataTableFilters } from "../typesV7"; +import type { PlDataTableFilters } from "../typesV8"; import { createPTableDefV3 } from "./createPTableDefV3"; +import { ColumnLazyImpl } from "../../../columns"; export function createPTableDefV2(params: { - columns: PColumn[]; - labelColumns: PColumn[]; + columns: PColumn[]; + labelColumns: PColumn[]; coreJoinType: "inner" | "full"; filters: null | PlDataTableFilters; sorting: PTableSorting[]; coreColumnPredicate?: (spec: PColumnIdAndSpec) => boolean; -}): PTableDefV2>> { +}): PTableDefV2 { let coreColumns = params.columns; const secondaryColumns: typeof params.columns = []; @@ -33,8 +33,8 @@ export function createPTableDefV2(params: { secondaryColumns.push(...params.labelColumns); return createPTableDefV3({ - primary: coreColumns.map((column) => ({ column })), - secondary: secondaryColumns.map((column) => ({ entries: [{ column }] })), + primary: coreColumns.map((column) => ColumnLazyImpl.fromColumn(column)), + secondary: secondaryColumns.map((column) => ColumnLazyImpl.fromColumn(column)), primaryJoinType: params.coreJoinType, filters: params.filters, sorting: params.sorting, diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV3.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV3.ts index 215c6a7412..dc9ec1c926 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV3.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/createPTableDefV3.ts @@ -1,6 +1,7 @@ import type { AxisQualification, - PColumn, + ColumnUniversalId, + PObjectId, PTableColumnId, PTableSorting, PTableDefV2, @@ -8,53 +9,33 @@ import type { SpecQuery, SpecQueryExpression, SpecQueryJoinEntry, - PObjectId, } from "@milaboratories/pl-model-common"; import { isBooleanExpression } from "@milaboratories/pl-model-common"; -import type { PColumnDataUniversal } from "../../../render"; import { isNil } from "es-toolkit"; -import type { PlDataTableFilters } from "../typesV7"; +import type { PlDataTableFilters } from "../typesV8"; import { distillFilterSpec, filterSpecToSpecQueryExpr } from "../../../filters"; import type { Nil } from "@milaboratories/helpers"; +import type { ColumnRecipe } from "../../.."; +import { hitQualifications, queriesQualifications } from "../../../columns"; -/** Primary side — base row grid. */ -export type PrimaryEntry = { - column: PColumn; -}; - -/** Secondary side leaf — the hit column or a label column, optionally reached via a linker chain. */ -export type SecondaryEntry = { - column: PColumn; - /** For hit: `forHit`. For label/direct: omit. Applied to the outermost emitted join entry. */ - qualifications?: AxisQualification[]; - /** - * Linker chain leading to `column`, ordered from outermost to innermost. - * When present, the entry is emitted as nested `linkerJoin` operators — - * one per linker — wrapping the hit column. Binds this hit to this exact - * chain so the engine cannot reuse a sibling chain that happens to share - * axis name + domain. - */ - linkers?: PColumn[]; -}; - -/** Secondary group — one join subtree outer-joined onto primary. */ -export type SecondaryGroup = { - entries: SecondaryEntry[]; - /** Per-variant qualifications applied to the cloned primary anchors on this group's side. - * Keyed by `PrimaryEntry.column.id`. Omit → base primary used unqualified (labels, non-variant columns). */ - primaryQualifications?: Record; -}; - -export function createPTableDefV3(params: { +/** + * Assemble a ptable def directly from recipes. Each secondary recipe is its own + * outer-joined subtree: `getQuery()` encodes its linker chain, and its + * hit-/queries-qualifications are derived on the spot via {@link hitQualifications} + * / {@link queriesQualifications} — no pre-extracted DTO. Bare leaves yield empty + * qualifications, so plain {@link ColumnRecipe} arrays (e.g. label columns) work + * unchanged. + */ +export function createPTableDefV3(params: { primaryJoinType: "inner" | "full"; - primary: PrimaryEntry[]; - secondary: SecondaryGroup[]; + primary: ColumnRecipe[]; + secondary: ColumnRecipe[]; filters?: Nil | PlDataTableFilters; sorting?: Nil | PTableSorting[]; -}): PTableDefV2> { - let query: SpecQuery> = { +}): PTableDefV2 { + let query: SpecQuery = { type: params.primaryJoinType === "inner" ? "innerJoin" : "fullJoin", - entries: params.primary.map((a) => toLeaf(a.column, [])), + entries: params.primary.map((c) => columnToJoinEntry(c, [])), }; if (params.secondary.length > 0) { @@ -62,11 +43,12 @@ export function createPTableDefV3(params: { type: "outerJoin", primary: { entry: query, - qualifications: params.secondary.flatMap((g) => - params.primary.flatMap((p) => g.primaryQualifications?.[p.column.id] ?? []), - ), + qualifications: params.secondary.flatMap((s) => { + const quals = queriesQualifications(s); + return params.primary.flatMap((p) => quals[p.id as PObjectId] ?? []); + }), }, - secondary: params.secondary.flatMap((g) => g.entries.map((e) => toJoinEntry(e))), + secondary: params.secondary.map((c) => columnToJoinEntry(c, hitQualifications(c))), }; } @@ -109,26 +91,12 @@ function columnIdToExpr(col: PTableColumnId): SpecQueryExpression { : { type: "columnRef", value: col.id }; } -function toLeaf( - col: PColumn, - qs: AxisQualification[], -): SpecQueryJoinEntry> { +function columnToJoinEntry( + col: ColumnRecipe, + qualifications: readonly AxisQualification[], +): SpecQueryJoinEntry { return { - entry: { type: "column", column: col }, - qualifications: qs, + entry: col.getQuery(), + qualifications, }; } - -function toJoinEntry(e: SecondaryEntry): SpecQueryJoinEntry> { - const qs = e.qualifications ?? []; - if (isNil(e.linkers) || e.linkers.length === 0) return toLeaf(e.column, qs); - - const folded = e.linkers.reduceRight>>( - (inner, linker) => ({ - entry: { type: "linkerJoin", linker: { column: linker }, secondary: [inner] }, - qualifications: [], - }), - toLeaf(e.column, []), - ); - return { ...folded, qualifications: qs }; -} diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV2.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV2.ts index 2d69a6f967..5c177c7bf3 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV2.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV2.ts @@ -1,20 +1,14 @@ import type { - AxisId, + ColumnUniversalId, + FilterSpecNode, PColumn, - PObjectId, - PTableColumnId, - PTableColumnIdAxis, PTableColumnIdColumn, - CanonicalizedJson, + PTableSorting, } from "@milaboratories/pl-model-common"; import { Annotation, - canonicalizeJson, - getAxisId, getColumnIdAndSpec, isLinkerColumn, - uniqueBy, - parseJson, PColumnName, } from "@milaboratories/pl-model-common"; import type { @@ -25,14 +19,21 @@ import type { } from "../../../render"; import { allPColumnsReady, deriveLabels, PColumnCollection } from "../../../render"; import { identity } from "es-toolkit"; -import type { CreatePlDataTableOps, PlDataTableModel } from "../typesV7"; +import type { + CreatePlDataTableOps, + PlDataTableFilters, + PlDataTableFilterSpecLeaf, + PlDataTableModel, +} from "../typesV8"; import { upgradePlDataTableStateV2 } from "../state-migration"; import type { PlDataTableStateV2 } from "../state-migration"; import { getMatchingLabelColumns } from "../labels"; -import { collectFilterSpecColumns } from "../../../filters/traverse"; -import { isEmpty } from "es-toolkit/compat"; +import { collectFilterSpecColumns, traverseFilterSpec } from "../../../filters/traverse"; import { createPTableDefV2 } from "./createPTableDefV2"; import { isColumnOptional } from "./utils"; +import { ColumnLazyImpl } from "../../../columns"; +import { createColumnResolver, type ColumnResolver } from "../columnResolver"; +import { isNil, type Nil } from "@milaboratories/helpers"; /** * @deprecated This function is deprecated and will be removed in future. Please migrate to createPlDataTable with v3 options for improved column discovery and display configuration. See createPlDataTableOptionsV3 for details on the new options format and migration guidance. @@ -81,52 +82,32 @@ export function createPlDataTableV2( const fullColumns = [...columns, ...fullLabelColumns]; - const fullColumnsAxes = uniqueBy( - fullColumns.flatMap((c) => c.spec.axesSpec.map((a) => getAxisId(a))), - (a) => canonicalizeJson(a), + const resolver = createColumnResolver( + fullColumns.map((c) => ColumnLazyImpl.fromColumn(c)), + { warn: ctx.logWarn.bind(ctx) }, ); - const fullColumnsIds: PTableColumnId[] = [ - ...fullColumnsAxes.map((a) => ({ type: "axis", id: a }) satisfies PTableColumnIdAxis), - ...fullColumns.map((c) => ({ type: "column", id: c.id }) satisfies PTableColumnIdColumn), - ]; - const fullColumnsIdsSet = new Set(fullColumnsIds.map((c) => canonicalizeJson(c))); - const isValidColumnId = (id: string): boolean => - fullColumnsIdsSet.has(id as CanonicalizedJson); - - // -- Filtering validation -- - const filters = tableStateNormalized.pTableParams.filters; - const defaultFilters = options?.filters ?? undefined; - const filterColumns = filters !== null ? collectFilterSpecColumns(filters) : []; - const firstInvalidFilterColumn = filterColumns.find((col) => !isValidColumnId(col)); - if (firstInvalidFilterColumn) - throw new Error( - `Invalid filter column ${firstInvalidFilterColumn}: column reference does not match the table columns`, - ); - const defaultFilterColumns = - defaultFilters !== undefined ? collectFilterSpecColumns(defaultFilters) : []; - const firstInvalidDefaultFilterColumn = defaultFilterColumns.find((col) => !isValidColumnId(col)); - if (firstInvalidDefaultFilterColumn) - throw new Error( - `Invalid default filter column ${firstInvalidDefaultFilterColumn}: column reference does not match the table columns`, - ); - // -- Sorting validation -- - const userSorting = tableStateNormalized.pTableParams.sorting; - const sorting = (isEmpty(userSorting) ? options?.sorting : userSorting) ?? []; - const firstInvalidSortingColumn = sorting.find( - (s) => !isValidColumnId(canonicalizeJson(s.column)), + // -- Filtering: rewrite via resolver, throw if any reference is unresolved -- + const rawFilters = tableStateNormalized.pTableParams.filters; + const rawDefaultFilters = options?.filters ?? undefined; + const filters = remapFiltersThrowing(rawFilters, resolver, "filter"); + const defaultFilters = remapFiltersThrowing( + rawDefaultFilters ?? null, + resolver, + "default filter", ); - if (firstInvalidSortingColumn) - throw new Error( - `Invalid sorting column ${JSON.stringify(firstInvalidSortingColumn.column)}: column reference does not match the table columns`, - ); + + // -- Sorting: rewrite via resolver, throw if any reference is unresolved -- + const userSorting = tableStateNormalized.pTableParams.sorting; + const rawSorting = (isNil(userSorting) ? options?.sorting : userSorting) ?? []; + const sorting = remapSortingThrowing(rawSorting, resolver); const coreJoinType = options?.coreJoinType ?? "full"; const fullDef = createPTableDefV2({ columns, labelColumns: fullLabelColumns, coreJoinType, - filters, + filters: filters ?? null, sorting, coreColumnPredicate: options?.coreColumnPredicate, }); @@ -135,8 +116,8 @@ export function createPlDataTableV2( const pframeHandle = ctx.createPFrame(fullColumns); if (!fullHandle || !pframeHandle) return undefined; - const hiddenColumns = new Set( - ((): PObjectId[] => { + const hiddenColumns = new Set( + ((): ColumnUniversalId[] => { // Inner join works as a filter - all columns must be present if (coreJoinType === "inner") return []; @@ -172,10 +153,7 @@ export function createPlDataTableV2( // Preserve filter columns from being hidden if (filters) { collectFilterSpecColumns(filters) - .flatMap((c) => { - const obj = parseJson(c); - return obj.type === "column" ? [obj.id] : []; - }) + .flatMap((c) => (c.type === "column" ? [c.id] : [])) .forEach((c) => hiddenColumns.delete(c)); } @@ -192,7 +170,7 @@ export function createPlDataTableV2( columns: visibleColumns, labelColumns: visibleLabelColumns, coreJoinType, - filters, + filters: filters ?? null, sorting, coreColumnPredicate, }); @@ -205,13 +183,62 @@ export function createPlDataTableV2( fullTableHandle: fullHandle, fullPframeHandle: pframeHandle, visibleTableHandle: visibleHandle, - defaultFilters, + defaultFilters: defaultFilters ?? undefined, } satisfies PlDataTableModel; } +/** Rewrite filter column refs through the resolver; throw on any unresolved ref. */ +function remapFiltersThrowing( + filters: Nil | PlDataTableFilters, + resolver: ColumnResolver, + label: string, +): Nil | PlDataTableFilters { + if (isNil(filters)) return filters; + + type Node = FilterSpecNode; + return traverseFilterSpec(filters, { + leaf: (leaf): Node => { + if (leaf.type === undefined) return leaf; + const result = { ...leaf }; + if ("column" in result) { + const c = resolver(result.column); + if (isNil(c)) + throw new Error( + `Invalid ${label} column ${JSON.stringify(result.column)}: column reference does not match the table columns`, + ); + result.column = c; + } + if ("rhs" in result) { + const c = resolver(result.rhs); + if (isNil(c)) + throw new Error( + `Invalid ${label} column ${JSON.stringify(result.rhs)}: column reference does not match the table columns`, + ); + result.rhs = c; + } + return result; + }, + and: (results): Node => ({ type: "and", filters: results }), + or: (results): Node => ({ type: "or", filters: results }), + not: (result): Node => ({ type: "not", filter: result }), + }) as PlDataTableFilters; +} + +/** Rewrite sorting column refs through the resolver; throw on any unresolved ref. */ +function remapSortingThrowing(sorting: PTableSorting[], resolver: ColumnResolver): PTableSorting[] { + return sorting.map((s) => { + const c = resolver(s.column); + if (isNil(c)) + throw new Error( + `Invalid sorting column ${JSON.stringify(s.column)}: column reference does not match the table columns`, + ); + return { ...s, column: c }; + }); +} + function getAllLabelColumns( resultPool: AxisLabelProvider & ColumnProvider, -): PColumn[] | undefined { +): undefined | PColumn[] { return new PColumnCollection() .addAxisLabelProvider(resultPool) .addColumnProvider(resultPool) diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts index c74bc71adb..be4c674436 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts @@ -1,53 +1,68 @@ import type { - AxisId, - CanonicalizedJson, + ColumnUniversalId, FilterSpecNode, - PColumn, PObjectId, PTableColumnId, - PTableColumnIdAxis, PTableColumnIdColumn, PTableSorting, - PColumnSpec, MultiColumnSelector, - PFrameSpecDriver, - DiscoveredPColumnId, + ColumnSelector, } from "@milaboratories/pl-model-common"; -import { canonicalizeJson, getAxisId, parseJson, uniqueBy } from "@milaboratories/pl-model-common"; -import { collectFilterSpecColumns, traverseFilterSpec } from "../../../filters/traverse"; -import type { RenderCtxBase, PColumnDataUniversal } from "../../../render"; -import { isEmpty } from "es-toolkit/compat"; -import type { PlDataTableFilters, PlDataTableFilterSpecLeaf, PlDataTableModel } from "../typesV7"; +import { + canonicalizeAxisId, + dedupColumns, + extractPObjectId, + uniqueBy, +} from "@milaboratories/pl-model-common"; +import { collectFilterSpecColumns } from "../../../filters/traverse"; +import { createColumnResolver, type ColumnResolver } from "../columnResolver"; +import type { RenderCtxBase } from "../../../render"; +import type { + PlDataTableColumnsMeta, + PlDataTableFilters, + PlDataTableFilterSpecLeaf, + PlDataTableModel, +} from "../typesV8"; import { upgradePlDataTableStateV2 } from "../state-migration"; import type { PlDataTableStateV2 } from "../state-migration"; -import type { ColumnSelector, ColumnSnapshot, ColumnVariant, MatchingMode } from "../../../columns"; +import type { MatchingMode } from "@milaboratories/pl-model-common"; +import { + isLeafColumn, + hitQualifications, + collectLinkerColumns, + queriesQualifications, + type ColumnRecipe, +} from "../../../columns"; import type { DeriveLabelsOptions } from "../../../labels/derive_distinct_labels"; import { deriveAllLabels, - deriveAllTooltips, evaluateRules, + getEffectiveVisibility, + getOrderPriority, isColumnHidden, - isColumnOptional, - withHidenAxesAnnotations, - withLabelAnnotations, - withTableVisualAnnotations, - withInfoAnnotations, - withDataStatusAnnotations, + buildDataStatusMap, + toRuleColumn, } from "./utils"; -import type { PrimaryEntry, SecondaryGroup } from "./createPTableDefV3"; import { createPTableDefV3 } from "./createPTableDefV3"; import { - discoverLabelColumnVariants, - discoverTableColumnSnaphots, + discoverLabelColumns, + discoverTableColumns, type DiscoverTableColumnOptions, } from "./discoverColumns"; -import { getField, isNil, isPlainObject, throwError, type Nil } from "@milaboratories/helpers"; -import { flow } from "es-toolkit"; +import { isNil, isPlainObject, type Nil } from "@milaboratories/helpers"; +import { uniq } from "es-toolkit"; -export type createPlDataTableOptionsV3 = { +export type createPlDataTableOptionsV3 = ( + | { + columns: Nil | DiscoverTableColumnOptions; + } + | { + primaryColumns: ColumnRecipe[]; + columns: Nil | ColumnRecipe[]; + } +) & { tableState?: PlDataTableStateV2; - columns: Nil | DiscoverTableColumnOptions | TableColumnVariant[]; filters?: PlDataTableFilters; sorting?: PTableSorting[]; primaryJoinType?: "inner" | "full"; @@ -56,7 +71,7 @@ export type createPlDataTableOptionsV3 = { displayOptions?: ColumnsDisplayOptions; }; -/** Structured source config — selectors/anchors instead of raw ColumnSource. */ +/** Structured source config — selectors/anchors instead of raw ColumnsSource. */ export type ColumnsSelectorConfig = { include?: MultiColumnSelector | MultiColumnSelector[]; exclude?: MultiColumnSelector | MultiColumnSelector[]; @@ -72,309 +87,285 @@ export type ColumnsDisplayOptions = { }; export type ColumnOrderRule = { - match: ColumnMatcher | ColumnSelector; + match: ColumnSelector; /** Higher number = further left in table */ priority: number; }; export type ColumnVisibilityRule = { - match: ColumnMatcher | ColumnSelector; + match: ColumnSelector; visibility: "default" | "optional" | "hidden"; }; -export type ColumnMatcher = (spec: PColumnSpec) => boolean; - export function createPlDataTableV3( ctx: RenderCtxBase, options: createPlDataTableOptionsV3, ): PlDataTableModel | undefined { - const pframeSpec = ctx.getService("pframeSpec"); const state = upgradePlDataTableStateV2(options.tableState); const primaryJoinType = options.primaryJoinType ?? "full"; - const discovered = isPlainObject(options.columns) - ? discoverTableColumnSnaphots(ctx, options.columns) - : isNil(options.columns) - ? options.columns - : uniqueBy( - [...options.columns, ...discoverLabelColumnVariants(ctx, options.columns)], - (c) => c.column.id, - ); - if (isNil(discovered) || discovered.length === 0) return undefined; + const resolved = resolveInputColumns(ctx, options); + if (resolved === undefined) return undefined; + const { primary, secondary } = resolved; + if (primary.length === 0) return undefined; - const splited = splitDiscoveredColumns(discovered); + const { direct, linked } = splitByTopology(secondary); + const allColumns = [...primary, ...secondary]; const derivedLabels = deriveAllLabels({ - columns: discovered - .map((dc) => ({ - id: dc.column.id, - spec: dc.column.spec, - linkerPath: dc.path, - qualifications: dc.qualifications, - })) - .filter((v) => !isColumnHidden(v.spec)), + // Skip hidden columns when deriving labels — they don't appear in the + // table, so they shouldn't influence label disambiguation (#1623). + columns: allColumns.filter((c) => !isColumnHidden(c.getSpec())).map(toLabelableColumn), deriveLabelsOptions: { includeNativeLabel: true, ...options.labelsOptions, }, }); - const derivedTooltips = deriveAllTooltips({ - columns: discovered.map((dc) => ({ - id: dc.column.id, - originalId: getField(dc, "originalId"), - spec: dc.column.spec, - linkerPath: dc.path, - qualifications: dc.qualifications, - })), - }); - - const annotated = annotateColumnGroups({ - pframeSpec, - ...splited, - derivedLabels, - derivedTooltips, - displayOptions: options.displayOptions, - }); - - const primarySnapshots = annotated.direct.filter((c) => c.isPrimary); - const secondarySnapshots = annotated.direct.filter((c) => !c.isPrimary); - - if (primarySnapshots.length === 0) return undefined; + // Rule-based visibility/order maps (keyed by bare PObjectId). Computed once + // here: `computeHiddenColumns` needs visibility to decide the visible set, + // and `buildColumnsMeta` (below, once the visible set is known) reuses both. + const allColumnsForRules = [...primary, ...direct, ...linked, ...collectLinkerSnapshots(linked)]; + const visibilityByColId = evaluateRules( + options.displayOptions?.visibility ?? [], + allColumnsForRules, + ); + const orderByColId = evaluateRules(options.displayOptions?.ordering ?? [], allColumnsForRules); - const columnIsAvailable = createColumnValidationById([ - ...annotated.direct.map((v) => v.column), - ...annotated.linked.flatMap((lc) => [...(lc.path ?? []).map((s) => s.linker), lc.column]), - ]); + const resolver = createColumnResolver( + [...primary, ...direct, ...linked.flatMap((lc) => [...collectLinkerColumns(lc), lc])], + { warn: ctx.logWarn.bind(ctx) }, + ); - const remapedDefaultFilters = remapFilterColumnIds(options.filters, discovered); + const remapedDefaultFilters = remapFilterColumnIds(options.filters, resolver); const filters = filterFilters( concatFilters( state.pTableParams.filters, state.pTableParams.defaultFilters ?? remapedDefaultFilters, ), - columnIsAvailable, + resolver, ); const sorting = filterSorting( - resolveSorting(state.pTableParams.sorting, remapSortingColumnIds(options.sorting, discovered)), - columnIsAvailable, + resolveSorting(state.pTableParams.sorting, remapSortingColumnIds(options.sorting, resolver)), + resolver, ); - const primaryEntries: PrimaryEntry[] = primarySnapshots.map( - (v) => ({ column: resolveSnapshot(v.column) }), - ); - const secondaryGroups: SecondaryGroup[] = buildSecondaryGroups( - secondarySnapshots, - annotated.linked, - ); const fullDef = createPTableDefV3({ primaryJoinType, - primary: primaryEntries, - secondary: secondaryGroups, + primary, + secondary: [...direct, ...linked], filters, sorting, }); const fullHandle = ctx.createPTableV2(fullDef); // TODO: is workaround for dropdown suggestions. - // Pframe have not equivalent data for columns relativly to Ptable - const pframeHandle = ctx.createPFrame([ - ...annotated.direct.map((v) => resolveSnapshot(v.column)), - ...annotated.linked.map((v) => resolveSnapshot(v.column)), - ...collectLinkerSnapshots(annotated.linked).map(resolveSnapshot), - ]); + // Pframe have not equivalent data for columns relativly to Ptable. + // PFrame is the physical column registry — one entry per bare PObjectId, + // so strip the rich recipe ids down to their physical leaf id and dedupe: + // multiple discovered variants of the same hit collapse to the same bare id. + const pframeHandle = ctx.createPFrame( + uniq([ + ...primary.map((v) => extractPObjectId(v.id)), + ...direct.map((v) => extractPObjectId(v.id)), + ...linked.map((v) => extractPObjectId(v.id)), + ]), + ); const hiddenSpecs = state.pTableParams.hiddenColIds; - const hiddenColumnIds = computeHiddenColumns( - [...annotated.direct, ...annotated.linked].map((v) => v.column), + const hiddenColumnIds = computeHiddenColumns({ + columns: [...primary, ...direct, ...linked], + visibilityByColId, sorting, filters, hiddenSpecs, - ); + }); - const visible = buildVisibleColumns(annotated, hiddenColumnIds); + const visible = { + primary, + direct: direct.filter((c) => !hiddenColumnIds.has(c.id)), + linked: linked.filter((c) => !hiddenColumnIds.has(c.id)), + }; const visibleDef = createPTableDefV3({ primaryJoinType, - primary: primaryEntries, - secondary: buildSecondaryGroups( - visible.direct.filter((c) => !c.isPrimary), - visible.linked, - ), + primary, + secondary: [...visible.direct, ...visible.linked], filters, sorting, }); const visibleHandle = ctx.createPTableV2(visibleDef); + // Built here, where the visible set is known: per-column data status is + // computed for visible columns only (the probe is meaningful only for what + // the user sees), while label/visibility/order/axes cover all columns. + const columnsMeta = buildColumnsMeta({ + fullColumns: [...primary, ...direct, ...linked], + visibleColumns: [...visible.primary, ...visible.direct, ...visible.linked], + primaryColumns: primary, + derivedLabels, + visibilityByColId, + orderByColId, + }); + return { sourceId: state.pTableParams.sourceId, fullTableHandle: fullHandle, fullPframeHandle: pframeHandle, visibleTableHandle: visibleHandle, defaultFilters: remapedDefaultFilters, + columnsMeta, } satisfies PlDataTableModel; } -export type TableColumnVariant = ( - | ColumnVariant - | (ColumnVariant & { readonly originalId: PObjectId }) -) & { - readonly isPrimary?: boolean; -}; - -type SplitDiscoveredColumns = { - readonly direct: TableColumnVariant[]; - readonly linked: TableColumnVariant[]; +type ResolvedColumns = { + readonly primary: ColumnRecipe[]; + readonly secondary: ColumnRecipe[]; }; -type AnnotatedColumnGroups = { - readonly direct: TableColumnVariant[]; - readonly linked: TableColumnVariant[]; -}; - -type VisibleColumns = { - readonly direct: TableColumnVariant[]; - readonly linked: TableColumnVariant[]; -}; +/** Normalize either option branch into a {primary, secondary} pair of recipes. */ +function resolveInputColumns( + ctx: RenderCtxBase, + options: createPlDataTableOptionsV3, +): ResolvedColumns | undefined { + if ("primaryColumns" in options) { + const primary = options.primaryColumns; + const secondary = options.columns ?? []; + const labels = discoverLabelColumns(ctx, primary, [...primary, ...secondary]); + // Exclude from secondary anything already present in primary: a label + // column the block hands in as primary can be re-discovered here as a + // label for that same axis, landing in the table twice with the same id. + const primaryIds = new Set(primary.map((c) => c.id)); + return { + primary, + secondary: dedupColumns( + [...secondary, ...labels].filter((c) => !primaryIds.has(c.id)), + (c) => c.id, + (c) => c.getSpec(), + ), + }; + } + + if (isPlainObject(options.columns)) { + return discoverTableColumns(ctx, options.columns); + } + + return undefined; +} -/** Split discovered columns into direct (no linker path) and linked (with linker path). */ -function splitDiscoveredColumns(columns: TableColumnVariant[]): SplitDiscoveredColumns { - const direct = columns.filter((dc) => (dc.path?.length ?? 0) === 0); - const linked = columns.filter((dc) => (dc.path?.length ?? 0) > 0); +/** Split secondary recipes by query topology: leaves (no linker chain) vs joined. */ +function splitByTopology(columns: ColumnRecipe[]): { + direct: ColumnRecipe[]; + linked: ColumnRecipe[]; +} { + const direct: ColumnRecipe[] = []; + const linked: ColumnRecipe[] = []; + for (const c of columns) { + if (isLeafColumn(c)) direct.push(c); + else linked.push(c); + } return { direct, linked }; } -/** All linker snapshots across the given linked columns, deduped by id. */ -function collectLinkerSnapshots(linked: TableColumnVariant[]): ColumnSnapshot[] { +/** All linker recipes across the given linked columns, deduped by id. */ +function collectLinkerSnapshots(linked: ColumnRecipe[]): ColumnRecipe[] { return uniqueBy( - linked.flatMap((lc) => (lc.path ?? []).map((s) => s.linker)), + linked.flatMap((c) => collectLinkerColumns(c)), (c) => c.id, ); } -/** - * Annotate all column groups with derived labels and display-rule annotations. - * Evaluates `displayOptions` rules against all discovered columns (direct, - * linked, labels, linkers) and writes the winning visibility/priority into - * column annotations via `withTableVisualAnnotations`. - */ -function annotateColumnGroups(params: { - direct: TableColumnVariant[]; - linked: TableColumnVariant[]; - derivedLabels: Record; - derivedTooltips: Record; - displayOptions?: ColumnsDisplayOptions; - pframeSpec: PFrameSpecDriver; -}): AnnotatedColumnGroups { - const { direct, linked, derivedLabels, derivedTooltips, displayOptions, pframeSpec } = params; - - const allColumnsForRules = [ - ...direct.map((v) => v.column), - ...linked.map((v) => v.column), - ...collectLinkerSnapshots(linked), - ]; - const visibilityByColId = evaluateRules( - displayOptions?.visibility ?? [], - allColumnsForRules, - pframeSpec, - ); - const orderByColId = evaluateRules( - displayOptions?.ordering ?? [], - allColumnsForRules, - pframeSpec, - ); - - const directAnnotated = liftToVariantColumns( - direct, - flow( - (cols) => withDataStatusAnnotations(cols), - (cols) => withLabelAnnotations(derivedLabels, cols), - (cols) => withInfoAnnotations(derivedTooltips, cols), - (cols) => withTableVisualAnnotations(visibilityByColId, orderByColId, cols), - ), - ); - - const linkedAnnotated = liftToVariantColumns( - linked, - flow( - (cols) => withDataStatusAnnotations(cols), - (cols) => withHidenAxesAnnotations(cols), - (cols) => withLabelAnnotations(derivedLabels, cols), - (cols) => withInfoAnnotations(derivedTooltips, cols), - (cols) => withTableVisualAnnotations(visibilityByColId, orderByColId, cols), - ), - ).map((lc) => ({ ...lc, path: annotateLinkerPath(derivedLabels, lc.path) })); - +function toLabelableColumn(col: ColumnRecipe) { return { - direct: directAnnotated, - linked: linkedAnnotated, + id: col.id, + spec: col.getSpec(), + linkerPath: collectLinkerColumns(col).map((linker) => ({ + linker: { spec: linker.getSpec() }, + })), + qualifications: { + forHit: [...hitQualifications(col)], + forQueries: queriesQualifications(col), + }, }; } -/** Lift a snapshot-array transform so it runs on the inner `column` of each variant. */ -function liftToVariantColumns< - V extends { readonly column: ColumnSnapshot }, ->( - variants: V[], - fn: ( - cols: ColumnSnapshot[], - ) => ColumnSnapshot[], -): V[] { - const cols = fn(variants.map((v) => v.column)); - if (cols.length !== variants.length) - throw new Error( - `liftToVariantColumns: fn must preserve array length (got ${cols.length}, expected ${variants.length})`, - ); - return variants.map((v, i) => ({ ...v, column: cols[i] })); -} - -function annotateLinkerPath( - derivedLabels: Record, - path: TableColumnVariant["path"], -): TableColumnVariant["path"] { - if (isNil(path) || path.length === 0) return path; - const annotatedLinkers = withHidenAxesAnnotations( - withLabelAnnotations( - derivedLabels, - path.map((s) => s.linker), - ), +/** + * Compute display metadata as a sidecar keyed by each emitted column's + * `ColumnUniversalId` (and axis `AxisId`), instead of baking it into the specs + * via `withSpecs`. Spec overrides change a recipe's id, so the same physical + * column reached two ways would diverge into two ids and render twice; keeping + * meta out of the spec preserves identity and lets dedup collapse such cases. + * The UI overlays this onto the engine-emitted specs at render time. + */ +function buildColumnsMeta(params: { + /** Every column in the table (primary + secondary) — drives `columns` and `axes`. */ + fullColumns: ColumnRecipe[]; + /** The visible subset — `status` is probed for these only. */ + visibleColumns: ColumnRecipe[]; + /** The primary join columns — their axes are the visible index; all other axes are hidden. */ + primaryColumns: ColumnRecipe[]; + derivedLabels: Record; + visibilityByColId: Map; + orderByColId: Map; +}): PlDataTableColumnsMeta { + const { fullColumns, visibleColumns, primaryColumns, derivedLabels } = params; + const { visibilityByColId, orderByColId } = params; + // Status only for visible columns — the probe is meaningful only for what the + // user actually sees; non-visible columns get no `status`. + const visibleStatus = buildDataStatusMap(visibleColumns); + + const columns = fullColumns.reduce((acc, c) => { + const rc = toRuleColumn(c); + acc[c.id] = { + label: derivedLabels[c.id], + visibility: getEffectiveVisibility(rc, visibilityByColId), + order: getOrderPriority(rc, orderByColId), + status: visibleStatus[c.id], + }; + return acc; + }, {}); + + // Only primary columns declare the table's visible axes. Any axis introduced + // solely by a secondary column (a linker-bridge axis, or a distinct-domain + // copy) is flagged hidden — otherwise it renders as a second index axis. + const primaryAxisKeys = new Set( + primaryColumns.flatMap((c) => c.getSpec().axesSpec.map((a) => canonicalizeAxisId(a))), ); - return path.map((s, i) => ({ ...s, linker: annotatedLinkers[i] })); -} - -/** Build an index of all valid column IDs (axes + columns) for filter/sorting validation. */ -function createColumnValidationById( - fullColumns: { readonly id: PObjectId; readonly spec: PColumnSpec }[], -) { - const axisIds = uniqueBy( - fullColumns.flatMap((c) => c.spec.axesSpec.map(getAxisId)), - (a) => canonicalizeJson(a), + const axes = fullColumns.reduce( + (acc, c) => + c.getSpec().axesSpec.reduce((inner, ax) => { + const key = canonicalizeAxisId(ax); + if (inner[key] === undefined) inner[key] = { hidden: !primaryAxisKeys.has(key) }; + return inner; + }, acc), + {}, ); - const allIds: PTableColumnId[] = [ - ...axisIds.map((a) => ({ type: "axis", id: a }) satisfies PTableColumnIdAxis), - ...fullColumns.map((c) => ({ type: "column", id: c.id }) satisfies PTableColumnIdColumn), - ]; - - const validIdSet = new Set(allIds.map((c) => canonicalizeJson(c))); - - return (id: string): boolean => { - return validIdSet.has(id as CanonicalizedJson); - }; + return { columns, axes }; } -/** Drop filter leaves whose column references are not available in the table. */ +/** Drop filter leaves whose column references cannot be resolved; rewrite the + * resolvable ones through the resolver. Prune empty and/or/not groups. */ function filterFilters( filters: Nil | PlDataTableFilters, - isValidColumnId: (id: string) => boolean, + resolver: ColumnResolver, ): Nil | PlDataTableFilters { if (isNil(filters)) return filters; - const isLeafValid = (leaf: PlDataTableFilterSpecLeaf): boolean => { - if (leaf.type === undefined) return true; - if ("column" in leaf && !isValidColumnId(leaf.column)) return false; - if ("rhs" in leaf && !isValidColumnId(leaf.rhs)) return false; - return true; + const rewriteLeaf = (leaf: PlDataTableFilterSpecLeaf): Nil | PlDataTableFilterSpecLeaf => { + if (leaf.type === undefined) return leaf; + const result = { ...leaf }; + if ("column" in result) { + const c = resolver(result.column); + if (isNil(c)) return undefined; + result.column = c; + } + if ("rhs" in result) { + const c = resolver(result.rhs); + if (isNil(c)) return undefined; + result.rhs = c; + } + return result; }; const prune = (node: PlDataTableFilterNode): Nil | PlDataTableFilterNode => { @@ -382,13 +373,14 @@ function filterFilters( const kept = node.filters .map((f) => prune(f)) .filter((f): f is PlDataTableFilterNode => !isNil(f)); + if (kept.length === 0) return undefined; return { type: node.type, filters: kept }; } if (node.type === "not") { const inner = prune(node.filter); return isNil(inner) ? undefined : { type: "not", filter: inner }; } - return isLeafValid(node) ? node : undefined; + return rewriteLeaf(node); }; return prune(filters) as Nil | PlDataTableFilters; @@ -404,59 +396,41 @@ function concatFilters( return { ...a, filters: [...a.filters, ...b.filters] }; } -/** Pick user sorting from state if non-empty, otherwise fall back to options default. */ +/** Pick user sorting from state if set, otherwise fall back to options default. + * null = user has not touched sorting → use default; + * [] = user has explicitly cleared sorting → no sorting (default ignored). */ function resolveSorting( - userSorting: PTableSorting[], + userSorting: Nil | PTableSorting[], defaultSorting: Nil | PTableSorting[], ): PTableSorting[] { - return (isEmpty(userSorting) ? defaultSorting : userSorting) ?? []; + return (isNil(userSorting) ? defaultSorting : userSorting) ?? []; } -/** Drop sorting entries whose column is not available in the table. */ -function filterSorting( - sorting: PTableSorting[], - isValidColumnId: (id: string) => boolean, -): PTableSorting[] { - return sorting.filter((s) => isValidColumnId(canonicalizeJson(s.column))); -} - -function buildSecondaryGroups( - direct: TableColumnVariant[], - linked: TableColumnVariant[], -): SecondaryGroup[] { - return [ - ...direct.map( - (c): SecondaryGroup => ({ - entries: [{ column: resolveSnapshot(c.column), qualifications: c.qualifications?.forHit }], - primaryQualifications: c.qualifications?.forQueries, - }), - ), - ...linked.map( - (lc): SecondaryGroup => ({ - entries: [ - { - column: resolveSnapshot(lc.column), - linkers: lc.path?.map((s) => resolveSnapshot(s.linker)), - qualifications: lc.qualifications?.forHit, - }, - ], - primaryQualifications: lc.qualifications?.forQueries, - }), - ), - ]; +/** Rewrite sorting entries through the resolver; drop those that fail to resolve. */ +function filterSorting(sorting: PTableSorting[], resolver: ColumnResolver): PTableSorting[] { + return sorting.flatMap((s) => { + const col = resolver(s.column); + return isNil(col) ? [] : [{ ...s, column: col }]; + }); } -/** Determine which columns should be hidden based on state or optional-column defaults. */ -function computeHiddenColumns( - columns: { readonly id: PObjectId; readonly spec: PColumnSpec }[], - sorting: Nil | PTableSorting[], - filters: Nil | PlDataTableFilters, - hiddenSpecs: Nil | PTableColumnId[], -): Set { - const alwaysHidden = columns.filter((c) => isColumnHidden(c.spec)).map((c) => c.id); +/** Determine which columns should be hidden based on state or optional-column defaults. + * Keyed by the recipe's logical {@link ColumnUniversalId} (rich) — variants of + * the same physical column are independently visible/hidden. */ +function computeHiddenColumns(params: { + columns: ColumnRecipe[]; + visibilityByColId: Map; + sorting: Nil | PTableSorting[]; + filters: Nil | PlDataTableFilters; + hiddenSpecs: Nil | PTableColumnId[]; +}): Set { + const { columns, visibilityByColId, sorting, filters, hiddenSpecs } = params; + const visibilityOf = (c: ColumnRecipe) => + getEffectiveVisibility(toRuleColumn(c), visibilityByColId); + const alwaysHidden = columns.filter((c) => visibilityOf(c) === "hidden").map((c) => c.id); const optionalHidden = !isNil(hiddenSpecs) ? hiddenSpecs.filter((s): s is PTableColumnIdColumn => s.type === "column").map((s) => s.id) - : columns.filter((c) => isColumnOptional(c.spec)).map((c) => c.id); + : columns.filter((c) => visibilityOf(c) === "optional").map((c) => c.id); const initial = [...alwaysHidden, ...optionalHidden]; const preserved = collectPreservedColumnIds(sorting, filters); @@ -467,99 +441,87 @@ function computeHiddenColumns( function collectPreservedColumnIds( sorting: Nil | PTableSorting[], filters: Nil | PlDataTableFilters, -): Set { +): Set { const sortedIds = (sorting ?? []) .map((s) => s.column) .filter((c): c is PTableColumnIdColumn => c.type === "column") .map((c) => c.id); const filterIds = !isNil(filters) - ? collectFilterSpecColumns(filters).flatMap((c) => { - const obj = parseJson(c); - return obj.type === "column" ? [obj.id] : []; - }) + ? collectFilterSpecColumns(filters).flatMap((c) => (c.type === "column" ? [c.id] : [])) : []; - return new Set([...sortedIds, ...filterIds]); + return new Set([...sortedIds, ...filterIds]); } -/** Filter annotated columns to only visible ones, re-matching label columns for the visible subset. */ -function buildVisibleColumns( - annotated: AnnotatedColumnGroups, - hiddenColumns: Set, -): VisibleColumns { - const direct = annotated.direct.filter((c) => !hiddenColumns.has(c.column.id)); - const linked = annotated.linked.filter((c) => !hiddenColumns.has(c.column.id)); - return { direct, linked }; -} - -/** Resolve a ColumnSnapshot to a PColumn with lazily-evaluated data. */ -function resolveSnapshot( - snap: ColumnSnapshot, -): PColumn { - return { id: snap.id, spec: snap.spec, data: snap.data?.get() }; -} - -/** Remap column references in sorting entries. */ +/** Remap column references in sorting entries through the resolver. + * Unresolved entries are dropped with a warning (best-effort contract). */ function remapSortingColumnIds( sorting: Nil | PTableSorting[], - columns: TableColumnVariant[], + resolver: ColumnResolver, ): Nil | PTableSorting[] { return sorting?.flatMap((s) => { - if (s.column.type === "axis") return [s]; // Axis references are unaffected by column ID remapping - - const id = s.column.id; - const column = columns.find((c) => (getField(c, "originalId") ?? c.column.id) === id); - if (column === undefined) return []; - - return [ - { - ...s, - column: { - type: "column" as const, - id: column.column.id, - }, - }, - ]; + const col = resolver(s.column); + if (isNil(col)) { + console.warn( + `Sorting column ${JSON.stringify(s.column)} does not match any discovered column — dropped.`, + ); + return []; + } + return [{ ...s, column: col }]; }); } type PlDataTableFilterNode = FilterSpecNode; -/** Remap column references in a filter tree. */ +/** Remap column references in a filter tree through the resolver. + * Unresolved leaves are dropped with a warning; empty groups are pruned. */ function remapFilterColumnIds( filters: Nil | PlDataTableFilters, - columns: TableColumnVariant[], + resolver: ColumnResolver, ): Nil | PlDataTableFilters { if (isNil(filters)) return filters; - const map = ( - tableColumnId: CanonicalizedJson, - ): CanonicalizedJson => { - const parsed = parseJson(tableColumnId); - if (parsed.type === "axis") return tableColumnId; // Axis references are unaffected by column ID remapping - - const originalId = parsed.id; - const column = - columns.find((c) => (getField(c, "originalId") ?? c.column.id) === originalId) ?? - throwError(`Column ID "${parsed.id}" in filters does not match any discovered column`); - - return canonicalizeJson({ - type: "column", - id: column.column.id, - }); + const mapLeaf = (leaf: PlDataTableFilterSpecLeaf): Nil | PlDataTableFilterSpecLeaf => { + if (leaf.type === undefined) return leaf; + const result = { ...leaf }; + if ("column" in result) { + const c = resolver(result.column); + if (isNil(c)) { + console.warn( + `Filter column ${JSON.stringify(result.column)} does not match any discovered column — dropped.`, + ); + return undefined; + } + result.column = c; + } + if ("rhs" in result) { + const c = resolver(result.rhs); + if (isNil(c)) { + console.warn( + `Filter rhs ${JSON.stringify(result.rhs)} does not match any discovered column — dropped.`, + ); + return undefined; + } + result.rhs = c; + } + return result; }; - return traverseFilterSpec(filters, { - leaf: (leaf): PlDataTableFilterNode => { - if (leaf.type === undefined) return leaf; - const result = { ...leaf }; - if ("column" in result) result.column = map(result.column); - if ("rhs" in result) result.rhs = map(result.rhs); - return result; - }, - and: (results): PlDataTableFilterNode => ({ type: "and", filters: results }), - or: (results): PlDataTableFilterNode => ({ type: "or", filters: results }), - not: (result): PlDataTableFilterNode => ({ type: "not", filter: result }), - }) as PlDataTableFilters; + const prune = (node: PlDataTableFilterNode): Nil | PlDataTableFilterNode => { + if (node.type === "and" || node.type === "or") { + const kept = node.filters + .map((f) => prune(f)) + .filter((f): f is PlDataTableFilterNode => !isNil(f)); + if (kept.length === 0) return undefined; + return { type: node.type, filters: kept }; + } + if (node.type === "not") { + const inner = prune(node.filter); + return isNil(inner) ? undefined : { type: "not", filter: inner }; + } + return mapLeaf(node); + }; + + return prune(filters) as Nil | PlDataTableFilters; } diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/discoverColumns.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/discoverColumns.ts index 4d8b2f4fa0..9a8b6c6968 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/discoverColumns.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/discoverColumns.ts @@ -1,171 +1,75 @@ -import type { PColumnSpec, PlRef, PObjectId } from "@milaboratories/pl-model-common"; -import { createDiscoveredPColumnId, isPlRef, PColumnName } from "@milaboratories/pl-model-common"; -import type { RenderCtxBase } from "../../../render"; import type { - ColumnSource, - ColumnVariant, + PColumnSpec, + PlRef, + PObjectId, RelaxedColumnSelector, - ColumnSnapshotProvider, - ColumnSnapshot, -} from "../../../columns"; -import { ColumnCollectionBuilder } from "../../../columns"; -import { toColumnSnapshotProvider } from "../../../columns/column_snapshot_provider"; -import { collectCtxColumnSnapshotProviders } from "../../../columns/ctx_column_sources"; -import { throwError } from "@milaboratories/helpers"; -import type { ColumnsSelectorConfig, TableColumnVariant } from "./createPlDataTableV3"; +} from "@milaboratories/pl-model-common"; +import { PColumnName } from "@milaboratories/pl-model-common"; +import type { RenderCtxBase } from "../../../render"; +import type { ColumnRecipe, ColumnsSource } from "../../../columns"; +import { ColumnsCollection, collectLinkerIds, isLeafColumn } from "../../../columns"; +import type { ColumnsSelectorConfig } from "./createPlDataTableV3"; export type DiscoverTableColumnOptions = { - sources?: ColumnSource[]; + sources?: ColumnsSource[]; anchors: Record; selector: ColumnsSelectorConfig; }; -/** Discover columns from sources/anchors and normalize into a flat TableColumnVariant list. */ -export function discoverTableColumnSnaphots( +/** Split of discovered columns into anchor-matching (primary) and the rest. */ +export type DiscoveredTableColumns = { + readonly primary: ColumnRecipe[]; + readonly secondary: ColumnRecipe[]; +}; + +/** + * Discover columns from sources/anchors and split them into primary + * (direct anchor hits — zero-hop, query is a bare column) and secondary + * (reached via a linker chain, query carries a join). + */ +export function discoverTableColumns( ctx: RenderCtxBase, options: DiscoverTableColumnOptions, -): TableColumnVariant[] | undefined { - // Resolve PlRef anchors to PColumnSpec - const resolvedOptions = { - ...options, - anchors: resolveAnchors(ctx, options.anchors), - }; - - // Resolve providers - const providers = resolveProviders(ctx, resolvedOptions.sources); - if (providers.length === 0) return undefined; - - // Build collection (anchored or plain) - const collection = new ColumnCollectionBuilder(ctx.getService("pframeSpec")) - .addSources(providers) - .build(resolvedOptions); - if (collection === undefined) return undefined; +): undefined | DiscoveredTableColumns { + const discoveredColumns = ColumnsCollection(options.sources, { ctx: ctx.ctx }) + .discover({ ...options.selector, anchors: options.anchors }) + .getColumns(); - try { - const variants = collection.findColumnVariants(resolvedOptions.selector); - const anchors = collection.getAnchors(); - return mapToTableColumnVariants(variants, anchors); - } finally { - collection.dispose(); + const primary: ColumnRecipe[] = []; + const secondary: ColumnRecipe[] = []; + for (const col of discoveredColumns) { + if (isLeafColumn(col)) primary.push(col); + else secondary.push(col); } + return { primary, secondary }; } /** * Discover label columns matching the axes of the given value columns from - * ctx providers. Returns label snapshots wrapped as direct TableColumnVariants - * (path: [], empty qualifications, isPrimary: false). + * ctx providers. Returns a list of label {@link ColumnRecipe}s. + * + * Primary columns are used as anchors for axis-aware label discovery. + * `maxHops` is taken from the longest linker chain present in `columns`. */ -export function discoverLabelColumnVariants( +export function discoverLabelColumns( ctx: RenderCtxBase, - columns: TableColumnVariant[], -): TableColumnVariant[] { + primary: ColumnRecipe[], + columns: ColumnRecipe[], +): ColumnRecipe[] { if (columns.length === 0) return []; - const collection = new ColumnCollectionBuilder(ctx.getService("pframeSpec")) - .addSources(collectCtxColumnSnapshotProviders(ctx)) - .addSource(columns.map((c) => c.column)) - .build({ - allowPartialColumnList: true, - anchors: Object.fromEntries( - columns - .filter((col) => (col.path?.length ?? 0) === 0 && col.isPrimary) - .map((col, i) => [`anchor_${i}`, col.column.spec]), - ), - }); - try { - const axes = columns.flatMap((col) => col.column.spec.axesSpec); - return collection - .findColumnVariants({ + const axes = columns.flatMap((col) => col.getSpec().axesSpec); + return ( + ColumnsCollection(undefined, { ctx: ctx.ctx }) + // .addSource({ columns, isFinal: true }) + .discover({ include: axes.map((a) => ({ name: { type: "exact", value: PColumnName.Label }, axes: [{ name: { type: "exact", value: a.name } }], })), - maxHops: columns.reduce((acc, c) => Math.max(acc, c.path?.length ?? 0), 0), + anchors: Object.fromEntries(primary.map((col, i) => [`anchor_${i}`, col.getSpec()])), + maxHops: columns.reduce((acc, c) => Math.max(acc, collectLinkerIds(c).length), 0), }) - .map((variant) => ({ - ...variant, - column: { - ...variant.column, - id: createDiscoveredPColumnId({ - column: variant.column.id, - path: variant.path?.map((p) => ({ - type: "linker", - column: p.linker.id, - })), - columnQualifications: variant.qualifications?.forHit, - queriesQualifications: variant.qualifications?.forQueries, - }), - originalId: variant.column.id, - }, - })); - } finally { - collection.dispose(); - } -} - -/** Resolve PlRef values in anchors to PColumnSpec via the result pool. */ -function resolveAnchors( - ctx: RenderCtxBase, - anchors: Record, -): Record { - const result: Record = {}; - for (const [key, value] of Object.entries(anchors)) { - if (isPlRef(value)) { - result[key] = - ctx.resultPool.getPColumnSpecByRef(value) ?? - throwError( - `Anchor ${key} with ref ${JSON.stringify(value)} could not be resolved to a PColumnSpec`, - ); - } else { - result[key] = value; - } - } - return result; -} - -/** Resolve column snapshot providers from explicit sources or context. */ -function resolveProviders( - ctx: RenderCtxBase, - sources: undefined | ColumnSource[], -): ColumnSnapshotProvider[] { - return sources !== undefined - ? sources.map(toColumnSnapshotProvider) - : collectCtxColumnSnapshotProviders(ctx); -} - -/** Map column variants into TableColumnVariant list with anchor-derived isPrimary flag. */ -function mapToTableColumnVariants( - variants: readonly ColumnVariant[], - anchors: Map>, -): TableColumnVariant[] { - const columnIdToAnchorName = new Map( - Array.from(anchors.entries(), ([key, { id }]) => [id, key] as const), + .getColumns() ); - - return variants.map((variant): TableColumnVariant => { - const snap = variant.column; - const isPrimary = columnIdToAnchorName.get(snap.id) !== undefined; - - const discoveredId = createDiscoveredPColumnId({ - column: snap.id, - path: variant.path?.map((p) => ({ - type: "linker", - column: p.linker.id, - })), - columnQualifications: variant.qualifications?.forHit, - queriesQualifications: variant.qualifications?.forQueries, - }); - return { - column: { - id: discoveredId, - spec: snap.spec, - data: snap.data, - dataStatus: snap.dataStatus, - }, - path: variant.path, - qualifications: variant.qualifications, - originalId: snap.id, - isPrimary, - }; - }); } diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/index.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/index.ts index c97bec2a48..484f2e6ffc 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/index.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/index.ts @@ -1,5 +1,5 @@ import type { RenderCtxBase } from "../../../render"; -import type { PlDataTableModel } from "../typesV7"; +import type { PlDataTableModel } from "../typesV8"; import { createPlDataTableOptionsV2, createPlDataTableV2 } from "./createPlDataTableV2"; import { createPlDataTableV3 } from "./createPlDataTableV3"; import type { createPlDataTableOptionsV3 } from "./createPlDataTableV3"; @@ -26,4 +26,4 @@ export function createPlDataTable( } } -export { discoverTableColumnSnaphots } from "./discoverColumns"; +export { discoverTableColumns as discoverTableColumnSnaphots } from "./discoverColumns"; diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/utils.test.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/utils.test.ts index 5f366885da..9edac76fa8 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/utils.test.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/utils.test.ts @@ -1,8 +1,17 @@ -import { Annotation, type PColumnSpec, type PObjectId } from "@milaboratories/pl-model-common"; -import { SpecDriver } from "@milaboratories/pf-spec-driver"; -import { describe, expect, test } from "vitest"; -import { deriveAllLabels, evaluateRules, type LabelableColumn, type RuleColumn } from "./utils"; +import { + Annotation, + createGlobalPObjectId, + type PColumnSpec, + type PObjectId, +} from "@milaboratories/pl-model-common"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { deriveAllLabels, evaluateRules, type LabelableColumn } from "./utils"; import type { ColumnOrderRule, ColumnVisibilityRule } from "./createPlDataTableV3"; +import { ColumnLazy } from "../../../columns"; +import { + createTestCollectionDriver, + type TestCollectionDriverHandle, +} from "../../../columns/__test_helpers__/collection_driver"; // --------------------------------------------------------------------------- // Helpers @@ -100,11 +109,6 @@ describe("deriveAllLabels", () => { expect(result["c2"]).toBe("Score 2"); }); - // Regression: LabelableColumn.linkerPath used to be `linkerPath` while - // deriveDistinctLabels.Entry expected `linkersPath`. Structural typing + - // optional fields hid the mismatch from the type checker — the field flowed - // through as "extra" and disambiguation silently broke. This asserts the - // linker label reaches the output. test("linkerPath flows into deriveDistinctLabels and produces the 'via' suffix", () => { const linkerSpec = makeSpec({ name: "linker", @@ -136,92 +140,79 @@ describe("deriveAllLabels", () => { // evaluateRules // --------------------------------------------------------------------------- -function makeRuleColumn(id: string, spec: Partial = {}): RuleColumn { - return { - id: id as PObjectId, - spec: makeSpec({ axesSpec: [{ name: "id", type: "String" }], ...spec }), - }; +let driverHandle: TestCollectionDriverHandle; + +beforeEach(() => { + driverHandle = createTestCollectionDriver(); + driverHandle.installAmbientCtx(); +}); + +afterEach(async () => { + driverHandle.uninstallAmbientCtx(); + await driverHandle.dispose(); +}); + +function gid(name: string): PObjectId { + return createGlobalPObjectId("test-block", name); +} + +function makeLazyColumn(name: string, spec: Partial = {}): ColumnLazy { + const s = makeSpec({ axesSpec: [{ name: "id", type: "String" }], ...spec }); + const id = gid(name); + driverHandle.register([{ id, spec: s }]); + return ColumnLazy.fromColumn({ id, spec: s, data: undefined as never }); } describe("evaluateRules", () => { test("returns empty map when rules or columns are empty", () => { - const driver = new SpecDriver(); - expect(evaluateRules([], [makeRuleColumn("c1")], driver).size).toBe(0); + expect(evaluateRules([], [makeLazyColumn("c1")]).size).toBe(0); expect( - evaluateRules([{ match: () => true, visibility: "hidden" }], [], driver) - .size, + evaluateRules( + [{ match: { name: "anything" }, visibility: "hidden" }], + [], + ).size, ).toBe(0); }); - test("evaluates predicate rules without touching the driver", () => { - const driver = new Proxy({} as SpecDriver, { - get() { - throw new Error("driver should not be called for predicate-only rules"); - }, - }); - - const rules: ColumnOrderRule[] = [ - { match: (spec) => spec.name === "alpha", priority: 10 }, - { match: (spec) => spec.name === "beta", priority: 5 }, - ]; - const result = evaluateRules( - rules, - [ - makeRuleColumn("a", { name: "alpha" }), - makeRuleColumn("b", { name: "beta" }), - makeRuleColumn("c", { name: "gamma" }), - ], - driver, - ); - - expect(result.get("a" as PObjectId)?.priority).toBe(10); - expect(result.get("b" as PObjectId)?.priority).toBe(5); - expect(result.has("c" as PObjectId)).toBe(false); - }); - - test("evaluates selector rules via PFrameSpec.discoverColumns", () => { - const driver = new SpecDriver(); + test("evaluates selector rules via the columnsCollection service", () => { const rules: ColumnVisibilityRule[] = [ { match: { name: "^note$" }, visibility: "hidden" }, { match: { name: "^score$" }, visibility: "optional" }, ]; const columns = [ - makeRuleColumn("n", { name: "note" }), - makeRuleColumn("s", { name: "score" }), - makeRuleColumn("x", { name: "other" }), + makeLazyColumn("n", { name: "note" }), + makeLazyColumn("s", { name: "score" }), + makeLazyColumn("x", { name: "other" }), ]; - const result = evaluateRules(rules, columns, driver); + const result = evaluateRules(rules, columns); - expect(result.get("n" as PObjectId)?.visibility).toBe("hidden"); - expect(result.get("s" as PObjectId)?.visibility).toBe("optional"); - expect(result.has("x" as PObjectId)).toBe(false); + expect(result.get(gid("n"))?.visibility).toBe("hidden"); + expect(result.get(gid("s"))?.visibility).toBe("optional"); + expect(result.has(gid("x"))).toBe(false); }); - test("preserves original rule order when predicate and selector rules are mixed", () => { - const driver = new SpecDriver(); + test("preserves original rule order with overlapping selector rules", () => { const rules: ColumnOrderRule[] = [ - { match: (spec) => spec.name === "alpha", priority: 100 }, - { match: { name: "^alpha$" }, priority: 1 }, // shadowed by the predicate above + { match: { name: "^alpha$" }, priority: 100 }, + { match: { name: "^alpha$" }, priority: 1 }, // shadowed by the earlier rule { match: { name: "^beta$" }, priority: 50 }, ]; - const result = evaluateRules( - rules, - [makeRuleColumn("a", { name: "alpha" }), makeRuleColumn("b", { name: "beta" })], - driver, - ); - - expect(result.get("a" as PObjectId)?.priority).toBe(100); - expect(result.get("b" as PObjectId)?.priority).toBe(50); + const result = evaluateRules(rules, [ + makeLazyColumn("a", { name: "alpha" }), + makeLazyColumn("b", { name: "beta" }), + ]); + + expect(result.get(gid("a"))?.priority).toBe(100); + expect(result.get(gid("b"))?.priority).toBe(50); }); test("dedupes columns by id before building spec frame (no duplicate-key crash)", () => { - const driver = new SpecDriver(); const rules: ColumnVisibilityRule[] = [{ match: { name: "^dup$" }, visibility: "hidden" }]; - const dup = makeRuleColumn("d", { name: "dup" }); + const dup = makeLazyColumn("d", { name: "dup" }); - const result = evaluateRules(rules, [dup, dup, dup], driver); + const result = evaluateRules(rules, [dup, dup, dup]); - expect(result.get("d" as PObjectId)?.visibility).toBe("hidden"); + expect(result.get(gid("d"))?.visibility).toBe("hidden"); }); }); diff --git a/sdk/model/src/components/PlDataTable/createPlDataTable/utils.ts b/sdk/model/src/components/PlDataTable/createPlDataTable/utils.ts index 9e5ba590e8..ad2635900b 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTable/utils.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTable/utils.ts @@ -1,11 +1,11 @@ import { + type AnnotationDataStatus, + type ColumnUniversalId, type PColumn, type PColumnSpec, - type PFrameSpecDriver, type PObjectId, Annotation, - canonicalizeAxisId, - DiscoveredPColumnId, + extractPObjectId, readAnnotation, } from "@milaboratories/pl-model-common"; import { @@ -16,13 +16,18 @@ import { deriveDistinctTooltips, type TooltipEntry, } from "../../../labels/derive_distinct_tooltips"; -import type { ColumnDataStatus, MatchQualifications, MatchVariant } from "../../../columns"; -import type { ColumnMatcher, ColumnOrderRule, ColumnVisibilityRule } from "./createPlDataTableV3"; -import type { ColumnSelector } from "../../../columns"; -import { ArrayColumnProvider, ColumnCollectionBuilder } from "../../../columns"; -import { isNil } from "es-toolkit"; +import type { ColumnOrderRule, ColumnVisibilityRule } from "./createPlDataTableV3"; +import type { ColumnSelector, MatchQualifications } from "@milaboratories/pl-model-common"; +import { ColumnsCollection, hasColumnData, type ColumnRecipe } from "../../../columns"; +import type { GlobalCfgRenderCtx } from "../../../render/internal"; +import { isNil, uniqBy } from "es-toolkit"; import { getField } from "@milaboratories/helpers"; +/** Adapt a {@link ColumnRecipe} to the plain {@link RuleColumn} shape. */ +export function toRuleColumn(col: ColumnRecipe): RuleColumn { + return { id: extractPObjectId(col.id), spec: col.getSpec() }; +} + /** Check if column should be omitted from the table */ export function isColumnHidden(spec: { annotations?: Annotation }): boolean { return readAnnotation(spec, Annotation.Table.Visibility) === "hidden"; @@ -64,47 +69,36 @@ export function getOrderPriority( * (first-match-wins, preserving original rule order). * * Predicate-based rules (`ColumnMatcher`) are evaluated directly on the spec. - * Selector-based rules (`ColumnSelector`) are matched via `PFrameSpecDriver.discoverColumns` - * using the same engine as `ColumnCollection.findColumns` — no client-side matcher. + * Selector-based rules (`ColumnSelector`) are matched via the + * `columnsCollection` service — the inline column ids must be resolvable in + * the active render ctx for the host to read their specs. */ -export function evaluateRules( +export function evaluateRules( rules: R[], - columns: RuleColumn[], - pframeSpec: PFrameSpecDriver, + columns: ColumnRecipe[], ): Map { const result = new Map(); if (rules.length === 0 || columns.length === 0) return result; - const hasSelectorRules = rules.some((rule) => typeof rule.match !== "function"); + const selectorRules = rules.filter((rule) => typeof rule.match !== "function"); const selectorHitsByRule = new Map>(); - if (hasSelectorRules) { - const dedupedColumns = dedupeById(columns); - const pColumns = dedupedColumns.map((c) => ({ id: c.id, spec: c.spec, data: undefined })); - const collection = new ColumnCollectionBuilder(pframeSpec) - .addSource(new ArrayColumnProvider(pColumns)) - .build(); - if (collection === undefined) return result; - - try { - for (const rule of rules) { - if (typeof rule.match === "function") continue; - const hits = collection.findColumns({ include: rule.match }); - selectorHitsByRule.set(rule, new Set(hits.map((h) => h.id))); - } - } finally { - collection.dispose(); + if (selectorRules.length > 0) { + const baseColumns = uniqBy(columns, (col) => col.id); + const baseCollection = ColumnsCollection([{ columns: baseColumns, isFinal: true }]); + for (const rule of selectorRules) { + const hitIds = baseCollection.filter({ include: rule.match }).getColumnIds(); + if (hitIds.length === 0) return result; + selectorHitsByRule.set(rule, new Set(hitIds.map((id) => extractPObjectId(id)))); } } for (const col of columns) { + const colKey = extractPObjectId(col.id); for (const rule of rules) { - const matches = - typeof rule.match === "function" - ? rule.match(col.spec) - : (selectorHitsByRule.get(rule)?.has(col.id) ?? false); + const matches = selectorHitsByRule.get(rule)?.has(colKey) ?? false; if (matches) { - result.set(col.id, rule); + result.set(colKey, rule); break; } } @@ -112,146 +106,46 @@ export function evaluateRules(); - const result: RuleColumn[] = []; - for (const col of columns) { - if (seen.has(col.id)) continue; - seen.add(col.id); - result.push(col); - } - return result; -} - /** - * Writes derived labels into column and axis annotations. - * Returns new column objects with modified specs — original columns are not mutated. + * Build a sidecar map `columnId → AnnotationDataStatus` derived from each + * recipe's own status. This is *rendering* metadata — kept out of the spec + * so it cannot perturb column identity (id-bearing spec overrides would + * diverge the visible PTable's column ids from the full table / pframe). * - * For each column: writes derived label into Annotation.Label (if present in derivedLabels). - * For each axis in column specs: writes derived axis label into AxisSpec annotations. - */ -export function withLabelAnnotations< - T extends { - readonly id: PObjectId; - readonly spec: PColumnSpec; - }, ->(derivedLabels: undefined | Record, columns: T[]): T[] { - if (derivedLabels === undefined) return columns; - return columns.map((col) => { - const colLabel = derivedLabels[col.id]; - return { - ...col, - spec: { - ...col.spec, - ...(isNil(colLabel) - ? {} - : { annotations: { ...col.spec.annotations, [Annotation.Label]: colLabel } }), - axesSpec: col.spec.axesSpec.map((axis) => { - const label = derivedLabels[canonicalizeAxisId(axis)]; - return isNil(label) - ? axis - : { ...axis, annotations: { ...axis.annotations, [Annotation.Label]: label } }; - }), - }, - } as T; - }); -} - -export function withDataStatusAnnotations< - T extends { - readonly spec: PColumnSpec; - readonly dataStatus: ColumnDataStatus; - }, ->(columns: T[]): T[] { - return columns.map((col) => { - return { - ...col, - spec: { - ...col.spec, - annotations: { - ...col.spec.annotations, - [Annotation.DataStatus]: col.dataStatus, - }, - }, - } as T; - }); -} - -/** - * Writes effective display properties (OrderPriority, Visibility) from precomputed rule maps - * into column annotations. Returns new column objects — originals are not mutated. - */ -export function withTableVisualAnnotations< - T extends { readonly id: PObjectId; readonly spec: PColumnSpec }, ->( - visibilityByColId: undefined | Map, - orderByColId: undefined | Map, - columns: T[], -): T[] { - if (visibilityByColId === undefined && orderByColId === undefined) return columns; - return columns.map((col) => { - const annotations = { ...col.spec.annotations }; - - const visibility = getEffectiveVisibility(col, visibilityByColId); - if (!isNil(visibility)) annotations[Annotation.Table.Visibility] = visibility; - - const orderPriority = getOrderPriority(col, orderByColId); - if (!isNil(orderPriority)) annotations[Annotation.Table.OrderPriority] = String(orderPriority); - - return { - ...col, - spec: { - ...col.spec, - annotations: annotations, - }, - } as T; - }); -} - -/** - * Writes derived info annotations into column annotations. - * Columns without an info entry are passed through unchanged. + * Mapping `ColumnFieldStatus` × actual byte-presence → `AnnotationDataStatus`: + * - `absent` → `"absent"` + * - `resolving` → `"computing"` + * - `present` + `hasColumnData()` → `"ready"` + * - `present` + no bytes yet → `"computing"` + * + * Intentionally scoped to visible columns — the probe is meaningful only for + * what the user will actually see. */ -export function withInfoAnnotations< - T extends { readonly id: PObjectId; readonly spec: PColumnSpec }, ->(infoById: undefined | Record, columns: T[]): T[] { - if (isNil(infoById)) return columns; - return columns.map((col) => { - const info = infoById[col.id]; - if (isNil(info)) return col; - return { - ...col, - spec: { - ...col.spec, - annotations: { ...col.spec.annotations, [Annotation.Table.Info]: info }, - }, - } as T; - }); +export function buildDataStatusMap( + columns: ColumnRecipe[], + opts: { ctx?: GlobalCfgRenderCtx } = {}, +): Record { + return columns.reduce( + (acc, c) => ((acc[c.id] = deriveDataStatus(c, opts)), acc), + {} as Record, + ); } -export function withHidenAxesAnnotations( - columns: T[], -): T[] { - return columns.map( - (col) => - ({ - ...col, - spec: { - ...col.spec, - axesSpec: col.spec.axesSpec.map((axis) => ({ - ...axis, - annotations: { ...axis.annotations, [Annotation.Table.Visibility]: "hidden" }, - })), - }, - }) as T, - ); +function deriveDataStatus( + column: ColumnRecipe, + opts: { ctx?: GlobalCfgRenderCtx }, +): AnnotationDataStatus { + const field = column.getDataStatus(); + if (field === "absent") return "absent"; + if (field === "resolving") return "computing"; + return hasColumnData(column, opts) ? "ready" : "computing"; } /** Column shape required by label derivation. */ export type LabelableColumn = { - readonly id: PObjectId; + readonly id: ColumnUniversalId; readonly spec: PColumnSpec; - readonly linkerPath?: MatchVariant["path"]; + readonly linkerPath?: ReadonlyArray<{ readonly linker: { readonly spec: PColumnSpec } }>; readonly qualifications?: MatchQualifications; }; @@ -263,9 +157,7 @@ export function deriveAllLabels(options: { const { columns, deriveLabelsOptions } = options; const entries = columns.map((c) => ({ spec: c.spec, - linkerPath: c.linkerPath?.map((step) => ({ - spec: step.linker.spec, - })), + linkerPath: c.linkerPath?.map((step) => ({ spec: step.linker.spec })), qualifications: c.qualifications, })); const columnLabels = deriveDistinctLabels(entries, deriveLabelsOptions).reduce( @@ -277,20 +169,20 @@ export function deriveAllLabels(options: { /** Column shape required by tooltip derivation. */ export type TooltipableColumn = { - readonly id: PObjectId | DiscoveredPColumnId; + readonly id: ColumnUniversalId; readonly spec: PColumnSpec; readonly originalId?: PObjectId; - readonly linkerPath?: MatchVariant["path"]; + readonly linkerPath?: ReadonlyArray<{ readonly linker: { readonly spec: PColumnSpec } }>; readonly qualifications?: MatchQualifications; }; /** Derive origin tooltips for columns whose qualifications or linker path carry info. */ export function deriveAllTooltips(options: { columns: TooltipableColumn[]; -}): Record { +}): Record { const { columns } = options; - const variantCountByOriginal = columns.reduce>((acc, c) => { + const variantCountByOriginal = columns.reduce>((acc, c) => { return acc.set(getField(c, "originalId") ?? c.id, (acc.get(c.originalId ?? c.id) ?? 0) + 1); }, new Map()); @@ -311,7 +203,7 @@ export function deriveAllTooltips(options: { return { entries, variantSeen }; }, - { entries: [] as TooltipEntry[], variantSeen: new Map() }, + { entries: [] as TooltipEntry[], variantSeen: new Map() }, ); const tooltips = deriveDistinctTooltips(entries); diff --git a/sdk/model/src/components/PlDataTable/createPlDataTableSheet.ts b/sdk/model/src/components/PlDataTable/createPlDataTableSheet.ts index 4903050215..1be3b83626 100644 --- a/sdk/model/src/components/PlDataTable/createPlDataTableSheet.ts +++ b/sdk/model/src/components/PlDataTable/createPlDataTableSheet.ts @@ -1,6 +1,6 @@ import type { AxisSpec } from "@milaboratories/pl-model-common"; import type { RenderCtxBase } from "../../render"; -import type { PlDataTableSheet } from "./typesV7"; +import type { PlDataTableSheet } from "./typesV8"; /** Create sheet entries for PlDataTable */ export function createPlDataTableSheet( diff --git a/sdk/model/src/components/PlDataTable/index.ts b/sdk/model/src/components/PlDataTable/index.ts index 09e803e04e..4a30d9d560 100644 --- a/sdk/model/src/components/PlDataTable/index.ts +++ b/sdk/model/src/components/PlDataTable/index.ts @@ -9,11 +9,14 @@ export type { PTableParamsV2, PlDataTableStateV2Normalized, PlDataTableModel, + PlDataTableColumnMeta, + PlDataTableAxisMeta, + PlDataTableColumnsMeta, PlDataTableFilterSpecLeaf, PlDataTableFilterMeta, PlDataTableFilters, PlDataTableFiltersWithMeta, -} from "./typesV7"; +} from "./typesV8"; export type { PlDataTableStateV2 } from "./state-migration"; export { @@ -37,7 +40,6 @@ export type { ColumnsDisplayOptions, ColumnOrderRule, ColumnVisibilityRule, - ColumnMatcher, ColumnsSelectorConfig, createPlDataTableOptionsV3, } from "./createPlDataTable/createPlDataTableV3"; diff --git a/sdk/model/src/components/PlDataTable/labels.ts b/sdk/model/src/components/PlDataTable/labels.ts index c31d23767e..8d2cc7f28b 100644 --- a/sdk/model/src/components/PlDataTable/labels.ts +++ b/sdk/model/src/components/PlDataTable/labels.ts @@ -1,41 +1,12 @@ import type { AxisId, PColumn, PColumnSpec, PObjectId } from "@milaboratories/pl-model-common"; -import { - getAxisId, - isLabelColumn, - matchAxisId, - PColumnName, -} from "@milaboratories/pl-model-common"; -import type { PColumnDataUniversal, RenderCtxBase } from "../../render"; -import { ColumnCollectionBuilder, collectCtxColumnSnapshotProviders } from "../../columns"; - -/** - * Get all label columns visible in the current render context - * (result pool + block outputs + prerun). - */ -export function getAllLabelColumns( - ctx: RenderCtxBase, -): PColumn[] { - const pframeSpec = ctx.getService("pframeSpec"); - const collection = new ColumnCollectionBuilder(pframeSpec) - .addSources(collectCtxColumnSnapshotProviders(ctx)) - .build({ allowPartialColumnList: true }); - try { - return collection - .findColumns({ include: { name: PColumnName.Label, axes: [] } }) - .reduce[]>((acc, hit) => { - const data = hit.data?.get(); - return data === undefined ? acc : [...acc, { id: hit.id, spec: hit.spec, data }]; - }, []); - } finally { - collection.dispose(); - } -} +import { getAxisId, isLabelColumn, matchAxisId } from "@milaboratories/pl-model-common"; +import type { PColumnDataUniversal } from "../../render"; /** Get label columns matching the provided columns from the result pool */ export function getMatchingLabelColumns( columns: { spec: PColumnSpec }[], - allLabelColumns: PColumn[], -): PColumn[] { + allLabelColumns: PColumn[], +): PColumn[] { // split input columns into label and value columns const inputLabelColumns: typeof columns = []; const inputValueColumns: typeof columns = []; diff --git a/sdk/model/src/components/PlDataTable/state-migration.ts b/sdk/model/src/components/PlDataTable/state-migration.ts index 578bef7276..78a3253a2a 100644 --- a/sdk/model/src/components/PlDataTable/state-migration.ts +++ b/sdk/model/src/components/PlDataTable/state-migration.ts @@ -15,13 +15,21 @@ import { import { distillFilterSpec } from "../../filters"; import type { PlDataTableFilterState, PlTableFilter } from "./typesV4"; import type { + PlDataTableFilters, PlDataTableFiltersWithMeta, - PlDataTableGridStateCore, PlDataTableSheetState, PlDataTableStateV2CacheEntry, PlDataTableStateV2Normalized, - PlTableColumnIdJson, PTableParamsV2, +} from "./typesV8"; +import type { + PlDataTableFilters as PlDataTableFiltersV7, + PlDataTableFiltersWithMeta as PlDataTableFiltersWithMetaV7, + PlDataTableGridStateCore as PlDataTableGridStateCoreV7, + PlDataTableStateV2CacheEntry as PlDataTableStateV2V7CacheEntry, + PlDataTableStateV2Normalized as PlDataTableStateV2V7, + PlTableColumnIdJson, + PTableParamsV2 as PTableParamsV2V7, } from "./typesV7"; import type { PlDataTableGridStateV6, @@ -111,7 +119,7 @@ export type PlDataTableStateV2 = sheetsState: PlDataTableSheetState[]; filtersState: PlDataTableFilterState[]; }[]; - pTableParams: PTableParamsV2; + pTableParams: PTableParamsV2V7; } | { version: 4; @@ -135,15 +143,18 @@ export type PlDataTableStateV2 = sourceId: string; gridState: PlDataTableGridStateV5; sheetsState: PlDataTableSheetState[]; - filtersState: null | PlDataTableFiltersWithMeta; - defaultFiltersState: null | PlDataTableFiltersWithMeta; + filtersState: null | PlDataTableFiltersWithMetaV7; + defaultFiltersState: null | PlDataTableFiltersWithMetaV7; searchString?: string; }[]; - pTableParams: PTableParamsV2; + pTableParams: PTableParamsV2V7; } // v6 stored colIds as canonicalized full `PTableColumnSpec` (including // annotations + `pl7.app/trace`). v7 strips down to `PTableColumnId`. | PlDataTableStateV2V6 + // v7 still had filter leaves serialised as `CanonicalizedJson` + // strings and `sorting: []` for the "untouched" case. + | PlDataTableStateV2V7 // Normalized state | PlDataTableStateV2Normalized; @@ -168,7 +179,7 @@ export function upgradePlDataTableStateV2( ...entry, filtersState: [], })), - pTableParams: createDefaultPTableParams(), + pTableParams: createDefaultPTableParamsV7(), }; } // v3 -> v4 @@ -182,7 +193,7 @@ export function upgradePlDataTableStateV2( if (state.version === 4) { state = migrateV4toV6(state); } - // v5 -> v6: unwrap `{source, labeled}` colIds in gridState back to bare PTableColumnSpec. + // v5 -> v6: unwrap `{source, labeled}` colIds in gridState. if (state.version === 5) { state = migrateV5toV6(state); } @@ -190,12 +201,17 @@ export function upgradePlDataTableStateV2( if (state.version === 6) { state = migrateV6toV7(state); } + // v7 -> v8: parse CanonicalizedJson filter leaves into object + // form; translate `sorting: []` into `sorting: null`. + if (state.version === 7) { + state = migrateV7toV8(state); + } return state; } /** Migrate v5 to v6: unwrap `{source, labeled}` colIds in gridState. */ function migrateV5toV6(state: Extract): PlDataTableStateV2V6 { - // pTableParams reset: v5 stored DiscoveredPColumnId-based hiddenColIds with + // pTableParams reset: v5 stored ColumnDiscoveredId-based hiddenColIds with // empty-array fields (e.g. `{column, path: [], columnQualifications: [], ...}`). // v6 distills empty fields, so the same logical column serialises differently // and lookups would silently miss every previously-hidden discovered column. @@ -206,7 +222,7 @@ function migrateV5toV6(state: Extract): PlDa ...entry, gridState: unwrapV5GridState(entry.gridState), })), - pTableParams: createDefaultPTableParams(), + pTableParams: createDefaultPTableParamsV7(), }; } @@ -242,11 +258,11 @@ function unwrapV5GridState(gridState: PlDataTableGridStateV5): PlDataTableGridSt /** Migrate v6 to v7: rewrite each colId from a full PTableColumnSpec to its * compact PTableColumnId (drops annotations/spec body, ~16× smaller per column). */ -function migrateV6toV7(state: PlDataTableStateV2V6): PlDataTableStateV2Normalized { +function migrateV6toV7(state: PlDataTableStateV2V6): PlDataTableStateV2V7 { return { version: 7, stateCache: state.stateCache.map( - (entry): PlDataTableStateV2CacheEntry => ({ + (entry): PlDataTableStateV2V7CacheEntry => ({ ...entry, gridState: shrinkV6GridState(entry.gridState), }), @@ -266,7 +282,7 @@ function shrinkV6ColId(json: PlDataTableV6ColIdJson): PlTableColumnIdJson { return json as unknown as PlTableColumnIdJson; } -function shrinkV6GridState(gridState: PlDataTableGridStateV6): PlDataTableGridStateCore { +function shrinkV6GridState(gridState: PlDataTableGridStateV6): PlDataTableGridStateCoreV7 { return { columnOrder: gridState.columnOrder ? { orderedColIds: gridState.columnOrder.orderedColIds.map(shrinkV6ColId) } @@ -291,14 +307,13 @@ function migrateV4toV6(state: Extract): PlDa const nextId = () => ++idCounter; const migratedCache: PlDataTableStateV2V6CacheEntry[] = state.stateCache.map((entry) => { - const leaves: PlDataTableFiltersWithMeta["filters"] = []; + const leaves: PlDataTableFiltersWithMetaV7["filters"] = []; for (const f of entry.filtersState) { if (f.filter !== null && !f.filter.disabled) { - const column = canonicalizeJson(f.id); - leaves.push(migrateTableFilter(column, f.filter.value, nextId)); + leaves.push(migrateTableFilter(canonicalizeJson(f.id), f.filter.value, nextId)); } } - const filtersState: PlDataTableFiltersWithMeta | null = + const filtersState: PlDataTableFiltersWithMetaV7 | null = leaves.length > 0 ? { id: nextId(), type: "and", filters: leaves } : null; return { @@ -329,16 +344,17 @@ function migrateV4toV6(state: Extract): PlDa defaultFilters: null, sorting: state.pTableParams.sorting, } - : createDefaultPTableParams(), + : createDefaultPTableParamsV7(), }; } -/** Migrate a single per-column PlTableFilter to a tree-based FilterSpec node */ +/** Migrate a single per-column PlTableFilter to a v7-form tree node (legacy + * string column ids). v7 → v8 later converts these to object form. */ function migrateTableFilter( column: CanonicalizedJson, filter: PlTableFilter, nextId: () => number, -): PlDataTableFiltersWithMeta["filters"][number] { +): PlDataTableFiltersWithMetaV7["filters"][number] { const id = nextId(); switch (filter.type) { case "isNA": @@ -404,7 +420,9 @@ function migrateTableFilter( } } -export function createDefaultPTableParams(): PTableParamsV2 { +/** Default pTableParams in the v7 shape (sorting: []). Used by intermediate + * migration steps that produce v6/v7 state before the v7 → v8 conversion. */ +function createDefaultPTableParamsV7(): PTableParamsV2V7 { return { sourceId: null, hiddenColIds: null, @@ -414,10 +432,121 @@ export function createDefaultPTableParams(): PTableParamsV2 { }; } +export function createDefaultPTableParams(): PTableParamsV2 { + return { + sourceId: null, + hiddenColIds: null, + filters: null, + defaultFilters: null, + sorting: null, + }; +} + export function createPlDataTableStateV2(): PlDataTableStateV2Normalized { return { - version: 7, + version: 8, stateCache: [], pTableParams: createDefaultPTableParams(), }; } + +// --- v7 -> v8 migration ---------------------------------------------------- +// Parses CanonicalizedJson filter leaves into the new object +// form. Malformed leaves are dropped; groups that become empty after pruning +// are dropped; cache entries whose tree fully prunes get filtersState: null. +// Also translates `sorting: []` (legacy "untouched") into `sorting: null`. + +function migrateV7toV8(state: PlDataTableStateV2V7): PlDataTableStateV2Normalized { + const newCache: PlDataTableStateV2CacheEntry[] = state.stateCache.map((entry) => ({ + sourceId: entry.sourceId, + gridState: entry.gridState, + sheetsState: entry.sheetsState, + filtersState: convertFiltersWithMeta(entry.filtersState), + defaultFiltersState: convertFiltersWithMeta(entry.defaultFiltersState), + searchString: entry.searchString, + })); + + const params = state.pTableParams; + let pTableParams: PTableParamsV2; + if (params.sourceId === null) { + pTableParams = createDefaultPTableParams(); + } else { + pTableParams = { + sourceId: params.sourceId, + hiddenColIds: params.hiddenColIds, + // Empty array in legacy state means "not touched" — translate to null so the + // model falls back to default sorting; non-empty stays as user's explicit value. + sorting: params.sorting.length === 0 ? null : params.sorting, + filters: convertFiltersPlain(params.filters), + defaultFilters: convertFiltersPlain(params.defaultFilters), + }; + } + + return { version: 8, stateCache: newCache, pTableParams }; +} + +function convertFiltersWithMeta( + node: null | PlDataTableFiltersWithMetaV7, +): null | PlDataTableFiltersWithMeta { + if (node === null) return null; + const result = pruneFilterNode(node as unknown); + if (result === undefined) return null; + return result as PlDataTableFiltersWithMeta; +} + +function convertFiltersPlain(node: null | PlDataTableFiltersV7): null | PlDataTableFilters { + if (node === null) return null; + const result = pruneFilterNode(node as unknown); + if (result === undefined) return null; + return result as PlDataTableFilters; +} + +/** Recursively converts and prunes a legacy v7 filter tree. Preserves meta + * fields (id/isExpanded/source/isSuppressed) on non-leaf nodes. Returns + * `undefined` when the subtree is empty after pruning. */ +function pruneFilterNode(node: unknown): unknown | undefined { + if (!node || typeof node !== "object") return undefined; + const n = node as { type?: unknown }; + if (n.type === "and" || n.type === "or") { + const filters = (n as { filters?: unknown[] }).filters ?? []; + const kept = filters.map((f) => pruneFilterNode(f)).filter((f): f is object => f !== undefined); + if (kept.length === 0) return undefined; + return { ...(n as object), type: n.type, filters: kept }; + } + if (n.type === "not") { + const inner = pruneFilterNode((n as { filter?: unknown }).filter); + if (inner === undefined) return undefined; + return { ...(n as object), type: "not", filter: inner }; + } + return convertLegacyLeaf(n); +} + +function convertLegacyLeaf(leaf: { + type?: unknown; + column?: unknown; + rhs?: unknown; +}): unknown | undefined { + if (leaf.type === undefined) return leaf; + const result: Record = { ...leaf }; + if ("column" in result) { + const c = parseLegacyLeafColumn(result.column); + if (c === undefined) return undefined; + result.column = c; + } + if ("rhs" in result) { + const c = parseLegacyLeafColumn(result.rhs); + if (c === undefined) return undefined; + result.rhs = c; + } + return result; +} + +function parseLegacyLeafColumn(s: unknown): PTableColumnId | undefined { + if (typeof s !== "string") return undefined; + const parsed = parseJsonSafely(s as CanonicalizedJson); + if (!parsed || typeof parsed !== "object") return undefined; + if (!("type" in parsed) || !("id" in parsed)) return undefined; + const t = (parsed as { type: unknown }).type; + if (t !== "axis" && t !== "column") return undefined; + return parsed as PTableColumnId; +} diff --git a/sdk/model/src/components/PlDataTable/typesV8.ts b/sdk/model/src/components/PlDataTable/typesV8.ts new file mode 100644 index 0000000000..4cade3bd9a --- /dev/null +++ b/sdk/model/src/components/PlDataTable/typesV8.ts @@ -0,0 +1,188 @@ +import type { + AnnotationDataStatus, + AxisId, + AxisSpec, + CanonicalizedJson, + ColumnUniversalId, + ListOptionBase, + PTableSorting, + PColumnIdAndSpec, + PTableHandle, + RootFilterSpec, + PTableColumnId, + PFrameHandle, +} from "@milaboratories/pl-model-common"; +import type { FilterSpecLeaf } from "../../filters"; +import { Nil } from "@milaboratories/helpers"; + +export type PlTableColumnIdJson = CanonicalizedJson; + +export type PlDataTableGridStateCore = { + /** Includes column ordering */ + columnOrder?: { + /** All colIds in order */ + orderedColIds: PlTableColumnIdJson[]; + }; + /** Includes current sort columns and direction */ + sort?: { + /** Sorted columns and directions in order */ + sortModel: { + /** Column Id to apply the sort to. */ + colId: PlTableColumnIdJson; + /** Sort direction */ + sort: "asc" | "desc"; + }[]; + }; + /** Includes column visibility */ + columnVisibility?: { + /** All colIds which were hidden */ + hiddenColIds: PlTableColumnIdJson[]; + }; +}; + +export type PlDataTableSheet = { + /** spec of the axis to use */ + axis: AxisSpec; + /** options to show in the filter dropdown */ + options: ListOptionBase[]; + /** default (selected) value */ + defaultValue?: string | number; +}; + +export type PlDataTableSheetState = { + /** id of the axis */ + axisId: AxisId; + /** selected value */ + value: string | number; +}; + +/** Tree-based filter state compatible with PlAdvancedFilter's RootFilter */ +export type PlDataTableFilterMeta = { + id: number; + source?: "table-filter" | "table-search"; + isExpanded?: boolean; + isSuppressed?: boolean; +}; +export type PlDataTableFilterSpecLeaf = FilterSpecLeaf; +export type PlDataTableFilters = RootFilterSpec; +export type PlDataTableFiltersWithMeta = RootFilterSpec< + PlDataTableFilterSpecLeaf, + PlDataTableFilterMeta +>; + +export type PlDataTableStateV2CacheEntry = { + /** DataSource identifier for state management */ + sourceId: string; + /** Internal ag-grid state */ + gridState: PlDataTableGridStateCore; + /** Sheets state */ + sheetsState: PlDataTableSheetState[]; + /** User filters state (tree-based, compatible with PlAdvancedFilter) */ + filtersState: null | PlDataTableFiltersWithMeta; + /** Default filters state from model (snapshot of defaults) */ + defaultFiltersState: null | PlDataTableFiltersWithMeta; + /** Fast search string */ + searchString?: string; +}; + +export type PTableParamsV2 = + | { + sourceId: null; + hiddenColIds: null; + sorting: null; + filters: null; + defaultFilters: null; + } + | { + sourceId: string; + hiddenColIds: null | PTableColumnId[]; + sorting: null | PTableSorting[]; + filters: null | PlDataTableFilters; + defaultFilters: null | PlDataTableFilters; + }; + +export type PlDataTableStateV2Normalized = { + /** Version for upgrades */ + version: 8; + /** Internal states, LRU cache for 5 sourceId-s */ + stateCache: PlDataTableStateV2CacheEntry[]; + /** PTable params derived from the cache state for the current sourceId */ + pTableParams: PTableParamsV2; +}; + +/** PlAgDataTable model */ +export type PlDataTableModel = { + /** DataSource identifier for state management */ + sourceId: null | string; + /** p-table including all columns, used to show the full specification of the table */ + fullTableHandle?: PTableHandle; + /** p-frame handle */ + fullPframeHandle?: PFrameHandle; + /** p-table including only visible columns, used to get the data */ + visibleTableHandle?: PTableHandle; + /** Default filters from model options, surfaced for UI display */ + defaultFilters?: Nil | PlDataTableFilters; + /** + * Sidecar rendering metadata kept out of the column specs: baking + * label/visibility/order/status into a spec via overrides changes the + * recipe's `ColumnUniversalId`, so the same physical column reached two ways + * (e.g. as a primary column and as a discovered axis label) would diverge + * into two ids and render twice. The UI overlays this onto the + * engine-emitted specs at render time. + */ + columnsMeta?: PlDataTableColumnsMeta; +}; + +/** Display metadata for one column, applied by the UI over the emitted spec. */ +export type PlDataTableColumnMeta = { + /** Disambiguated label (overrides the spec's intrinsic `pl7.app/label`). */ + label?: string; + /** Effective visibility from rules + intrinsic annotations. */ + visibility?: "default" | "optional" | "hidden"; + /** Effective order priority (higher = further left). */ + order?: number; + /** Per-column data status (set for visible columns only). */ + status?: AnnotationDataStatus; +}; + +/** Display metadata for one axis, applied by the UI over the emitted spec. */ +export type PlDataTableAxisMeta = { + /** Whether the UI must hide this axis (axes introduced only by secondary columns). */ + hidden: boolean; +}; + +/** Sidecar display metadata for a table, keyed by emitted column/axis identity. */ +export type PlDataTableColumnsMeta = { + /** Per-column metadata keyed by the emitted `ColumnUniversalId`. */ + columns: Record; + /** Per-axis metadata keyed by the canonicalized `AxisId`. */ + axes: Record, PlDataTableAxisMeta>; +}; + +export type CreatePlDataTableOps = { + /** Filters for columns and non-partitioned axes */ + filters?: PlDataTableFilters; + + /** Sorting to columns hidden from user */ + sorting?: PTableSorting[]; + + /** + * Selects columns for which will be inner-joined to the table. + * + * Default behaviour: all columns are considered to be core + */ + coreColumnPredicate?: (spec: PColumnIdAndSpec) => boolean; + + /** + * Determines how core columns should be joined together: + * inner - so user will only see records present in all core columns + * full - so user will only see records present in any of the core columns + * + * All non-core columns will be left joined to the table produced by the core + * columns, in other words records form the pool of non-core columns will only + * make their way into the final table if core table contains corresponding key. + * + * Default: 'full' + */ + coreJoinType?: "inner" | "full"; +}; diff --git a/sdk/model/src/components/PlDatasetSelector/build_dataset_options.ts b/sdk/model/src/components/PlDatasetSelector/build_dataset_options.ts index c116211026..2c15e0ba5c 100644 --- a/sdk/model/src/components/PlDatasetSelector/build_dataset_options.ts +++ b/sdk/model/src/components/PlDatasetSelector/build_dataset_options.ts @@ -2,15 +2,10 @@ import type { MultiColumnSelector, Option, PObjectSpec } from "@milaboratories/p import { multiColumnSelectorsToPredicate } from "@milaboratories/pl-model-common"; import type { DeriveLabelsOptions } from "../../labels/derive_distinct_labels"; import type { RenderCtxBase } from "../../render"; -import type { AnchoredColumnCollection } from "../../columns/column_collection_builder"; -import { ColumnCollectionBuilder } from "../../columns/column_collection_builder"; -import { - ResultPoolColumnSnapshotProvider, - collectCtxColumnSnapshotProviders, -} from "../../columns/ctx_column_sources"; import type { DatasetOption } from "./dataset_selection"; import { buildRefMap, filterMatchesToOptions, findFilterColumns } from "./filter_discovery"; import { enrichmentVariantsToRefs, findEnrichmentColumns } from "./enrichment_discovery"; +import { ColumnsProvider, getCtxProviders } from "../../columns"; type SpecPredicateOption = | MultiColumnSelector @@ -63,70 +58,52 @@ export function buildDatasetOptions( if (options.length === 0) return []; const refMap = buildRefMap(ctx.resultPool.getSpecs().entries); - const pframeSpec = ctx.getService("pframeSpec"); const withEnrichments = opts?.withEnrichments ?? false; - const filterSource = new ResultPoolColumnSnapshotProvider(ctx.resultPool); - // Hoisted out of the per-option loop: collectCtxColumnSnapshotProviders - // walks the entire output tree, so calling it once per dataset option would - // be O(N × tree). - const enrichmentSources = withEnrichments ? collectCtxColumnSnapshotProviders(ctx) : undefined; + const filterSource = ColumnsProvider(ctx.ctx.getUpstreamBlockCtx()); + // Hoist providers once: getCtxProviders walks the full output tree, so + // calling it per dataset option would be O(N × tree). + const enrichmentSources = withEnrichments ? getCtxProviders(ctx) : undefined; return options.map((primary: Option): DatasetOption => { const datasetSpec = ctx.resultPool.getPColumnSpecByRef(primary.ref); if (!datasetSpec) return { primary }; - // Allocations happen inside try so a throw on the second build() - // still disposes the first collection. - let filterCollection: AnchoredColumnCollection | undefined; - let enrichmentCollection: AnchoredColumnCollection | undefined; - try { - // ResultPoolColumnSnapshotProvider is always complete; - // allowPartialColumnList narrows the return type to non-undefined. - filterCollection = new ColumnCollectionBuilder(pframeSpec) - .addSource(filterSource) - .build({ anchors: { main: datasetSpec }, allowPartialColumnList: true }); + // ResultPoolColumnsProvider's completeness is per-block; allowPartialColumnList + // tells discoverColumns to return partial results while pool entries flap. + const filterMatches = findFilterColumns({ + sources: [filterSource], + anchorSpec: datasetSpec, + }).filter((m) => filterPredicate(m.getSpec()!)); - enrichmentCollection = - enrichmentSources !== undefined - ? new ColumnCollectionBuilder(pframeSpec) - .addSources(enrichmentSources) - .build({ anchors: { main: datasetSpec } }) - : undefined; + const filters = + filterMatches.length === 0 + ? undefined + : filterMatchesToOptions(filterMatches, { + refsByObjectId: refMap, + datasetSpec, + labelOptions: opts?.labelOptions, + }); - const filterMatches = findFilterColumns(filterCollection).filter((m) => - filterPredicate(m.column.spec), - ); - const filters = - filterMatches.length === 0 - ? undefined - : filterMatchesToOptions(filterMatches, { - refsByObjectId: refMap, - datasetSpec, - labelOptions: opts?.labelOptions, - }); - - let enrichments; - if (enrichmentCollection && withEnrichments) { - const enrichmentVariants = findEnrichmentColumns(enrichmentCollection, { - maxHops: opts?.enrichmentMaxHops, - ...(typeof withEnrichments === "function" - ? { predicate: withEnrichments } - : { include: withEnrichments }), - }); - if (enrichmentVariants.length > 0) { - enrichments = enrichmentVariantsToRefs(enrichmentVariants, opts?.labelOptions); - } + let enrichments; + if (enrichmentSources && withEnrichments) { + const enrichmentVariants = findEnrichmentColumns({ + sources: enrichmentSources, + anchorSpec: datasetSpec, + maxHops: opts?.enrichmentMaxHops, + ...(typeof withEnrichments === "function" + ? { predicate: withEnrichments } + : { include: withEnrichments }), + }); + if (enrichmentVariants.length > 0) { + enrichments = enrichmentVariantsToRefs(enrichmentVariants, opts?.labelOptions); } - - return { - primary, - ...(filters !== undefined && filters.length > 0 ? { filters } : {}), - ...(enrichments !== undefined && enrichments.length > 0 ? { enrichments } : {}), - }; - } finally { - filterCollection?.dispose(); - enrichmentCollection?.dispose(); } + + return { + primary, + ...(filters !== undefined && filters.length > 0 ? { filters } : {}), + ...(enrichments !== undefined && enrichments.length > 0 ? { enrichments } : {}), + }; }); } diff --git a/sdk/model/src/components/PlDatasetSelector/enrichment_discovery.ts b/sdk/model/src/components/PlDatasetSelector/enrichment_discovery.ts index 2e146d2d7f..8e0654d436 100644 --- a/sdk/model/src/components/PlDatasetSelector/enrichment_discovery.ts +++ b/sdk/model/src/components/PlDatasetSelector/enrichment_discovery.ts @@ -1,41 +1,41 @@ -import { Annotation, createEnrichmentRef } from "@milaboratories/pl-model-common"; +import { + Annotation, + createEnrichmentRef, + extractPObjectId, + isGlobalPObjectId, +} from "@milaboratories/pl-model-common"; import type { + ColumnsCollectionDriverModel, EnrichmentStep, LabeledEnrichmentRef, LabeledEnrichmentRefs, MultiColumnSelector, - PObjectId, + PColumnSpec, PObjectSpec, } from "@milaboratories/pl-model-common"; -import type { - AnchoredColumnCollection, - ColumnVariant, -} from "../../columns/column_collection_builder"; +import type { ColumnRecipe, ColumnsSource } from "../../columns"; +import { + isLeafColumn, + ColumnsCollection, + collectLinkerIds, + hitQualifications, + collectLinkerColumns, + queriesQualifications, +} from "../../columns"; import { deriveDistinctLabels, type DeriveLabelsOptions, type Entry, } from "../../labels/derive_distinct_labels"; -/** - * True for global-form ids — `canonicalize({__isRef: true, blockId, name})` — - * which the workflow can resolve via bquery. Local-form ids (`resolvePath`) - * fail this check and are excluded from auto-discovery; prerun/outputs hops - * must be supplied as resolved `{spec, data}` instead. - */ -function isGloballyAddressable(id: PObjectId): boolean { - try { - const decoded = JSON.parse(id); - return ( - typeof decoded === "object" && - decoded !== null && - decoded.__isRef === true && - typeof decoded.blockId === "string" && - typeof decoded.name === "string" - ); - } catch { - return false; - } +export interface FindEnrichmentColumnsOptions { + sources: ReadonlyArray; + anchorSpec: PColumnSpec; + maxHops?: number; + include?: MultiColumnSelector | MultiColumnSelector[]; + predicate?: (spec: PObjectSpec) => boolean; + /** Override the resolved `columnsCollection` service (primarily for tests). */ + driver?: ColumnsCollectionDriverModel; } /** @@ -43,36 +43,34 @@ function isGloballyAddressable(id: PObjectId): boolean { * (filters / the primary itself) and structural hits (subset, linker, label * columns). Narrow further with `include` selectors or a `predicate`. */ -export function findEnrichmentColumns( - collection: AnchoredColumnCollection, - options?: { - maxHops?: number; - include?: MultiColumnSelector | MultiColumnSelector[]; - predicate?: (spec: PObjectSpec) => boolean; - }, -): ColumnVariant[] { +export function findEnrichmentColumns(options: FindEnrichmentColumnsOptions): ColumnRecipe[] { const include = - options?.include === undefined + options.include === undefined ? undefined : Array.isArray(options.include) ? options.include : [options.include]; - const variants = collection.findColumnVariants({ - mode: "enrichment", - maxHops: options?.maxHops ?? 4, - include, - exclude: [ - { annotations: { [Annotation.IsSubset]: "true" } }, - { annotations: { [Annotation.IsLinkerColumn]: "true" } }, - { name: Annotation.Label }, - ], - }); - const predicate = options?.predicate; - return variants.filter((v) => { - if ((v.path?.length ?? 0) === 0) return false; - if (predicate !== undefined && !predicate(v.column.spec)) return false; - if (!isGloballyAddressable(v.column.id)) return false; - if (v.path?.some((p) => !isGloballyAddressable(p.linker.id))) return false; + + const columns = ColumnsCollection([...options.sources], { driver: options.driver }) + .discover({ + anchors: { main: options.anchorSpec }, + mode: "enrichment", + maxHops: options.maxHops ?? 4, + include, + exclude: [ + { annotations: { [Annotation.IsSubset]: "true" } }, + { annotations: { [Annotation.IsLinkerColumn]: "true" } }, + { name: Annotation.Label }, + ], + }) + .getColumns(); + + const predicate = options.predicate; + return columns.filter((v) => { + if (isLeafColumn(v)) return false; + if (predicate !== undefined && !predicate(v.getSpec())) return false; + if (!isGlobalPObjectId(extractPObjectId(v.id))) return false; + if (collectLinkerIds(v).some((id) => !isGlobalPObjectId(id))) return false; return true; }); } @@ -83,27 +81,33 @@ export function findEnrichmentColumns( * `qualifications.forHit`; `forQueries` is re-derived by the table builder. */ export function enrichmentVariantsToRefs( - variants: ColumnVariant[], + variants: ColumnRecipe[], labelOptions?: DeriveLabelsOptions, ): LabeledEnrichmentRefs { if (variants.length === 0) return []; - const entries: Entry[] = variants.map((variant) => ({ - spec: variant.column.spec, - linkerPath: variant.path?.map((p) => ({ spec: p.linker.spec })), - qualifications: variant.qualifications, + const linkerIdsByVariant = variants.map((v) => collectLinkerIds(v)); + const hitQualsByVariant = variants.map((v) => [...hitQualifications(v)]); + + const entries: Entry[] = variants.map((variant, i) => ({ + spec: variant.getSpec(), + linkerPath: collectLinkerColumns(variant).map((linker) => ({ spec: linker.getSpec() })), + qualifications: { + forHit: hitQualsByVariant[i], + forQueries: queriesQualifications(variant), + }, })); const labels = deriveDistinctLabels(entries, labelOptions); return variants.map((variant, i): LabeledEnrichmentRef => { - const path: undefined | EnrichmentStep[] = variant.path?.map((step) => ({ - type: "linker", - linker: step.linker.id, - })); + const path: undefined | EnrichmentStep[] = + linkerIdsByVariant[i].length === 0 + ? undefined + : linkerIdsByVariant[i].map((id) => ({ type: "linker", linker: id })); return { - ref: createEnrichmentRef(variant.column.id, { + ref: createEnrichmentRef(extractPObjectId(variant.id), { path, - qualifications: variant.qualifications?.forHit, + qualifications: hitQualsByVariant[i], }), label: labels[i], }; diff --git a/sdk/model/src/components/PlDatasetSelector/filter_discovery.test.ts b/sdk/model/src/components/PlDatasetSelector/filter_discovery.test.ts index c4e9e11cd4..d90105168c 100644 --- a/sdk/model/src/components/PlDatasetSelector/filter_discovery.test.ts +++ b/sdk/model/src/components/PlDatasetSelector/filter_discovery.test.ts @@ -1,25 +1,36 @@ -import { Annotation, createPlRef } from "@milaboratories/pl-model-common"; -import type { AxisSpec, PColumnSpec, PObjectId } from "@milaboratories/pl-model-common"; -import { SpecDriver } from "@milaboratories/pf-spec-driver"; +import { Annotation, createGlobalPObjectId, createPlRef } from "@milaboratories/pl-model-common"; +import type { + AxisSpec, + PColumn, + PColumnSpec, + PlRef, + PObjectId, +} from "@milaboratories/pl-model-common"; import canonicalize from "canonicalize"; -import { afterEach, describe, expect, test } from "vitest"; -import type { ColumnSnapshot } from "../../columns/column_snapshot"; -import { ColumnCollectionBuilder } from "../../columns/column_collection_builder"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import type { PColumnDataUniversal } from "../../render/internal"; import { buildRefMap, filterMatchesToOptions, findFilterColumns } from "./filter_discovery"; +import { + createTestCollectionDriver, + type TestCollectionDriverHandle, +} from "../../columns/__test_helpers__/collection_driver"; -const drivers: SpecDriver[] = []; +let driverHandle: TestCollectionDriverHandle; -function createSpecFrameCtx() { - const driver = new SpecDriver(); - drivers.push(driver); - return driver; -} +beforeEach(() => { + driverHandle = createTestCollectionDriver(); + driverHandle.installAmbientCtx(); +}); afterEach(async () => { - for (const driver of drivers) await driver.dispose(); - drivers.length = 0; + driverHandle.uninstallAmbientCtx(); + await driverHandle.dispose(); }); +function registerCols(cols: PColumn[]): void { + driverHandle.register(cols.map((c) => ({ id: c.id, spec: c.spec }))); +} + function axis(name: string): AxisSpec { return { name, type: "String" } as AxisSpec; } @@ -32,52 +43,68 @@ function spec( return { kind: "PColumn", name, valueType: "Int", axesSpec, annotations } as PColumnSpec; } -function snap(id: string, s: PColumnSpec): ColumnSnapshot { - return { id: id as PObjectId, spec: s, dataStatus: "ready", data: { get: () => ({}) as never } }; +function col(id: PObjectId, s: PColumnSpec): PColumn { + return { + id, + spec: s, + data: {} as never, + }; +} + +/** Canonical global PObjectId — required so downstream `extractPObjectId` accepts it. */ +function gid(name: string): PObjectId { + return createGlobalPObjectId("test-block", name); } // anchor defines the key space: [sample, gene] const anchorAxes = [axis("sample"), axis("gene")]; const anchorSpec = spec("anchor", anchorAxes); -const anchorSnap = snap("anchor-id", anchorSpec); +const anchorSnap = col(gid("anchor"), anchorSpec); describe("findFilterColumns", () => { test("returns columns with pl7.app/isSubset annotation", () => { - const filter = snap("f1", spec("filter1", [axis("sample")], { [Annotation.IsSubset]: "true" })); - const regular = snap("r1", spec("regular1", [axis("sample")])); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([filter, regular, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; + const filter = col( + gid("f1"), + spec("filter1", [axis("sample")], { [Annotation.IsSubset]: "true" }), + ); + const regular = col(gid("r1"), spec("regular1", [axis("sample")])); + registerCols([filter, regular, anchorSnap]); - const results = findFilterColumns(collection); - expect(results.every((m) => m.column.spec.name !== "regular1")).toBe(true); - expect(results.some((m) => m.column.spec.name === "filter1")).toBe(true); + const results = findFilterColumns({ + sources: [{ columns: [filter, regular, anchorSnap], isFinal: true }], + anchorSpec, + driver: driverHandle.driver, + }); + expect(results.every((m) => m.getSpec().name !== "regular1")).toBe(true); + expect(results.some((m) => m.getSpec().name === "filter1")).toBe(true); }); test("axes subset: excludes filter whose axes are not a subset of anchor axes", () => { // filter with axis "other" — not a subset of anchor axes [sample, gene] - const badFilter = snap( - "f2", + const badFilter = col( + gid("f2"), spec("bad-filter", [axis("other")], { [Annotation.IsSubset]: "true" }), ); + registerCols([badFilter, anchorSnap]); - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([badFilter, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const results = findFilterColumns(collection); - expect(results.every((m) => m.column.spec.name !== "bad-filter")).toBe(true); + const results = findFilterColumns({ + sources: [{ columns: [badFilter, anchorSnap], isFinal: true }], + anchorSpec, + driver: driverHandle.driver, + }); + expect(results.every((m) => m.getSpec().name !== "bad-filter")).toBe(true); }); test("empty result when no filters exist", () => { - const regular = snap("r1", spec("regular1", [axis("sample")])); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([regular, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; + const regular = col(gid("r1"), spec("regular1", [axis("sample")])); + registerCols([regular, anchorSnap]); - expect(findFilterColumns(collection)).toHaveLength(0); + const results = findFilterColumns({ + sources: [{ columns: [regular, anchorSnap], isFinal: true }], + anchorSpec, + driver: driverHandle.driver, + }); + expect(results).toHaveLength(0); }); }); @@ -101,18 +128,25 @@ describe("filterMatchesToOptions", () => { const filterRef1 = createPlRef("b1", "filter-top1000"); const filterRef2 = createPlRef("b1", "filter-highconf"); - const refMap = buildRefMap([{ ref: filterRef1 }, { ref: filterRef2 }]); + // Build ref map from entries (simulating result pool) + const refMap = buildRefMap([ + { ref: anchorSnap.id as unknown as PlRef }, + { ref: filterRef1 }, + { ref: filterRef2 }, + ]); + const filterSpec1 = spec("filter1", [axis("sample")], { [Annotation.IsSubset]: "true" }); const filterSpec2 = spec("filter2", [axis("sample")], { [Annotation.IsSubset]: "true" }); - const f1Snap = snap(canonicalize(filterRef1)! as string, filterSpec1); - const f2Snap = snap(canonicalize(filterRef2)! as string, filterSpec2); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([f1Snap, f2Snap, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; + const f1Snap = col(canonicalize(filterRef1)! as PObjectId, filterSpec1); + const f2Snap = col(canonicalize(filterRef2)! as PObjectId, filterSpec2); + registerCols([f1Snap, f2Snap, anchorSnap]); - const matches = findFilterColumns(collection); + const matches = findFilterColumns({ + sources: [{ columns: [f1Snap, f2Snap, anchorSnap], isFinal: true }], + anchorSpec, + driver: driverHandle.driver, + }); expect(matches.length).toBe(2); const options = filterMatchesToOptions(matches, { @@ -120,7 +154,6 @@ describe("filterMatchesToOptions", () => { datasetSpec: anchorSpec, }); expect(options).toHaveLength(2); - // Each option has a ref and label for (const opt of options) { expect(opt.ref).toBeDefined(); expect(opt.label).toBeDefined(); @@ -153,13 +186,15 @@ describe("filterMatchesToOptions", () => { { type: "milaboratories.antibody-tcr-lead-selection", label: "Top 10", importance: 30 }, ]), }); - const fSnap = snap(canonicalize(filterRef)! as string, filterSpec); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([fSnap, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const matches = findFilterColumns(collection); + const fSnap = col(canonicalize(filterRef)! as PObjectId, filterSpec); + const dsSnap = col(gid("dataset"), datasetSpec); + registerCols([fSnap, dsSnap]); + + const matches = findFilterColumns({ + sources: [{ columns: [fSnap, dsSnap], isFinal: true }], + anchorSpec: datasetSpec, + driver: driverHandle.driver, + }); expect(matches).toHaveLength(1); const options = filterMatchesToOptions(matches, { refsByObjectId: refMap, datasetSpec }); @@ -215,14 +250,16 @@ describe("filterMatchesToOptions", () => { { type: "milaboratories.antibody-tcr-lead-selection", label: "Top 11", importance: 30 }, ]), }); - const f1Snap = snap(canonicalize(ref1)! as string, filterSpec1); - const f2Snap = snap(canonicalize(ref2)! as string, filterSpec2); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([f1Snap, f2Snap, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const matches = findFilterColumns(collection); + const f1Snap = col(canonicalize(ref1)! as PObjectId, filterSpec1); + const f2Snap = col(canonicalize(ref2)! as PObjectId, filterSpec2); + const dsSnap = col(gid("dataset"), datasetSpec); + registerCols([f1Snap, f2Snap, dsSnap]); + + const matches = findFilterColumns({ + sources: [{ columns: [f1Snap, f2Snap, dsSnap], isFinal: true }], + anchorSpec: datasetSpec, + driver: driverHandle.driver, + }); expect(matches).toHaveLength(2); const options = filterMatchesToOptions(matches, { refsByObjectId: refMap, datasetSpec }); @@ -239,14 +276,15 @@ describe("filterMatchesToOptions", () => { const knownRef = createPlRef("b1", "known"); const knownSpec = spec("known", [axis("sample")], { [Annotation.IsSubset]: "true" }); const orphanSpec = spec("orphan", [axis("sample")], { [Annotation.IsSubset]: "true" }); - const knownSnap = snap(canonicalize(knownRef)! as string, knownSpec); - const orphanSnap = snap("orphan-id", orphanSpec); - - const builder = new ColumnCollectionBuilder(createSpecFrameCtx()); - builder.addSource([knownSnap, orphanSnap, anchorSnap]); - const collection = builder.build({ anchors: { main: anchorSpec } })!; - - const matches = findFilterColumns(collection); + const knownSnap = col(canonicalize(knownRef)! as PObjectId, knownSpec); + const orphanSnap = col(gid("orphan"), orphanSpec); + registerCols([knownSnap, orphanSnap, anchorSnap]); + + const matches = findFilterColumns({ + sources: [{ columns: [knownSnap, orphanSnap, anchorSnap], isFinal: true }], + anchorSpec, + driver: driverHandle.driver, + }); expect(matches.length).toBe(2); const refMap = buildRefMap([{ ref: knownRef }]); const options = filterMatchesToOptions(matches, { diff --git a/sdk/model/src/components/PlDatasetSelector/filter_discovery.ts b/sdk/model/src/components/PlDatasetSelector/filter_discovery.ts index c02fefd307..180e0baef4 100644 --- a/sdk/model/src/components/PlDatasetSelector/filter_discovery.ts +++ b/sdk/model/src/components/PlDatasetSelector/filter_discovery.ts @@ -1,27 +1,44 @@ -import { Annotation } from "@milaboratories/pl-model-common"; -import type { Option, PlRef, PObjectId, PObjectSpec } from "@milaboratories/pl-model-common"; -import canonicalize from "canonicalize"; +import { Annotation, extractPObjectId } from "@milaboratories/pl-model-common"; import type { - AnchoredColumnCollection, - ColumnMatch, -} from "../../columns/column_collection_builder"; + ColumnsCollectionDriverModel, + Option, + PColumnSpec, + PlRef, + PObjectId, + PObjectSpec, +} from "@milaboratories/pl-model-common"; +import canonicalize from "canonicalize"; +import type { ColumnRecipe, ColumnsSource } from "../../columns"; +import { ColumnsCollection, collectLinkerColumns } from "../../columns"; import { deriveDistinctLabels, type DeriveLabelsOptions, type Entry, } from "../../labels/derive_distinct_labels"; +export interface FindFilterColumnsOptions { + sources: ReadonlyArray; + anchorSpec: PColumnSpec; + /** Override the resolved `columnsCollection` service (primarily for tests). */ + driver?: ColumnsCollectionDriverModel; +} + /** - * Columns annotated `pl7.app/isSubset: "true"` whose axes ⊆ anchor axes. - * The axes-subset constraint comes from `mode: "enrichment"`. + * Matches columns annotated `pl7.app/isSubset: "true"` whose axes ⊆ anchor axes. + * + * The axes-subset constraint is enforced by `mode: "enrichment"`, which sets + * `allowFloatingHitAxes: false` — every axis of the matched column must be + * present in the anchor's axes. See `matchingModeToConstraints()` in + * `discover_columns.ts`. */ -export function findFilterColumns(collection: AnchoredColumnCollection): ColumnMatch[] { - return collection.findColumns({ - mode: "enrichment", - include: { - annotations: { [Annotation.IsSubset]: "true" }, - }, - }); +export function findFilterColumns(options: FindFilterColumnsOptions): ColumnRecipe[] { + return ColumnsCollection([...options.sources], { driver: options.driver }) + .discover({ + anchors: { main: options.anchorSpec }, + mode: "enrichment", + include: { annotations: { [Annotation.IsSubset]: "true" } }, + }) + .getColumns(); } export type FilterMatchOptions = { @@ -39,30 +56,32 @@ export type FilterMatchOptions = { }; /** - * Derive labels for filter column matches (for `DatasetOption.filters`). - * Matches whose column id is missing from `refsByObjectId` are silently - * dropped — they cannot be exposed as selectable options. + * Derive labeled options from filter column variants, for use in DatasetOption.filters. + * + * Entries whose column id has no PlRef in `refsByObjectId` are silently + * skipped — they cannot be exposed as user-selectable options. */ export function filterMatchesToOptions( - matches: ColumnMatch[], + variants: ColumnRecipe[], options: FilterMatchOptions, ): Option[] { - if (matches.length === 0) return []; + if (variants.length === 0) return []; const { refsByObjectId, datasetSpec, labelOptions } = options; - // One entry per match-variant (different paths to the same column are - // exposed as separate Options). The `ref` field rides along on the - // Entry-shaped objects via structural typing; `deriveDistinctLabels` - // ignores extra fields. - const entries = matches.flatMap((match): (Entry & { ref: PlRef })[] => { - const ref = refsByObjectId.get(match.column.id); + // One Option per variant — `deriveDistinctLabels` disambiguates labels by + // path. All variants of the same column share a terminal PObjectId, so the + // ref lookup happens once per variant. + const entries: (Entry & { ref: PlRef })[] = variants.flatMap((variant) => { + const ref = refsByObjectId.get(extractPObjectId(variant.id)); if (ref === undefined) return []; - return match.variants.map((variant) => ({ - ref, - spec: match.column.spec, - linkerPath: variant.path.map((p) => ({ spec: p.linker.spec })), - })); + return [ + { + ref, + spec: variant.getSpec(), + linkerPath: collectLinkerColumns(variant).map((linker) => ({ spec: linker.getSpec() })), + }, + ]; }); // Appending the dataset forces a discriminating trace step into every diff --git a/sdk/model/src/filters/converters/filterToQuery.test.ts b/sdk/model/src/filters/converters/filterToQuery.test.ts index 3c4158c37f..d0d8f62d12 100644 --- a/sdk/model/src/filters/converters/filterToQuery.test.ts +++ b/sdk/model/src/filters/converters/filterToQuery.test.ts @@ -1,17 +1,24 @@ import { describe, expect, it } from "vitest"; -import type { FilterSpec, FilterSpecLeaf } from "@milaboratories/pl-model-common"; +import type { + AxisId, + FilterSpec, + FilterSpecLeaf, + PObjectId, + PTableColumnId, +} from "@milaboratories/pl-model-common"; import { filterSpecToSpecQueryExpr } from "./filterToQuery"; -type QFilterSpec = FilterSpec>; +type QFilterSpec = FilterSpec>; -/** Helper: creates a CanonicalizedJson for a regular column. */ -function colRef(id: string): string { - return JSON.stringify({ type: "column", id }); +/** Helper: creates a PTableColumnId for a regular column. */ +function colRef(id: string): PTableColumnId { + return { type: "column", id: id as PObjectId }; } -/** Helper: creates a CanonicalizedJson for an axis. */ -function axisRef(id: string): string { - return JSON.stringify({ type: "axis", id }); +/** Helper: creates a PTableColumnId for an axis. AxisId stays as a string-shaped + * value here because the converter passes `id` through to SingleAxisSelector. */ +function axisRef(id: string): PTableColumnId { + return { type: "axis", id: id as unknown as AxisId }; } describe("filterSpecToSpecQueryExpr", () => { diff --git a/sdk/model/src/filters/converters/filterToQuery.ts b/sdk/model/src/filters/converters/filterToQuery.ts index 3e203a43a7..3b690dbb8b 100644 --- a/sdk/model/src/filters/converters/filterToQuery.ts +++ b/sdk/model/src/filters/converters/filterToQuery.ts @@ -1,23 +1,36 @@ import { assertNever } from "@milaboratories/pl-model-common"; import type { + AxisId, + ColumnUniversalId, FilterSpec, FilterSpecLeaf, - PTableColumnId, SingleAxisSelector, SpecQueryExpression, } from "@milaboratories/pl-model-common"; import { traverseFilterSpec } from "../traverse"; -/** Parses a CanonicalizedJson string into a SpecQueryExpression reference. */ -function resolveColumnRef(columnStr: string): SpecQueryExpression { - const parsed = JSON.parse(columnStr) as PTableColumnId; - return parsed.type === "axis" - ? { type: "axisRef", value: parsed.id as SingleAxisSelector } - : { type: "columnRef", value: parsed.id }; +export type QueryColumnIdAxis = { + type: "axis"; + id: AxisId; +}; + +export type QueryColumnIdColumn = { + type: "column"; + /** May be a rich {@link ColumnUniversalId} (Discovered / Overrided / Filtered) or bare PObjectId. */ + id: ColumnUniversalId; +}; + +export type QueryColumnId = QueryColumnIdAxis | QueryColumnIdColumn; + +/** Converts a QueryColumnId object into a SpecQueryExpression reference. */ +function resolveColumnRef(col: QueryColumnId): SpecQueryExpression { + return col.type === "axis" + ? { type: "axisRef", value: col.id as SingleAxisSelector } + : { type: "columnRef", value: col.id }; } /** Converts a FilterSpec tree into a SpecQueryExpression. */ -export function filterSpecToSpecQueryExpr>( +export function filterSpecToSpecQueryExpr>( filter: FilterSpec, ): SpecQueryExpression { return traverseFilterSpec(filter, { @@ -38,7 +51,7 @@ export function filterSpecToSpecQueryExpr>( }); } -function leafToSpecQueryExpr>( +function leafToSpecQueryExpr>( filter: Leaf, ): SpecQueryExpression { switch (filter.type) { diff --git a/sdk/model/src/filters/converters/filterUiToExpressionImpl.test.ts b/sdk/model/src/filters/converters/filterUiToExpressionImpl.test.ts index d7b5b09693..384625e042 100644 --- a/sdk/model/src/filters/converters/filterUiToExpressionImpl.test.ts +++ b/sdk/model/src/filters/converters/filterUiToExpressionImpl.test.ts @@ -1,4 +1,3 @@ -import type { SUniversalPColumnId } from "@milaboratories/pl-model-common"; import { describe, expect, it } from "vitest"; import { convertFilterUiToExpressions } from "./filterUiToExpressionImpl"; import { FilterSpec } from "../types"; @@ -8,8 +7,8 @@ describe("convertFilterUiToExpressions", () => { const uiFilter: FilterSpec = { type: "or", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, - { type: "patternEquals", column: "colB" as unknown as SUniversalPColumnId, value: "test" }, + { type: "isNA", column: "colA" }, + { type: "patternEquals", column: "colB", value: "test" }, ], }; const result = convertFilterUiToExpressions(uiFilter); @@ -23,8 +22,8 @@ describe("convertFilterUiToExpressions", () => { const uiFilter: FilterSpec = { type: "and", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, - { type: "greaterThan", column: "colNum" as unknown as SUniversalPColumnId, x: 10 }, + { type: "isNA", column: "colA" }, + { type: "greaterThan", column: "colNum", x: 10 }, ], }; const result = convertFilterUiToExpressions(uiFilter); @@ -37,7 +36,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "not" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "not", - filter: { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + filter: { type: "isNA", column: "colA" }, }; const result = convertFilterUiToExpressions(uiFilter); expect(result.type).toBe("not"); @@ -47,7 +46,7 @@ describe("convertFilterUiToExpressions", () => { }); it('should compile "isNA" filter to ptabler expression', () => { - const uiFilter: FilterSpec = { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }; + const uiFilter: FilterSpec = { type: "isNA", column: "colA" }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ type: "is_na", @@ -58,7 +57,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "isNotNA" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "isNotNA", - column: "colA" as unknown as SUniversalPColumnId, + column: "colA", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -70,7 +69,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "patternEquals" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "patternEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "abc", }; const result = convertFilterUiToExpressions(uiFilter); @@ -84,7 +83,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "patternNotEquals" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "patternNotEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "abc", }; const result = convertFilterUiToExpressions(uiFilter); @@ -98,7 +97,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "patternContainSubsequence" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "patternContainSubsequence", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", value: "sub", }; const result = convertFilterUiToExpressions(uiFilter); @@ -110,7 +109,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "patternNotContainSubsequence" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "patternNotContainSubsequence", - column: "colC" as unknown as SUniversalPColumnId, + column: "colC", value: "sub", }; const result = convertFilterUiToExpressions(uiFilter); @@ -132,7 +131,7 @@ describe("convertFilterUiToExpressions", () => { testCases.forEach(({ type, expected }) => { const uiFilter: FilterSpec = { type, - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", x: 10, }; const result = convertFilterUiToExpressions(uiFilter); @@ -147,8 +146,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "lessThanColumn" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "lessThanColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -161,8 +160,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "lessThanColumn" filter with minDiff to ptabler expression', () => { const uiFilter: FilterSpec = { type: "lessThanColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 5, }; const result = convertFilterUiToExpressions(uiFilter); @@ -180,8 +179,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "greaterThanColumn" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "greaterThanColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -194,8 +193,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "greaterThanColumn" filter with minDiff to ptabler expression', () => { const uiFilter: FilterSpec = { type: "greaterThanColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 7, }; const result = convertFilterUiToExpressions(uiFilter); @@ -213,8 +212,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "equalToColumn" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "equalToColumn", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -227,8 +226,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "greaterThanColumnOrEqual" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "greaterThanColumnOrEqual", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -241,8 +240,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "greaterThanColumnOrEqual" filter with minDiff to ptabler expression', () => { const uiFilter: FilterSpec = { type: "greaterThanColumnOrEqual", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 2, }; const result = convertFilterUiToExpressions(uiFilter); @@ -260,8 +259,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "lessThanColumnOrEqual" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "lessThanColumnOrEqual", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", }; const result = convertFilterUiToExpressions(uiFilter); expect(result as any).toEqual({ @@ -274,8 +273,8 @@ describe("convertFilterUiToExpressions", () => { it('should compile "lessThanColumnOrEqual" filter with minDiff to ptabler expression', () => { const uiFilter: FilterSpec = { type: "lessThanColumnOrEqual", - column: "colNum1" as unknown as SUniversalPColumnId, - rhs: "colNum2" as unknown as SUniversalPColumnId, + column: "colNum1", + rhs: "colNum2", minDiff: 3, }; const result = convertFilterUiToExpressions(uiFilter); @@ -293,7 +292,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "topN" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "topN", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", n: 5, }; const result = convertFilterUiToExpressions(uiFilter); @@ -312,7 +311,7 @@ describe("convertFilterUiToExpressions", () => { it('should compile "bottomN" filter to ptabler expression', () => { const uiFilter: FilterSpec = { type: "bottomN", - column: "colNum" as unknown as SUniversalPColumnId, + column: "colNum", n: 3, }; const result = convertFilterUiToExpressions(uiFilter); @@ -335,15 +334,15 @@ describe("convertFilterUiToExpressions", () => { { type: "or", filters: [ - { type: "isNA", column: "colA" as unknown as SUniversalPColumnId }, + { type: "isNA", column: "colA" }, { type: "patternEquals", - column: "colB" as unknown as SUniversalPColumnId, + column: "colB", value: "test", }, ], }, - { type: "greaterThan", column: "colNum" as unknown as SUniversalPColumnId, x: 10 }, + { type: "greaterThan", column: "colNum", x: 10 }, ], }; const result = convertFilterUiToExpressions(uiFilter); diff --git a/sdk/model/src/labels/derive_distinct_labels.ts b/sdk/model/src/labels/derive_distinct_labels.ts index b027d5be7b..cefb1dd1c7 100644 --- a/sdk/model/src/labels/derive_distinct_labels.ts +++ b/sdk/model/src/labels/derive_distinct_labels.ts @@ -3,6 +3,7 @@ import { parseJson, readAnnotation, type AxisQualification, + type MatchQualifications, type PObjectId, type PObjectSpec, type StringifiedJson, @@ -10,7 +11,6 @@ import { } from "@milaboratories/pl-model-common"; import { throwError } from "@milaboratories/helpers"; import { isFunction, isNil } from "es-toolkit"; -import type { MatchQualifications } from "../columns/column_collection_builder"; export type { Trace, TraceEntry } from "@milaboratories/pl-model-common"; diff --git a/sdk/model/src/labels/derive_distinct_tooltips.test.ts b/sdk/model/src/labels/derive_distinct_tooltips.test.ts index b65e8390f6..04c4c70c04 100644 --- a/sdk/model/src/labels/derive_distinct_tooltips.test.ts +++ b/sdk/model/src/labels/derive_distinct_tooltips.test.ts @@ -1,12 +1,13 @@ import { Annotation, type AxisQualification, + type PColumn, type PColumnSpec, type PObjectId, } from "@milaboratories/pl-model-common"; import { describe, expect, test } from "vitest"; import { deriveDistinctTooltips, type TooltipEntry } from "./derive_distinct_tooltips"; -import type { ColumnSnapshot, MatchVariant } from "../columns"; +import type { PColumnDataUniversal } from "../render/internal"; function createSpec(name: string, label?: string): PColumnSpec { return { @@ -25,16 +26,18 @@ function axisQualification( return { axis: { name: axisName }, contextDomain }; } -function linkerSnapshot(name: string, label?: string): ColumnSnapshot { +function linkerSnapshot(name: string, label?: string): PColumn { return { id: `linker-${name}` as PObjectId, spec: createSpec(name, label), - dataStatus: "ready", data: undefined, }; } -function pathStep(linkerName: string, label?: string): MatchVariant["path"][number] { +function pathStep( + linkerName: string, + label?: string, +): NonNullable[number] { return { linker: linkerSnapshot(linkerName, label) }; } diff --git a/sdk/model/src/labels/derive_distinct_tooltips.ts b/sdk/model/src/labels/derive_distinct_tooltips.ts index b766fa9318..44da944c09 100644 --- a/sdk/model/src/labels/derive_distinct_tooltips.ts +++ b/sdk/model/src/labels/derive_distinct_tooltips.ts @@ -3,10 +3,10 @@ import { PObjectId, readAnnotation, type AxisQualification, + type MatchQualifications, type PColumnSpec, } from "@milaboratories/pl-model-common"; import { isNil } from "es-toolkit"; -import type { MatchQualifications, MatchVariant } from "../columns"; export type TooltipEntry = { /** Main column spec — used for column-name fallback when no label. */ @@ -14,7 +14,7 @@ export type TooltipEntry = { /** Qualifications carried by this variant. */ qualifications?: MatchQualifications; /** Linker steps traversed to reach the hit column. */ - linkerPath?: MatchVariant["path"]; + linkerPath?: ReadonlyArray<{ readonly linker: { readonly spec: PColumnSpec } }>; /** Position of this variant within the same physical column (1-based). */ variantIndex?: number; /** Total variants for the same physical column. */ diff --git a/sdk/model/src/pframe_utils/axes.ts b/sdk/model/src/pframe_utils/axes.ts index dbf23a170c..b6392c6e2c 100644 --- a/sdk/model/src/pframe_utils/axes.ts +++ b/sdk/model/src/pframe_utils/axes.ts @@ -71,7 +71,7 @@ function getKeysCombinations(idsLists: AxisId[][]) { } export function getAvailableWithLinkersAxes( - linkerColumns: PColumn[], + linkerColumns: PColumn[], blockAxes: AxesVault, ): AxesVault { const linkerMap = LinkerMap.fromColumns(linkerColumns.map(getColumnIdAndSpec)); @@ -89,7 +89,7 @@ export function getAvailableWithLinkersAxes( } /** Add columns with fully compatible axes created from partial compatible ones */ -export function enrichCompatible, "data">>( +export function enrichCompatible, "data">>( blockAxes: AxesVault, columns: T[], ): T[] { diff --git a/sdk/model/src/pframe_utils/columns.ts b/sdk/model/src/pframe_utils/columns.ts index a9522ab28b..b955551504 100644 --- a/sdk/model/src/pframe_utils/columns.ts +++ b/sdk/model/src/pframe_utils/columns.ts @@ -1,4 +1,4 @@ -import type { PColumn, PColumnSpec, PColumnLazy, PFrameDef } from "@milaboratories/pl-model-common"; +import type { PColumn, PColumnSpec, PFrameDef } from "@milaboratories/pl-model-common"; import { getNormalizedAxesList, getAxisId, @@ -15,7 +15,7 @@ import { PColumnCollection } from "../render"; export function getAllRelatedColumns( ctx: RenderCtxBase, predicate: (spec: PColumnSpec) => boolean, -): PFrameDef | PColumnLazy> { +): PFrameDef> { // if current block doesn't produce own columns then use all columns from result pool const columns = new PColumnCollection(); columns.addColumnProvider(ctx.resultPool); @@ -46,10 +46,10 @@ export function getRelatedColumns( columns: rootColumns, predicate, }: { - columns: PColumn[]; + columns: PColumn[]; predicate: (spec: PColumnSpec) => boolean; }, -): PFrameDef | PColumnLazy> { +): PFrameDef> { // if current block has its own columns then take from result pool only compatible with them const columns = new PColumnCollection(); columns.addColumnProvider(ctx.resultPool); diff --git a/sdk/model/src/pframe_utils/index.ts b/sdk/model/src/pframe_utils/index.ts index e820542253..c229704bc7 100644 --- a/sdk/model/src/pframe_utils/index.ts +++ b/sdk/model/src/pframe_utils/index.ts @@ -310,7 +310,7 @@ export async function getColumnsFull( export async function getColumnOrAxisValueLabelsId( handle: PFrameHandle, strAxisId: CanonicalizedJson, -): Promise { +): Promise { const labelColumns = await getColumnsFull(handle, { selectedSources: [], strictlyCompatible: false, diff --git a/sdk/model/src/render/accessor.ts b/sdk/model/src/render/accessor.ts index 1cfa01cb4d..16c7f84a33 100644 --- a/sdk/model/src/render/accessor.ts +++ b/sdk/model/src/render/accessor.ts @@ -1,16 +1,17 @@ -import type { - AnyLogHandle, - ImportProgress, - LocalBlobHandleAndSize, +import { + type AnyLogHandle, + type ImportProgress, + type LocalBlobHandleAndSize, + type RemoteBlobHandleAndSize, + type FolderURL, + type ArchiveFormat, + type ProgressLogWithInfo, + type RangeBytes, + isPColumn, + mapPObjectData, PColumn, PObject, - RemoteBlobHandleAndSize, - FolderURL, - ArchiveFormat, - ProgressLogWithInfo, - RangeBytes, } from "@milaboratories/pl-model-common"; -import { isPColumn, mapPObjectData } from "@milaboratories/pl-model-common"; import { getCfgRenderCtx } from "../internal"; import { FutureRef } from "./future"; import type { AccessorHandle } from "./internal"; @@ -108,15 +109,11 @@ export class TreeNodeAccessor { return this.resolveWithCommon({}, ...transformedSteps); } - public resolveAny( - ...steps: [ - Omit & { - errorIfFieldNotAssigned: true; - }, - ] + public traverse( + ...steps: [Omit & { errorIfFieldNotAssigned: true }] ): TreeNodeAccessor; - public resolveAny(...steps: (FieldTraversalStep | string)[]): TreeNodeAccessor | undefined; - public resolveAny(...steps: (FieldTraversalStep | string)[]): TreeNodeAccessor | undefined { + public traverse(...steps: (FieldTraversalStep | string)[]): TreeNodeAccessor | undefined; + public traverse(...steps: (FieldTraversalStep | string)[]): TreeNodeAccessor | undefined { return this.resolveWithCommon({}, ...steps); } @@ -188,6 +185,10 @@ export class TreeNodeAccessor { return JSON.parse(content); } + public hasData(): boolean { + return getCfgRenderCtx().hasData(this.handle); + } + public getDataBase64(): string | undefined { return getCfgRenderCtx().getDataBase64(this.handle); } @@ -202,9 +203,21 @@ export class TreeNodeAccessor { return JSON.parse(content); } - /** - * - */ + public getFileContentAsBase64(range?: RangeBytes): FutureRef { + return new FutureRef(getCfgRenderCtx().getBlobContentAsBase64(this.handle, range)); + } + + public getFileContentAsString(range?: RangeBytes): FutureRef { + return new FutureRef(getCfgRenderCtx().getBlobContentAsString(this.handle, range)); + } + + public getFileContentAsJson(range?: RangeBytes): FutureRef { + return new FutureRef( + getCfgRenderCtx().getBlobContentAsString(this.handle, range), + ).mapDefined((v) => JSON.parse(v) as T); + } + + /** @deprecated */ public getPColumns( errorOnUnknownField: boolean = false, prefix: string = "", @@ -220,9 +233,7 @@ export class TreeNodeAccessor { return pf; } - /** - * - */ + /** @deprecated */ public parsePObjectCollection( errorOnUnknownField: boolean = false, prefix: string = "", @@ -242,20 +253,6 @@ export class TreeNodeAccessor { return result; } - public getFileContentAsBase64(range?: RangeBytes): FutureRef { - return new FutureRef(getCfgRenderCtx().getBlobContentAsBase64(this.handle, range)); - } - - public getFileContentAsString(range?: RangeBytes): FutureRef { - return new FutureRef(getCfgRenderCtx().getBlobContentAsString(this.handle, range)); - } - - public getFileContentAsJson(range?: RangeBytes): FutureRef { - return new FutureRef( - getCfgRenderCtx().getBlobContentAsString(this.handle, range), - ).mapDefined((v) => JSON.parse(v) as T); - } - /** * @deprecated use getFileContentAsBase64 */ diff --git a/sdk/model/src/render/api.ts b/sdk/model/src/render/api.ts index 9d076d04fe..01e3d24459 100644 --- a/sdk/model/src/render/api.ts +++ b/sdk/model/src/render/api.ts @@ -6,16 +6,15 @@ import type { ModelServices, Option, PColumn, - PColumnLazy, PColumnSelector, PColumnSpec, PColumnValues, PFrameDef, PFrameHandle, + ColumnUniversalId, PObject, PObjectId, PObjectSpec, - PSpecPredicate, PTableDef, PTableDefV2, PTableHandle, @@ -45,6 +44,7 @@ import { withEnrichments, legacyColumnSelectorsToPredicate, extractAllColumns, + visitDataInfo, } from "@milaboratories/pl-model-common"; import canonicalize from "canonicalize"; import type { Optional } from "utility-types"; @@ -72,6 +72,8 @@ import { patchInSetFilters } from "./util/pframe_upgraders"; import type { PColumnDataUniversal } from "./internal"; import { getService } from "../services"; import { allPColumnsReady } from "./util"; +import { throwError } from "@milaboratories/helpers"; +import { isString } from "es-toolkit"; /** * Helper function to match domain objects @@ -92,19 +94,25 @@ export type UniversalColumnOption = { label: string; value: SUniversalPColumnId /** * Transforms PColumn data into the internal representation expected by the platform - * @param data Data from a PColumn to transform + * @param column Data from a PColumn to transform * @returns Transformed data compatible with platform API */ -function transformPColumnData( - data: PColumn | PColumnLazy, +export function finalizePColumnData( + column: PColumn>, ): PColumn> { - return mapPObjectData(data, (d) => { - if (d instanceof TreeNodeAccessor) { - return d.handle; - } else if (isDataInfo(d)) { - return mapDataInfo(d, (accessor) => accessor.handle); + return mapPObjectData(column, (data) => { + if (data instanceof TreeNodeAccessor) { + return data.handle; + } else if (isDataInfo(data)) { + let ready = true; + visitDataInfo(data, (accessor) => (ready &&= accessor?.getIsReadyOrError() ?? false)); + if (!ready) return []; + return mapDataInfo( + data, + (accessor) => accessor?.handle ?? throwError("Accessor handle is undefined"), + ); } else { - return d ?? []; + return data ?? []; } }); } @@ -130,16 +138,22 @@ type GetOptionsOpts = { label?: ((spec: PObjectSpec, ref: PlRef) => string) | LabelDerivationOps; }; +/** + * @deprecated use the direct `ctx.*` methods on `RenderCtxBase` instead. + * + * Migration map: + * - `getSpecs` → `ctx.getSpecs` + * - `getSpecByRef` / `getPColumnSpecByRef` → `ctx.getSpecByRef` / `ctx.getPColumnSpecByRef` + * - `getDataByRef` / `getPColumnByRef` → `ctx.getDataByRef` / `ctx.getPColumnByRef` + * - `getOptions` → `ctx.getOptions` + * + * Eager methods `selectColumns` / `getData` / `getDataWithErrors` should be + * replaced by `ctx.getSpecs` + on-demand `ctx.getStatusByRef` / + * `ctx.getPColumnByRef` so that the data resource is not subscribed up-front. + */ export class ResultPool implements ColumnProvider, AxisLabelProvider { private readonly ctx: GlobalCfgRenderCtx = getCfgRenderCtx(); - /** - * @deprecated use getOptions() - */ - public calculateOptions(predicate: PSpecPredicate): Option[] { - return this.ctx.calculateOptions(predicate); - } - public getOptions( predicateOrSelector: ((spec: PObjectSpec) => boolean) | PColumnSelector | PColumnSelector[], opts?: GetOptionsOpts, @@ -221,7 +235,7 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider { | APColumnSelectorWithSplit | APColumnSelectorWithSplit[], opts?: UniversalPColumnOpts, - ): PColumn[] | undefined { + ): undefined | PColumn[] { const anchorCtx = this.resolveAnchorCtx(anchorsOrCtx); if (!anchorCtx) return undefined; return new PColumnCollection() @@ -351,11 +365,6 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider { * @returns data associated with the ref */ public getDataByRef(ref: PlRef): PObject | undefined { - // @TODO remove after 1 Jan 2025; forward compatibility - if (typeof this.ctx.getDataFromResultPoolByRef === "undefined") - return this.getData().entries.find( - (f) => f.ref.blockId === ref.blockId && f.ref.name === ref.name, - )?.obj; const data = this.ctx.getDataFromResultPoolByRef(ref.blockId, ref.name); // Keep original call // Need to handle undefined case before mapping if (!data) return undefined; @@ -491,6 +500,9 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider { * * @param selectors - A predicate function, a single selector, or an array of selectors. * @returns An array of PColumn objects matching the selectors. Data is loaded on first access. + * @deprecated use `RenderCtxBase.rawResultPool` + `ResultPoolColumnsProvider` + + * `discoverColumns` instead. This eager path materializes column data + * unnecessarily for callers that only filter by spec. */ public selectColumns( selectors: ((spec: PColumnSpec) => boolean) | PColumnSelector | PColumnSelector[], @@ -552,7 +564,7 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider { /** Main entry point to the API available within model lambdas (like outputs, sections, etc..) */ export abstract class RenderCtxBase { - protected readonly ctx: GlobalCfgRenderCtx; + public readonly ctx: GlobalCfgRenderCtx; constructor() { this.ctx = getCfgRenderCtx(); @@ -611,30 +623,9 @@ export abstract class RenderCtxBase { return this.getNamedAccessor(MainAccessorName); } + /** @deprecated use `rawResultPool` + `ResultPoolColumnsProvider` instead. */ public readonly resultPool = new ResultPool(); - /** - * Find labels data for a given axis id. It will search for a label column and return its data as a map. - * @returns a map of axis value => label - * @deprecated Use resultPool.findLabels instead - */ - public findLabels(axis: AxisId): Record | undefined { - return this.resultPool.findLabels(axis); - } - - private verifyInlineAndExplicitColumnsSupport( - columns: (PColumn | PColumnLazy)[], - ) { - const hasInlineColumns = columns.some( - (c) => !(c.data instanceof TreeNodeAccessor) || isDataInfo(c.data), - ); // Updated check for DataInfo - const inlineColumnsSupport = this.ctx.featureFlags?.inlineColumnsSupport === true; - if (hasInlineColumns && !inlineColumnsSupport) - throw Error(`Inline or explicit columns not supported`); // Combined check - - // Removed redundant explicitColumns check - } - private patchPTableDef( def: PTableDef>, ): PTableDef> { @@ -656,13 +647,10 @@ export abstract class RenderCtxBase { return def; } - // TODO remove all non-PColumn fields public createPFrame( - def: PFrameDef< - PColumn | PColumnLazy - >, + def: PFrameDef>, ): PFrameHandle | undefined { - return this.ctx.createPFrame(def.map((c) => transformPColumnData(c))); + return this.ctx.createPFrame(def.map((v) => (isString(v) ? v : finalizePColumnData(v)))); } // TODO remove all non-PColumn fields @@ -698,20 +686,16 @@ export abstract class RenderCtxBase { rawDef = this.patchPTableDef(def); } const columns = extractAllColumns(rawDef.src); - this.verifyInlineAndExplicitColumnsSupport(columns); if (!allPColumnsReady(columns)) return undefined; - return this.ctx.createPTable(mapPTableDef(rawDef, (po) => transformPColumnData(po))); + return this.ctx.createPTable(mapPTableDef(rawDef, finalizePColumnData)); } public createPTableV2( - def: PTableDefV2>, + def: PTableDefV2>, ): PTableHandle | undefined { - return this.ctx.createPTableV2(mapPTableDefV2(def, (po) => transformPColumnData(po))); - } - - /** @deprecated scheduled for removal from SDK */ - public getBlockLabel(blockId: string): string { - return this.ctx.getBlockLabel(blockId); + return this.ctx.createPTableV2( + mapPTableDefV2(def, { column: (v) => (isString(v) ? v : finalizePColumnData(v)) }), + ); } public getCurrentUnstableMarker(): string | undefined { diff --git a/sdk/model/src/render/index.ts b/sdk/model/src/render/index.ts index b4466f919b..9b3e25791d 100644 --- a/sdk/model/src/render/index.ts +++ b/sdk/model/src/render/index.ts @@ -3,5 +3,6 @@ export * from "./traversal_ops"; export * from "./accessor"; export * from "./util"; export { FutureRef } from "./future"; -export type { PColumnDataUniversal } from "./internal"; +export type { AccessorHandle, PColumnDataUniversal } from "./internal"; +export { MainAccessorName, StagingAccessorName } from "./internal"; export type { ExtractFutureRefType } from "./future"; diff --git a/sdk/model/src/render/internal.ts b/sdk/model/src/render/internal.ts index a2088be9b9..1b058a802c 100644 --- a/sdk/model/src/render/internal.ts +++ b/sdk/model/src/render/internal.ts @@ -1,17 +1,22 @@ import type { Optional } from "utility-types"; -import type { Branded, StringifiedJson } from "@milaboratories/pl-model-common"; +import type { + AccessorHandle, + Branded, + PObjectId, + StringifiedJson, + SUniversalPColumnId, +} from "@milaboratories/pl-model-common"; +export type { AccessorHandle }; import type { CommonFieldTraverseOps, FieldTraversalStep, ResourceType } from "./traversal_ops"; import type { ArchiveFormat, AnyFunction, - Option, PColumn, PColumnValues, PFrameDef, PFrameHandle, PObject, PObjectSpec, - PSpecPredicate, PTableDef, PTableDefV2, PTableHandle, @@ -23,10 +28,9 @@ import type { import type { TreeNodeAccessor } from "./accessor"; import type { ServiceDispatch } from "@milaboratories/pl-model-common"; -export const StagingAccessorName = "staging"; export const MainAccessorName = "main"; +export const StagingAccessorName = "staging"; -export type AccessorHandle = Branded; export type FutureHandle = Branded; export type PColumnDataUniversal = @@ -34,6 +38,10 @@ export type PColumnDataUniversal = | DataInfo | PColumnValues; +// Raw upstream-block ctx shape lives in `@milaboratories/pl-model-common` +// (`UpstreamBlockCtx`). Import directly from there at use sites. +import type { UpstreamBlockCtx } from "@milaboratories/pl-model-common"; + export interface GlobalCfgRenderCtxMethods { // // Root accessor creation @@ -73,11 +81,13 @@ export interface GlobalCfgRenderCtxMethods | undefined; - calculateOptions(predicate: PSpecPredicate): Option[]; + // + // Raw result pool — list of upstream block ctx accessor handles. + // SDK-side providers compose enumerate/status/data themselves on top of these. + // + + /** + * For each upstream block in the staging graph, returns its prod/staging ctx + * accessor handles (when present). Sandbox uses these handles with + * `getPObjectEntryDataHandle` to build + * column snapshots; spec/status/data merge with prod-vs-staging precedence + * lives sandbox-side. + */ + getUpstreamBlockCtx(): ReadonlyArray>; // // PFrame / PTable // - createPFrame(def: PFrameDef>>): PFrameHandle; + createPFrame( + def: PFrameDef< + PObjectId | SUniversalPColumnId | PColumn> + >, + ): PFrameHandle; - createPTable(def: PTableDef>>): PTableHandle; + createPTable( + def: PTableDef< + PObjectId | SUniversalPColumnId | PColumn> + >, + ): PTableHandle; createPTableV2( - def: PTableDefV2>>, + def: PTableDefV2< + PObjectId | SUniversalPColumnId | PColumn> + >, ): PTableHandle; // @@ -181,6 +213,7 @@ export const GlobalCfgRenderCtxFeatureFlags = { activeArgs: true as const, pTablePartitionFiltersSupport: true as const, pFrameInSetFilterSupport: true as const, + lazyColumnStatusSupport: true as const, }; export interface GlobalCfgRenderCtx extends GlobalCfgRenderCtxMethods, ServiceDispatch { diff --git a/sdk/model/src/render/util/axis_filtering.ts b/sdk/model/src/render/util/axis_filtering.ts index ed941345cf..01b032fcb6 100644 --- a/sdk/model/src/render/util/axis_filtering.ts +++ b/sdk/model/src/render/util/axis_filtering.ts @@ -69,7 +69,6 @@ export function filterDataInfoEntries( break; } default: - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions throw new Error(`Unsupported data info type: ${type satisfies never}`); } diff --git a/sdk/model/src/render/util/column_collection.ts b/sdk/model/src/render/util/column_collection.ts index 2800a647ce..d200f84469 100644 --- a/sdk/model/src/render/util/column_collection.ts +++ b/sdk/model/src/render/util/column_collection.ts @@ -7,7 +7,6 @@ import type { NativePObjectId, PartitionedDataInfoEntries, PColumn, - PColumnLazy, PColumnSelector, PColumnSpec, PColumnValues, @@ -60,7 +59,7 @@ export interface AxisLabelProvider { /** * A simple implementation of {@link ColumnProvider} backed by a pre-defined array of columns. */ -class ArrayColumnProvider implements ColumnProvider { +class ArrayColumnsProvider implements ColumnProvider { constructor(private readonly columns: PColumn[]) {} selectColumns( @@ -76,21 +75,21 @@ class ArrayColumnProvider implements ColumnProvider { } /** Lazy calculates the data, returns undefined if data is not ready. */ -export type PColumnLazyWithLabel = PColumnLazy & { +export type PColumnWithLabel = PColumn & { label: string; }; /** Universal column is a column that uses a universal column id, and always have label. */ -export type PColumnLazyUniversal = PColumnLazyWithLabel & { +export type PColumnUniversal = PColumnWithLabel & { id: SUniversalPColumnId; }; -/** @deprecated Use PColumnLazyWithLabel instead. */ -export type PColumnEntryWithLabel = PColumnLazy & { +/** @deprecated Use PColumnWithLabel instead. */ +export type PColumnEntryWithLabel = PColumn & { label: string; }; -/** @deprecated Use PColumnLazyUniversal instead. */ +/** @deprecated Use PColumnUniversal instead. */ export type PColumnEntryUniversal = PColumnEntryWithLabel & { id: SUniversalPColumnId; }; @@ -220,7 +219,7 @@ type UniversalPColumnOpts = UniversalPColumnOptsNoDeriver & { export class PColumnCollection { private readonly defaultProviderStore: PColumn[] = []; private readonly providers: ColumnProvider[] = [ - new ArrayColumnProvider(this.defaultProviderStore), + new ArrayColumnsProvider(this.defaultProviderStore), ]; private readonly axisLabelProviders: AxisLabelProvider[] = []; @@ -498,12 +497,13 @@ export class PColumnCollection { } result.push({ - id: finalId, + id: finalId as PObjectId, spec: finalSpec, - data: () => - entry.type === "split" + get data() { + return entry.type === "split" ? entriesToDataInfo(filterDataInfoEntries(entry.dataEntries, axisFiltersTuple!)) - : entry.originalColumn.data, + : entry.originalColumn.data; + }, label: label, }); } @@ -555,30 +555,30 @@ export class PColumnCollection { | APColumnSelectorWithSplit | APColumnSelectorWithSplit[], opts: UniversalPColumnOpts, - ): PColumn[] | undefined; + ): PColumn[] | undefined; public getColumns( predicateOrSelectors: | ((spec: PColumnSpec) => boolean) | PColumnSelectorWithSplit | PColumnSelectorWithSplit[], opts?: UniversalPColumnOptsNoDeriver, - ): PColumn[] | undefined; + ): PColumn[] | undefined; public getColumns( predicateOrSelectors: | ((spec: PColumnSpec) => boolean) | APColumnSelectorWithSplit | APColumnSelectorWithSplit[], opts?: Optional, - ): PColumn[] | undefined { + ): PColumn[] | undefined { const entries = this.getUniversalEntries(predicateOrSelectors, { overrideLabelAnnotation: true, // default for getColumns ...opts, } as UniversalPColumnOpts); if (!entries) return undefined; - const columns: PColumn[] = []; + const columns: PColumn[] = []; for (const entry of entries) { - const data = entry.data(); + const data = entry.data; if (!data) { if (opts?.dontWaitAllData) continue; return undefined; diff --git a/sdk/model/src/render/util/pcolumn_data.ts b/sdk/model/src/render/util/pcolumn_data.ts index ad88c1aa37..7384cb6d93 100644 --- a/sdk/model/src/render/util/pcolumn_data.ts +++ b/sdk/model/src/render/util/pcolumn_data.ts @@ -2,7 +2,6 @@ import type { DataInfo, PartitionedDataInfoEntries, PColumn, - PColumnLazy, PColumnValues, } from "@milaboratories/pl-model-common"; import { @@ -546,14 +545,14 @@ export function convertOrParsePColumnData( } export function isPColumnReady( - c: PColumn | PColumnLazy, -): c is PColumn | PColumnLazy { + c: PColumn, +): c is PColumn { const isValues = (d: PColumnDataUniversal): d is PColumnValues => Array.isArray(d); const isAccessor = (d: PColumnDataUniversal): d is TreeNodeAccessor => d instanceof TreeNodeAccessor; let ready = true; - const data = typeof c.data === "function" ? c.data() : c.data; + const data = c.data; if (data == null) { return false; } else if (isAccessor(data)) { @@ -567,10 +566,7 @@ export function isPColumnReady( } export function allPColumnsReady( - columns: ( - | PColumn - | PColumnLazy - )[], -): columns is (PColumn | PColumnLazy)[] { + columns: PColumn[], +): columns is PColumn[] { return columns.every(isPColumnReady); } diff --git a/sdk/model/src/services/block_services.ts b/sdk/model/src/services/block_services.ts index 716d68161a..d81c34bacd 100644 --- a/sdk/model/src/services/block_services.ts +++ b/sdk/model/src/services/block_services.ts @@ -12,6 +12,7 @@ export const BLOCK_SERVICE_FLAGS = { requiresPFrameSpec: true, requiresPFrame: true, requiresDialog: true, + requiresColumnsCollection: true, } as const satisfies Partial; export type BlockServiceFlags = typeof BLOCK_SERVICE_FLAGS; diff --git a/sdk/model/src/services/get_services.ts b/sdk/model/src/services/get_services.ts index 8326e5637f..7de676a601 100644 --- a/sdk/model/src/services/get_services.ts +++ b/sdk/model/src/services/get_services.ts @@ -9,8 +9,11 @@ const cachedServices = new WeakMap< Map> >(); -export function getService(name: T): ModelServices[T] { - const ctx = getCfgRenderCtx(); +export function getService( + name: T, + deps?: { ctx?: GlobalCfgRenderCtx }, +): ModelServices[T] { + const ctx = deps?.ctx ?? getCfgRenderCtx(); const map = cachedServices.has(ctx) ? cachedServices.get(ctx)! diff --git a/sdk/ui-vue/package.json b/sdk/ui-vue/package.json index 815b19842f..cd8872b594 100644 --- a/sdk/ui-vue/package.json +++ b/sdk/ui-vue/package.json @@ -23,6 +23,7 @@ "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz" }, "dependencies": { + "@milaboratories/columns-collection-driver": "workspace:*", "@milaboratories/pf-spec-driver": "workspace:*", "@milaboratories/pl-model-common": "workspace:*", "@milaboratories/uikit": "workspace:*", diff --git a/sdk/ui-vue/src/components/PlAdvancedFilter/FilterEditor.vue b/sdk/ui-vue/src/components/PlAdvancedFilter/FilterEditor.vue index 1562fa37dc..292e50d593 100644 --- a/sdk/ui-vue/src/components/PlAdvancedFilter/FilterEditor.vue +++ b/sdk/ui-vue/src/components/PlAdvancedFilter/FilterEditor.vue @@ -13,6 +13,7 @@ import type { AnchoredPColumnId, AxisFilterByIdx, AxisFilterValue, + PTableColumnId, SUniversalPColumnId, } from "@platforma-sdk/model"; import { @@ -102,7 +103,9 @@ function changeSourceId(newSourceId?: PlAdvancedFilterColumnId) { if (!newSourceId) { return; } - const newSourceInfo = props.columnOptions.find((v) => v.id === getSourceId(newSourceId)); + const newSourceInfo = props.columnOptions.find((v) => + columnIdEquals(v.id, getSourceId(newSourceId)), + ); if (!newSourceInfo) { return; } @@ -119,8 +122,8 @@ function changeSourceId(newSourceId?: PlAdvancedFilterColumnId) { } const inconsistentSourceSelected = computed(() => { - const selectedOption = props.columnOptions.find( - (op) => op.id === getSourceId(props.filter.column), + const selectedOption = props.columnOptions.find((op) => + columnIdEquals(op.id, getSourceId(props.filter.column)), ); return selectedOption === undefined; }); @@ -132,7 +135,13 @@ const sourceOptions = computed(() => { return options; }); +function isPTableColumnIdObject(c: PlAdvancedFilterColumnId): c is PTableColumnId { + return typeof c === "object" && c !== null; +} + function getSourceId(column: PlAdvancedFilterColumnId): PlAdvancedFilterColumnId { + // PTableColumnId object form: no axisFilters embedded, return as-is. + if (isPTableColumnIdObject(column)) return column; try { const parsedColumnId = parseColumnId(column as SUniversalPColumnId); if (isFilteredPColumn(parsedColumnId)) { @@ -154,7 +163,7 @@ function getColumnAsSourceAndFixedAxes( column: PlAdvancedFilterColumnId, ): ColumnAsSourceAndFixedAxes { const sourceId = getSourceId(column); - const option = props.columnOptions.find((op) => op.id === sourceId); + const option = props.columnOptions.find((op) => columnIdEquals(op.id, sourceId)); const axesToBeFixed = (option?.axesToBeFixed ?? []).reduce( (res, item) => { res[item.idx] = undefined; @@ -162,15 +171,22 @@ function getColumnAsSourceAndFixedAxes( }, {} as Record, ); + // PTableColumnId object form has no axis filters baked in. + if (isPTableColumnIdObject(column)) { + return { source: column, axisFiltersByIndex: axesToBeFixed }; + } try { const parsedColumnId = parseColumnId(column as SUniversalPColumnId); if (isFilteredPColumn(parsedColumnId)) { return { source: sourceId, - axisFiltersByIndex: parsedColumnId.axisFilters.reduce((res, item) => { - res[item[0]] = item[1]; - return res; - }, axesToBeFixed), + axisFiltersByIndex: (parsedColumnId.axisFilters as AxisFilterByIdx[]).reduce( + (res, item) => { + res[item[0]] = item[1]; + return res; + }, + axesToBeFixed, + ), }; } } catch { @@ -179,12 +195,24 @@ function getColumnAsSourceAndFixedAxes( return { source: column, axisFiltersByIndex: axesToBeFixed }; } +/** Identity comparison for PlAdvancedFilterColumnId values — handles both string + * and object (PTableColumnId) variants. */ +function columnIdEquals(a: PlAdvancedFilterColumnId, b: PlAdvancedFilterColumnId): boolean { + if (a === b) return true; + if (typeof a === "object" && typeof b === "object" && a !== null && b !== null) { + return JSON.stringify(a) === JSON.stringify(b); + } + return false; +} + function stringifyColumn(value: ColumnAsSourceAndFixedAxes): PlAdvancedFilterColumnId { if (Object.keys(value.axisFiltersByIndex).length === 0) { return value.source; } + // PTableColumnId object form can't carry axisFilters — return as-is. + if (isPTableColumnIdObject(value.source)) return value.source; return stringifyColumnId({ - source: parseColumnId(value.source as SUniversalPColumnId) as AnchoredPColumnId, + source: parseColumnId(value.source as SUniversalPColumnId) as unknown as AnchoredPColumnId, axisFilters: Object.entries(value.axisFiltersByIndex).map( ([idx, value]) => [Number(idx), value] as AxisFilterByIdx, ), @@ -207,7 +235,7 @@ function updateAxisFilterValue(idx: number, value: AxisFilterValue | undefined) } const currentOption = computed(() => - props.columnOptions.find((op) => op.id === columnAsSourceAndFixedAxes.value.source), + props.columnOptions.find((op) => columnIdEquals(op.id, columnAsSourceAndFixedAxes.value.source)), ); const currentSpec = computed(() => currentOption.value?.spec ? getNormalizedSpec(currentOption.value.spec) : null, diff --git a/sdk/ui-vue/src/components/PlAdvancedFilter/types.ts b/sdk/ui-vue/src/components/PlAdvancedFilter/types.ts index cf27c45851..96ad5edd8a 100644 --- a/sdk/ui-vue/src/components/PlAdvancedFilter/types.ts +++ b/sdk/ui-vue/src/components/PlAdvancedFilter/types.ts @@ -13,11 +13,12 @@ import { PartialBy, RequiredBy } from "@milaboratories/helpers"; export type Operand = "or" | "and"; -// Can be any of string type, but for better type safety we use union of specific types +// Either a string column id (SUniversalPColumnId / canonicalized axis JSON) or +// a structured PTableColumnId object (axis | column ref). export type PlAdvancedFilterColumnId = | SUniversalPColumnId | CanonicalizedJson - | CanonicalizedJson; + | PTableColumnId; export type RequiredMeta = { id: number; diff --git a/sdk/ui-vue/src/components/PlAdvancedFilter/utils.ts b/sdk/ui-vue/src/components/PlAdvancedFilter/utils.ts index 7bf6029d03..69630a05d7 100644 --- a/sdk/ui-vue/src/components/PlAdvancedFilter/utils.ts +++ b/sdk/ui-vue/src/components/PlAdvancedFilter/utils.ts @@ -29,7 +29,7 @@ export function getNewId() { return Date.now(); } -export function createNewGroup(selectedSourceId: string): NodeFilter { +export function createNewGroup(selectedSourceId: PlAdvancedFilterColumnId): NodeFilter { return { id: getNewId(), isExpanded: true, diff --git a/sdk/ui-vue/src/components/PlAgDataTable/PlAgDataTableV2.vue b/sdk/ui-vue/src/components/PlAgDataTable/PlAgDataTableV2.vue index ed5d8ecdc3..011fcf82e5 100644 --- a/sdk/ui-vue/src/components/PlAgDataTable/PlAgDataTableV2.vue +++ b/sdk/ui-vue/src/components/PlAgDataTable/PlAgDataTableV2.vue @@ -2,6 +2,7 @@ import { promiseTimeout, isJsonEqual } from "@milaboratories/helpers"; import type { AxisId, + PlDataTableColumnsMeta, PlDataTableGridStateCore, PlDataTableStateV2, PlSelectionModel, @@ -16,7 +17,6 @@ import { getAxisId, canonicalizeJson, isAbortError, - isColumnOptional, } from "@platforma-sdk/model"; import type { CellRendererSelectorFunc, GridApi, GridState } from "ag-grid-enterprise"; import { AgGridVue } from "ag-grid-vue3"; @@ -30,7 +30,7 @@ import PlAgRowCount from "./PlAgRowCount.vue"; import { DeferredCircular, ensureNodeVisible } from "./sources/focus-row"; import { PlAgDataTableRowNumberColId } from "./sources/row-number"; import type { PlAgCellButtonAxisParams } from "./sources/table-source-v2"; -import { calculateGridOptions } from "./sources/table-source-v2"; +import { calculateGridOptions, effectiveVisibility } from "./sources/table-source-v2"; import { useTableState } from "./sources/table-state-v2"; import type { PlAgDataTableV2Controller, @@ -133,6 +133,7 @@ gridOptions.value.onGridPreDestroyed = (event) => { makePartialState(event.api.getState()), gridState.value, event.api, + currentColumnsMeta(), ); } gridApi.value = null; @@ -145,6 +146,7 @@ gridOptions.value.onStateUpdated = (event) => { makePartialState(event.state), gridState.value, event.api, + currentColumnsMeta(), ); // We have to keep initialState synchronized with gridState for gridState recovery after key updating. gridOptions.value.initialState = gridState.value = partialState; @@ -219,6 +221,7 @@ function normalizeColumnVisibility( partialState: PlDataTableGridStateCore, prevState: PlDataTableGridStateCore, api: GridApi, + columnsMeta: PlDataTableColumnsMeta | undefined, ): PlDataTableGridStateCore { if (partialState.columnVisibility !== undefined) return partialState; @@ -229,7 +232,7 @@ function normalizeColumnVisibility( // No previous explicit state → compute defaults from current columns // to replicate: hide: hiddenColIds?.includes(colId) ?? isColumnOptional(spec.spec) - const defaultHidden = getDefaultHiddenColIds(api); + const defaultHidden = getDefaultHiddenColIds(api, columnsMeta); if (defaultHidden.length > 0) { return { ...partialState, columnVisibility: { hiddenColIds: defaultHidden } }; } @@ -237,14 +240,26 @@ function normalizeColumnVisibility( return partialState; } -function getDefaultHiddenColIds(api: GridApi): PlTableColumnIdJson[] { +/** Sidecar display meta is only on the `model` branch of the settings union. */ +function currentColumnsMeta(): PlDataTableColumnsMeta | undefined { + return "model" in props.settings ? props.settings.model?.columnsMeta : undefined; +} + +function getDefaultHiddenColIds( + api: GridApi, + columnsMeta: PlDataTableColumnsMeta | undefined, +): PlTableColumnIdJson[] { const cols = api.getAllGridColumns(); if (!cols) return []; return cols .filter((col) => { const spec = col.getColDef().context as PTableColumnSpec | undefined; - return spec !== undefined && spec.type === "column" && isColumnOptional(spec.spec); + return ( + spec !== undefined && + spec.type === "column" && + effectiveVisibility(spec, columnsMeta) === "optional" + ); }) .map((col) => col.getColId() as PlTableColumnIdJson); } @@ -426,6 +441,7 @@ watch( pfDriver: getRawPlatformaInstance().pFrameDriver, fullTableHandle: settings.model?.fullTableHandle, visibleTableHandle: settings.model?.visibleTableHandle, + columnsMeta: settings.model?.columnsMeta, sheets: settings.sheets ?? [], dataRenderedTracker, hiddenColIds: gridState.value.columnVisibility?.hiddenColIds, diff --git a/sdk/ui-vue/src/components/PlAgDataTable/sources/table-source-v2.ts b/sdk/ui-vue/src/components/PlAgDataTable/sources/table-source-v2.ts index 4994ebb3c9..66d27f949d 100644 --- a/sdk/ui-vue/src/components/PlAgDataTable/sources/table-source-v2.ts +++ b/sdk/ui-vue/src/components/PlAgDataTable/sources/table-source-v2.ts @@ -1,5 +1,7 @@ import type { + AnnotationDataStatus, AxesSpec, + PlDataTableColumnsMeta, PTableColumnId, PTableColumnSpecAxis, PTableColumnSpecColumn, @@ -50,10 +52,6 @@ import { isJsonEqual } from "@milaboratories/helpers"; import type { DeferredCircular } from "./focus-row"; import { isNil, uniq } from "es-toolkit"; -export function isLabelColumn(column: PTableColumnSpec): column is PTableColumnSpecColumn { - return column.type === "column" && isLabelColumnSpec(column.spec); -} - /** Convert columnar data from the driver to rows, used by ag-grid */ function columns2rows( fields: number[], @@ -88,6 +86,7 @@ export async function calculateGridOptions({ sheets, fullTableHandle, visibleTableHandle, + columnsMeta, dataRenderedTracker, hiddenColIds, cellButtonAxisParams, @@ -97,6 +96,8 @@ export async function calculateGridOptions({ generation: Ref; fullTableHandle: PTableHandle; visibleTableHandle: PTableHandle; + /** Sidecar display metadata (label/visibility/order/status/hidden axes) from the model. */ + columnsMeta?: PlDataTableColumnsMeta; dataRenderedTracker: DeferredCircular>; hiddenColIds?: PlTableColumnIdJson[]; cellButtonAxisParams?: PlAgCellButtonAxisParams; @@ -108,13 +109,19 @@ export async function calculateGridOptions({ const stateGeneration = generation.value; // get specs of the full table - const [tableSpecs, visibleTableSpecs] = await Promise.all([ + const [rawTableSpecs, rawVisibleTableSpecs] = await Promise.all([ pfDriver.getSpec(fullTableHandle), pfDriver.getSpec(visibleTableHandle), ]); if (stateGeneration !== generation.value) throw new Error("table state generation changed"); + // Engine specs flow through untouched; readers below merge `columnsMeta` at + // their use sites via the `effective*` helpers — keeps spec === what the + // engine produced (matters for `colDef.context` consumers). + const tableSpecs = rawTableSpecs; + const visibleTableSpecs = rawVisibleTableSpecs; + // index mapping from full specs to visible subset (hidden columns → -1) const specsToVisibleSpecsMapping = buildSpecsToVisibleSpecsMapping(tableSpecs, visibleTableSpecs); @@ -125,18 +132,33 @@ export async function calculateGridOptions({ // displayable column indices ordered: axes first, then columns by OrderPriority const fields = sortIndicesByTypeAndPriority( - selectDisplayableIndices(tableSpecs, isPartitionedAxis, getLabelColumnIndex), + selectDisplayableIndices(tableSpecs, isPartitionedAxis, getLabelColumnIndex, columnsMeta), tableSpecs, + columnsMeta, ); // default hidden columns derived from Optional annotation when no saved state - const resolvedHiddenColIds = hiddenColIds ?? computeDefaultHiddenColIds(fields, tableSpecs); + const resolvedHiddenColIds = + hiddenColIds ?? computeDefaultHiddenColIds(fields, tableSpecs, columnsMeta); const columnDefs: ColDef[] = [ makeRowNumberColDef(), - ...fields.map((field) => - makeColDef(field, tableSpecs[field], resolvedHiddenColIds, cellButtonAxisParams), - ), + ...fields.map((field) => { + const spec = tableSpecs[field]; + // Only visible columns get a status (hidden columns → undefined → no + // "computing/absent" placeholder rendering). + const visible = (specsToVisibleSpecsMapping.get(field) ?? -1) >= 0; + const status = + visible && spec.type === "column" ? columnsMeta?.columns[spec.id]?.status : undefined; + return makeColDef({ + iCol: field, + spec, + dataStatus: status, + hiddenColIds: resolvedHiddenColIds, + cellButtonAxisParams, + columnsMeta, + }); + }), ]; // axes — taken directly from visible table (always present as part of join) @@ -229,15 +251,26 @@ export type PlAgCellButtonAxisParams = { /** * Calculates column definition for a given p-table column */ -export function makeColDef( - iCol: number, - spec: PTableColumnSpec, - hiddenColIds: PlTableColumnIdJson[] | undefined, - cellButtonAxisParams?: PlAgCellButtonAxisParams, -): ColDef { +export function makeColDef({ + iCol, + spec, + dataStatus, + hiddenColIds, + cellButtonAxisParams, + columnsMeta, +}: { + iCol: number; + spec: PTableColumnSpec; + /** Per-column rendering status from the model; `undefined` for hidden cols. */ + dataStatus: AnnotationDataStatus | undefined; + hiddenColIds: PlTableColumnIdJson[] | undefined; + cellButtonAxisParams?: PlAgCellButtonAxisParams; + /** Sidecar meta used to resolve the effective header label. */ + columnsMeta?: PlDataTableColumnsMeta; +}): ColDef { const colId = canonicalizeJson(getPTableColumnId(spec)); const valueType = spec.type === "axis" ? spec.spec.type : spec.spec.valueType; - const columnRenderingSpec = getColumnRenderingSpec(spec); + const columnRenderingSpec = getColumnRenderingSpec(spec, dataStatus); const cellStyle: CellStyle = {}; if (columnRenderingSpec.fontFamily) { if (columnRenderingSpec.fontFamily === "monospace") { @@ -247,8 +280,7 @@ export function makeColDef( cellStyle.fontFamily = columnRenderingSpec.fontFamily; } } - const headerName = - readAnnotation(spec.spec, Annotation.Label)?.trim() ?? `Unlabeled ${spec.type} ${iCol}`; + const headerName = effectiveLabel(spec, columnsMeta)?.trim() ?? `Unlabeled ${spec.type} ${iCol}`; return { colId, @@ -318,6 +350,70 @@ export function makeColDef( }; } +export function isLabelColumn(column: PTableColumnSpec): column is PTableColumnSpecColumn { + return column.type === "column" && isLabelColumnSpec(column.spec); +} + +/** + * Effective column label: sidecar override wins over intrinsic spec + * annotation. Axes have no label override in the sidecar — they fall through + * to `pl7.app/label` on the axis spec. + */ +export function effectiveLabel( + spec: PTableColumnSpec, + meta: PlDataTableColumnsMeta | undefined, +): string | undefined { + if (spec.type === "column") { + const override = meta?.columns[spec.id]?.label; + if (override !== undefined) return override; + } + return readAnnotation(spec.spec, Annotation.Label); +} + +/** + * Effective visibility. For columns: sidecar override wins over the intrinsic + * `pl7.app/table/visibility` annotation. For axes: `meta.axes[id].hidden` + * forces hidden; otherwise the intrinsic axis annotation rules. + * + * Kept out of the spec on purpose — baking visibility into a column spec via + * override changes its `ColumnUniversalId`, so the same physical column reached + * two ways (e.g. as primary and as a discovered label) would diverge into two + * ids and render twice. The sidecar preserves identity; readers apply this at + * use sites. + */ +export function effectiveVisibility( + spec: PTableColumnSpec, + meta: PlDataTableColumnsMeta | undefined, +): "default" | "optional" | "hidden" { + if (spec.type === "axis") { + if (meta?.axes[canonicalizeJson(getAxisId(spec.id))]?.hidden === true) return "hidden"; + } else { + const override = meta?.columns[spec.id]?.visibility; + if (override !== undefined) return override; + } + if (isColumnHidden(spec.spec)) return "hidden"; + if (isColumnOptional(spec.spec)) return "optional"; + return "default"; +} + +/** + * Effective order priority — number suitable for left-to-right comparison + * (higher = further left). Sidecar override on columns wins over the + * intrinsic `pl7.app/table/orderPriority`; axes have no override. + * `undefined` when neither source supplied a value (consumer decides default). + */ +export function effectiveOrder( + spec: PTableColumnSpec, + meta: PlDataTableColumnsMeta | undefined, +): number | undefined { + if (spec.type === "column") { + const override = meta?.columns[spec.id]?.order; + if (override !== undefined) return override; + } + const raw = readAnnotationJson(spec.spec, Annotation.Table.OrderPriority); + return raw as number | undefined; +} + /** Build index mapping from full tableSpecs to their position in visibleTableSpecs (missing → -1). */ function buildSpecsToVisibleSpecsMapping( tableSpecs: PTableColumnSpec[], @@ -370,6 +466,7 @@ function selectDisplayableIndices( tableSpecs: PTableColumnSpec[], isPartitionedAxis: (axisId: AxisId) => boolean, getLabelColumnIndex: (axisId: AxisId) => number, + meta: PlDataTableColumnsMeta | undefined, ): number[] { return tableSpecs .entries() @@ -377,11 +474,12 @@ function selectDisplayableIndices( switch (spec.type) { case "axis": return ( - !(getLabelColumnIndex(spec.id) > -1 ? true : isColumnHidden(spec.spec)) && - !isPartitionedAxis(spec.id) + !(getLabelColumnIndex(spec.id) > -1 + ? true + : effectiveVisibility(spec, meta) === "hidden") && !isPartitionedAxis(spec.id) ); case "column": - return !isColumnHidden(spec.spec) && !isLinkerColumnSpec(spec.spec); + return effectiveVisibility(spec, meta) !== "hidden" && !isLinkerColumnSpec(spec.spec); } }) .map(([i]) => i) @@ -389,14 +487,15 @@ function selectDisplayableIndices( } /** Sort: axes first, then columns by OrderPriority annotation (higher priority = further left). */ -function sortIndicesByTypeAndPriority(indices: number[], tableSpecs: PTableColumnSpec[]): number[] { +function sortIndicesByTypeAndPriority( + indices: number[], + tableSpecs: PTableColumnSpec[], + meta: PlDataTableColumnsMeta | undefined, +): number[] { const priorityOf = (i: number): number => { const spec = tableSpecs[i]; - const prior = - spec.type === "axis" || isLabelColumnSpec(spec.spec) - ? Infinity - : Number(readAnnotationJson(spec.spec, Annotation.Table.OrderPriority)); - + if (spec.type === "axis" || isLabelColumnSpec(spec.spec)) return Infinity; + const prior = Number(effectiveOrder(spec, meta)); return isNaN(prior) ? 0 : prior; }; return [...indices].sort((a, b) => priorityOf(b) - priorityOf(a)); @@ -406,12 +505,14 @@ function sortIndicesByTypeAndPriority(indices: number[], tableSpecs: PTableColum function computeDefaultHiddenColIds( fields: number[], tableSpecs: PTableColumnSpec[], + meta: PlDataTableColumnsMeta | undefined, ): PlTableColumnIdJson[] { return fields.reduce((acc, field) => { const spec = tableSpecs[field]; - return spec.type === "column" && isColumnOptional(spec.spec) - ? [...acc, canonicalizeJson(getPTableColumnId(spec))] - : acc; + if (spec.type === "column" && effectiveVisibility(spec, meta) === "optional") { + acc.push(canonicalizeJson(getPTableColumnId(spec))); + } + return acc; }, []); } diff --git a/sdk/ui-vue/src/components/PlAgDataTable/sources/table-state-v2.ts b/sdk/ui-vue/src/components/PlAgDataTable/sources/table-state-v2.ts index 1eeb49d201..9d026b73c3 100644 --- a/sdk/ui-vue/src/components/PlAgDataTable/sources/table-state-v2.ts +++ b/sdk/ui-vue/src/components/PlAgDataTable/sources/table-state-v2.ts @@ -1,7 +1,6 @@ import { createDefaultPTableParams, parseJson, - canonicalizeJson, upgradePlDataTableStateV2, type FilterSpec, type FilterSpecLeaf, @@ -20,7 +19,6 @@ import { distillFilterSpec, PlDataTableFiltersWithMeta, getPTableColumnId, - CanonicalizedJson, } from "@platforma-sdk/model"; import { computed, type Ref, type WritableComputedRef } from "vue"; import type { PlDataTableSettingsV2 } from "../types"; @@ -333,16 +331,16 @@ function stripSuppressedFilters(node: PlDataTableFiltersWithMeta): PlDataTableFi function createSearchFilterNode( columns: PTableColumnSpec[], search: null | undefined | string, -): null | FilterSpec>> { +): null | FilterSpec> { const trimmed = search?.trim(); if (isNil(trimmed) || trimmed.length === 0) return null; - const parts: FilterSpec>>[] = []; + const parts: FilterSpec>[] = []; const numericValue = Number(trimmed); const isValidNumber = trimmed.length > 0 && !isNaN(numericValue) && isFinite(numericValue); for (const col of columns) { - const column = canonicalizeJson(getPTableColumnId(col)); + const column = getPTableColumnId(col); const spec = col.spec; if (isStringValueType(spec)) { @@ -405,9 +403,9 @@ function annotateNodeWithIds( function convertPartitionFiltersToFilterSpec( sheetsState: PlDataTableSheetState[], -): FilterSpec>>[] { +): FilterSpec>[] { return sheetsState.map((s) => { - const column = canonicalizeJson({ type: "axis", id: s.axisId }); + const column: PTableColumnId = { type: "axis", id: s.axisId }; return typeof s.value === "number" ? { type: "equal" as const, column, x: s.value } : { type: "patternEquals" as const, column, value: s.value }; diff --git a/sdk/ui-vue/src/components/PlAgDataTable/sources/value-rendering.ts b/sdk/ui-vue/src/components/PlAgDataTable/sources/value-rendering.ts index a5518e5398..75d94ebb6f 100644 --- a/sdk/ui-vue/src/components/PlAgDataTable/sources/value-rendering.ts +++ b/sdk/ui-vue/src/components/PlAgDataTable/sources/value-rendering.ts @@ -1,5 +1,6 @@ import { Annotation, + AnnotationDataStatus, PTableNA, readAnnotation, ValueType, @@ -14,7 +15,7 @@ import type { PlAgDataTableV2Row } from "../types"; export function formatSpecialValues( value: undefined | PTableValue | PTableHidden, - dataStatus: undefined | "absent" | "error" | "computing" | "ready", + dataStatus: undefined | AnnotationDataStatus, ): string | undefined { if (dataStatus === "absent") { return "absent"; @@ -36,7 +37,10 @@ export type ColumnRenderingSpec = { fontFamily?: string; }; -export function getColumnRenderingSpec(spec: PTableColumnSpec): ColumnRenderingSpec { +export function getColumnRenderingSpec( + spec: PTableColumnSpec, + dataStatus: undefined | AnnotationDataStatus, +): ColumnRenderingSpec { const valueType = spec.type === "axis" ? spec.spec.type : spec.spec.valueType; let renderSpec: ColumnRenderingSpec; switch (valueType) { @@ -45,7 +49,6 @@ export function getColumnRenderingSpec(spec: PTableColumnSpec): ColumnRenderingS case ValueType.Float: case ValueType.Double: { const format = readAnnotation(spec.spec, Annotation.Format); - const dataStatus = readAnnotation(spec.spec, Annotation.DataStatus); const formatFn = format ? d3.format(format) : undefined; renderSpec = { valueFormatter: (params) => { @@ -59,10 +62,7 @@ export function getColumnRenderingSpec(spec: PTableColumnSpec): ColumnRenderingS default: renderSpec = { valueFormatter: (params) => { - const formatted = formatSpecialValues( - params.value, - readAnnotation(spec.spec, Annotation.DataStatus), - ); + const formatted = formatSpecialValues(params.value, dataStatus); if (formatted !== undefined) return formatted; return params.value?.toString() ?? ""; }, diff --git a/sdk/ui-vue/src/components/PlTableFilters/PlTableFiltersV2.vue b/sdk/ui-vue/src/components/PlTableFilters/PlTableFiltersV2.vue index 3656aeaca7..f259db0dee 100644 --- a/sdk/ui-vue/src/components/PlTableFilters/PlTableFiltersV2.vue +++ b/sdk/ui-vue/src/components/PlTableFilters/PlTableFiltersV2.vue @@ -4,16 +4,14 @@ import type { PlDataTableFiltersWithMeta, PFrameHandle, PTableColumnId, - CanonicalizedJson, } from "@platforma-sdk/model"; import { - canonicalizeJson, Annotation, Domain, readAnnotation, readDomain, getUniqueSourceValuesWithLabels, - parseJson, + extractPObjectId, getPTableColumnId, } from "@platforma-sdk/model"; import { computed, ref } from "vue"; @@ -68,7 +66,7 @@ const onUpdateFilters = (_value: PlAdvancedFilter) => { const options = computed(() => { return props.columns.map((col, idx) => { - const id = makeFilterColumnId(col); + const id = getPTableColumnId(col); const label = readAnnotation(col.spec, Annotation.Label)?.trim() ?? `Unlabeled ${col.type} ${idx}`; const alphabet = @@ -112,25 +110,23 @@ function handleSuggestOptions(params: { return []; } - const strId = params.columnId as CanonicalizedJson; - const tableColumnId = parseJson(strId); - - if (tableColumnId.type !== "column") { + const tableColumnId = params.columnId as PTableColumnId; + if ( + typeof tableColumnId !== "object" || + tableColumnId === null || + tableColumnId.type !== "column" + ) { throw new Error("ColumnId should be of type 'column' for suggest options"); } return getUniqueSourceValuesWithLabels(props.pframeHandle, { - columnId: tableColumnId.id, + columnId: extractPObjectId(tableColumnId.id), axisIdx: params.axisIdx, limit: 100, searchQuery: params.searchType === "label" ? params.searchStr : undefined, searchQueryValue: params.searchType === "value" ? params.searchStr : undefined, }).then((v) => v.values); } - -function makeFilterColumnId(spec: PTableColumnSpec): CanonicalizedJson { - return canonicalizeJson(getPTableColumnId(spec)); -}