diff --git a/.changeset/resource-signature-support.md b/.changeset/resource-signature-support.md new file mode 100644 index 0000000000..038e5891bf --- /dev/null +++ b/.changeset/resource-signature-support.md @@ -0,0 +1,11 @@ +--- +"@milaboratories/pl-client": minor +"@milaboratories/pl-drivers": minor +"@milaboratories/pl-tree": minor +--- + +Add resource signature propagation for server-side access control. + +- `pl-client`: cross-transaction `SignatureCache`, automatic signature tracking in `PlTransaction` (store/retrieve signatures for all resource and field operations), `setDefaultColor` for color proof on resource creation, `PermissionDeniedError` error type +- `pl-drivers`: pass `resourceSignature` through proxied APIs (download, upload, logs, progress, ls), encode signatures in remote blob and log handles +- `pl-tree`: propagate `resourceSignature` in `ResourceInfo`, `ResourceSnapshot`, and `PlTreeResource` state diff --git a/lib/node/pl-client/src/core/client.ts b/lib/node/pl-client/src/core/client.ts index d8cd067a3a..677a062e43 100644 --- a/lib/node/pl-client/src/core/client.ts +++ b/lib/node/pl-client/src/core/client.ts @@ -1,15 +1,16 @@ import type { AuthOps, PlClientConfig, PlConnectionStatusListener, wireProtocol } from "./config"; import type { PlCallOps } from "./ll_client"; import { LLPlClient } from "./ll_client"; -import type { AnyResourceRef } from "./transaction"; +import type { AnyResourceRef, SignatureResolver } from "./transaction"; import { PlTransaction, toGlobalResourceId, TxCommitConflict } from "./transaction"; import { createHash } from "node:crypto"; -import type { OptionalResourceId, ResourceId } from "./types"; +import type { GlobalResourceId, OptionalResourceId, ResourceId } from "./types"; import { bigintToResourceId, ensureResourceIdNotNull, isNullResourceId, NullResourceId, + toResourceSignature, } from "./types"; import { ClientRoot } from "../helpers/pl"; import { isUnimplementedError } from "./errors"; @@ -38,6 +39,7 @@ export type TxOps = PlCallOps & { retryOptions?: RetryOptions; name?: string; lockId?: string; + signatureResolver?: SignatureResolver; }; const defaultTxOps = { @@ -94,7 +96,7 @@ export class PlClient { public readonly finalPredicate: FinalResourceDataPredicate; /** Resource data cache, to minimize redundant data rereading from remote db */ - private readonly resourceDataCache: LRUCache; + private readonly resourceDataCache: LRUCache; private constructor( configOrAddress: PlClientConfig | string, @@ -236,7 +238,10 @@ export class PlClient { const responses = await this._ll.listUserResources({ limit: 1 }); for (const msg of responses) { if (msg.entry.oneofKind === "userRoot") { - rootFromServer = bigintToResourceId(msg.entry.userRoot.resourceId); + rootFromServer = bigintToResourceId( + msg.entry.userRoot.resourceId, + toResourceSignature(msg.entry.userRoot.resourceSignature), + ); break; } } @@ -268,6 +273,9 @@ export class PlClient { return await altRoot.globalId; }, + rootFromServer.signature !== undefined + ? { signatureResolver: (id) => (id.id === rootFromServer!.id ? rootFromServer!.signature : undefined) } + : undefined, ); } } else { @@ -341,14 +349,17 @@ export class PlClient { // opening low-level tx const llTx = this.ll.createTx(writable, ops); // wrapping it into high-level tx (this also asynchronously sends initialization message) - const tx = new PlTransaction( - llTx, - name, - writable, - clientRoot, - this.finalPredicate, - this.resourceDataCache, - ); + const tx = new PlTransaction(llTx, name, writable, clientRoot, { + finalPredicate: this.finalPredicate, + resourceDataCache: this.resourceDataCache, + signatureResolver: ops?.signatureResolver, + }); + + // Auto-set default color proof so that resource creation (write TXs) + // and name lookups (read TXs) carry the correct access color. + if (!isNullResourceId(clientRoot) && clientRoot.signature && writable) { + tx.setDefaultColor(clientRoot.signature); + } let ok = false; let result: T | undefined = undefined; diff --git a/lib/node/pl-client/src/core/errors.ts b/lib/node/pl-client/src/core/errors.ts index 57f01ad8b8..39f1f1e52b 100644 --- a/lib/node/pl-client/src/core/errors.ts +++ b/lib/node/pl-client/src/core/errors.ts @@ -25,6 +25,18 @@ export function isUnauthenticated(err: unknown, nested: boolean = false): boolea return false; } +export function isPermissionDenied(err: unknown, nested: boolean = false): boolean { + if (err === undefined || err === null) return false; + + if (err instanceof PermissionDeniedError) return true; + if ((err as any).name === "RpcError" && (err as any).code === "PERMISSION_DENIED") return true; + if ((err as any).name === "RESTError" && (err as any).status.code === Code.PERMISSION_DENIED) + return true; + if ((err as any).cause !== undefined && !nested) + return isPermissionDenied((err as any).cause, true); + return false; +} + export function isTimeoutError(err: unknown, nested: boolean = false): boolean { if (err === undefined || err === null) return false; @@ -125,6 +137,13 @@ export class UnauthenticatedError extends Error { } } +export class PermissionDeniedError extends Error { + name = "PermissionDeniedError"; + constructor(message: string) { + super("PermissionDenied: " + message); + } +} + export class DisconnectedError extends Error { name = "DisconnectedError"; constructor(message: string) { @@ -144,6 +163,10 @@ export function rethrowMeaningfulError(error: any, wrapIfUnknown: boolean = fals if (error instanceof UnauthenticatedError) throw error; throw new UnauthenticatedError(error.message); } + if (isPermissionDenied(error)) { + if (error instanceof PermissionDeniedError) throw error; + throw new PermissionDeniedError(error.message); + } if (isConnectionProblem(error)) { if (error instanceof DisconnectedError) throw error; throw new DisconnectedError(error.message); diff --git a/lib/node/pl-client/src/core/ll_client.test.ts b/lib/node/pl-client/src/core/ll_client.test.ts index 3a57756fb5..88ce986ad4 100644 --- a/lib/node/pl-client/src/core/ll_client.test.ts +++ b/lib/node/pl-client/src/core/ll_client.test.ts @@ -146,7 +146,6 @@ test("test https call via proxy", async () => { test("list user resources returns user root", async () => { const client = await getTestLLClient(); const responses = await client.listUserResources({ limit: 1 }); - expect(responses.length).toBe(1); - const msg = responses[0]; - expect(msg.entry.oneofKind).toBe("userRoot"); + const userRoots = responses.filter((r) => r.entry.oneofKind === "userRoot"); + expect(userRoots.length).toBe(1); }); diff --git a/lib/node/pl-client/src/core/ll_transaction.test.ts b/lib/node/pl-client/src/core/ll_transaction.test.ts index c2a3eca09b..e37b92cf25 100644 --- a/lib/node/pl-client/src/core/ll_transaction.test.ts +++ b/lib/node/pl-client/src/core/ll_transaction.test.ts @@ -5,6 +5,18 @@ import { test, expect } from "vitest"; import { isTimeoutOrCancelError } from "./errors"; import { Aborted } from "@milaboratories/ts-helpers"; +import type { LLPlClient } from "./ll_client"; + +/** Get root resource signature from ListUserResources for use as color proof in tests. */ +async function getRootSignature(client: LLPlClient): Promise { + const responses = await client.listUserResources({ limit: 1 }); + for (const msg of responses) { + if (msg.entry.oneofKind === "userRoot" && msg.entry.userRoot.resourceSignature) { + return msg.entry.userRoot.resourceSignature; + } + } + return new Uint8Array(0); +} test("check successful transaction", async () => { const client = await getTestLLClient(); @@ -80,6 +92,7 @@ test("check timeout error type (passive)", async () => { test("check timeout error type (active)", async () => { const client = await getTestLLClient(); + const rootSig = await getRootSignature(client); const tx = client.createTx(true, { timeout: 500 }); try { @@ -96,6 +109,12 @@ test("check timeout error type (active)", async () => { ); expect(openResponse.txOpen.tx?.isValid).toBeTruthy(); + // Set default color so resource creation succeeds in strict mode + await tx.send( + { oneofKind: "setDefaultColor", setDefaultColor: { colorProof: rootSig } }, + false, + ); + const rData = Uint8Array.from([ (Math.random() * 256) & 0xff, (Math.random() * 256) & 0xff, @@ -115,12 +134,13 @@ test("check timeout error type (active)", async () => { type: { name: "TestValue", version: "1" }, data: rData, errorIfExists: false, - colorProof: new Uint8Array(0), }, }, false, ); - const id = (await createResponse).resourceCreateValue.resourceId; + const createResp = (await createResponse).resourceCreateValue; + const id = createResp.resourceId; + const resourceSignature = createResp.resourceSignature ?? new Uint8Array(0); while (true) { const vr = await tx.send( @@ -129,7 +149,7 @@ test("check timeout error type (active)", async () => { resourceGet: { resourceId: id, loadFields: false, - resourceSignature: new Uint8Array(0), + resourceSignature, }, }, false, @@ -144,6 +164,7 @@ test("check timeout error type (active)", async () => { test("check is abort error (active)", async () => { const client = await getTestLLClient(); + const rootSig = await getRootSignature(client); const tx = client.createTx(true, { abortSignal: AbortSignal.timeout(100) }); try { @@ -160,6 +181,12 @@ test("check is abort error (active)", async () => { ); expect(openResponse.txOpen.tx?.isValid).toBeTruthy(); + // Set default color so resource creation succeeds in strict mode + await tx.send( + { oneofKind: "setDefaultColor", setDefaultColor: { colorProof: rootSig } }, + false, + ); + const rData = Uint8Array.from([ Math.random() & 0xff, Math.random() & 0xff, @@ -179,12 +206,13 @@ test("check is abort error (active)", async () => { type: { name: "TestValue", version: "1" }, data: rData, errorIfExists: false, - colorProof: new Uint8Array(0), }, }, false, ); - const id = (await createResponse).resourceCreateValue.resourceId; + const createResp = (await createResponse).resourceCreateValue; + const id = createResp.resourceId; + const resourceSignature = createResp.resourceSignature ?? new Uint8Array(0); while (true) { const vr = await tx.send( @@ -193,7 +221,7 @@ test("check is abort error (active)", async () => { resourceGet: { resourceId: id, loadFields: false, - resourceSignature: new Uint8Array(0), + resourceSignature, }, }, false, diff --git a/lib/node/pl-client/src/core/transaction.ts b/lib/node/pl-client/src/core/transaction.ts index 6ac67812a7..77cc3a779f 100644 --- a/lib/node/pl-client/src/core/transaction.ts +++ b/lib/node/pl-client/src/core/transaction.ts @@ -2,6 +2,8 @@ /* eslint-disable no-prototype-builtins */ import type { AnyResourceId, + ColorProof, + GlobalResourceId, LocalResourceId, OptionalResourceId, BasicResourceData, @@ -9,6 +11,7 @@ import type { FieldType, ResourceData, ResourceId, + ResourceSignature, ResourceType, FutureFieldType, } from "./types"; @@ -19,6 +22,7 @@ import { isLocalResourceId, extractBasicResourceData, isNullResourceId, + toResourceSignature, } from "./types"; import type { ClientMessageRequest, @@ -82,7 +86,7 @@ export type FieldRef = _FieldId; export type LocalFieldId = _FieldId; export type AnyFieldId = FieldId | LocalFieldId; -export type AnyResourceRef = ResourceRef | ResourceId; +export type AnyResourceRef = ResourceRef | AnyResourceId; export type AnyFieldRef = _FieldId; // FieldRef | FieldId export type AnyRef = AnyResourceRef | AnyFieldRef; @@ -91,42 +95,53 @@ export function isField(ref: AnyRef): ref is AnyFieldRef { } export function isResource(ref: AnyRef): ref is AnyResourceRef { - return ( - typeof ref === "bigint" || (ref.hasOwnProperty("globalId") && ref.hasOwnProperty("localId")) - ); + if (typeof ref === "bigint") return true; // LocalResourceId or NullResourceId + return isResourceRef(ref) || isResourceId(ref); } export function isResourceId(ref: AnyRef): ref is ResourceId { - return typeof ref === "bigint" && !isLocalResourceId(ref) && !isNullResourceId(ref); + return ( + typeof ref === "object" && + ref !== null && + "id" in ref && + !("globalId" in ref) && + !("resourceId" in ref) + ); } export function isFieldRef(ref: AnyFieldRef): ref is FieldRef { return isResourceRef(ref.resourceId); } -export function isResourceRef(ref: AnyResourceRef): ref is ResourceRef { - return ref.hasOwnProperty("globalId") && ref.hasOwnProperty("localId"); +export function isResourceRef(ref: AnyRef): ref is ResourceRef { + return ( + typeof ref === "object" && + ref !== null && + ref.hasOwnProperty("globalId") && + ref.hasOwnProperty("localId") + ); } + export function toFieldId(ref: AnyFieldRef): AnyFieldId { if (isFieldRef(ref)) return { resourceId: ref.resourceId.localId, fieldName: ref.fieldName }; - else return ref as FieldId; + return ref as AnyFieldId; } export async function toGlobalFieldId(ref: AnyFieldRef): Promise { if (isFieldRef(ref)) return { resourceId: await ref.resourceId.globalId, fieldName: ref.fieldName }; - else return ref as FieldId; + return { resourceId: ref.resourceId as ResourceId, fieldName: ref.fieldName }; } export function toResourceId(ref: AnyResourceRef): AnyResourceId { if (isResourceRef(ref)) return ref.localId; - else return ref; + return ref; } export async function toGlobalResourceId(ref: AnyResourceRef): Promise { if (isResourceRef(ref)) return await ref.globalId; - else return ref; + return ref as ResourceId; } export function field(resourceId: AnyResourceRef, fieldName: string): AnyFieldRef { @@ -147,6 +162,15 @@ async function notFoundToUndefined(cb: () => Promise): Promise ResourceSignature | undefined; + +export type PlTransactionOptions = { + finalPredicate: FinalResourceDataPredicate; + signatureResolver?: SignatureResolver; + resourceDataCache: LRUCache; + enableFormattedErrors?: boolean; +}; + /** * Each platform transaction has 3 stages: * - initialization (txOpen message -> txInfo response) @@ -164,6 +188,12 @@ export class PlTransaction { private localResourceIdCounter = 0; + /** Maps local resource ids to their signatures until the resource is globalized */ + private readonly signatureStore = new Map(); + + /** Default color proof to include in resource creation requests */ + private defaultColorProof?: ColorProof; + /** Store logical tx open / closed state to prevent invalid sequence of requests. * True means output stream was completed. * Contract: there must be no async operations between setting this field to true and sending complete signal to stream. */ @@ -182,22 +212,30 @@ export class PlTransaction { }; } + private readonly finalPredicate: FinalResourceDataPredicate; + private readonly sharedResourceDataCache: LRUCache; + private readonly signatureResolver?: SignatureResolver; + private readonly enableFormattedErrors: boolean; + constructor( private readonly ll: LLPlTransaction, public readonly name: string, public readonly writable: boolean, private readonly _clientRoot: OptionalResourceId, - private readonly finalPredicate: FinalResourceDataPredicate, - private readonly sharedResourceDataCache: LRUCache, - private readonly enableFormattedErrors: boolean = false, + options: PlTransactionOptions, ) { + this.finalPredicate = options.finalPredicate; + this.sharedResourceDataCache = options.resourceDataCache; + this.signatureResolver = options.signatureResolver; + this.enableFormattedErrors = options.enableFormattedErrors ?? false; + // initiating transaction this.globalTxId = this.sendSingleAndParse( { oneofKind: "txOpen", txOpen: { name, - enableFormattedErrors, + enableFormattedErrors: this.enableFormattedErrors, writable: writable ? TxAPI_Open_Request_WritableTx.WRITABLE : TxAPI_Open_Request_WritableTx.NOT_WRITABLE, @@ -276,6 +314,51 @@ export class PlTransaction { void this.track(this.sendVoidSync(r)); } + private storeSignature(id: LocalResourceId, sig?: Uint8Array): void { + const rs = toResourceSignature(sig); + if (rs && !this.signatureStore.has(id)) { + this.signatureStore.set(id, rs); + } + } + + private getSignature(rId: AnyResourceId): ResourceSignature | undefined { + if (typeof rId !== "bigint") { + // ResourceId (object): prefer embedded signature, then resolver + return rId.signature ?? this.signatureStore.get(rId.id) ?? this.signatureResolver?.(rId); + } + // LocalResourceId: look in store + return this.signatureStore.get(rId); + } + + private toSignedResourceId(rId: AnyResourceRef): { resourceId: bigint; resourceSignature?: ResourceSignature } { + const resourceId = toResourceId(rId); + const rawId: bigint = typeof resourceId === "bigint" ? resourceId : resourceId.id; + return { resourceId: rawId, resourceSignature: this.getSignature(resourceId) }; + } + + private toSignedFieldId(fId: AnyFieldRef): { resourceId: bigint; resourceSignature?: ResourceSignature; fieldName: string } { + const base = toFieldId(fId); + const rawId: bigint = typeof base.resourceId === "bigint" ? base.resourceId : base.resourceId.id; + return { resourceId: rawId, fieldName: base.fieldName, resourceSignature: this.getSignature(base.resourceId) }; + } + + private toSignedErrorRef(rId: AnyResourceRef): { + errorResourceId: bigint; + errorResourceSignature?: ResourceSignature; + } { + const { resourceId, resourceSignature } = this.toSignedResourceId(rId); + return { errorResourceId: resourceId, errorResourceSignature: resourceSignature }; + } + + /** Set default color proof for subsequent resource creation requests */ + public setDefaultColor(colorProof: ColorProof): void { + this.defaultColorProof = colorProof; + this.sendVoidAsync({ + oneofKind: "setDefaultColor", + setDefaultColor: { colorProof }, + }); + } + private checkTxOpen() { if (this._completed) throw new Error("Transaction already closed"); } @@ -355,26 +438,23 @@ export class PlTransaction { name: string, type: ResourceType, errorIfExists: boolean = false, + colorProof?: ColorProof, ): ResourceRef { - const localId = this.nextLocalResourceId(false); - - const globalId = this.sendSingleAndParse( - { + return this.createResource( + false, + (localId) => ({ oneofKind: "resourceCreateSingleton", resourceCreateSingleton: { type, id: localId, data: Buffer.from(name), errorIfExists, - colorProof: new Uint8Array(0), + colorProof: colorProof ?? this.defaultColorProof, }, - }, - (r) => r.resourceCreateSingleton.resourceId as ResourceId, + }), + (r) => r.resourceCreateSingleton.resourceId, + (r) => r.resourceCreateSingleton.resourceSignature, ); - - void this.track(globalId); - - return { globalId, localId }; } public getSingleton(name: string, loadFields: true): Promise; @@ -399,10 +479,18 @@ export class PlTransaction { root: boolean, req: (localId: LocalResourceId) => OneOfKind, parser: (resp: OneOfKind) => bigint, + sigExtractor?: (resp: OneOfKind) => Uint8Array | undefined, ): ResourceRef { const localId = this.nextLocalResourceId(root); - const globalId = this.sendSingleAndParse(req(localId), (r) => parser(r) as ResourceId); + const globalId = this.sendSingleAndParse(req(localId), (r) => { + const rawId = parser(r); + const sig = sigExtractor ? toResourceSignature(sigExtractor(r)) : undefined; + if (sig) { + this.storeSignature(localId, sig); + } + return { id: rawId as GlobalResourceId, signature: sig }; + }); void this.track(globalId); @@ -415,10 +503,15 @@ export class PlTransaction { true, (localId) => ({ oneofKind: "resourceCreateRoot", resourceCreateRoot: { type, id: localId } }), (r) => r.resourceCreateRoot.resourceId, + (r) => r.resourceCreateRoot.resourceSignature, ); } - public createStruct(type: ResourceType, data?: Uint8Array | string): ResourceRef { + public createStruct( + type: ResourceType, + data?: Uint8Array | string, + colorProof?: ColorProof, + ): ResourceRef { this._stat.structsCreated++; this._stat.structsCreatedDataBytes += data?.length ?? 0; return this.createResource( @@ -430,14 +523,19 @@ export class PlTransaction { id: localId, data: data === undefined ? undefined : typeof data === "string" ? Buffer.from(data) : data, - colorProof: new Uint8Array(0), + colorProof: colorProof ?? this.defaultColorProof, }, }), (r) => r.resourceCreateStruct.resourceId, + (r) => r.resourceCreateStruct.resourceSignature, ); } - public createEphemeral(type: ResourceType, data?: Uint8Array | string): ResourceRef { + public createEphemeral( + type: ResourceType, + data?: Uint8Array | string, + colorProof?: ColorProof, + ): ResourceRef { this._stat.ephemeralsCreated++; this._stat.ephemeralsCreatedDataBytes += data?.length ?? 0; return this.createResource( @@ -449,10 +547,11 @@ export class PlTransaction { id: localId, data: data === undefined ? undefined : typeof data === "string" ? Buffer.from(data) : data, - colorProof: new Uint8Array(0), + colorProof: colorProof ?? this.defaultColorProof, }, }), (r) => r.resourceCreateEphemeral.resourceId, + (r) => r.resourceCreateEphemeral.resourceSignature, ); } @@ -460,6 +559,7 @@ export class PlTransaction { type: ResourceType, data: Uint8Array | string, errorIfExists: boolean = false, + colorProof?: ColorProof, ): ResourceRef { this._stat.valuesCreated++; this._stat.valuesCreatedDataBytes += data?.length ?? 0; @@ -472,10 +572,11 @@ export class PlTransaction { id: localId, data: typeof data === "string" ? Buffer.from(data) : data, errorIfExists, - colorProof: new Uint8Array(0), + colorProof: colorProof ?? this.defaultColorProof, }, }), (r) => r.resourceCreateValue.resourceId, + (r) => r.resourceCreateValue.resourceSignature, ); } @@ -499,7 +600,7 @@ export class PlTransaction { public setResourceName(name: string, rId: AnyResourceRef): void { this.sendVoidAsync({ oneofKind: "resourceNameSet", - resourceNameSet: { resourceId: toResourceId(rId), name }, + resourceNameSet: { ...this.toSignedResourceId(rId), name }, }); } @@ -510,7 +611,12 @@ export class PlTransaction { public getResourceByName(name: string): Promise { return this.sendSingleAndParse( { oneofKind: "resourceNameGet", resourceNameGet: { name } }, - (r) => ensureResourceIdNotNull(r.resourceNameGet.resourceId as OptionalResourceId), + (r) => { + const rawId = r.resourceNameGet.resourceId; + if (rawId === 0n) throw new Error("null resource id from getResourceByName"); + const sig = toResourceSignature(r.resourceNameGet.resourceSignature); + return { id: rawId as GlobalResourceId, signature: sig }; + }, ); } @@ -524,7 +630,7 @@ export class PlTransaction { public removeResource(rId: ResourceId): void { this.sendVoidAsync({ oneofKind: "resourceRemove", - resourceRemove: { resourceId: rId }, + resourceRemove: this.toSignedResourceId(rId), }); } @@ -532,7 +638,7 @@ export class PlTransaction { return this.sendSingleAndParse( { oneofKind: "resourceExists", - resourceExists: { resourceId: rId }, + resourceExists: this.toSignedResourceId(rId), }, (r) => r.resourceExists.exists, ); @@ -571,9 +677,9 @@ export class PlTransaction { ignoreCache: boolean = false, ): Promise { return this.track(async () => { - if (!ignoreCache && !isResourceRef(rId) && !isLocalResourceId(rId)) { + if (!ignoreCache && isResourceId(rId)) { // checking if we can return result from cache - const fromCache = this.sharedResourceDataCache.get(rId); + const fromCache = this.sharedResourceDataCache.get(rId.id); if (fromCache && fromCache.cacheTxOpenTimestamp < this.txOpenTimestamp) { if (!loadFields) { this._stat.rGetDataCacheHits++; @@ -592,7 +698,7 @@ export class PlTransaction { { oneofKind: "resourceGet", resourceGet: { - resourceId: toResourceId(rId), + ...this.toSignedResourceId(rId), loadFields: loadFields, }, }, @@ -605,9 +711,9 @@ export class PlTransaction { // we will cache only final resource data states // caching result even if we were ignore the cache - if (!isResourceRef(rId) && !isLocalResourceId(rId) && this.finalPredicate(result)) { + if (isResourceId(rId) && this.finalPredicate(result)) { deepFreeze(result); - const fromCache = this.sharedResourceDataCache.get(rId); + const fromCache = this.sharedResourceDataCache.get(rId.id); if (fromCache) { if (loadFields && !fromCache.data) { fromCache.data = result; @@ -618,13 +724,13 @@ export class PlTransaction { const basicData = extractBasicResourceData(result); deepFreeze(basicData); if (loadFields) - this.sharedResourceDataCache.set(rId, { + this.sharedResourceDataCache.set(rId.id, { basicData, data: result, cacheTxOpenTimestamp: this.txOpenTimestamp, }); else - this.sharedResourceDataCache.set(rId, { + this.sharedResourceDataCache.set(rId.id, { basicData, data: undefined, cacheTxOpenTimestamp: this.txOpenTimestamp, @@ -660,8 +766,8 @@ export class PlTransaction { ); // cleaning cache record if resource was removed from the db - if (result === undefined && !isResourceRef(rId) && !isLocalResourceId(rId)) - this.sharedResourceDataCache.delete(rId); + if (result === undefined && isResourceId(rId)) + this.sharedResourceDataCache.delete(rId.id); return result; }); @@ -678,7 +784,7 @@ export class PlTransaction { this._stat.inputsLocked++; this.sendVoidAsync({ oneofKind: "resourceLockInputs", - resourceLockInputs: { resourceId: toResourceId(rId), resourceSignature: new Uint8Array(0) }, + resourceLockInputs: this.toSignedResourceId(rId), }); } @@ -690,7 +796,7 @@ export class PlTransaction { this._stat.outputsLocked++; this.sendVoidAsync({ oneofKind: "resourceLockOutputs", - resourceLockOutputs: { resourceId: toResourceId(rId), resourceSignature: new Uint8Array(0) }, + resourceLockOutputs: this.toSignedResourceId(rId), }); } @@ -703,8 +809,8 @@ export class PlTransaction { this.sendVoidAsync({ oneofKind: "resourceSetError", resourceSetError: { - resourceId: toResourceId(rId), - errorResourceId: toResourceId(ref), + ...this.toSignedResourceId(rId), + ...this.toSignedErrorRef(ref), }, }); } @@ -717,7 +823,10 @@ export class PlTransaction { this._stat.fieldsCreated++; this.sendVoidAsync({ oneofKind: "fieldCreate", - fieldCreate: { type: fieldTypeToProto(fieldType), id: toFieldId(fId) }, + fieldCreate: { + type: fieldTypeToProto(fieldType), + id: this.toSignedFieldId(fId), + }, }); if (value !== undefined) this.setField(fId, value); } @@ -726,7 +835,7 @@ export class PlTransaction { return this.sendSingleAndParse( { oneofKind: "fieldExists", - fieldExists: { field: toFieldId(fId) }, + fieldExists: { field: this.toSignedFieldId(fId) }, }, (r) => r.fieldExists.exists, ); @@ -734,25 +843,26 @@ export class PlTransaction { public setField(fId: AnyFieldRef, ref: AnyRef): void { this._stat.fieldsSet++; - if (isResource(ref)) + if (isResource(ref)) { this.sendVoidAsync({ oneofKind: "fieldSet", fieldSet: { - field: toFieldId(fId), + field: this.toSignedFieldId(fId), value: { - resourceId: toResourceId(ref), + ...this.toSignedResourceId(ref), fieldName: "", // default value, read as undefined }, }, }); - else + } else { this.sendVoidAsync({ oneofKind: "fieldSet", fieldSet: { - field: toFieldId(fId), - value: toFieldId(ref), + field: this.toSignedFieldId(fId), + value: this.toSignedFieldId(ref), }, }); + } } public setFieldError(fId: AnyFieldRef, ref: AnyResourceRef): void { @@ -760,8 +870,8 @@ export class PlTransaction { this.sendVoidAsync({ oneofKind: "fieldSetError", fieldSetError: { - field: toFieldId(fId), - errorResourceId: toResourceId(ref), + field: this.toSignedFieldId(fId), + ...this.toSignedErrorRef(ref), }, }); } @@ -769,7 +879,7 @@ export class PlTransaction { public getField(fId: AnyFieldRef): Promise { this._stat.fieldsGet++; return this.sendSingleAndParse( - { oneofKind: "fieldGet", fieldGet: { field: toFieldId(fId) } }, + { oneofKind: "fieldGet", fieldGet: { field: this.toSignedFieldId(fId) } }, (r) => protoToField(notEmpty(r.fieldGet.field)), ); } @@ -779,11 +889,17 @@ export class PlTransaction { } public resetField(fId: AnyFieldRef): void { - this.sendVoidAsync({ oneofKind: "fieldReset", fieldReset: { field: toFieldId(fId) } }); + this.sendVoidAsync({ + oneofKind: "fieldReset", + fieldReset: { field: this.toSignedFieldId(fId) }, + }); } public removeField(fId: AnyFieldRef): void { - this.sendVoidAsync({ oneofKind: "fieldRemove", fieldRemove: { field: toFieldId(fId) } }); + this.sendVoidAsync({ + oneofKind: "fieldRemove", + fieldRemove: { field: this.toSignedFieldId(fId) }, + }); } // @@ -796,7 +912,7 @@ export class PlTransaction { { oneofKind: "resourceKeyValueList", resourceKeyValueList: { - resourceId: toResourceId(rId), + ...this.toSignedResourceId(rId), startFrom: "", limit: 0, }, @@ -837,8 +953,7 @@ export class PlTransaction { this.sendVoidAsync({ oneofKind: "resourceKeyValueSet", resourceKeyValueSet: { - resourceId: toResourceId(rId), - resourceSignature: new Uint8Array(0), + ...this.toSignedResourceId(rId), key, value: toBytes(value), }, @@ -849,8 +964,7 @@ export class PlTransaction { this.sendVoidAsync({ oneofKind: "resourceKeyValueDelete", resourceKeyValueDelete: { - resourceId: toResourceId(rId), - resourceSignature: new Uint8Array(0), + ...this.toSignedResourceId(rId), key, }, }); @@ -862,7 +976,7 @@ export class PlTransaction { { oneofKind: "resourceKeyValueGet", resourceKeyValueGet: { - resourceId: toResourceId(rId), + ...this.toSignedResourceId(rId), key, }, }, @@ -893,7 +1007,7 @@ export class PlTransaction { { oneofKind: "resourceKeyValueGetIfExists", resourceKeyValueGetIfExists: { - resourceId: toResourceId(rId), + ...this.toSignedResourceId(rId), key, }, }, diff --git a/lib/node/pl-client/src/core/type_conversion.ts b/lib/node/pl-client/src/core/type_conversion.ts index f08b62eb19..58d84b540b 100644 --- a/lib/node/pl-client/src/core/type_conversion.ts +++ b/lib/node/pl-client/src/core/type_conversion.ts @@ -15,8 +15,9 @@ import type { ResourceData, ResourceId, ResourceKind, + GlobalResourceId, } from "./types"; -import { NullResourceId } from "./types"; +import { NullResourceId, toResourceSignature } from "./types"; import { assertNever, notEmpty } from "@milaboratories/ts-helpers"; import { throwPlNotFoundError } from "./errors"; @@ -26,12 +27,17 @@ function resourceIsDeleted(proto: Resource): boolean { return proto.deletedTime !== undefined && proto.deletedTime.seconds !== 0n; } +function protoIdToOptionalResourceId(id: bigint, signature?: Uint8Array): OptionalResourceId { + if (id === 0n) return NullResourceId; + return { id: id as GlobalResourceId, signature: toResourceSignature(signature) }; +} + /** Throws "native" pl not found error, if resource is marked as deleted. */ export function protoToResource(proto: Resource): ResourceData { if (resourceIsDeleted(proto)) throwPlNotFoundError("resource deleted"); return { - id: proto.resourceId as ResourceId, - originalResourceId: proto.originalResourceId as OptionalResourceId, + id: { id: proto.resourceId as GlobalResourceId, signature: toResourceSignature(proto.resourceSignature) }, + originalResourceId: protoIdToOptionalResourceId(proto.originalResourceId), type: notEmpty(proto.type), data: proto.data, inputsLocked: proto.inputsLocked, @@ -57,7 +63,10 @@ function protoToResourceKind(proto: Resource_Kind): ResourceKind { function protoToError(proto: Resource): OptionalResourceId { const f = proto.fields.find((f) => f?.id?.fieldName === ResourceErrorField); - return (f?.error ?? NullResourceId) as OptionalResourceId; + if (!f) return NullResourceId; + const errId = f.error ?? 0n; + if (errId === 0n) return NullResourceId; + return { id: errId as GlobalResourceId, signature: toResourceSignature(f.errorSignature) }; } export function protoToField(proto: Field): FieldData { @@ -65,8 +74,8 @@ export function protoToField(proto: Field): FieldData { name: notEmpty(proto.id?.fieldName), type: protoToFieldType(proto.type), status: protoToFieldStatus(proto.valueStatus), - value: proto.value as OptionalResourceId, - error: proto.error as OptionalResourceId, + value: protoIdToOptionalResourceId(proto.value, proto.valueSignature), + error: protoIdToOptionalResourceId(proto.error, proto.errorSignature), valueIsFinal: proto.valueIsFinal, }; } diff --git a/lib/node/pl-client/src/core/types.ts b/lib/node/pl-client/src/core/types.ts index ceb30c49c2..0b26fde357 100644 --- a/lib/node/pl-client/src/core/types.ts +++ b/lib/node/pl-client/src/core/types.ts @@ -4,15 +4,25 @@ import { cachedDeserialize, notEmpty } from "@milaboratories/ts-helpers"; declare const __resource_id_type__: unique symbol; type BrandResourceId = bigint & { [__resource_id_type__]: B }; -/** Global resource id */ -export type ResourceId = BrandResourceId<"global">; - +/** Opaque authorization signature attached to a resource. */ +declare const __resource_signature_type__: unique symbol; +export type ResourceSignature = Uint8Array & { readonly [__resource_signature_type__]: true }; /** Null resource id */ export type NullResourceId = BrandResourceId<"null">; +/** Global resource id — always signed (received from server). */ +export type GlobalResourceId = BrandResourceId<"global"> + /** Local resource id */ export type LocalResourceId = BrandResourceId<"local">; +export type SignedResourceId = { + id: GlobalResourceId, + signature?: ResourceSignature +} + +export type ResourceId = SignedResourceId + /** Any non-null resource id */ export type AnyResourceId = ResourceId | LocalResourceId; @@ -24,12 +34,12 @@ export type OptionalAnyResourceId = NullResourceId | ResourceId | LocalResourceI export const NullResourceId = 0n as NullResourceId; -export function isNullResourceId(resourceId: bigint): resourceId is NullResourceId { - return resourceId === NullResourceId; +export function isNullResourceId(resourceId: OptionalAnyResourceId): resourceId is NullResourceId { + return typeof resourceId === "bigint" && resourceId === NullResourceId; } export function isNotNullResourceId(resourceId: OptionalResourceId): resourceId is ResourceId { - return resourceId !== NullResourceId; + return typeof resourceId !== "bigint"; } export function ensureResourceIdNotNull(resourceId: OptionalResourceId): ResourceId { @@ -37,8 +47,9 @@ export function ensureResourceIdNotNull(resourceId: OptionalResourceId): Resourc return resourceId; } -export function isAnyResourceId(resourceId: bigint): resourceId is AnyResourceId { - return resourceId !== 0n; +export function isAnyResourceId(resourceId: OptionalAnyResourceId): resourceId is AnyResourceId { + if (typeof resourceId === "bigint") return resourceId !== 0n; + return true; } // see local / global resource logic below... @@ -68,6 +79,29 @@ export function resourceTypesEqual(type1: ResourceType, type2: ResourceType): bo return type1.name === type2.name && type1.version === type2.version; } +/** Color proof used for resource creation requests (alias for ResourceSignature). */ +export type ColorProof = ResourceSignature; + +/** Encode resource signature to standard base64 for REST API bodies. */ +export function signatureToBase64(sig?: ResourceSignature): string { + return sig ? Buffer.from(sig).toString("base64") : ""; +} + +/** Encode resource signature to base64url for embedding in URL-based handles. */ +export function signatureToBase64Url(sig?: ResourceSignature): string { + return sig ? Buffer.from(sig).toString("base64url") : ""; +} + +/** Cast raw bytes to a branded ResourceSignature, returning undefined for empty/missing input. */ +export function toResourceSignature(raw?: Uint8Array): ResourceSignature | undefined { + return raw && raw.length > 0 ? (raw as ResourceSignature) : undefined; +} + +/** Decode base64url-encoded string back to a branded ResourceSignature. */ +export function base64UrlToSignature(str: string): ResourceSignature { + return toResourceSignature(Buffer.from(str, "base64url"))!; +} + /** Readonly fields here marks properties of resource that can't change according to pl's state machine. */ export type BasicResourceData = { readonly id: ResourceId; @@ -186,8 +220,9 @@ export function createLocalResourceId( (BigInt(localTxId) << LocalResourceIdTxIdOffset)) as LocalResourceId; } -export function createGlobalResourceId(isRoot: boolean, unmaskedId: bigint): ResourceId { - return ((isRoot ? ResourceIdRootMask : 0n) | unmaskedId) as ResourceId; +export function createGlobalResourceId(isRoot: boolean, unmaskedId: bigint, signature?: ResourceSignature): ResourceId { + const id = ((isRoot ? ResourceIdRootMask : 0n) | unmaskedId) as GlobalResourceId; + return { id, signature }; } export function extractTxId(localResourceId: LocalResourceId): number { @@ -195,6 +230,7 @@ export function extractTxId(localResourceId: LocalResourceId): number { } export function checkLocalityOfResourceId(resourceId: AnyResourceId, expectedTxId: number): void { + if (typeof resourceId !== "bigint") return; // ResourceId (object) is never local if (!isLocalResourceId(resourceId)) return; if (extractTxId(resourceId) !== expectedTxId) throw Error( @@ -203,21 +239,22 @@ export function checkLocalityOfResourceId(resourceId: AnyResourceId, expectedTxI } export function resourceIdToString(resourceId: OptionalAnyResourceId): string { - if (isNullResourceId(resourceId)) return "XX:0x0"; - if (isLocalResourceId(resourceId)) + const raw: bigint = typeof resourceId === "bigint" ? resourceId : resourceId.id; + if (raw === 0n) return "XX:0x0"; + if ((raw & ResourceIdLocalMask) !== 0n) return ( - (isRootResourceId(resourceId) ? "R" : "N") + + (isRootResourceId(raw) ? "R" : "N") + "L:0x" + - (LocalIdMask & resourceId).toString(16) + + (LocalIdMask & raw).toString(16) + "[0x" + - extractTxId(resourceId).toString(16) + + extractTxId(raw as LocalResourceId).toString(16) + "]" ); else return ( - (isRootResourceId(resourceId) ? "R" : "N") + + (isRootResourceId(raw) ? "R" : "N") + "G:0x" + - (NoFlagsIdMask & resourceId).toString(16) + (NoFlagsIdMask & raw).toString(16) ); } @@ -235,15 +272,23 @@ export function resourceIdFromString(str: string): OptionalAnyResourceId | undef } /** Converts bigint to global resource id */ -export function bigintToResourceId(resourceId: bigint): ResourceId { +export function bigintToResourceId(resourceId: bigint, signature?: ResourceSignature): ResourceId { if (isLocalResourceId(resourceId)) - throw new Error(`Local resource id: ${resourceIdToString(resourceId)}`); - if (isNullResourceId(resourceId)) throw new Error(`Null resource id.`); - return resourceId as ResourceId; + throw new Error(`Local resource id: ${resourceIdToString(resourceId as LocalResourceId)}`); + if (isNullResourceId(resourceId as NullResourceId)) throw new Error(`Null resource id.`); + return { id: resourceId as GlobalResourceId, signature }; } export function stringifyWithResourceId(object: unknown): string { - return JSON.stringify(object, (key, value) => - typeof value === "bigint" ? resourceIdToString(value as OptionalAnyResourceId) : value, - ); + return JSON.stringify(object, (key, value) => { + if (typeof value === "bigint") return resourceIdToString(value as LocalResourceId); + if ( + typeof value === "object" && + value !== null && + "id" in value && + typeof (value as { id: unknown }).id === "bigint" + ) + return resourceIdToString(value as ResourceId); + return value; + }); } diff --git a/lib/node/pl-client/src/test/test_config.ts b/lib/node/pl-client/src/test/test_config.ts index 0546f56ea9..3ce37f0ffd 100644 --- a/lib/node/pl-client/src/test/test_config.ts +++ b/lib/node/pl-client/src/test/test_config.ts @@ -89,8 +89,11 @@ function saveAuthInfoCallback(tConf: TestConfig): (authInformation: AuthInformat } const cleanAuthInfoCallback = () => { - console.warn(`Removing: ${getFullAuthDataFilePath()}`); - fs.rmSync(getFullAuthDataFilePath()); + const p = getFullAuthDataFilePath(); + if (fs.existsSync(p)) { + console.warn(`Removing: ${p}`); + fs.rmSync(p); + } }; export async function getTestClientConf(): Promise<{ conf: PlClientConfig; auth: AuthOps }> { diff --git a/lib/node/pl-drivers/src/clients/download.ts b/lib/node/pl-drivers/src/clients/download.ts index 611c48c149..ac933466ec 100644 --- a/lib/node/pl-drivers/src/clients/download.ts +++ b/lib/node/pl-drivers/src/clients/download.ts @@ -3,6 +3,7 @@ import type { WireClientProvider, WireClientProviderFactory } from "@milaborator import { addRTypeToMetadata, stringifyWithResourceId, + signatureToBase64, RestAPI, createRTypeRoutingHeader, } from "@milaboratories/pl-client"; @@ -139,15 +140,15 @@ export class ClientDownload { const client = this.wire.get(); if (client instanceof DownloadClient) { return await client.getDownloadURL( - { resourceId: id, isInternalUse: false }, + { resourceId: id.id, resourceSignature: id.signature, isInternalUse: false }, addRTypeToMetadata(type, withAbort), ).response; } else { return ( await client.POST("/v1/get-download-url", { body: { - resourceId: id.toString(), - resourceSignature: "", + resourceId: id.id.toString(), + resourceSignature: signatureToBase64(id.signature), isInternalUse: false, }, headers: { ...createRTypeRoutingHeader(type) }, diff --git a/lib/node/pl-drivers/src/clients/logs.ts b/lib/node/pl-drivers/src/clients/logs.ts index 5683bd4292..85f8a68d06 100644 --- a/lib/node/pl-drivers/src/clients/logs.ts +++ b/lib/node/pl-drivers/src/clients/logs.ts @@ -3,7 +3,12 @@ import type { MiLogger } from "@milaboratories/ts-helpers"; import { notEmpty } from "@milaboratories/ts-helpers"; import type { Dispatcher } from "undici"; import type { WireClientProvider, WireClientProviderFactory } from "@milaboratories/pl-client"; -import { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from "@milaboratories/pl-client"; +import { + addRTypeToMetadata, + createRTypeRoutingHeader, + signatureToBase64, + RestAPI, +} from "@milaboratories/pl-client"; import type { StreamingAPI_Response } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol"; import { StreamingClient } from "../proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client"; import type { StreamingApiPaths, StreamingRestClientType } from "../proto-rest"; @@ -47,7 +52,13 @@ export class ClientLogs { if (client instanceof StreamingClient) { return ( await client.lastLines( - { resourceId: rId, lineCount: lineCount, offset: offsetBytes, search: searchStr }, + { + resourceId: rId.id, + resourceSignature: rId.signature, + lineCount: lineCount, + offset: offsetBytes, + search: searchStr, + }, addRTypeToMetadata(rType, options), ) ).response; @@ -56,8 +67,8 @@ export class ClientLogs { const resp = ( await client.POST("/v1/last-lines", { body: { - resourceId: rId.toString(), - resourceSignature: "", + resourceId: rId.id.toString(), + resourceSignature: signatureToBase64(rId.signature), lineCount: lineCount, offset: offsetBytes.toString(), search: searchStr ?? "", @@ -90,7 +101,8 @@ export class ClientLogs { return ( await client.readText( { - resourceId: notEmpty(rId), + resourceId: rId.id, + resourceSignature: rId.signature, readLimit: BigInt(lineCount), offset: offsetBytes, search: searchStr, @@ -103,8 +115,8 @@ export class ClientLogs { const resp = ( await client.POST("/v1/read/text", { body: { - resourceId: rId.toString(), - resourceSignature: "", + resourceId: rId.id.toString(), + resourceSignature: signatureToBase64(rId.signature), readLimit: lineCount.toString(), offset: offsetBytes.toString(), search: searchStr ?? "", diff --git a/lib/node/pl-drivers/src/clients/ls_api.ts b/lib/node/pl-drivers/src/clients/ls_api.ts index 4375033f57..fdfe2ad5e1 100644 --- a/lib/node/pl-drivers/src/clients/ls_api.ts +++ b/lib/node/pl-drivers/src/clients/ls_api.ts @@ -2,7 +2,11 @@ import type { MiLogger } from "@milaboratories/ts-helpers"; import type { RpcOptions } from "@protobuf-ts/runtime-rpc"; import type { WireClientProvider, WireClientProviderFactory } from "@milaboratories/pl-client"; import { RestAPI } from "@milaboratories/pl-client"; -import { addRTypeToMetadata, createRTypeRoutingHeader } from "@milaboratories/pl-client"; +import { + addRTypeToMetadata, + createRTypeRoutingHeader, + signatureToBase64, +} from "@milaboratories/pl-client"; import type { LsAPI_List_Response, LsAPI_ListItem, @@ -45,7 +49,8 @@ export class ClientLs { if (client instanceof LSClient) { return await client.list( { - resourceId: rInfo.id, + resourceId: rInfo.id.id, + resourceSignature: rInfo.id.signature, location: path, }, addRTypeToMetadata(rInfo.type, options), @@ -54,8 +59,8 @@ export class ClientLs { const resp = ( await client.POST("/v1/list", { body: { - resourceId: rInfo.id.toString(), - resourceSignature: "", + resourceId: rInfo.id.id.toString(), + resourceSignature: signatureToBase64(rInfo.id.signature), location: path, }, headers: { ...createRTypeRoutingHeader(rInfo.type) }, diff --git a/lib/node/pl-drivers/src/clients/progress.ts b/lib/node/pl-drivers/src/clients/progress.ts index ce594ccb6c..c877ac8a29 100644 --- a/lib/node/pl-drivers/src/clients/progress.ts +++ b/lib/node/pl-drivers/src/clients/progress.ts @@ -4,7 +4,12 @@ import type { WireClientProviderFactory, PlClient, } from "@milaboratories/pl-client"; -import { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from "@milaboratories/pl-client"; +import { + addRTypeToMetadata, + createRTypeRoutingHeader, + signatureToBase64, + RestAPI, +} from "@milaboratories/pl-client"; import type { MiLogger } from "@milaboratories/ts-helpers"; import { notEmpty } from "@milaboratories/ts-helpers"; import type { Dispatcher } from "undici"; @@ -50,19 +55,29 @@ export class ClientProgress { close() {} /** getStatus gets a progress status by given rId and rType. */ - async getStatus({ id, type }: ResourceInfo, options?: RpcOptions): Promise { + async getStatus( + { id, type }: ResourceInfo, + options?: RpcOptions, + ): Promise { const client = this.wire.get(); let report: ProgressAPI_Report; if (client instanceof ProgressClient) { report = notEmpty( - (await client.getStatus({ resourceId: id }, addRTypeToMetadata(type, options)).response) - .report, + ( + await client.getStatus( + { resourceId: id.id, resourceSignature: id.signature }, + addRTypeToMetadata(type, options), + ).response + ).report, ); } else { const resp = ( await client.POST("/v1/get-progress", { - body: { resourceId: id.toString(), resourceSignature: "" }, + body: { + resourceId: id.id.toString(), + resourceSignature: signatureToBase64(id.signature), + }, headers: { ...createRTypeRoutingHeader(type) }, }) ).data!.report; diff --git a/lib/node/pl-drivers/src/clients/upload.ts b/lib/node/pl-drivers/src/clients/upload.ts index fcc0687dc5..f96631974f 100644 --- a/lib/node/pl-drivers/src/clients/upload.ts +++ b/lib/node/pl-drivers/src/clients/upload.ts @@ -3,7 +3,12 @@ import type { WireClientProviderFactory, PlClient, } from "@milaboratories/pl-client"; -import { addRTypeToMetadata, createRTypeRoutingHeader, RestAPI } from "@milaboratories/pl-client"; +import { + addRTypeToMetadata, + createRTypeRoutingHeader, + signatureToBase64, + RestAPI, +} from "@milaboratories/pl-client"; import type { ResourceInfo } from "@milaboratories/pl-tree"; import type { MiLogger } from "@milaboratories/ts-helpers"; import type { RpcOptions } from "@protobuf-ts/runtime-rpc"; @@ -81,8 +86,9 @@ export class ClientUpload { const client = this.wire.get(); if (client instanceof UploadClient) { - const init = (await client.init({ resourceId: id }, addRTypeToMetadata(type, options))) - .response; + const init = ( + await client.init({ resourceId: id.id, resourceSignature: id.signature }, addRTypeToMetadata(type, options)) + ).response; return { overall: init.partsCount, @@ -95,8 +101,8 @@ export class ClientUpload { const init = ( await client.POST("/v1/upload/init", { body: { - resourceId: id.toString(), - resourceSignature: "", + resourceId: id.id.toString(), + resourceSignature: signatureToBase64(id.signature), }, headers: { ...createRTypeRoutingHeader(type) }, }) @@ -127,7 +133,8 @@ export class ClientUpload { info = ( await client.getPartURL( { - resourceId: id, + resourceId: id.id, + resourceSignature: id.signature, partNumber, uploadedPartSize: 0n, isInternalUse: false, @@ -140,8 +147,8 @@ export class ClientUpload { const resp = ( await client.POST("/v1/upload/get-part-url", { body: { - resourceId: id.toString(), - resourceSignature: "", + resourceId: id.id.toString(), + resourceSignature: signatureToBase64(id.signature), partNumber: partNumber.toString(), uploadedPartSize: "0", isInternalUse: false, @@ -212,7 +219,11 @@ export class ClientUpload { ); } - await this.updateProgress({ id, type }, BigInt(info.chunkEnd - info.chunkStart), options); + await this.updateProgress( + { id, type }, + BigInt(info.chunkEnd - info.chunkStart), + options, + ); } public async finalize(info: ResourceInfo, options?: RpcOptions) { @@ -221,7 +232,8 @@ export class ClientUpload { if (client instanceof UploadClient) { await client.finalize( { - resourceId: info.id, + resourceId: info.id.id, + resourceSignature: info.id.signature, checksumAlgorithm: UploadAPI_ChecksumAlgorithm.UNSPECIFIED, checksum: new Uint8Array(0), }, @@ -230,8 +242,8 @@ export class ClientUpload { } else { await client.POST("/v1/upload/finalize", { body: { - resourceId: info.id.toString(), - resourceSignature: "", + resourceId: info.id.id.toString(), + resourceSignature: signatureToBase64(info.id.signature), checksumAlgorithm: 0, checksum: "", }, @@ -263,7 +275,8 @@ export class ClientUpload { if (client instanceof UploadClient) { await client.updateProgress( { - resourceId: id, + resourceId: id.id, + resourceSignature: id.signature, bytesProcessed, }, addRTypeToMetadata(type, options), @@ -273,8 +286,8 @@ export class ClientUpload { await client.POST("/v1/upload/update-progress", { body: { - resourceId: id.toString(), - resourceSignature: "", + resourceId: id.id.toString(), + resourceSignature: signatureToBase64(id.signature), bytesProcessed: bytesProcessed.toString(), }, headers: { ...createRTypeRoutingHeader(type) }, diff --git a/lib/node/pl-drivers/src/drivers/download_blob/blob_key.ts b/lib/node/pl-drivers/src/drivers/download_blob/blob_key.ts index f026181544..885226fad2 100644 --- a/lib/node/pl-drivers/src/drivers/download_blob/blob_key.ts +++ b/lib/node/pl-drivers/src/drivers/download_blob/blob_key.ts @@ -2,7 +2,7 @@ import { bigintToResourceId, ResourceId } from "@milaboratories/pl-client"; import * as path from "node:path"; export function blobKey(rId: ResourceId): string { - return `${BigInt(rId)}`; + return `${rId.id}`; } export function pathToKey(fPath: string): string { diff --git a/lib/node/pl-drivers/src/drivers/download_blob/download_blob.test.ts b/lib/node/pl-drivers/src/drivers/download_blob/download_blob.test.ts index 7cad2f97c3..132121b9eb 100644 --- a/lib/node/pl-drivers/src/drivers/download_blob/download_blob.test.ts +++ b/lib/node/pl-drivers/src/drivers/download_blob/download_blob.test.ts @@ -304,6 +304,7 @@ async function makeDownloadableBlobFromAssets(client: PlClient, fileName: string return { id: download.id, type: download.type, + resourceSignature: download.resourceSignature, data: undefined, fields: undefined, kv: { diff --git a/lib/node/pl-drivers/src/drivers/download_blob_url/driver.ts b/lib/node/pl-drivers/src/drivers/download_blob_url/driver.ts index f4ba1985ae..cd3505807b 100644 --- a/lib/node/pl-drivers/src/drivers/download_blob_url/driver.ts +++ b/lib/node/pl-drivers/src/drivers/download_blob_url/driver.ts @@ -225,6 +225,6 @@ export class DownloadBlobToURLDriver implements BlobToURLDriver { } private getFilePath(id: ResourceId, format: ArchiveFormat): string { - return path.join(this.saveDir, `${String(BigInt(id))}_${format}`); + return path.join(this.saveDir, `${String(id.id)}_${format}`); } } diff --git a/lib/node/pl-drivers/src/drivers/download_blob_url/driver_id.ts b/lib/node/pl-drivers/src/drivers/download_blob_url/driver_id.ts index 6899ce7922..7616b9a0fc 100644 --- a/lib/node/pl-drivers/src/drivers/download_blob_url/driver_id.ts +++ b/lib/node/pl-drivers/src/drivers/download_blob_url/driver_id.ts @@ -5,7 +5,7 @@ import type { ArchiveFormat } from "@milaboratories/pl-model-common"; export type Id = string; export function newId(id: ResourceId, format: ArchiveFormat): Id { - return `id:${String(BigInt(id))}-${format}`; + return `id:${String(id.id)}-${format}`; } // export function diff --git a/lib/node/pl-drivers/src/drivers/download_url/task.ts b/lib/node/pl-drivers/src/drivers/download_url/task.ts index d47334281f..e24fc57fff 100644 --- a/lib/node/pl-drivers/src/drivers/download_url/task.ts +++ b/lib/node/pl-drivers/src/drivers/download_url/task.ts @@ -64,6 +64,16 @@ export class DownloadByUrlTask { return; } + // If our abort was triggered, treat any resulting error (e.g. ClientDestroyedError + // thrown when the client is closed before the abort signal propagates to undici) + // as an abort rather than a recoverable error that would cause infinite retries. + if (this.signalCtl.signal.aborted) { + this.setError(this.signalCtl.signal.reason); + this.change.markChanged(`download of ${this.url} aborted`); + await rmRFDir(this.path); + return; + } + throw e; } } diff --git a/lib/node/pl-drivers/src/drivers/helpers/download_remote_handle.ts b/lib/node/pl-drivers/src/drivers/helpers/download_remote_handle.ts index 6f06c6d645..a8b2397dee 100644 --- a/lib/node/pl-drivers/src/drivers/helpers/download_remote_handle.ts +++ b/lib/node/pl-drivers/src/drivers/helpers/download_remote_handle.ts @@ -4,19 +4,26 @@ import type { Signer } from "@milaboratories/ts-helpers"; import type { OnDemandBlobResourceSnapshot } from "../types"; import type { RemoteBlobHandle } from "@milaboratories/pl-model-common"; -import { bigintToResourceId } from "@milaboratories/pl-client"; +import { + bigintToResourceId, + signatureToBase64Url, + base64UrlToSignature, +} from "@milaboratories/pl-client"; import { ResourceInfo } from "@milaboratories/pl-tree"; import { getSize } from "../types"; -// https://regex101.com/r/Q4YdTa/5 const remoteHandleRegex = - /^blob\+remote:\/\/download\/(?(?.+)\/(?.+?)\/(?\d+?)\/(?\d+?))#(?.*)$/; + /^blob\+remote:\/\/download\/(?(?.+)\/(?[^/]+)\/(?\d+)\/(?\d+)(?:\/(?[A-Za-z0-9_-]*))?)#(?.*)$/; export function newRemoteHandle( rInfo: OnDemandBlobResourceSnapshot, signer: Signer, ): RemoteBlobHandle { - let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`; + let content = `${rInfo.type.name}/${rInfo.type.version}/${rInfo.id.id}/${getSize(rInfo)}`; + const sigStr = signatureToBase64Url(rInfo.id.signature); + if (sigStr) { + content += `/${sigStr}`; + } return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle; } @@ -37,13 +44,14 @@ export function parseRemoteHandle( throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`); } - const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!; + const { content, resourceType, resourceVersion, resourceId, size, resourceSig, signature } = + parsed.groups!; signer.verify(content, signature, `Signature verification failed for ${handle}`); return { info: { - id: bigintToResourceId(BigInt(resourceId)), + id: bigintToResourceId(BigInt(resourceId), resourceSig ? base64UrlToSignature(resourceSig) : undefined), type: { name: resourceType, version: resourceVersion }, }, size: Number(size), diff --git a/lib/node/pl-drivers/src/drivers/helpers/logs_handle.ts b/lib/node/pl-drivers/src/drivers/helpers/logs_handle.ts index f5f4c5f79e..5f6fe5836e 100644 --- a/lib/node/pl-drivers/src/drivers/helpers/logs_handle.ts +++ b/lib/node/pl-drivers/src/drivers/helpers/logs_handle.ts @@ -3,14 +3,20 @@ import type { ResourceInfo } from "@milaboratories/pl-tree"; import type * as sdk from "@milaboratories/pl-model-common"; -import { bigintToResourceId } from "@milaboratories/pl-client"; +import { + bigintToResourceId, + signatureToBase64Url, + base64UrlToSignature, +} from "@milaboratories/pl-client"; export function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle { + const sigStr = signatureToBase64Url(rInfo.id.signature); + const resSig = sigStr ? `/${sigStr}` : ""; if (live) { - return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.LiveLogHandle; + return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${rInfo.id.id}${resSig}` as sdk.LiveLogHandle; } - return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.ReadyLogHandle; + return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${rInfo.id.id}${resSig}` as sdk.ReadyLogHandle; } /** Handle of the live logs of a program. @@ -18,7 +24,7 @@ export function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHand * in this case the handle should be refreshed. */ export const liveHandleRegex = - /^log\+live:\/\/log\/(?.*)\/(?.*)\/(?.*)$/; + /^log\+live:\/\/log\/(?.+)\/(?[^/]+)\/(?\d+)(?:\/(?[A-Za-z0-9_-]*))?$/; export function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle { return liveHandleRegex.test(handle); @@ -27,7 +33,7 @@ export function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle { /** Handle of the ready logs of a program. */ export const readyHandleRegex = - /^log\+ready:\/\/log\/(?.*)\/(?.*)\/(?.*)$/; + /^log\+ready:\/\/log\/(?.+)\/(?[^/]+)\/(?\d+)(?:\/(?[A-Za-z0-9_-]*))?$/; export function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle { return readyHandleRegex.test(handle); @@ -43,10 +49,10 @@ export function getResourceInfoFromLogHandle(handle: sdk.AnyLogHandle): Resource } else throw new Error(`Log handle is malformed: ${handle}`); if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`); - const { resourceType, resourceVersion, resourceId } = parsed.groups!; + const { resourceType, resourceVersion, resourceId, resourceSig } = parsed.groups!; return { - id: bigintToResourceId(BigInt(resourceId)), + id: bigintToResourceId(BigInt(resourceId), resourceSig ? base64UrlToSignature(resourceSig) : undefined), type: { name: resourceType, version: resourceVersion }, }; } diff --git a/lib/node/pl-drivers/src/drivers/helpers/ls_storage_entry.ts b/lib/node/pl-drivers/src/drivers/helpers/ls_storage_entry.ts index 40b2a5f424..3a3a2ca08c 100644 --- a/lib/node/pl-drivers/src/drivers/helpers/ls_storage_entry.ts +++ b/lib/node/pl-drivers/src/drivers/helpers/ls_storage_entry.ts @@ -71,7 +71,7 @@ export function isRemoteStorageHandle( } export function createRemoteStorageHandle(name: string, rId: ResourceId): sdk.StorageHandleRemote { - return `remote://${name}/${BigInt(rId)}`; + return `remote://${name}/${rId.id}`; } function parseRemoteStorageHandle(handle: string): RemoteStorageHandleData { diff --git a/lib/node/pl-drivers/src/drivers/ls.ts b/lib/node/pl-drivers/src/drivers/ls.ts index a8570e6fb9..b217443522 100644 --- a/lib/node/pl-drivers/src/drivers/ls.ts +++ b/lib/node/pl-drivers/src/drivers/ls.ts @@ -23,6 +23,7 @@ import { parseUploadHandle, } from "./helpers/ls_remote_import_handle"; import { + type StorageHandleData, createLocalStorageHandle, createRemoteStorageHandle, parseStorageHandle, @@ -30,6 +31,7 @@ import { import type { LocalStorageProjection, VirtualLocalStorageSpec } from "./types"; import { DefaultVirtualLocalStorages } from "./virtual_storages"; + /** * Extends public and safe SDK's driver API with methods used internally in the middle * layer and in tests. @@ -65,7 +67,7 @@ export class LsDriver implements InternalLsDriver { private constructor( private readonly logger: MiLogger, private readonly lsClient: ClientLs, - /** Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. */ + /** Pl storage id, to resource id (with embedded signature). */ private readonly storageIdToResourceId: Record, private readonly signer: Signer, /** Virtual storages by name */ @@ -191,14 +193,12 @@ export class LsDriver implements InternalLsDriver { initialFullPath: s.initialPath, })); - const otherStorages = Object.entries(this.storageIdToResourceId!).map( - ([storageId, resourceId]) => ({ - name: storageId, - handle: createRemoteStorageHandle(storageId, resourceId), - initialFullPath: "", // we don't have any additional information from where to start browsing remote storages - isInitialPathHome: false, - }), - ); + const otherStorages = Object.entries(this.storageIdToResourceId!).map(([storageId, resourceId]) => ({ + name: storageId, + handle: createRemoteStorageHandle(storageId, resourceId), + initialFullPath: "", // we don't have any additional information from where to start browsing remote storages + isInitialPathHome: false, + })); // root must be a storage so we can index any file, // but for UI it's enough @@ -209,11 +209,20 @@ export class LsDriver implements InternalLsDriver { return [...virtualStorages, ...noRoot]; } + /** Enrich parsed storage data with the signature embedded in the stored ResourceId */ + private withSignature(storageData: StorageHandleData): StorageHandleData { + if (storageData.isRemote) { + const sig = this.storageIdToResourceId[storageData.name]?.signature; + if (sig) return { ...storageData, id: { ...storageData.id, signature: sig } }; + } + return storageData; + } + public async listFiles( storageHandle: sdk.StorageHandle, fullPath: string, ): Promise { - const storageData = parseStorageHandle(storageHandle); + const storageData = this.withSignature(parseStorageHandle(storageHandle)); if (storageData.isRemote) { const response = await this.lsClient.list(storageData, fullPath); @@ -258,7 +267,7 @@ export class LsDriver implements InternalLsDriver { storageHandle: sdk.StorageHandle, fullPath: string, ): Promise { - const storageData = parseStorageHandle(storageHandle); + const storageData = this.withSignature(parseStorageHandle(storageHandle)); if (!storageData.isRemote) { throw new Error(`Storage ${storageData.name} is not remote`); } @@ -333,7 +342,7 @@ async function doGetAvailableStorageIds(client: PlClient): Promise { return Object.fromEntries( provider.fields .filter((f) => f.type == "Dynamic" && isNotNullResourceId(f.value)) diff --git a/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts b/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts index d86a4f2795..6d9eef29c2 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts @@ -1,4 +1,4 @@ -import type { PlClient, ResourceId } from "@milaboratories/pl-client"; +import type { GlobalResourceId, PlClient, ResourceId } from "@milaboratories/pl-client"; import { field, isNotNullResourceId, @@ -216,7 +216,7 @@ export class MiddleLayer { // Projects // - private readonly openedProjectsByRid = new Map(); + private readonly openedProjectsByRid = new Map(); private async projectIdToResourceId(id: string): Promise { return await this.pl.withReadTx("Project id to resource id", async (tx) => { @@ -234,31 +234,31 @@ export class MiddleLayer { /** Opens a project, and starts corresponding project maintenance loop. */ public async openProject(id: ResourceId | string) { const rid = await this.ensureProjectRid(id); - if (this.openedProjectsByRid.has(rid)) throw new Error(`Project ${rid} already opened`); - this.openedProjectsByRid.set(rid, await Project.init(this.env, rid)); - this.openedProjectsList.setValue([...this.openedProjectsByRid.keys()]); + if (this.openedProjectsByRid.has(rid.id)) throw new Error(`Project ${rid} already opened`); + this.openedProjectsByRid.set(rid.id, { rid, project: await Project.init(this.env, rid) }); + this.openedProjectsList.setValue([...this.openedProjectsByRid.values()].map((v) => v.rid)); } /** Closes the project, and deallocate all corresponding resources. */ public async closeProject(rid: ResourceId): Promise { - const prj = this.openedProjectsByRid.get(rid); - if (prj === undefined) throw new Error(`Project ${rid} not found among opened projects`); - this.openedProjectsByRid.delete(rid); - await prj.destroy(); - this.openedProjectsList.setValue([...this.openedProjectsByRid.keys()]); + const entry = this.openedProjectsByRid.get(rid.id); + if (entry === undefined) throw new Error(`Project ${rid} not found among opened projects`); + this.openedProjectsByRid.delete(rid.id); + await entry.project.destroy(); + this.openedProjectsList.setValue([...this.openedProjectsByRid.values()].map((v) => v.rid)); } /** Returns a project access object for opened project, for the given project * resource id. */ public getOpenedProject(rid: ResourceId): Project { - const prj = this.openedProjectsByRid.get(rid); - if (prj === undefined) throw new Error(`Project ${rid} not found among opened projects`); - return prj; + const entry = this.openedProjectsByRid.get(rid.id); + if (entry === undefined) throw new Error(`Project ${rid} not found among opened projects`); + return entry.project; } /** Returns true if project with given resource id is currently opened. */ public isProjectOpened(rid: ResourceId): boolean { - return this.openedProjectsByRid.has(rid); + return this.openedProjectsByRid.has(rid.id); } /** @@ -267,7 +267,7 @@ export class MiddleLayer { * them. */ public async close() { - await Promise.all([...this.openedProjectsByRid.values()].map((prj) => prj.destroy())); + await Promise.all([...this.openedProjectsByRid.values()].map((entry) => entry.project.destroy())); // this.env.quickJs; await this.projectListTree.terminate(); await this.env.dispose(); diff --git a/lib/node/pl-middle-layer/src/middle_layer/project.ts b/lib/node/pl-middle-layer/src/middle_layer/project.ts index f34554ae4b..f2ec2b3e35 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/project.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/project.ts @@ -120,7 +120,7 @@ export class Project { } get projectLockId(): string { - return "project:" + this.rid.toString(); + return "project:" + this.rid.id.toString(); } private async refreshLoop(): Promise { diff --git a/lib/node/pl-middle-layer/src/middle_layer/project_list.ts b/lib/node/pl-middle-layer/src/middle_layer/project_list.ts index 8e97c4766c..00737b6a74 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/project_list.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/project_list.ts @@ -55,7 +55,7 @@ export async function createProjectList( rid: prj.id, created: new Date(created), lastModified: new Date(lastModified), - opened: oProjects.indexOf(prj.id) >= 0, + opened: oProjects.some((id) => id.id === prj.id.id), meta, }); } diff --git a/lib/node/pl-middle-layer/src/model/project_model.ts b/lib/node/pl-middle-layer/src/model/project_model.ts index 57c1e6714b..4d28e22cfd 100644 --- a/lib/node/pl-middle-layer/src/model/project_model.ts +++ b/lib/node/pl-middle-layer/src/model/project_model.ts @@ -5,7 +5,7 @@ import type { } from "@milaboratories/pl-model-middle-layer"; import type { BlockRenderingMode } from "@platforma-sdk/model"; -export interface ProjectListEntry extends ProjectListEntryFromModel { +export interface ProjectListEntry extends Omit { /** Project resource ID. */ rid: ResourceId; } diff --git a/lib/node/pl-tree/src/accessors.ts b/lib/node/pl-tree/src/accessors.ts index 04b633c06d..4724c5608d 100644 --- a/lib/node/pl-tree/src/accessors.ts +++ b/lib/node/pl-tree/src/accessors.ts @@ -5,7 +5,11 @@ import type { ComputableHooks, UsageGuard, } from "@milaboratories/computable"; -import type { ResourceId, ResourceType, OptionalResourceId } from "@milaboratories/pl-client"; +import type { + ResourceId, + ResourceType, + OptionalResourceId, +} from "@milaboratories/pl-client"; import { resourceIdToString, resourceTypesEqual, @@ -196,7 +200,10 @@ export class PlTreeNodeAccessor { } public get resourceInfo(): ResourceInfo { - return { id: this.id, type: this.resourceType }; + return { + id: this.id, + type: this.resourceType, + }; } private getResourceFromTree(rid: ResourceId, ops: ResourceTraversalOps): PlTreeNodeAccessor { diff --git a/lib/node/pl-tree/src/snapshot.test.ts b/lib/node/pl-tree/src/snapshot.test.ts index 19da642848..cfc9a5f3a8 100644 --- a/lib/node/pl-tree/src/snapshot.test.ts +++ b/lib/node/pl-tree/src/snapshot.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "vitest"; import { Computable } from "@milaboratories/computable"; -import { DefaultFinalResourceDataPredicate, ResourceId } from "@milaboratories/pl-client"; +import { bigintToResourceId, DefaultFinalResourceDataPredicate } from "@milaboratories/pl-client"; import { z } from "zod"; import { InferSnapshot, makeResourceSnapshot, rsSchema } from "./snapshot"; import { PlTreeState } from "./state"; @@ -38,16 +38,16 @@ test("simple snapshot test", async () => { }); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(1n))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], data: new TextEncoder().encode(`{"jf": 0}`), }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), }, ]); @@ -58,8 +58,8 @@ test("simple snapshot test", async () => { tree.updateFromResourceData([ { ...TestValueResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], data: new TextEncoder().encode(`{"jf": 0}`), kv: [{ key: "thekey", value: Buffer.from('"thevalue"') }], }, @@ -67,13 +67,13 @@ test("simple snapshot test", async () => { expect(c1.isChanged()).toBeTruthy(); expect(await c1.getValue()).toMatchObject({ - id: rid(1n), + id: bigintToResourceId(1n), type: TestStructuralResourceType1, data: { jf: 0, }, fields: { - b: rid(2n), + b: bigintToResourceId(2n), c: undefined, }, kv: { @@ -85,8 +85,8 @@ test("simple snapshot test", async () => { tree.updateFromResourceData([ { ...TestValueResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], data: new TextEncoder().encode(`{"jf": 0}`), kv: [{ key: "thekey", value: Buffer.from("123") }], // thekey type changed to number (invalid accordig to zod schema) }, @@ -96,7 +96,3 @@ test("simple snapshot test", async () => { expect((await c1.getValueOrError()).type).toStrictEqual("error"); expect(c1.isChanged()).toBeFalsy(); }); - -function rid(id: bigint): ResourceId { - return id as ResourceId; -} diff --git a/lib/node/pl-tree/src/state.test.ts b/lib/node/pl-tree/src/state.test.ts index 655e73e2d0..30aa5b0888 100644 --- a/lib/node/pl-tree/src/state.test.ts +++ b/lib/node/pl-tree/src/state.test.ts @@ -1,9 +1,9 @@ import { expect, test } from "vitest"; import { Computable } from "@milaboratories/computable"; import { + bigintToResourceId, DefaultFinalResourceDataPredicate, NullResourceId, - ResourceId, } from "@milaboratories/pl-client"; import { isPlTreeEntry, isPlTreeEntryAccessor, isPlTreeNodeAccessor } from "./accessors"; import { PlTreeState } from "./state"; @@ -18,10 +18,6 @@ import { TestValueResourceState1, } from "./test_utils"; -function rid(id: bigint): ResourceId { - return id as ResourceId; -} - test("simple tree test 1", async () => { const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate); const entry = tree.entry(); @@ -54,11 +50,11 @@ test("simple tree test 1", async () => { expect(c1.isChanged()).toBeFalsy(); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(rid(1n)))] }, - { ...TestStructuralResourceState1, id: rid(rid(1n)), fields: [iField("b", rid(rid(2n)))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, + { ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [iField("b", bigintToResourceId(2n))] }, { ...TestValueResourceState1, - id: rid(rid(2n)), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); @@ -85,11 +81,11 @@ test("simple tree kv test", async () => { expect(c1.isChanged()).toBeFalsy(); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(rid(1n)))] }, - { ...TestStructuralResourceState1, id: rid(rid(1n)), fields: [iField("b", rid(rid(2n)))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, + { ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [iField("b", bigintToResourceId(2n))] }, { ...TestValueResourceState1, - id: rid(rid(2n)), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); @@ -101,7 +97,7 @@ test("simple tree kv test", async () => { tree.updateFromResourceData([ { ...TestValueResourceState1, - id: rid(rid(2n)), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), kv: [{ key: "thekey", value: Buffer.from("thevalue") }], }, @@ -114,7 +110,7 @@ test("simple tree kv test", async () => { tree.updateFromResourceData([ { ...TestValueResourceState1, - id: rid(rid(2n)), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), kv: [], }, @@ -143,11 +139,11 @@ test("partial tree update", async () => { expect(c1.isChanged()).toBeFalsy(); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(1n))] }, - { ...TestStructuralResourceState1, id: rid(1n), fields: [dField("b", rid(2n))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, + { ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [dField("b", bigintToResourceId(2n))] }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); @@ -155,7 +151,7 @@ test("partial tree update", async () => { expect(await c1.getValue()).toStrictEqual("Test1"); expect(c1.isChanged()).toBeFalsy(); - tree.updateFromResourceData([{ ...TestStructuralResourceState1, id: rid(1n), fields: [] }]); + tree.updateFromResourceData([{ ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [] }]); expect(c1.isChanged()).toBeTruthy(); expect(await c1.getValue()).toBeUndefined(); expect(c1.isChanged()).toBeFalsy(); @@ -172,10 +168,10 @@ test("resource error", async () => { expect(c1.isChanged()).toBeFalsy(); tree.updateFromResourceData([ - { ...TestDynamicRootState1, error: rid(7n), fields: [] }, + { ...TestDynamicRootState1, error: bigintToResourceId(7n), fields: [] }, { ...TestErrorResourceState2, - id: rid(7n), + id: bigintToResourceId(7n), data: Buffer.from('"error"'), fields: [], }, @@ -197,11 +193,11 @@ test("field error", async () => { tree.updateFromResourceData([ { ...TestDynamicRootState1, - fields: [dField("b", NullResourceId, rid(7n))], + fields: [dField("b", NullResourceId, bigintToResourceId(7n))], }, { ...TestErrorResourceState2, - id: rid(7n), + id: bigintToResourceId(7n), data: Buffer.from('"error"'), fields: [], }, @@ -214,17 +210,17 @@ test("exception - deletion of input field", () => { const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(1n))] }, - { ...TestStructuralResourceState1, id: rid(1n), fields: [iField("b", rid(2n))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, + { ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [iField("b", bigintToResourceId(2n))] }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); expect(() => - tree.updateFromResourceData([{ ...TestStructuralResourceState1, id: rid(1n), fields: [] }]), + tree.updateFromResourceData([{ ...TestStructuralResourceState1, id: bigintToResourceId(1n), fields: [] }]), ).toThrow(/removal of Input field/); }); @@ -232,16 +228,16 @@ test("exception - addition of input field", () => { const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(1n))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], ...ResourceReady, }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); @@ -250,8 +246,8 @@ test("exception - addition of input field", () => { tree.updateFromResourceData([ { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n)), iField("df")], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n)), iField("df")], ...ResourceReady, }, ]), @@ -265,17 +261,17 @@ test("exception - ready without locks 1", () => { tree.updateFromResourceData([ { ...TestDynamicRootState1, - fields: [dField("b"), dField("a", rid(1n))], + fields: [dField("b"), dField("a", bigintToResourceId(1n))], }, { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], resourceReady: true, }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]), @@ -286,15 +282,15 @@ test("exception - ready without locks 2", () => { const tree = new PlTreeState(TestDynamicRootId1, DefaultFinalResourceDataPredicate); tree.updateFromResourceData([ - { ...TestDynamicRootState1, fields: [dField("b"), dField("a", rid(1n))] }, + { ...TestDynamicRootState1, fields: [dField("b"), dField("a", bigintToResourceId(1n))] }, { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]); @@ -303,17 +299,17 @@ test("exception - ready without locks 2", () => { tree.updateFromResourceData([ { ...TestDynamicRootState1, - fields: [dField("b"), dField("a", rid(1n))], + fields: [dField("b"), dField("a", bigintToResourceId(1n))], }, { ...TestStructuralResourceState1, - id: rid(1n), - fields: [iField("b", rid(2n))], + id: bigintToResourceId(1n), + fields: [iField("b", bigintToResourceId(2n))], resourceReady: true, }, { ...TestValueResourceState1, - id: rid(2n), + id: bigintToResourceId(2n), data: new TextEncoder().encode("Test1"), }, ]), diff --git a/lib/node/pl-tree/src/state.ts b/lib/node/pl-tree/src/state.ts index 8067006798..8740499696 100644 --- a/lib/node/pl-tree/src/state.ts +++ b/lib/node/pl-tree/src/state.ts @@ -3,11 +3,13 @@ import type { FieldData, FieldStatus, FieldType, + GlobalResourceId, KeyValue, OptionalResourceId, ResourceData, ResourceId, ResourceKind, + ResourceSignature, ResourceType, } from "@milaboratories/pl-client"; import { @@ -17,6 +19,12 @@ import { resourceIdToString, stringifyWithResourceId, } from "@milaboratories/pl-client"; + +function sameOptionalResourceId(a: OptionalResourceId, b: OptionalResourceId): boolean { + if (isNullResourceId(a)) return isNullResourceId(b); + if (isNullResourceId(b)) return false; + return a.id === b.id; +} import type { Watcher } from "@milaboratories/computable"; import { ChangeSource, KeyedChangeSource } from "@milaboratories/computable"; import { PlTreeEntry } from "./accessors"; @@ -387,7 +395,7 @@ export class PlTreeResource implements ResourceDataWithFinalState { export class PlTreeState { /** resource heap */ - private resources: Map = new Map(); + private resources: Map = new Map(); private readonly resourcesAdded = new ChangeSource(); /** Resets to false if any invalid state transitions are registered, * after that tree will produce errors for any read or write operations */ @@ -410,7 +418,7 @@ export class PlTreeState { public get(watcher: Watcher, rid: ResourceId): PlTreeResource { this.checkValid(); - const res = this.resources.get(rid); + const res = this.resources.get(rid.id); if (res === undefined) { // to make recovery from resource not found possible, considering some // race conditions, where computable is created before tree is updated @@ -425,12 +433,12 @@ export class PlTreeState { this.checkValid(); // All resources for which recount should be incremented, first are aggregated in this list - const incrementRefs: ResourceId[] = []; - const decrementRefs: ResourceId[] = []; + const incrementRefs: GlobalResourceId[] = []; + const decrementRefs: GlobalResourceId[] = []; // patching / creating resources for (const rd of resourceData) { - let resource = this.resources.get(rd.id); + let resource = this.resources.get(rd.id.id); const statBeforeMutation = resource?.basicState; const unexpectedTransitionError = (reason: string): never => { @@ -455,7 +463,7 @@ export class PlTreeState { resource.version += 1; // duplicate / original - if (resource.originalResourceId !== rd.originalResourceId) { + if (!sameOptionalResourceId(resource.originalResourceId, rd.originalResourceId)) { if (resource.originalResourceId !== NullResourceId) unexpectedTransitionError("originalResourceId can't change after it is set"); resource.originalResourceId = rd.originalResourceId; @@ -467,11 +475,11 @@ export class PlTreeState { } // error - if (resource.error !== rd.error) { + if (!sameOptionalResourceId(resource.error, rd.error)) { if (isNotNullResourceId(resource.error)) unexpectedTransitionError("resource can't change attached error after it is set"); resource.error = rd.error; - incrementRefs.push(resource.error as ResourceId); + if (isNotNullResourceId(resource.error)) incrementRefs.push(resource.error.id); notEmpty(resource.resourceStateChange).markChanged( `error changed for ${resourceIdToString(resource.id)}`, ); @@ -494,8 +502,8 @@ export class PlTreeState { fd.valueIsFinal, resource.version, ); - if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value); - if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error); + if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value.id); + if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error.id); if (fd.type === "Input" || fd.type === "Service") { if (resource.inputsLocked) @@ -558,10 +566,10 @@ export class PlTreeState { } // field value - if (field.value !== fd.value) { - if (isNotNullResourceId(field.value)) decrementRefs.push(field.value); + if (!sameOptionalResourceId(field.value, fd.value)) { + if (isNotNullResourceId(field.value)) decrementRefs.push(field.value.id); field.value = fd.value; - if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value); + if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value.id); field.change.markChanged( `field ${fd.name} value changed in ${resourceIdToString(resource.id)}`, ); @@ -569,10 +577,10 @@ export class PlTreeState { } // field error - if (field.error !== fd.error) { - if (isNotNullResourceId(field.error)) decrementRefs.push(field.error); + if (!sameOptionalResourceId(field.error, fd.error)) { + if (isNotNullResourceId(field.error)) decrementRefs.push(field.error.id); field.error = fd.error; - if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error); + if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error.id); field.change.markChanged( `field ${fd.name} error changed in ${resourceIdToString(resource.id)}`, ); @@ -611,8 +619,8 @@ export class PlTreeState { ); fields.delete(fieldName); - if (isNotNullResourceId(field.value)) decrementRefs.push(field.value); - if (isNotNullResourceId(field.error)) decrementRefs.push(field.error); + if (isNotNullResourceId(field.value)) decrementRefs.push(field.value.id); + if (isNotNullResourceId(field.error)) decrementRefs.push(field.error.id); notEmpty(resource!.dynamicFieldListChanged).markChanged( `dynamic field ${fieldName} removed from ${resourceIdToString(resource!.id)}`, @@ -700,7 +708,7 @@ export class PlTreeState { resource = new PlTreeResource(rd); resource.verifyReadyState(); - if (isNotNullResourceId(resource.error)) incrementRefs.push(resource.error); + if (isNotNullResourceId(resource.error)) incrementRefs.push(resource.error.id); for (const fd of rd.fields) { const field = new PlTreeField( fd.name, @@ -711,8 +719,8 @@ export class PlTreeState { fd.valueIsFinal, InitialResourceVersion, ); - if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value); - if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error); + if (isNotNullResourceId(fd.value)) incrementRefs.push(fd.value.id); + if (isNotNullResourceId(fd.error)) incrementRefs.push(fd.error.id); resource.fieldsMap.set(fd.name, field); } @@ -723,7 +731,7 @@ export class PlTreeState { if (this.isFinalPredicate(resource)) resource.markFinal(); // adding the resource to the heap - this.resources.set(resource.id, resource); + this.resources.set(resource.id.id, resource); this.resourcesAdded.markChanged(`new resource ${resourceIdToString(resource.id)} added`); } } @@ -741,7 +749,7 @@ export class PlTreeState { // recursively applying refCount decrements / doing garbage collection let currentRefs = decrementRefs; while (currentRefs.length > 0) { - const nextRefs: ResourceId[] = []; + const nextRefs: GlobalResourceId[] = []; for (const rid of currentRefs) { const res = this.resources.get(rid); if (!res) { @@ -751,16 +759,16 @@ export class PlTreeState { res.refCount--; // garbage collection - if (res.refCount === 0 && res.id !== this.root) { + if (res.refCount === 0 && res.id.id !== this.root.id) { // removing fields res.fieldsMap.forEach((field) => { - if (isNotNullResourceId(field.value)) nextRefs.push(field.value); - if (isNotNullResourceId(field.error)) nextRefs.push(field.error); + if (isNotNullResourceId(field.value)) nextRefs.push(field.value.id); + if (isNotNullResourceId(field.error)) nextRefs.push(field.error.id); field.change.markChanged( `field ${field.name} removed during garbage collection of ${resourceIdToString(res.id)}`, ); }); - if (isNotNullResourceId(res.error)) nextRefs.push(res.error); + if (isNotNullResourceId(res.error)) nextRefs.push(res.error.id); res.resourceRemoved.markChanged( `resource removed during garbage collection: ${resourceIdToString(res.id)}`, ); @@ -773,7 +781,7 @@ export class PlTreeState { // checking for orphans (maybe removed in the future) if (!allowOrphanInputs) { for (const rd of resourceData) { - if (!this.resources.has(rd.id)) { + if (!this.resources.has(rd.id.id)) { this.invalidateTree(); throw new TreeStateUpdateError(`orphan input resource ${rd.id}`); } @@ -787,6 +795,10 @@ export class PlTreeState { return this.entry(rid); } + public getResourceSignature(rid: ResourceId): ResourceSignature | undefined { + return this.resources.get(rid.id)?.id.signature; + } + public entry(rid: ResourceId = this.root): PlTreeEntry { this.checkValid(); return new PlTreeEntry({ treeProvider: () => this }, rid); diff --git a/lib/node/pl-tree/src/synchronized_tree.ts b/lib/node/pl-tree/src/synchronized_tree.ts index 55e205d124..89ae36b9b3 100644 --- a/lib/node/pl-tree/src/synchronized_tree.ts +++ b/lib/node/pl-tree/src/synchronized_tree.ts @@ -124,12 +124,14 @@ export class SynchronizedTreeState { private async refresh(stats?: TreeLoadingStat, txOps?: TxOps): Promise { if (this.terminated) throw new Error("tree synchronization is terminated"); const request = constructTreeLoadingRequest(this.state, this.pruning); + const treeState = this.state; const data = await this.pl.withReadTx( "ReadingTree", - async (tx) => { - return await loadTreeState(tx, request, stats); + async (tx) => loadTreeState(tx, request, stats), + { + ...txOps, + signatureResolver: (id) => treeState.getResourceSignature(id), }, - txOps, ); this.state.updateFromResourceData(data, true); } diff --git a/lib/node/pl-tree/src/test_utils.ts b/lib/node/pl-tree/src/test_utils.ts index c79cfac549..774e858152 100644 --- a/lib/node/pl-tree/src/test_utils.ts +++ b/lib/node/pl-tree/src/test_utils.ts @@ -6,7 +6,7 @@ import type { ResourceId, ResourceType, } from "@milaboratories/pl-client"; -import { NullResourceId } from "@milaboratories/pl-client"; +import { NullResourceId, bigintToResourceId } from "@milaboratories/pl-client"; import type { ExtendedResourceData } from "./state"; export const TestRootType1: ResourceType = { @@ -100,7 +100,7 @@ export const TestErrorResourceState2: Omit type: TestErrorResourceType1, }; -export const TestDynamicRootId1 = 1000001n as ResourceId; +export const TestDynamicRootId1 = bigintToResourceId(1000001n); export const TestDynamicRootState1: Omit = { ...InitialStructuralResourceState, inputsLocked: true, @@ -110,7 +110,7 @@ export const TestDynamicRootState1: Omit = { id: TestDynamicRootId1, }; -export const TestDynamicRootId2 = 1000002n as ResourceId; +export const TestDynamicRootId2 = bigintToResourceId(1000002n); export const TestDynamicRootState2: Omit = { ...InitialStructuralResourceState, inputsLocked: true,