From c2816da1d94029511514f1502ae8f66fc3bffe6e Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 4 Jun 2026 11:06:49 -0500 Subject: [PATCH] change(web): remove unused transition-edit field Build-bot: skip build:web Test-bot: skip --- .../main/correction/context-tokenization.ts | 20 -------- .../context/context-state.tests.ts | 1 + .../context/context-tokenization.tests.ts | 51 +------------------ .../context/transition-helpers.tests.ts | 1 - ...ine-suggestion-context-transition.tests.ts | 1 - 5 files changed, 3 insertions(+), 71 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 175c0eb202e..cb5af5a4954 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -108,18 +108,6 @@ export class ContextTokenization { */ readonly tokens: ContextToken[]; - /** - * Denotes whether or not the transition to this tokenization added or deleted - * any tokens. - */ - readonly transitionEdits?: { - addedNewTokens: boolean, - removedOldTokens: boolean, - // NOTE: slated for removal in an upcoming PR. Exists in this form to - // facilitate factorization of the changes into smaller bodies of work. - editedTokenCount: number - }; - /** * The portion of edits from the true input keystroke that are not part of the * final entry in `token`. If `null`, all edits are considered part of the @@ -147,18 +135,10 @@ export class ContextTokenization { throw new Error("ContextTokenization requires at least one existing ContextToken"); } this.tokens = [].concat(tokens); - if(tokenizationPath) { - this.transitionEdits = { - addedNewTokens: tokenizationPath?.inputs[0].sample.has(1) ?? false, - removedOldTokens: (tokenizationPath?.alignment.removedTokenCount ?? 0) > 0, - editedTokenCount: tokenizationPath?.inputs[0].sample.size - } - } this.taillessTrueKeystroke = taillessTrueKeystroke; } else { const priorToClone = param1; this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry)); - this.transitionEdits = priorToClone.transitionEdits ? {...priorToClone.transitionEdits} : null; this.taillessTrueKeystroke = priorToClone.taillessTrueKeystroke; } } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index 677b541181a..23750ea44a0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -248,6 +248,7 @@ describe('ContextState', () => { assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 44ad985578f..1353255e75f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -26,7 +26,6 @@ import { ExtendedEditOperation, generateSubsetId, models, - TransitionEdge, SearchQuotientSpur, traceInsertEdits, LegacyQuotientSpur @@ -100,7 +99,6 @@ describe('ContextTokenization', function() { let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text)))); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isNotOk(tokenization.transitionEdits); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -108,36 +106,11 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - // We _could_ flesh this out a bit more... but it's not really needed for this test. - const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); - let transitionEdits: TransitionEdge = { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, - removedTokenCount: 0 - }, - inputs: [{sample: (() => { - const map = new Map(); - map.set(0, emptyTransform); - return map; - })(), p: 1}], - inputSubsetId: generateSubsetId() - }; - - let tokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let tokenization = new ContextTokenization(tokens, null, null /* dummy val */); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isOk(tokenization.transitionEdits); - assert.deepEqual(tokenization.transitionEdits, { - addedNewTokens: false, - removedOldTokens: false, - editedTokenCount: 1 - }); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -145,27 +118,7 @@ describe('ContextTokenization', function() { it('clones', () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - - // We _could_ flesh this out a bit more... but it's not really needed for this test. - const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); - let transitionEdits: TransitionEdge = { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, - removedTokenCount: 0 - }, - inputs: [{sample: (() => { - const map = new Map(); - map.set(0, emptyTransform); - return map; - })(), p: 1}], - inputSubsetId: generateSubsetId() - }; - - let baseTokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let baseTokenization = new ContextTokenization(tokens, null, null /* dummy val */); let cloned = new ContextTokenization(baseTokenization); assert.sameOrderedMembers( diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 882fc5a9d40..7e5c0ef8350 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -354,7 +354,6 @@ function assertMatchingToken(actual: ContextToken, expected: ContextToken, msg: function assertMatchingTokenization(actual: ContextTokenization, expected: ContextTokenization, msg: string) { assert.equal(actual.tokens.length, expected.tokens.length, msg); assert.deepEqual(actual.exampleInput, expected.exampleInput, msg); - assert.deepEqual(actual.transitionEdits, expected.transitionEdits, msg); for(let j=0; j < actual.tokens.length; j++) { const nestedMsg = `${msg}, token ${j}`; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index d232d90b1a7..88479bc7475 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -104,7 +104,6 @@ describe('determineContextTransition', () => { assert.equal(transition, tracker.latest); assert.isFalse(warningEmitterSpy.called); assert.sameOrderedMembers(transition.final.displayTokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); - assert.isOk(transition.final.displayTokenization.transitionEdits); assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution);