From 89296771b8f06447d2cbd9bbb602b514450bb4f7 Mon Sep 17 00:00:00 2001 From: sanat-g Date: Thu, 4 Jun 2026 21:43:14 -0400 Subject: [PATCH 1/4] Add Fast Apply basic edit example --- README.md | 11 ++++++++ fast-apply/basic-edit/.env.example | 1 + fast-apply/basic-edit/README.md | 30 ++++++++++++++++++++ fast-apply/basic-edit/edit.ts | 44 ++++++++++++++++++++++++++++++ fast-apply/basic-edit/package.json | 14 ++++++++++ fast-apply/basic-edit/src/auth.ts | 32 ++++++++++++++++++++++ 6 files changed, 132 insertions(+) create mode 100644 fast-apply/basic-edit/.env.example create mode 100644 fast-apply/basic-edit/README.md create mode 100644 fast-apply/basic-edit/edit.ts create mode 100644 fast-apply/basic-edit/package.json create mode 100644 fast-apply/basic-edit/src/auth.ts diff --git a/README.md b/README.md index f30ca3c..8feb18b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Use this table to jump to the right place based on what you're building. |----------------|-------| | **Python** | [`warpgrep/python-agent`](./warpgrep/python-agent) — full protocol reference, no SDK | | **TypeScript** | [`warpgrep/basic-search`](./warpgrep/basic-search) — simplest starting point | +| **TypeScript + Fast Apply** | [`fast-apply/basic-edit`](./fast-apply/basic-edit) — apply a targeted code edit | | **TypeScript + Anthropic SDK** | [`warpgrep/anthropic-agent`](./warpgrep/anthropic-agent) — Claude in an agent loop | | **TypeScript + OpenAI SDK** | [`warpgrep/openai-agent`](./warpgrep/openai-agent) — GPT-4o in an agent loop | | **TypeScript + Vercel AI SDK** | [`warpgrep/vercel-agent`](./warpgrep/vercel-agent) — automatic tool loop | @@ -22,6 +23,7 @@ Use this table to jump to the right place based on what you're building. | I want to... | Go to | |--------------|-------| | Search a local repo | [`warpgrep/basic-search`](./warpgrep/basic-search) | +| Apply a targeted code edit | [`fast-apply/basic-edit`](./fast-apply/basic-edit) | | Search a GitHub repo (no clone) | [`warpgrep/github-search`](./warpgrep/github-search) | | Stream search progress | [`warpgrep/streaming`](./warpgrep/streaming) | | Run search in a Vercel Sandbox | [`warpgrep/vercel-sandbox`](./warpgrep/vercel-sandbox) | @@ -40,6 +42,14 @@ Use this table to jump to the right place based on what you're building. ## Examples +### Fast Apply — Code Editing + +All in [`fast-apply/`](./fast-apply): + +| Example | Language | Description | +|---------|----------|-------------| +| [basic-edit](./fast-apply/basic-edit) | TypeScript | Apply a targeted partial edit to a TypeScript file | + ### WarpGrep — Code Search Subagent All in [`warpgrep/`](./warpgrep): @@ -90,6 +100,7 @@ MORPH_API_KEY=your-key npx tsx search.ts "Find the main entry point" /path/to/re ## Docs +- [Fast Apply reference](https://docs.morphllm.com/sdk/components/fast-apply) - [WarpGrep overview](https://docs.morphllm.com/sdk/components/warp-grep) - [WarpGrep tool reference](https://docs.morphllm.com/sdk/components/warp-grep/tool) - [WarpGrep direct API](https://docs.morphllm.com/sdk/components/warp-grep/direct) diff --git a/fast-apply/basic-edit/.env.example b/fast-apply/basic-edit/.env.example new file mode 100644 index 0000000..185a879 --- /dev/null +++ b/fast-apply/basic-edit/.env.example @@ -0,0 +1 @@ +MORPH_API_KEY=your-key diff --git a/fast-apply/basic-edit/README.md b/fast-apply/basic-edit/README.md new file mode 100644 index 0000000..1b5bcac --- /dev/null +++ b/fast-apply/basic-edit/README.md @@ -0,0 +1,30 @@ +# Fast Apply: Basic Edit + +Apply a focused code edit to an existing TypeScript file. Fast Apply merges a partial `code_edit` into the full file, so coding agents can describe small changes without rewriting the whole file. + +## Setup + +```bash +npm install +``` + +Use `.env.example` as a template, then export your key or pass it inline: + +```bash +MORPH_API_KEY=your-key npx tsx edit.ts +``` + +## Run + +```bash +MORPH_API_KEY=your-key npx tsx edit.ts +``` + +## What it does + +1. Creates a `MorphClient` with your API key +2. Calls `morph.fastApply.execute()` for `src/auth.ts` +3. Sends a targeted `code_edit` with `// ... existing code ...` markers +4. Writes the merged edit to the file and prints the diff + +The example fixes a missing null check in `login()`. This is useful for coding agents because the agent only needs to specify the intended change and enough local context to place it correctly. diff --git a/fast-apply/basic-edit/edit.ts b/fast-apply/basic-edit/edit.ts new file mode 100644 index 0000000..8d7b5bb --- /dev/null +++ b/fast-apply/basic-edit/edit.ts @@ -0,0 +1,44 @@ +// # Basic Edit +// +// Apply a targeted partial edit to a TypeScript file with Fast Apply. +// +// Usage: +// MORPH_API_KEY=your-key npx tsx edit.ts + +import { MorphClient } from "@morphllm/morphsdk"; + +const morph = new MorphClient({ apiKey: process.env.MORPH_API_KEY }); + +const result = await morph.fastApply.execute( + { + target_filepath: "src/auth.ts", + instruction: + "I am adding a null check after findUserByEmail so login throws Invalid credentials when no user is found.", + code_edit: `// ... existing code ... +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + if (!user) { + throw new Error("Invalid credentials"); + } + + const passwordMatches = await verifyPassword(password, user.passwordHash); +// ... existing code ...`, + }, + { baseDir: "." } +); + +if (!result.success) { + console.error("Edit failed:", result.error); + process.exit(1); +} + +console.log(`Edited ${result.filepath}`); +console.log( + `Changes: +${result.changes.linesAdded} -${result.changes.linesRemoved} ~${result.changes.linesModified}` +); + +if (result.udiff) { + console.log("\nDiff:\n"); + console.log(result.udiff); +} diff --git a/fast-apply/basic-edit/package.json b/fast-apply/basic-edit/package.json new file mode 100644 index 0000000..d52e243 --- /dev/null +++ b/fast-apply/basic-edit/package.json @@ -0,0 +1,14 @@ +{ + "name": "fast-apply-basic-edit", + "private": true, + "type": "module", + "scripts": { + "start": "npx tsx edit.ts" + }, + "dependencies": { + "@morphllm/morphsdk": "latest" + }, + "devDependencies": { + "tsx": "^4.0.0" + } +} diff --git a/fast-apply/basic-edit/src/auth.ts b/fast-apply/basic-edit/src/auth.ts new file mode 100644 index 0000000..de4f3ae --- /dev/null +++ b/fast-apply/basic-edit/src/auth.ts @@ -0,0 +1,32 @@ +export interface User { + id: string; + email: string; + passwordHash: string; +} + +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + const passwordMatches = await verifyPassword(password, user.passwordHash); + if (!passwordMatches) { + throw new Error("Invalid credentials"); + } + + return user; +} + +async function findUserByEmail(email: string): Promise { + if (email === "admin@example.com") { + return { + id: "user_123", + email, + passwordHash: "stored-hash", + }; + } + + return null; +} + +async function verifyPassword(password: string, passwordHash: string): Promise { + return password.length > 0 && passwordHash === "stored-hash"; +} From 1245d897088392bdc1f198bd11cf151735254759 Mon Sep 17 00:00:00 2001 From: sanat-g Date: Thu, 4 Jun 2026 22:01:40 -0400 Subject: [PATCH 2/4] Add Fast Apply direct API example --- README.md | 3 ++ fast-apply/direct-api/.env.example | 1 + fast-apply/direct-api/README.md | 30 ++++++++++++ fast-apply/direct-api/apply.ts | 74 ++++++++++++++++++++++++++++++ fast-apply/direct-api/package.json | 14 ++++++ fast-apply/direct-api/src/auth.ts | 32 +++++++++++++ 6 files changed, 154 insertions(+) create mode 100644 fast-apply/direct-api/.env.example create mode 100644 fast-apply/direct-api/README.md create mode 100644 fast-apply/direct-api/apply.ts create mode 100644 fast-apply/direct-api/package.json create mode 100644 fast-apply/direct-api/src/auth.ts diff --git a/README.md b/README.md index 8feb18b..e008004 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Use this table to jump to the right place based on what you're building. | **Python** | [`warpgrep/python-agent`](./warpgrep/python-agent) — full protocol reference, no SDK | | **TypeScript** | [`warpgrep/basic-search`](./warpgrep/basic-search) — simplest starting point | | **TypeScript + Fast Apply** | [`fast-apply/basic-edit`](./fast-apply/basic-edit) — apply a targeted code edit | +| **TypeScript + Direct API** | [`fast-apply/direct-api`](./fast-apply/direct-api) — call Fast Apply over HTTP | | **TypeScript + Anthropic SDK** | [`warpgrep/anthropic-agent`](./warpgrep/anthropic-agent) — Claude in an agent loop | | **TypeScript + OpenAI SDK** | [`warpgrep/openai-agent`](./warpgrep/openai-agent) — GPT-4o in an agent loop | | **TypeScript + Vercel AI SDK** | [`warpgrep/vercel-agent`](./warpgrep/vercel-agent) — automatic tool loop | @@ -24,6 +25,7 @@ Use this table to jump to the right place based on what you're building. |--------------|-------| | Search a local repo | [`warpgrep/basic-search`](./warpgrep/basic-search) | | Apply a targeted code edit | [`fast-apply/basic-edit`](./fast-apply/basic-edit) | +| Call Fast Apply directly over HTTP | [`fast-apply/direct-api`](./fast-apply/direct-api) | | Search a GitHub repo (no clone) | [`warpgrep/github-search`](./warpgrep/github-search) | | Stream search progress | [`warpgrep/streaming`](./warpgrep/streaming) | | Run search in a Vercel Sandbox | [`warpgrep/vercel-sandbox`](./warpgrep/vercel-sandbox) | @@ -49,6 +51,7 @@ All in [`fast-apply/`](./fast-apply): | Example | Language | Description | |---------|----------|-------------| | [basic-edit](./fast-apply/basic-edit) | TypeScript | Apply a targeted partial edit to a TypeScript file | +| [direct-api](./fast-apply/direct-api) | TypeScript | Call Fast Apply directly through the OpenAI-compatible API | ### WarpGrep — Code Search Subagent diff --git a/fast-apply/direct-api/.env.example b/fast-apply/direct-api/.env.example new file mode 100644 index 0000000..185a879 --- /dev/null +++ b/fast-apply/direct-api/.env.example @@ -0,0 +1 @@ +MORPH_API_KEY=your-key diff --git a/fast-apply/direct-api/README.md b/fast-apply/direct-api/README.md new file mode 100644 index 0000000..83afdfa --- /dev/null +++ b/fast-apply/direct-api/README.md @@ -0,0 +1,30 @@ +# Fast Apply: Direct API + +Call Morph Fast Apply directly through the OpenAI-compatible chat completions API. This example does not use the Morph SDK. + +## Setup + +```bash +npm install +``` + +Use `.env.example` as a template, then export your key or pass it inline: + +```bash +MORPH_API_KEY=your-key npx tsx apply.ts +``` + +## Run + +```bash +MORPH_API_KEY=your-key npx tsx apply.ts +``` + +## What it does + +1. Reads `src/auth.ts` +2. Sends the original code, instruction, and partial `code_edit` to `https://api.morphllm.com/v1/chat/completions` +3. Uses `morph-v3-fast` to merge the partial edit into the full file +4. Writes the merged code back to `src/auth.ts` and prints a diff + +This is the lowest-level Fast Apply integration path. It is useful when you want direct API control or when you are integrating Morph into an existing model gateway. diff --git a/fast-apply/direct-api/apply.ts b/fast-apply/direct-api/apply.ts new file mode 100644 index 0000000..7044879 --- /dev/null +++ b/fast-apply/direct-api/apply.ts @@ -0,0 +1,74 @@ +// # Direct API +// +// Call Morph Fast Apply through the OpenAI-compatible chat completions API. +// +// Usage: +// MORPH_API_KEY=your-key npx tsx apply.ts + +import { readFile, writeFile } from "node:fs/promises"; +import { createTwoFilesPatch } from "diff"; + +const apiKey = process.env.MORPH_API_KEY; +const filepath = "src/auth.ts"; + +if (!apiKey) { + console.error("Missing MORPH_API_KEY"); + process.exit(1); +} + +const originalCode = await readFile(filepath, "utf-8"); +const instruction = + "I am adding a null check after findUserByEmail so login throws Invalid credentials when no user is found."; +const codeEdit = `// ... existing code ... +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + if (!user) { + throw new Error("Invalid credentials"); + } + + const passwordMatches = await verifyPassword(password, user.passwordHash); +// ... existing code ...`; + +const response = await fetch("https://api.morphllm.com/v1/chat/completions", { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "morph-v3-fast", + messages: [ + { + role: "user", + content: `${instruction}\n${originalCode}\n${codeEdit}`, + }, + ], + }), +}); + +if (!response.ok) { + const errorText = await response.text(); + console.error(`Fast Apply request failed: ${response.status} ${response.statusText}`); + console.error(errorText); + process.exit(1); +} + +const completion = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; +}; + +const mergedCode = completion.choices?.[0]?.message?.content; + +if (!mergedCode) { + console.error("Fast Apply response did not include merged code"); + process.exit(1); +} + +await writeFile(filepath, mergedCode); + +const diff = createTwoFilesPatch(filepath, filepath, originalCode, mergedCode, "Original", "Modified"); + +console.log(`Edited ${filepath}`); +console.log("\nDiff:\n"); +console.log(diff); diff --git a/fast-apply/direct-api/package.json b/fast-apply/direct-api/package.json new file mode 100644 index 0000000..30c1aa9 --- /dev/null +++ b/fast-apply/direct-api/package.json @@ -0,0 +1,14 @@ +{ + "name": "fast-apply-direct-api", + "private": true, + "type": "module", + "scripts": { + "start": "npx tsx apply.ts" + }, + "dependencies": { + "diff": "^7.0.0" + }, + "devDependencies": { + "tsx": "^4.0.0" + } +} diff --git a/fast-apply/direct-api/src/auth.ts b/fast-apply/direct-api/src/auth.ts new file mode 100644 index 0000000..de4f3ae --- /dev/null +++ b/fast-apply/direct-api/src/auth.ts @@ -0,0 +1,32 @@ +export interface User { + id: string; + email: string; + passwordHash: string; +} + +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + const passwordMatches = await verifyPassword(password, user.passwordHash); + if (!passwordMatches) { + throw new Error("Invalid credentials"); + } + + return user; +} + +async function findUserByEmail(email: string): Promise { + if (email === "admin@example.com") { + return { + id: "user_123", + email, + passwordHash: "stored-hash", + }; + } + + return null; +} + +async function verifyPassword(password: string, passwordHash: string): Promise { + return password.length > 0 && passwordHash === "stored-hash"; +} From e226823388611359797363971e69e6b6723a6982 Mon Sep 17 00:00:00 2001 From: sanat-g Date: Thu, 4 Jun 2026 22:10:08 -0400 Subject: [PATCH 3/4] Add Fast Apply code-in-code-out example --- README.md | 3 ++ fast-apply/code-in-code-out/.env.example | 1 + fast-apply/code-in-code-out/README.md | 30 ++++++++++++++++ fast-apply/code-in-code-out/apply.ts | 45 ++++++++++++++++++++++++ fast-apply/code-in-code-out/package.json | 14 ++++++++ fast-apply/code-in-code-out/src/auth.ts | 32 +++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 fast-apply/code-in-code-out/.env.example create mode 100644 fast-apply/code-in-code-out/README.md create mode 100644 fast-apply/code-in-code-out/apply.ts create mode 100644 fast-apply/code-in-code-out/package.json create mode 100644 fast-apply/code-in-code-out/src/auth.ts diff --git a/README.md b/README.md index e008004..95ee519 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Use this table to jump to the right place based on what you're building. | **TypeScript** | [`warpgrep/basic-search`](./warpgrep/basic-search) — simplest starting point | | **TypeScript + Fast Apply** | [`fast-apply/basic-edit`](./fast-apply/basic-edit) — apply a targeted code edit | | **TypeScript + Direct API** | [`fast-apply/direct-api`](./fast-apply/direct-api) — call Fast Apply over HTTP | +| **TypeScript + Sandbox** | [`fast-apply/code-in-code-out`](./fast-apply/code-in-code-out) — manage file I/O yourself | | **TypeScript + Anthropic SDK** | [`warpgrep/anthropic-agent`](./warpgrep/anthropic-agent) — Claude in an agent loop | | **TypeScript + OpenAI SDK** | [`warpgrep/openai-agent`](./warpgrep/openai-agent) — GPT-4o in an agent loop | | **TypeScript + Vercel AI SDK** | [`warpgrep/vercel-agent`](./warpgrep/vercel-agent) — automatic tool loop | @@ -26,6 +27,7 @@ Use this table to jump to the right place based on what you're building. | Search a local repo | [`warpgrep/basic-search`](./warpgrep/basic-search) | | Apply a targeted code edit | [`fast-apply/basic-edit`](./fast-apply/basic-edit) | | Call Fast Apply directly over HTTP | [`fast-apply/direct-api`](./fast-apply/direct-api) | +| Use Fast Apply in a sandbox or harness | [`fast-apply/code-in-code-out`](./fast-apply/code-in-code-out) | | Search a GitHub repo (no clone) | [`warpgrep/github-search`](./warpgrep/github-search) | | Stream search progress | [`warpgrep/streaming`](./warpgrep/streaming) | | Run search in a Vercel Sandbox | [`warpgrep/vercel-sandbox`](./warpgrep/vercel-sandbox) | @@ -52,6 +54,7 @@ All in [`fast-apply/`](./fast-apply): |---------|----------|-------------| | [basic-edit](./fast-apply/basic-edit) | TypeScript | Apply a targeted partial edit to a TypeScript file | | [direct-api](./fast-apply/direct-api) | TypeScript | Call Fast Apply directly through the OpenAI-compatible API | +| [code-in-code-out](./fast-apply/code-in-code-out) | TypeScript | Return merged code while managing file I/O yourself | ### WarpGrep — Code Search Subagent diff --git a/fast-apply/code-in-code-out/.env.example b/fast-apply/code-in-code-out/.env.example new file mode 100644 index 0000000..185a879 --- /dev/null +++ b/fast-apply/code-in-code-out/.env.example @@ -0,0 +1 @@ +MORPH_API_KEY=your-key diff --git a/fast-apply/code-in-code-out/README.md b/fast-apply/code-in-code-out/README.md new file mode 100644 index 0000000..daa3fc7 --- /dev/null +++ b/fast-apply/code-in-code-out/README.md @@ -0,0 +1,30 @@ +# Fast Apply: Code In, Code Out + +Apply a focused edit with Fast Apply while managing file reads and writes yourself. This is useful in sandboxes, harnesses, or hosted environments where your app controls the filesystem. + +## Setup + +```bash +npm install +``` + +Use `.env.example` as a template, then export your key or pass it inline: + +```bash +MORPH_API_KEY=your-key npx tsx apply.ts +``` + +## Run + +```bash +MORPH_API_KEY=your-key npx tsx apply.ts +``` + +## What it does + +1. Reads `src/auth.ts` +2. Calls `applyEdit()` with the original file contents and a partial `codeEdit` +3. Receives `mergedCode` instead of letting the SDK write to disk +4. Writes the merged code itself and prints the diff + +The example fixes a missing null check in `login()`. Use this pattern when Fast Apply should return code for a caller, sandbox, or harness to store explicitly. diff --git a/fast-apply/code-in-code-out/apply.ts b/fast-apply/code-in-code-out/apply.ts new file mode 100644 index 0000000..310960b --- /dev/null +++ b/fast-apply/code-in-code-out/apply.ts @@ -0,0 +1,45 @@ +// # Code In, Code Out +// +// Apply a Fast Apply edit while managing file reads and writes yourself. +// +// Usage: +// MORPH_API_KEY=your-key npx tsx apply.ts + +import { readFile, writeFile } from "node:fs/promises"; +import { applyEdit } from "@morphllm/morphsdk"; + +const filepath = "src/auth.ts"; +const originalCode = await readFile(filepath, "utf-8"); + +const result = await applyEdit({ + originalCode, + instruction: + "I am adding a null check after findUserByEmail so login throws Invalid credentials when no user is found.", + codeEdit: `// ... existing code ... +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + if (!user) { + throw new Error("Invalid credentials"); + } + + const passwordMatches = await verifyPassword(password, user.passwordHash); +// ... existing code ...`, +}); + +if (!result.success || !result.mergedCode) { + console.error("Edit failed:", result.error); + process.exit(1); +} + +await writeFile(filepath, result.mergedCode); + +console.log(`Edited ${filepath}`); +console.log( + `Changes: +${result.changes.linesAdded} -${result.changes.linesRemoved} ~${result.changes.linesModified}` +); + +if (result.udiff) { + console.log("\nDiff:\n"); + console.log(result.udiff); +} diff --git a/fast-apply/code-in-code-out/package.json b/fast-apply/code-in-code-out/package.json new file mode 100644 index 0000000..9b18b65 --- /dev/null +++ b/fast-apply/code-in-code-out/package.json @@ -0,0 +1,14 @@ +{ + "name": "fast-apply-code-in-code-out", + "private": true, + "type": "module", + "scripts": { + "start": "npx tsx apply.ts" + }, + "dependencies": { + "@morphllm/morphsdk": "latest" + }, + "devDependencies": { + "tsx": "^4.0.0" + } +} diff --git a/fast-apply/code-in-code-out/src/auth.ts b/fast-apply/code-in-code-out/src/auth.ts new file mode 100644 index 0000000..de4f3ae --- /dev/null +++ b/fast-apply/code-in-code-out/src/auth.ts @@ -0,0 +1,32 @@ +export interface User { + id: string; + email: string; + passwordHash: string; +} + +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + const passwordMatches = await verifyPassword(password, user.passwordHash); + if (!passwordMatches) { + throw new Error("Invalid credentials"); + } + + return user; +} + +async function findUserByEmail(email: string): Promise { + if (email === "admin@example.com") { + return { + id: "user_123", + email, + passwordHash: "stored-hash", + }; + } + + return null; +} + +async function verifyPassword(password: string, passwordHash: string): Promise { + return password.length > 0 && passwordHash === "stored-hash"; +} From e40b5650557d74e6bcfa6a8abbbe5d9e1b470f96 Mon Sep 17 00:00:00 2001 From: sanat-g Date: Thu, 4 Jun 2026 22:16:36 -0400 Subject: [PATCH 4/4] Add Fast Apply eval harness example --- README.md | 3 + fast-apply/eval-harness/.env.example | 1 + fast-apply/eval-harness/README.md | 30 +++++++ fast-apply/eval-harness/package.json | 15 ++++ fast-apply/eval-harness/run-eval.ts | 90 +++++++++++++++++++ .../tasks/add-validation/edit.txt | 13 +++ .../tasks/add-validation/expected.ts | 9 ++ .../tasks/add-validation/input.ts | 5 ++ .../eval-harness/tasks/null-check/edit.txt | 13 +++ .../eval-harness/tasks/null-check/expected.ts | 20 +++++ .../eval-harness/tasks/null-check/input.ts | 16 ++++ .../eval-harness/tasks/rename-field/edit.txt | 11 +++ .../tasks/rename-field/expected.ts | 8 ++ .../eval-harness/tasks/rename-field/input.ts | 8 ++ 14 files changed, 242 insertions(+) create mode 100644 fast-apply/eval-harness/.env.example create mode 100644 fast-apply/eval-harness/README.md create mode 100644 fast-apply/eval-harness/package.json create mode 100644 fast-apply/eval-harness/run-eval.ts create mode 100644 fast-apply/eval-harness/tasks/add-validation/edit.txt create mode 100644 fast-apply/eval-harness/tasks/add-validation/expected.ts create mode 100644 fast-apply/eval-harness/tasks/add-validation/input.ts create mode 100644 fast-apply/eval-harness/tasks/null-check/edit.txt create mode 100644 fast-apply/eval-harness/tasks/null-check/expected.ts create mode 100644 fast-apply/eval-harness/tasks/null-check/input.ts create mode 100644 fast-apply/eval-harness/tasks/rename-field/edit.txt create mode 100644 fast-apply/eval-harness/tasks/rename-field/expected.ts create mode 100644 fast-apply/eval-harness/tasks/rename-field/input.ts diff --git a/README.md b/README.md index 95ee519..af96b94 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Use this table to jump to the right place based on what you're building. | **TypeScript + Fast Apply** | [`fast-apply/basic-edit`](./fast-apply/basic-edit) — apply a targeted code edit | | **TypeScript + Direct API** | [`fast-apply/direct-api`](./fast-apply/direct-api) — call Fast Apply over HTTP | | **TypeScript + Sandbox** | [`fast-apply/code-in-code-out`](./fast-apply/code-in-code-out) — manage file I/O yourself | +| **TypeScript + Eval Harness** | [`fast-apply/eval-harness`](./fast-apply/eval-harness) — evaluate Fast Apply edits | | **TypeScript + Anthropic SDK** | [`warpgrep/anthropic-agent`](./warpgrep/anthropic-agent) — Claude in an agent loop | | **TypeScript + OpenAI SDK** | [`warpgrep/openai-agent`](./warpgrep/openai-agent) — GPT-4o in an agent loop | | **TypeScript + Vercel AI SDK** | [`warpgrep/vercel-agent`](./warpgrep/vercel-agent) — automatic tool loop | @@ -28,6 +29,7 @@ Use this table to jump to the right place based on what you're building. | Apply a targeted code edit | [`fast-apply/basic-edit`](./fast-apply/basic-edit) | | Call Fast Apply directly over HTTP | [`fast-apply/direct-api`](./fast-apply/direct-api) | | Use Fast Apply in a sandbox or harness | [`fast-apply/code-in-code-out`](./fast-apply/code-in-code-out) | +| Evaluate Fast Apply outputs | [`fast-apply/eval-harness`](./fast-apply/eval-harness) | | Search a GitHub repo (no clone) | [`warpgrep/github-search`](./warpgrep/github-search) | | Stream search progress | [`warpgrep/streaming`](./warpgrep/streaming) | | Run search in a Vercel Sandbox | [`warpgrep/vercel-sandbox`](./warpgrep/vercel-sandbox) | @@ -55,6 +57,7 @@ All in [`fast-apply/`](./fast-apply): | [basic-edit](./fast-apply/basic-edit) | TypeScript | Apply a targeted partial edit to a TypeScript file | | [direct-api](./fast-apply/direct-api) | TypeScript | Call Fast Apply directly through the OpenAI-compatible API | | [code-in-code-out](./fast-apply/code-in-code-out) | TypeScript | Return merged code while managing file I/O yourself | +| [eval-harness](./fast-apply/eval-harness) | TypeScript | Run an exact-match harness over multiple Fast Apply tasks | ### WarpGrep — Code Search Subagent diff --git a/fast-apply/eval-harness/.env.example b/fast-apply/eval-harness/.env.example new file mode 100644 index 0000000..185a879 --- /dev/null +++ b/fast-apply/eval-harness/.env.example @@ -0,0 +1 @@ +MORPH_API_KEY=your-key diff --git a/fast-apply/eval-harness/README.md b/fast-apply/eval-harness/README.md new file mode 100644 index 0000000..18e0dfd --- /dev/null +++ b/fast-apply/eval-harness/README.md @@ -0,0 +1,30 @@ +# Fast Apply: Eval Harness + +Run a tiny exact-match benchmark for Fast Apply edits. Each task includes an input file, a partial `codeEdit`, and the expected merged output. + +## Setup + +```bash +npm install +``` + +Use `.env.example` as a template, then export your key or pass it inline: + +```bash +MORPH_API_KEY=your-key npx tsx run-eval.ts +``` + +## Run + +```bash +MORPH_API_KEY=your-key npx tsx run-eval.ts +``` + +## What it does + +1. Loads every task in `tasks/` +2. Calls `applyEdit()` with `input.ts`, the instruction, and the partial `codeEdit` +3. Compares `mergedCode` to `expected.ts` +4. Prints pass/fail results and a diff for failures + +This is a small harness, not a full benchmark suite. It shows how to evaluate Fast Apply behavior in a repeatable way before putting it inside a larger coding-agent workflow. diff --git a/fast-apply/eval-harness/package.json b/fast-apply/eval-harness/package.json new file mode 100644 index 0000000..2cc496b --- /dev/null +++ b/fast-apply/eval-harness/package.json @@ -0,0 +1,15 @@ +{ + "name": "fast-apply-eval-harness", + "private": true, + "type": "module", + "scripts": { + "start": "npx tsx run-eval.ts" + }, + "dependencies": { + "@morphllm/morphsdk": "latest", + "diff": "^7.0.0" + }, + "devDependencies": { + "tsx": "^4.0.0" + } +} diff --git a/fast-apply/eval-harness/run-eval.ts b/fast-apply/eval-harness/run-eval.ts new file mode 100644 index 0000000..3900590 --- /dev/null +++ b/fast-apply/eval-harness/run-eval.ts @@ -0,0 +1,90 @@ +// # Fast Apply Eval Harness +// +// Run a tiny exact-match benchmark over several Fast Apply edit tasks. +// +// Usage: +// MORPH_API_KEY=your-key npx tsx run-eval.ts + +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { applyEdit } from "@morphllm/morphsdk"; +import { createTwoFilesPatch } from "diff"; + +const tasksDir = "tasks"; +const separator = "\n---CODE_EDIT---\n"; + +function normalize(code: string): string { + return code.trimEnd() + "\n"; +} + +async function loadTask(name: string) { + const taskDir = join(tasksDir, name); + const [input, expected, editFile] = await Promise.all([ + readFile(join(taskDir, "input.ts"), "utf-8"), + readFile(join(taskDir, "expected.ts"), "utf-8"), + readFile(join(taskDir, "edit.txt"), "utf-8"), + ]); + + const [instruction, codeEdit] = editFile.split(separator); + + if (!instruction || !codeEdit) { + throw new Error(`Task ${name} is missing ${separator.trim()} in edit.txt`); + } + + return { + name, + input, + expected, + instruction: instruction.trim(), + codeEdit: codeEdit.trim(), + }; +} + +const taskNames = (await readdir(tasksDir, { withFileTypes: true })) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); + +let passed = 0; + +for (const taskName of taskNames) { + const task = await loadTask(taskName); + + const result = await applyEdit({ + originalCode: task.input, + instruction: task.instruction, + codeEdit: task.codeEdit, + }); + + if (!result.success || !result.mergedCode) { + console.log(`FAIL ${task.name}: ${result.error ?? "missing mergedCode"}`); + continue; + } + + const actual = normalize(result.mergedCode); + const expected = normalize(task.expected); + + if (actual === expected) { + passed += 1; + console.log(`PASS ${task.name}`); + continue; + } + + console.log(`FAIL ${task.name}`); + console.log( + createTwoFilesPatch( + `${task.name}/expected.ts`, + `${task.name}/actual.ts`, + expected, + actual, + "Expected", + "Actual" + ) + ); +} + +console.log(`\n${passed}/${taskNames.length} tasks passed`); + +if (passed !== taskNames.length) { + process.exit(1); +} diff --git a/fast-apply/eval-harness/tasks/add-validation/edit.txt b/fast-apply/eval-harness/tasks/add-validation/edit.txt new file mode 100644 index 0000000..7009e00 --- /dev/null +++ b/fast-apply/eval-harness/tasks/add-validation/edit.txt @@ -0,0 +1,13 @@ +I am validating that the parsed port is an integer between 1 and 65535 before returning it. + +---CODE_EDIT--- +// ... existing code ... +export function parsePort(value: string): number { + const port = Number(value); + + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error("Invalid port"); + } + + return port; +} diff --git a/fast-apply/eval-harness/tasks/add-validation/expected.ts b/fast-apply/eval-harness/tasks/add-validation/expected.ts new file mode 100644 index 0000000..ac3fa1c --- /dev/null +++ b/fast-apply/eval-harness/tasks/add-validation/expected.ts @@ -0,0 +1,9 @@ +export function parsePort(value: string): number { + const port = Number(value); + + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error("Invalid port"); + } + + return port; +} diff --git a/fast-apply/eval-harness/tasks/add-validation/input.ts b/fast-apply/eval-harness/tasks/add-validation/input.ts new file mode 100644 index 0000000..30e5e46 --- /dev/null +++ b/fast-apply/eval-harness/tasks/add-validation/input.ts @@ -0,0 +1,5 @@ +export function parsePort(value: string): number { + const port = Number(value); + + return port; +} diff --git a/fast-apply/eval-harness/tasks/null-check/edit.txt b/fast-apply/eval-harness/tasks/null-check/edit.txt new file mode 100644 index 0000000..aab3da4 --- /dev/null +++ b/fast-apply/eval-harness/tasks/null-check/edit.txt @@ -0,0 +1,13 @@ +I am adding a null check after findUserByEmail so login throws Invalid credentials when no user is found. + +---CODE_EDIT--- +// ... existing code ... +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + if (!user) { + throw new Error("Invalid credentials"); + } + + const passwordMatches = await verifyPassword(password, user.passwordHash); +// ... existing code ... diff --git a/fast-apply/eval-harness/tasks/null-check/expected.ts b/fast-apply/eval-harness/tasks/null-check/expected.ts new file mode 100644 index 0000000..2d7090d --- /dev/null +++ b/fast-apply/eval-harness/tasks/null-check/expected.ts @@ -0,0 +1,20 @@ +export interface User { + id: string; + email: string; + passwordHash: string; +} + +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + if (!user) { + throw new Error("Invalid credentials"); + } + + const passwordMatches = await verifyPassword(password, user.passwordHash); + if (!passwordMatches) { + throw new Error("Invalid credentials"); + } + + return user; +} diff --git a/fast-apply/eval-harness/tasks/null-check/input.ts b/fast-apply/eval-harness/tasks/null-check/input.ts new file mode 100644 index 0000000..b19d287 --- /dev/null +++ b/fast-apply/eval-harness/tasks/null-check/input.ts @@ -0,0 +1,16 @@ +export interface User { + id: string; + email: string; + passwordHash: string; +} + +export async function login(email: string, password: string): Promise { + const user = await findUserByEmail(email); + + const passwordMatches = await verifyPassword(password, user.passwordHash); + if (!passwordMatches) { + throw new Error("Invalid credentials"); + } + + return user; +} diff --git a/fast-apply/eval-harness/tasks/rename-field/edit.txt b/fast-apply/eval-harness/tasks/rename-field/edit.txt new file mode 100644 index 0000000..8505e29 --- /dev/null +++ b/fast-apply/eval-harness/tasks/rename-field/edit.txt @@ -0,0 +1,11 @@ +I am renaming the Profile field displayName to fullName and updating formatProfile to use the new field. + +---CODE_EDIT--- +interface Profile { + id: string; + fullName: string; +} + +export function formatProfile(profile: Profile): string { + return `${profile.fullName} (${profile.id})`; +} diff --git a/fast-apply/eval-harness/tasks/rename-field/expected.ts b/fast-apply/eval-harness/tasks/rename-field/expected.ts new file mode 100644 index 0000000..11d0572 --- /dev/null +++ b/fast-apply/eval-harness/tasks/rename-field/expected.ts @@ -0,0 +1,8 @@ +interface Profile { + id: string; + fullName: string; +} + +export function formatProfile(profile: Profile): string { + return `${profile.fullName} (${profile.id})`; +} diff --git a/fast-apply/eval-harness/tasks/rename-field/input.ts b/fast-apply/eval-harness/tasks/rename-field/input.ts new file mode 100644 index 0000000..0582f2e --- /dev/null +++ b/fast-apply/eval-harness/tasks/rename-field/input.ts @@ -0,0 +1,8 @@ +interface Profile { + id: string; + displayName: string; +} + +export function formatProfile(profile: Profile): string { + return `${profile.displayName} (${profile.id})`; +}