From 817d1a17f2bc7301d4323756a8209f03fc4e92ce Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 1 Jun 2026 13:29:40 -0400 Subject: [PATCH 1/4] feat: add api.md generation support to Python emitter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add opt-in 'generate-api-md' emitter option that generates an api.md file containing the public API surface of the generated SDK. Pipeline: preprocess → codegen → black → apistubgen → api.md Implementation: - New 'generate-api-md' option (defaults to false, opt-in) - Runs apistubgen with --code-model-path for fast token generation directly from the YAML code model (no package install needed) - Python script converts APIView token JSON to markdown (port of Export-APIViewMarkdown.ps1, avoids pwsh dependency) - Failures emit a warning but never block SDK generation - Only runs in native Python path (not Pyodide) Requires apiview-stub-generator with --code-model-path support (azure-sdk-tools PR #15104). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http-client-python/emitter/src/emitter.ts | 55 +++++++++- .../http-client-python/emitter/src/lib.ts | 13 +++ .../scripts/setup/export_apiview_markdown.py | 101 ++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 packages/http-client-python/eng/scripts/setup/export_apiview_markdown.py diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 3ecd85c5f57..9f909e62cb4 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -1,6 +1,7 @@ import { createSdkContext } from "@azure-tools/typespec-client-generator-core"; import { EmitContext, emitFile, joinPaths, NoTarget } from "@typespec/compiler"; -import { execSync } from "child_process"; +import { execFileSync, execSync } from "child_process"; +import { randomUUID } from "crypto"; import fs from "fs"; import jsyaml from "js-yaml"; import os from "os"; @@ -330,11 +331,63 @@ async function onEmitMain(context: EmitContext) { `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`, ); await checkForPylintIssues(outputDir, excludePattern); + + if (resolvedOptions["generate-api-md"]) { + await generateApiMd(program, venvPath, yamlPath, outputDir, root); + } } } } } +async function generateApiMd( + program: any, + venvPath: string, + yamlPath: string, + outputDir: string, + root: string, +): Promise { + const apiviewOutDir = path.join(os.tmpdir(), `tsp-apiview-${randomUUID()}`); + try { + fs.mkdirSync(apiviewOutDir, { recursive: true }); + + // Run apistubgen with --code-model-path to generate token JSON + execFileSync(venvPath, [ + "-m", + "apistub", + "--code-model-path", + yamlPath, + "--out-path", + apiviewOutDir, + "--skip-pylint", + ]); + + // Find the generated token JSON file + const tokenFiles = fs.readdirSync(apiviewOutDir).filter((f: string) => f.endsWith(".json")); + if (tokenFiles.length === 0) { + reportDiagnostic(program, { + code: "api-md-generation-failed", + target: NoTarget, + format: { details: "apistubgen did not produce a token JSON file" }, + }); + return; + } + + // Convert token JSON to api.md using the Python conversion script + const tokenJsonPath = path.join(apiviewOutDir, tokenFiles[0]); + const mdScript = path.join(root, "eng", "scripts", "setup", "export_apiview_markdown.py"); + execFileSync(venvPath, [mdScript, tokenJsonPath, outputDir]); + } catch (e: any) { + reportDiagnostic(program, { + code: "api-md-generation-failed", + target: NoTarget, + format: { details: e.message || String(e) }, + }); + } finally { + fs.rmSync(apiviewOutDir, { recursive: true, force: true }); + } +} + const browserPyodidePromise: Promise | null = typeof window !== "undefined" ? setupPyodideCallBrowser() : null; diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index b267c836bf3..9a8ae28d3b4 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -26,6 +26,7 @@ export interface PythonEmitterOptions { "keep-setup-py"?: boolean; "clear-output-folder"?: boolean; "emit-yaml-only"?: boolean; + "generate-api-md"?: boolean; } export interface PythonSdkContext extends SdkContext { @@ -117,6 +118,12 @@ export const PythonEmitterOptionsSchema: JSONSchemaType = description: "Emit YAML code model only, without running Python generator. For batch processing.", }, + "generate-api-md": { + type: "boolean", + nullable: true, + description: + "Whether to generate an api.md file containing the public API surface. Defaults to `false`. Requires `apiview-stub-generator` to be installed.", + }, }, required: [], }; @@ -175,6 +182,12 @@ const libDef = { default: paramMessage`No valid continuation token in '${"direction"}' for operation '${"operationId"}'.`, }, }, + "api-md-generation-failed": { + severity: "warning", + messages: { + default: paramMessage`Failed to generate api.md: ${"details"}. SDK generation was not affected.`, + }, + }, }, emitter: { options: PythonEmitterOptionsSchema, diff --git a/packages/http-client-python/eng/scripts/setup/export_apiview_markdown.py b/packages/http-client-python/eng/scripts/setup/export_apiview_markdown.py new file mode 100644 index 00000000000..4051487121a --- /dev/null +++ b/packages/http-client-python/eng/scripts/setup/export_apiview_markdown.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""Convert an APIView token JSON file to a markdown file. + +This is a Python port of Export-APIViewMarkdown.ps1 from azure-sdk-tools, +so that api.md generation does not require PowerShell to be installed. +""" + +import json +import os +import sys + + +LANGUAGE_ALIASES = { + "python": "py", + "javascript": "js", + "typescript": "ts", +} + + +def render_token(token): + prefix = " " if token.get("HasPrefixSpace") else "" + suffix = " " if token.get("HasSuffixSpace") else "" + return f"{prefix}{token.get('Value', '')}{suffix}" + + +def render_review_lines(review_lines, indent_level=0): + result = [] + indent = " " * indent_level + + for line in review_lines: + tokens = line.get("Tokens", []) + if not tokens: + result.append("") + else: + line_text = "".join(render_token(t) for t in tokens) + if line_text.strip(): + result.append(f"{indent}{line_text}") + else: + result.append("") + + children = line.get("Children") + if children: + child_lines = render_review_lines(children, indent_level + 1) + result.extend(child_lines) + + return result + + +def main(): + if len(sys.argv) < 3: + print("Usage: export_apiview_markdown.py ", file=sys.stderr) + sys.exit(1) + + token_json_path = sys.argv[1] + output_path = sys.argv[2] + + if not os.path.exists(token_json_path): + print(f"Token JSON file not found: {token_json_path}", file=sys.stderr) + sys.exit(1) + + with open(token_json_path, "r", encoding="utf-8") as f: + token_json = json.load(f) + + review_lines = token_json.get("ReviewLines") + if not review_lines: + print("The token JSON file does not contain a 'ReviewLines' property.", file=sys.stderr) + sys.exit(1) + + # Resolve output path + if os.path.isdir(output_path): + output_path = os.path.join(output_path, "api.md") + elif not os.path.splitext(output_path)[1]: + output_path = os.path.join(output_path, "api.md") + + # Get language for code fence + language = (token_json.get("Language") or "").lower() + language = LANGUAGE_ALIASES.get(language, language) + + rendered_lines = render_review_lines(review_lines) + + output_lines = [f"```{language}"] + output_lines.extend(rendered_lines) + output_lines.append("```") + + output_dir = os.path.dirname(output_path) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + with open(output_path, "w", encoding="utf-8", newline="\n") as f: + f.write("\n".join(output_lines)) + + print(f"Generated markdown: {output_path}") + + +if __name__ == "__main__": + main() From ea96ce01b1cea03c76bf5e84e21f26ab705e64ce Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 2 Jun 2026 16:31:53 -0400 Subject: [PATCH 2/4] fix: address PR review comments for api.md generation - Use stable token filename ({package_name}_python.json) instead of scanning for any .json file in the output directory - Add emitter tests for export_apiview_markdown.py script - Add changeset for the feature Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../feat-python-api-md-2026-6-2-16-31-0.md | 14 ++ .../http-client-python/emitter/src/emitter.ts | 17 +- .../emitter/test/emitter.test.ts | 148 ++++++++++++++++++ 3 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 .chronus/changes/feat-python-api-md-2026-6-2-16-31-0.md create mode 100644 packages/http-client-python/emitter/test/emitter.test.ts diff --git a/.chronus/changes/feat-python-api-md-2026-6-2-16-31-0.md b/.chronus/changes/feat-python-api-md-2026-6-2-16-31-0.md new file mode 100644 index 00000000000..9ab4bc0743e --- /dev/null +++ b/.chronus/changes/feat-python-api-md-2026-6-2-16-31-0.md @@ -0,0 +1,14 @@ +--- +changeKind: feature +packages: + - "@typespec/http-client-python" +--- + +Add `generate-api-md` emitter option to generate an `api.md` file containing the public API surface. When enabled, the emitter runs `apiview-stub-generator` to produce a token JSON file and converts it to markdown. Requires `apiview-stub-generator` to be installed in the Python environment. + +```yaml +# tspconfig.yaml +options: + "@typespec/http-client-python": + generate-api-md: true +``` diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 9f909e62cb4..7efe46f7b45 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -333,7 +333,14 @@ async function onEmitMain(context: EmitContext) { await checkForPylintIssues(outputDir, excludePattern); if (resolvedOptions["generate-api-md"]) { - await generateApiMd(program, venvPath, yamlPath, outputDir, root); + await generateApiMd( + program, + venvPath, + yamlPath, + outputDir, + root, + resolvedOptions["package-name"]!, + ); } } } @@ -346,6 +353,7 @@ async function generateApiMd( yamlPath: string, outputDir: string, root: string, + packageName: string, ): Promise { const apiviewOutDir = path.join(os.tmpdir(), `tsp-apiview-${randomUUID()}`); try { @@ -362,9 +370,9 @@ async function generateApiMd( "--skip-pylint", ]); - // Find the generated token JSON file - const tokenFiles = fs.readdirSync(apiviewOutDir).filter((f: string) => f.endsWith(".json")); - if (tokenFiles.length === 0) { + // apistubgen outputs {package_name}_python.json + const tokenJsonPath = path.join(apiviewOutDir, `${packageName}_python.json`); + if (!fs.existsSync(tokenJsonPath)) { reportDiagnostic(program, { code: "api-md-generation-failed", target: NoTarget, @@ -374,7 +382,6 @@ async function generateApiMd( } // Convert token JSON to api.md using the Python conversion script - const tokenJsonPath = path.join(apiviewOutDir, tokenFiles[0]); const mdScript = path.join(root, "eng", "scripts", "setup", "export_apiview_markdown.py"); execFileSync(venvPath, [mdScript, tokenJsonPath, outputDir]); } catch (e: any) { diff --git a/packages/http-client-python/emitter/test/emitter.test.ts b/packages/http-client-python/emitter/test/emitter.test.ts new file mode 100644 index 00000000000..3b306178b4d --- /dev/null +++ b/packages/http-client-python/emitter/test/emitter.test.ts @@ -0,0 +1,148 @@ +import { ok, strictEqual } from "assert"; +import { execFileSync } from "child_process"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, it } from "vitest"; + +describe("export_apiview_markdown.py", () => { + const root = path.resolve(import.meta.dirname, "../.."); + const scriptPath = path.join(root, "eng/scripts/setup/export_apiview_markdown.py"); + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "apimd-test-")); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function writeTokenJson(data: object): string { + const tokenPath = path.join(tmpDir, "token.json"); + fs.writeFileSync(tokenPath, JSON.stringify(data)); + return tokenPath; + } + + it("generates api.md from a simple token file", () => { + const tokenPath = writeTokenJson({ + Language: "Python", + ReviewLines: [ + { + Tokens: [ + { Value: "class", HasSuffixSpace: true }, + { Value: "MyClient", HasPrefixSpace: false }, + ], + }, + { + Tokens: [ + { Value: "def", HasSuffixSpace: true }, + { Value: "send(self)", HasPrefixSpace: false }, + ], + Children: [ + { + Tokens: [{ Value: "..." }], + }, + ], + }, + ], + }); + + const outDir = path.join(tmpDir, "output"); + fs.mkdirSync(outDir); + execFileSync("python3", [scriptPath, tokenPath, outDir]); + + const apiMd = fs.readFileSync(path.join(outDir, "api.md"), "utf-8"); + ok(apiMd.startsWith("```py"), "Should start with python code fence"); + ok(apiMd.includes("class"), "Should contain class token"); + ok(apiMd.includes("MyClient"), "Should contain MyClient token"); + ok(apiMd.endsWith("```"), "Should end with code fence"); + }); + + it("writes api.md directly when output path is a .md file", () => { + const tokenPath = writeTokenJson({ + Language: "Python", + ReviewLines: [{ Tokens: [{ Value: "class Foo" }] }], + }); + + const outFile = path.join(tmpDir, "custom.md"); + execFileSync("python3", [scriptPath, tokenPath, outFile]); + ok(fs.existsSync(outFile), "Should write to the specified .md file"); + const content = fs.readFileSync(outFile, "utf-8"); + ok(content.includes("class Foo")); + }); + + it("exits with error for empty ReviewLines", () => { + const tokenPath = writeTokenJson({ + Language: "Python", + ReviewLines: [], + }); + + const outDir = path.join(tmpDir, "output"); + fs.mkdirSync(outDir); + // Empty ReviewLines is treated as missing by the script + let threw = false; + try { + execFileSync("python3", [scriptPath, tokenPath, outDir], { stdio: "pipe" }); + } catch { + threw = true; + } + ok(threw, "Should exit with error for empty ReviewLines"); + }); + + it("resolves language aliases correctly", () => { + const tokenPath = writeTokenJson({ + Language: "JavaScript", + ReviewLines: [{ Tokens: [{ Value: "function foo() {}" }] }], + }); + + const outDir = path.join(tmpDir, "output"); + fs.mkdirSync(outDir); + execFileSync("python3", [scriptPath, tokenPath, outDir]); + + const apiMd = fs.readFileSync(path.join(outDir, "api.md"), "utf-8"); + ok(apiMd.startsWith("```js"), "Should use 'js' alias for JavaScript"); + }); + + it("renders nested children with indentation", () => { + const tokenPath = writeTokenJson({ + Language: "Python", + ReviewLines: [ + { + Tokens: [{ Value: "class Foo:" }], + Children: [ + { + Tokens: [{ Value: "def bar(self):" }], + Children: [{ Tokens: [{ Value: "pass" }] }], + }, + ], + }, + ], + }); + + const outDir = path.join(tmpDir, "output"); + fs.mkdirSync(outDir); + execFileSync("python3", [scriptPath, tokenPath, outDir]); + + const apiMd = fs.readFileSync(path.join(outDir, "api.md"), "utf-8"); + const lines = apiMd.split("\n"); + // Children should be indented + ok( + lines.some((l: string) => l.startsWith(" ") && l.includes("def bar")), + "First-level children should have 4-space indent", + ); + ok( + lines.some((l: string) => l.startsWith(" ") && l.includes("pass")), + "Second-level children should have 8-space indent", + ); + }); +}); + +describe("generateApiMd token file lookup", () => { + it("expected token filename follows {package_name}_python.json pattern", () => { + // Verify the naming convention used by apistubgen + const packageName = "azure-ai-inference"; + const expectedFilename = `${packageName}_python.json`; + strictEqual(expectedFilename, "azure-ai-inference_python.json"); + }); +}); From d7812da04b44eed84b787e25b882beffb906f57f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 2 Jun 2026 16:34:42 -0400 Subject: [PATCH 3/4] feat: enable generate-api-md by default Default generate-api-md to true. Silently skip if apiview-stub-generator is not installed (only warn if it is installed but fails). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/http-client-python/emitter/src/emitter.ts | 9 +++++++++ packages/http-client-python/emitter/src/lib.ts | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 7efe46f7b45..fe9d1790058 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -30,6 +30,7 @@ function addDefaultOptions(sdkContext: PythonSdkContext) { const defaultOptions = { "package-version": "1.0.0b1", "generate-packaging-files": true, + "generate-api-md": true, "validate-versioning": true, "clear-output-folder": false, }; @@ -355,6 +356,14 @@ async function generateApiMd( root: string, packageName: string, ): Promise { + // Check if apiview-stub-generator is installed before attempting + try { + execFileSync(venvPath, ["-c", "import apistub"], { stdio: "pipe" }); + } catch { + // apiview-stub-generator not installed — silently skip + return; + } + const apiviewOutDir = path.join(os.tmpdir(), `tsp-apiview-${randomUUID()}`); try { fs.mkdirSync(apiviewOutDir, { recursive: true }); diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index 9a8ae28d3b4..eb252deeb4e 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -122,7 +122,7 @@ export const PythonEmitterOptionsSchema: JSONSchemaType = type: "boolean", nullable: true, description: - "Whether to generate an api.md file containing the public API surface. Defaults to `false`. Requires `apiview-stub-generator` to be installed.", + "Whether to generate an api.md file containing the public API surface. Defaults to `true`. Requires `apiview-stub-generator` to be installed.", }, }, required: [], From c2fb2f93f418c6bfa334cade76eed91345609613 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 2 Jun 2026 16:35:11 -0400 Subject: [PATCH 4/4] feat: install apiview-stub-generator in prepare.py Ensures api.md generation is always available when using native Python. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/http-client-python/eng/scripts/setup/prepare.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/http-client-python/eng/scripts/setup/prepare.py b/packages/http-client-python/eng/scripts/setup/prepare.py index 6e6de3d547c..1fa474daa0e 100644 --- a/packages/http-client-python/eng/scripts/setup/prepare.py +++ b/packages/http-client-python/eng/scripts/setup/prepare.py @@ -32,6 +32,12 @@ def main(): except FileNotFoundError as e: raise ValueError(e.filename) + # Install apiview-stub-generator for api.md generation + try: + install_packages(["apiview-stub-generator"], venv_context, cwd=_ROOT_DIR) + except Exception: + pass # Non-critical: api.md generation will be skipped if unavailable + if __name__ == "__main__": main()