diff --git a/.github/scripts/type_completeness.py b/.github/scripts/type_completeness.py new file mode 100644 index 00000000..4234c683 --- /dev/null +++ b/.github/scripts/type_completeness.py @@ -0,0 +1,363 @@ +""" +Builds a Markdown summary comparing pyright `--verifytypes` reports for a +PR's base and head commits, for posting as a PR comment. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import ( + Any, + Literal +) + +# Decorates the patch coverage table. +STATUS_ICON = {"known": "✅", "ambiguous": "⚠️", "unknown": "❌"} + + +def status_of(symbol: dict[str, Any]) -> Literal['known', 'ambiguous', 'unknown']: + """ + Maps the --verifytypes JSON boolean flags, isTypeKnown and isTypeAmbiguous, to internal + representation in script: known, ambiguous, and unknown. + + Parameters + ---------- + symbol: dict[str, Any] + A single entry for a symbol from typeCompleteless.symbols in the JSON output. + + Returns + ------- + Literal['known', 'ambiguous', 'unknown'] + The mapped symbol type. + + """ + if symbol["isTypeKnown"]: + return "known" + if symbol["isTypeAmbiguous"]: + return "ambiguous" + return "unknown" + + +def load_type_completeness(report_path: Path) -> dict[str, Any]: + """ + Loads the --verifytypes JSON output and extracts the typeCompleteness section. + Discards the rest. + + Parameters + ---------- + report_path: Path + The path to the --verifytypes JSON report to parse. + + Returns + ------- + dict[str, Any] + A dictionary representation of the typeCompleteness section of the --verifytypes JSON report. + + """ + with report_path.open() as f: + data = json.load(f) + return data["typeCompleteness"] + + +def exported_symbols(type_completeness: dict[str, Any]) -> dict[str, dict[str, Any]]: + """ + Parses the dict generated by load_type_completeness() and filters it on exported symbols. + + Parameters + ---------- + type_completeness: dict[str, Any] + The dict generated by load_type_completeness(). + + Returns + ------- + dict[str, dict[str, Any]] + A dict of exported symbols, indexed by symbol name. + + """ + return {s["name"]: s for s in type_completeness["symbols"] if s["isExported"]} + + +def other_symbol_counts(type_completeness: dict[str, Any]) -> dict[str, int]: + """ + Parses the dict generated by load_type_completeness(). Extract the section on otherSymbolCounts + and return as a summary dict. + + Parameters + ---------- + type_completeness: dict[str, Any] + The dict generated by load_type_completeness(). + + Returns + ------- + dict[str, int] + Summary dict containing counts of other symbols. + + """ + other = type_completeness["otherSymbolCounts"] + return { + "known": other["withKnownType"], + "ambiguous": other["withAmbiguousType"], + "unknown": other["withUnknownType"], + } + + +def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]: + """ + Parses the dict (transformed to list) produced by exported_symbols(). Loops through the symbols and counts + them by type. Return a dict summary of the counts. + + Parameters + ---------- + symbols: list[dict[str, Any]] + The output dict produced by exported_symbols(), converted to a list. + + Returns + ------- + dict[str, int] + A summary count of exported symbols by type. + + """ + counts = {"known": 0, "ambiguous": 0, "unknown": 0} + for s in symbols: + counts[status_of(s)] += 1 + return counts + + +def completeness_pct(counts: dict[str, int]) -> float: + """ + Calculates the type completeness score. Equals the percentage of known symbols + relative to total symbols. + + Parameters + ---------- + counts: dict[str, int] + A summary dictionary of symbol counts by type (known, ambiguous, unknown). + + Returns + ------- + float + A type completeness score. + + """ + total = sum(counts.values()) + return 100.0 * counts["known"] / total if total else 0.0 + + +def render_counts_table(rows: list[tuple[str, dict[str, int]]]) -> str: + """ + Generate a Markdown type completeness summary table, for the project base, used for PR comment. + + Parameters + ---------- + rows: list[tuple[str, dict[str, int]]] + List of tuples in the form of [("Section Name", symbol_counts)]. e.g., [("Patch", patch_counts)] + + + Returns + ------- + str + A Markdown table of the type completeness summary, used for PR comment. + + """ + lines = ["| | Known | Ambiguous | Unknown | Total |", "|---|---|---|---|---|"] + for label, counts in rows: + total = sum(counts.values()) + lines.append( + f"| {label} | {counts['known']} | {counts['ambiguous']} | " + f"{counts['unknown']} | {total} |" + ) + return "\n".join(lines) + + +def render_patch_detail( + patch_names: list[str], + removed_names: list[str], + base: dict[str, dict[str, Any]], + head: dict[str, dict[str, Any]], +) -> str: + """ + Generate a Markdown type completeness summary table, for the PR patch, used for PR comment. + + Parameters + ---------- + patch_names: list[str] + A list of symbol names whose status changed in the PR, or are newly exported. + removed_names: list[str] + A list of symbol names that were exported at base but are no longer exported at + head (e.g. de-exported, renamed, or deleted). + base: dict[str, dict[str, Any]] + dict generated by exported_symbols(), base case. + head: dict[str, dict[str, Any]] + dict generated by exported_symbols(), head case. + + Returns + ------- + str + A Markdown type completeness summary table, for the PR patch, used for PR comment. + """ + lines = ["| Symbol | Status | Change |", "|---|---|---|"] + for name in sorted(patch_names): + head_status = status_of(head[name]) + icon = STATUS_ICON[head_status] + if name not in base: + change = "new" + else: + base_status = status_of(base[name]) + change = f"changed (was {STATUS_ICON[base_status]} {base_status})" + lines.append(f"| `{name}` | {icon} {head_status} | {change} |") + for name in sorted(removed_names): + base_status = status_of(base[name]) + lines.append( + f"| `{name}` | — | no longer exported (was {STATUS_ICON[base_status]} {base_status}) |" + ) + return "\n".join(lines) + + +def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) -> str: + """ + Builds the full Markdown summary for the PR comment, combining the project-level + type completeness report with the patch-level diff between base and head. + + Parameters + ---------- + base_path: Path + The path to the --verifytypes JSON report for the PR's base (or merge-base) + commit. + head_path: Path + The path to the --verifytypes JSON report for the PR's head commit. + run_url: str | None + Optional link to the workflow run, e.g. its step summary. When provided, a + link to it is included near the top of the summary. Omitted entirely when + None. + + Returns + ------- + str + The rendered Markdown summary, ready to be written to a file or posted as + a PR comment. + + """ + # Extract the type completeness section of --verifytypes JSON, and filter on exported symbols + # for both the head and base case. + base_tc = load_type_completeness(base_path) + head_tc = load_type_completeness(head_path) + base = exported_symbols(base_tc) + head = exported_symbols(head_tc) + + # Count up the symbol types. + project_counts = counts_by_status(list(head.values())) + project_pct = completeness_pct(project_counts) + other_counts = other_symbol_counts(head_tc) + + # Gather the patch symbols, i.e., those whose status changed in the PR, or newly exported ones. + patch_names = [ + name + for name, symbol in head.items() + if name not in base or status_of(base[name]) != status_of(symbol) + ] + # Symbols exported at base but no longer exported at head (de-exported, renamed, + # or deleted). These can't be found by iterating head, since they're absent from + # it entirely - iterate base instead and check for absence from head. + removed_names = [name for name in base if name not in head] + + # Generate the Markdown report. + sections = [ + "## Pyright Type Completeness", + "", + ] + if run_url: + sections += [ + f"[View the full `pyright --verifytypes` output for this commit]({run_url})", + "", + ] + sections += [ + f"**Project (full `chainladder` package, at this PR's head):** " + f"{project_pct:.1f}% of exported symbols fully typed " + f"({project_counts['known']} / {sum(project_counts.values())})", + "", + render_counts_table([("Project (head)", project_counts)]), + "", + f"Other symbols referenced but not exported by `chainladder`: " + f"{sum(other_counts.values())}", + "", + render_counts_table([("Other (head)", other_counts)]), + "", + "Symbols without documentation:", + f"- Functions without docstring: {head_tc['missingFunctionDocStringCount']}", + f"- Functions without default param: {head_tc['missingDefaultParamCount']}", + f"- Classes without docstring: {head_tc['missingClassDocStringCount']}", + "", + ] + + if patch_names or removed_names: + patch_counts = ( + counts_by_status([head[n] for n in patch_names]) + if patch_names + else {"known": 0, "ambiguous": 0, "unknown": 0} + ) + parts = [] + if patch_names: + patch_pct = completeness_pct(patch_counts) + parts.append( + f"{patch_pct:.1f}% fully typed " + f"({patch_counts['known']} / {sum(patch_counts.values())})" + ) + if removed_names: + parts.append(f"{len(removed_names)} no longer exported") + sections += [ + "**Patch (exported symbols added or changed by this PR):** " + "; ".join(parts), + "", + render_counts_table([("Patch", patch_counts)]), + "", + "
", + "Patch symbol details", + "", + render_patch_detail(patch_names, removed_names, base, head), + "", + "
", + ] + else: + sections += [ + "**Patch (exported symbols added or changed by this PR):** " + "no exported symbol type-completeness changes detected.", + ] + + return "\n".join(sections) + + +def main() -> None: + """ + Parse -verifytypes JSON output and produce a Markdown summary report for PR comment. + + Accepts command line arguments: + + --base: Path to the --verifytypes JSON report for the PR's base (or merge-base) commit. + --head: Path to the --verifytypes JSON report for the PR's head commit. + --output: Path to write the rendered Markdown summary to. + --run-url: Optional link to the workflow run, e.g. its step summary. Omitted from the + summary entirely when not provided. + + Returns + ------- + None + + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, type=Path) + parser.add_argument("--head", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument( + "--run-url", + default=None, + help="Link to the workflow run, e.g. for its step summary.", + ) + args = parser.parse_args() + + summary = build_summary(args.base, args.head, run_url=args.run_url) + args.output.write_text(summary) + print(summary) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/type_completeness_output.sh b/.github/scripts/type_completeness_output.sh new file mode 100755 index 00000000..fc2970b1 --- /dev/null +++ b/.github/scripts/type_completeness_output.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Appends a pyright --verifytypes text report to $GITHUB_STEP_SUMMARY, +# wrapped in a collapsible
block. GitHub caps each step's summary +# at 1MiB and silently drops the whole upload if exceeded, so the report is +# truncated to a safe byte budget with a note when that happens, rather than +# risking the entire block vanishing. +# +# This script is called twice per workflow run (PR head and PR base). The +# budget below is half of the 1MiB cap, not the full cap, so the two calls +# stay safely under the limit even in the worst case where they end up +# sharing one combined budget instead of each getting their own per-step +# allotment (true today, per GitHub's docs, but cheap to hedge against). +set -euo pipefail + +report_file="$1" +label="$2" +max_bytes=450000 + +total_bytes="$(wc -c < "$report_file")" + +{ + echo "
Full pyright --verifytypes output (${label})" + echo "" + echo '```text' + head -c "$max_bytes" "$report_file" + if [ "$total_bytes" -gt "$max_bytes" ]; then + echo "" + echo "... (truncated: ${total_bytes} bytes total, GITHUB_STEP_SUMMARY caps each step at 1MiB)" + fi + echo '```' + echo "
" +} >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/type_completeness.yml b/.github/workflows/type_completeness.yml new file mode 100644 index 00000000..70a3e10f --- /dev/null +++ b/.github/workflows/type_completeness.yml @@ -0,0 +1,90 @@ +name: Pyright Type Completeness + +# Main workflow for generating type completeness report. Calculates type completeness scores on both a +# project basis and a patch basis. +# +# Results are fed into a subsequent workflow, +# defined in type_completeness_comment.yml. This workflow must be split into 2 files in order to work +# securely on PRs from external forks of the repo by splitting read/write permissions. +# +# Auxiliary Files +# --------------- +# type_completeness_comment.yml - Writes a comment on the PR summarizing the type completeness. +# ../scripts/type_completeness.py - Parses the resulting JSON to extract coverage statistics. +# ../scripts/type_completeness_output.sh - Appends the Pyright terminal output to the GitHub action for user review. + +on: + pull_request: + +permissions: + contents: read + actions: write + +jobs: + typecompleteness: + name: pyright --verifytypes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + python-version: "3.14" + + - name: Install dependencies + run: uv sync --extra dev + + - name: Run pyright --verifytypes on PR head + # Get type coverage on the PR head. + # head.json -> Machine-readable report + # head.txt -> human-readable report + run: | + uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > head.json || true + uv run pyright --verifytypes chainladder --ignoreexternal > head.txt || true + bash .github/scripts/type_completeness_output.sh head.txt "PR head" + + - name: Run pyright --verifytypes on PR base + # Get type coverage on main, append terminal output to GitHub actions. + # base.json -> Machine-readable report + # base.txt -> Human-readable report + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + git fetch origin "$BASE_SHA" + MERGE_BASE="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" + rm -rf chainladder + git checkout "$MERGE_BASE" -- chainladder + uv run pyright --outputjson --verifytypes chainladder --ignoreexternal > base.json || true + uv run pyright --verifytypes chainladder --ignoreexternal > base.txt || true + rm -rf chainladder + git checkout "$HEAD_SHA" -- chainladder + bash .github/scripts/type_completeness_output.sh base.txt "PR base" + + - name: Build summary + # Construct summary by feeding JSON files into type_completeness.py. + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + uv run python .github/scripts/type_completeness.py \ + --base base.json \ + --head head.json \ + --run-url "$RUN_URL" \ + --output summary.md + + - name: Save PR number + run: echo "${{ github.event.pull_request.number }}" > pr_number.txt + # Generate summary artifact to be read in by subsequent workflow type_completeness_comment.yml. + - name: Upload summary artifact + uses: actions/upload-artifact@v4 + with: + name: type-completeness-summary + path: | + summary.md + pr_number.txt + retention-days: 1 diff --git a/.github/workflows/type_completeness_comment.yml b/.github/workflows/type_completeness_comment.yml new file mode 100644 index 00000000..53a0ae58 --- /dev/null +++ b/.github/workflows/type_completeness_comment.yml @@ -0,0 +1,35 @@ +name: Pyright Type Completeness Comment + +# Generate a comment containing type completeness summary. Triggered after completion of +# preceding workflow Pyright Type Completeness. +on: + workflow_run: + workflows: ["Pyright Type Completeness"] + types: [completed] + +permissions: + pull-requests: write + actions: read + +jobs: + comment: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Download summary artifact + uses: actions/download-artifact@v4 + with: + name: type-completeness-summary + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR number + id: pr + run: echo "number=$(cat pr_number.txt)" >> "$GITHUB_OUTPUT" + + - name: Comment on PR + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: pyright-typecompleteness + number_force: ${{ steps.pr.outputs.number }} + path: summary.md