Type completeness workflow

.github/scripts/type_completeness.py

@@ -0,0 +1,363 @@
+"""
+Builds a Markdown summary comparing pyright `--verifytypes` reports for a
+PR's base and head commits, for posting as a PR comment.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import (
+    Any,
+    Literal
+)
+
+# Decorates the patch coverage table.
+STATUS_ICON = {"known": "✅", "ambiguous": "⚠️", "unknown": "❌"}
+
+
+def status_of(symbol: dict[str, Any]) -> Literal['known', 'ambiguous', 'unknown']:
+    """
+    Maps the --verifytypes JSON boolean flags, isTypeKnown and isTypeAmbiguous, to internal
+    representation in script: known, ambiguous, and unknown.
+
+    Parameters
+    ----------
+    symbol: dict[str, Any]
+        A single entry for a symbol from typeCompleteless.symbols in the JSON output.
+
+    Returns
+    -------
+    Literal['known', 'ambiguous', 'unknown']
+        The mapped symbol type.
+
+    """
+    if symbol["isTypeKnown"]:
+        return "known"
+    if symbol["isTypeAmbiguous"]:
+        return "ambiguous"
+    return "unknown"
+
+
+def load_type_completeness(report_path: Path) -> dict[str, Any]:
+    """
+    Loads the --verifytypes JSON output and extracts the typeCompleteness section.
+    Discards the rest.
+
+    Parameters
+    ----------
+    report_path: Path
+        The path to the --verifytypes JSON report to parse.
+
+    Returns
+    -------
+    dict[str, Any]
+        A dictionary representation of the typeCompleteness section of the --verifytypes JSON report.
+
+    """
+    with report_path.open() as f:
+        data = json.load(f)
+    return data["typeCompleteness"]
+
+
+def exported_symbols(type_completeness: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """
+    Parses the dict generated by load_type_completeness() and filters it on exported symbols.
+
+    Parameters
+    ----------
+    type_completeness: dict[str, Any]
+        The dict generated by load_type_completeness().
+
+    Returns
+    -------
+    dict[str, dict[str, Any]]
+        A dict of exported symbols, indexed by symbol name.
+
+    """
+    return {s["name"]: s for s in type_completeness["symbols"] if s["isExported"]}
+
+
+def other_symbol_counts(type_completeness: dict[str, Any]) -> dict[str, int]:
+    """
+    Parses the dict generated by load_type_completeness(). Extract the section on otherSymbolCounts
+    and return as a summary dict.
+
+    Parameters
+    ----------
+    type_completeness: dict[str, Any]
+        The dict generated by load_type_completeness().
+
+    Returns
+    -------
+    dict[str, int]
+        Summary dict containing counts of other symbols.
+
+    """
+    other = type_completeness["otherSymbolCounts"]
+    return {
+        "known": other["withKnownType"],
+        "ambiguous": other["withAmbiguousType"],
+        "unknown": other["withUnknownType"],
+    }
+
+
+def counts_by_status(symbols: list[dict[str, Any]]) -> dict[str, int]:
+    """
+    Parses the dict (transformed to list) produced by exported_symbols(). Loops through the symbols and counts
+    them by type. Return a dict summary of the counts.
+
+    Parameters
+    ----------
+    symbols: list[dict[str, Any]]
+        The output dict produced by exported_symbols(), converted to a list.
+
+    Returns
+    -------
+    dict[str, int]
+        A summary count of exported symbols by type.
+
+    """
+    counts = {"known": 0, "ambiguous": 0, "unknown": 0}
+    for s in symbols:
+        counts[status_of(s)] += 1
+    return counts
+
+
+def completeness_pct(counts: dict[str, int]) -> float:
+    """
+    Calculates the type completeness score. Equals the percentage of known symbols
+    relative to total symbols.
+
+    Parameters
+    ----------
+    counts: dict[str, int]
+        A summary dictionary of symbol counts by type (known, ambiguous, unknown).
+
+    Returns
+    -------
+    float
+        A type completeness score.
+
+    """
+    total = sum(counts.values())
+    return 100.0 * counts["known"] / total if total else 0.0
+
+
+def render_counts_table(rows: list[tuple[str, dict[str, int]]]) -> str:
+    """
+    Generate a Markdown type completeness summary table, for the project base, used for PR comment.
+
+    Parameters
+    ----------
+    rows: list[tuple[str, dict[str, int]]]
+        List of tuples in the form of [("Section Name", symbol_counts)]. e.g., [("Patch", patch_counts)]
+
+
+    Returns
+    -------
+    str
+        A Markdown table of the type completeness summary, used for PR comment.
+
+    """
+    lines = ["| | Known | Ambiguous | Unknown | Total |", "|---|---|---|---|---|"]
+    for label, counts in rows:
+        total = sum(counts.values())
+        lines.append(
+            f"| {label} | {counts['known']} | {counts['ambiguous']} | "
+            f"{counts['unknown']} | {total} |"
+        )
+    return "\n".join(lines)
+
+
+def render_patch_detail(
+    patch_names: list[str],
+    removed_names: list[str],
+    base: dict[str, dict[str, Any]],
+    head: dict[str, dict[str, Any]],
+) -> str:
+    """
+    Generate a Markdown type completeness summary table, for the PR patch, used for PR comment.
+
+    Parameters
+    ----------
+    patch_names: list[str]
+        A list of symbol names whose status changed in the PR, or are newly exported.
+    removed_names: list[str]
+        A list of symbol names that were exported at base but are no longer exported at
+        head (e.g. de-exported, renamed, or deleted).
+    base: dict[str, dict[str, Any]]
+        dict generated by exported_symbols(), base case.
+    head: dict[str, dict[str, Any]]
+        dict generated by exported_symbols(), head case.
+
+    Returns
+    -------
+    str
+        A Markdown type completeness summary table, for the PR patch, used for PR comment.
+    """
+    lines = ["| Symbol | Status | Change |", "|---|---|---|"]
+    for name in sorted(patch_names):
+        head_status = status_of(head[name])
+        icon = STATUS_ICON[head_status]
+        if name not in base:
+            change = "new"
+        else:
+            base_status = status_of(base[name])
+            change = f"changed (was {STATUS_ICON[base_status]} {base_status})"
+        lines.append(f"| `{name}` | {icon} {head_status} | {change} |")
+    for name in sorted(removed_names):
+        base_status = status_of(base[name])
+        lines.append(
+            f"| `{name}` | — | no longer exported (was {STATUS_ICON[base_status]} {base_status}) |"
+        )
+    return "\n".join(lines)
+
+
+def build_summary(base_path: Path, head_path: Path, run_url: str | None = None) -> str:
+    """
+    Builds the full Markdown summary for the PR comment, combining the project-level
+    type completeness report with the patch-level diff between base and head.
+
+    Parameters
+    ----------
+    base_path: Path
+        The path to the --verifytypes JSON report for the PR's base (or merge-base)
+        commit.
+    head_path: Path
+        The path to the --verifytypes JSON report for the PR's head commit.
+    run_url: str | None
+        Optional link to the workflow run, e.g. its step summary. When provided, a
+        link to it is included near the top of the summary. Omitted entirely when
+        None.
+
+    Returns
+    -------
+    str
+        The rendered Markdown summary, ready to be written to a file or posted as
+        a PR comment.
+
+    """
+    # Extract the type completeness section of --verifytypes JSON, and filter on exported symbols
+    # for both the head and base case.
+    base_tc = load_type_completeness(base_path)
+    head_tc = load_type_completeness(head_path)
+    base = exported_symbols(base_tc)
+    head = exported_symbols(head_tc)
+
+    # Count up the symbol types.
+    project_counts = counts_by_status(list(head.values()))
+    project_pct = completeness_pct(project_counts)
+    other_counts = other_symbol_counts(head_tc)
+
+    # Gather the patch symbols, i.e., those whose status changed in the PR, or newly exported ones.
+    patch_names = [
+        name
+        for name, symbol in head.items()
+        if name not in base or status_of(base[name]) != status_of(symbol)
+    ]
+    # Symbols exported at base but no longer exported at head (de-exported, renamed,
+    # or deleted). These can't be found by iterating head, since they're absent from
+    # it entirely - iterate base instead and check for absence from head.
+    removed_names = [name for name in base if name not in head]
+
+    # Generate the Markdown report.
+    sections = [
+        "## Pyright Type Completeness",
+        "",
+    ]
+    if run_url:
+        sections += [
+            f"[View the full `pyright --verifytypes` output for this commit]({run_url})",
+            "",
+        ]
+    sections += [
+        f"**Project (full `chainladder` package, at this PR's head):** "
+        f"{project_pct:.1f}% of exported symbols fully typed "
+        f"({project_counts['known']} / {sum(project_counts.values())})",
+        "",
+        render_counts_table([("Project (head)", project_counts)]),
+        "",
+        f"Other symbols referenced but not exported by `chainladder`: "
+        f"{sum(other_counts.values())}",
+        "",
+        render_counts_table([("Other (head)", other_counts)]),
+        "",
+        "Symbols without documentation:",
+        f"- Functions without docstring: {head_tc['missingFunctionDocStringCount']}",
+        f"- Functions without default param: {head_tc['missingDefaultParamCount']}",
+        f"- Classes without docstring: {head_tc['missingClassDocStringCount']}",
+        "",
+    ]
+
+    if patch_names or removed_names:
+        patch_counts = (
+            counts_by_status([head[n] for n in patch_names])
+            if patch_names
+            else {"known": 0, "ambiguous": 0, "unknown": 0}
+        )
+        parts = []
+        if patch_names:
+            patch_pct = completeness_pct(patch_counts)
+            parts.append(
+                f"{patch_pct:.1f}% fully typed "
+                f"({patch_counts['known']} / {sum(patch_counts.values())})"
+            )
+        if removed_names:
+            parts.append(f"{len(removed_names)} no longer exported")
+        sections += [
+            "**Patch (exported symbols added or changed by this PR):** " + "; ".join(parts),
+            "",
+            render_counts_table([("Patch", patch_counts)]),
+            "",
+            "<details>",
+            "<summary>Patch symbol details</summary>",
+            "",
+            render_patch_detail(patch_names, removed_names, base, head),
+            "",
+            "</details>",
+        ]
+    else:
+        sections += [
+            "**Patch (exported symbols added or changed by this PR):** "
+            "no exported symbol type-completeness changes detected.",
+        ]
+
+    return "\n".join(sections)
+
+
+def main() -> None:
+    """
+    Parse -verifytypes JSON output and produce a Markdown summary report for PR comment.
+
+    Accepts command line arguments:
+
+    --base: Path to the --verifytypes JSON report for the PR's base (or merge-base) commit.
+    --head: Path to the --verifytypes JSON report for the PR's head commit.
+    --output: Path to write the rendered Markdown summary to.
+    --run-url: Optional link to the workflow run, e.g. its step summary. Omitted from the
+        summary entirely when not provided.
+
+    Returns
+    -------
+    None
+
+    """
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base", required=True, type=Path)
+    parser.add_argument("--head", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument(
+        "--run-url",
+        default=None,
+        help="Link to the workflow run, e.g. for its step summary.",
+    )
+    args = parser.parse_args()
+
+    summary = build_summary(args.base, args.head, run_url=args.run_url)
+    args.output.write_text(summary)
+    print(summary)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/type_completeness_output.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Appends a pyright --verifytypes text report to $GITHUB_STEP_SUMMARY,
+# wrapped in a collapsible <details> block. GitHub caps each step's summary
+# at 1MiB and silently drops the whole upload if exceeded, so the report is
+# truncated to a safe byte budget with a note when that happens, rather than
+# risking the entire block vanishing.
+#
+# This script is called twice per workflow run (PR head and PR base). The
+# budget below is half of the 1MiB cap, not the full cap, so the two calls
+# stay safely under the limit even in the worst case where they end up
+# sharing one combined budget instead of each getting their own per-step
+# allotment (true today, per GitHub's docs, but cheap to hedge against).
+set -euo pipefail
+
+report_file="$1"
+label="$2"
+max_bytes=450000
+
+total_bytes="$(wc -c < "$report_file")"
+
+{
+  echo "<details><summary>Full pyright --verifytypes output (${label})</summary>"
+  echo ""
+  echo '```text'
+  head -c "$max_bytes" "$report_file"
+  if [ "$total_bytes" -gt "$max_bytes" ]; then
+    echo ""
+    echo "... (truncated: ${total_bytes} bytes total, GITHUB_STEP_SUMMARY caps each step at 1MiB)"
+  fi
+  echo '```'
+  echo "</details>"
+} >> "$GITHUB_STEP_SUMMARY"