Next.js Deploy Suite #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Next.js Deploy Suite | |
| on: | |
| schedule: | |
| # Nightly at 02:00 UTC against main | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| vinext-ref: | |
| description: vinext branch/ref to test | |
| required: false | |
| default: main | |
| type: string | |
| next-ref: | |
| description: Next.js ref to test against | |
| required: false | |
| default: v16.2.6 | |
| type: string | |
| suite-filter: | |
| description: Which suites to run (pages, app, or all) | |
| required: false | |
| default: all | |
| type: choice | |
| options: | |
| - app | |
| - pages | |
| - all | |
| test-path: | |
| description: Optional path filter — comma-separated, e.g. `test/e2e/edge-async-local-storage`. Empty = all. | |
| required: false | |
| default: "" | |
| type: string | |
| test-concurrency: | |
| description: Per-shard Next.js test concurrency | |
| required: false | |
| default: "2" | |
| type: string | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: nextjs-deploy-suite-${{ inputs.vinext-ref || github.ref }}-${{ inputs.next-ref || 'v16.2.6' }}-${{ inputs.suite-filter || 'all' }}-${{ inputs.test-path || 'all' }} | |
| cancel-in-progress: false | |
| jobs: | |
| build: | |
| name: Build vinext + Next.js | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout vinext | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.vinext-ref || github.ref }} | |
| - uses: ./.github/actions/setup | |
| with: | |
| node-version: "24" | |
| - name: Enable pnpm shim for Next.js scripts | |
| run: corepack enable pnpm | |
| - name: Checkout Next.js | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: vercel/next.js | |
| ref: ${{ inputs.next-ref || 'v16.2.6' }} | |
| path: next.js | |
| - name: Build vinext | |
| run: vp run vinext#build | |
| - name: Prepare Next.js checkout | |
| run: bash scripts/run-nextjs-deploy-suite.sh "$GITHUB_WORKSPACE/next.js" | |
| env: | |
| CI: true | |
| VINEXT_BUILD: "0" | |
| NEXTJS_PREPARE: "1" | |
| NEXTJS_PREPARE_ONLY: "1" | |
| - name: Generate deploy manifest | |
| env: | |
| TEST_PATH: ${{ inputs.test-path }} | |
| run: | | |
| args=( | |
| "$GITHUB_WORKSPACE/next.js" | |
| "$GITHUB_WORKSPACE/nextjs-deploy-manifest.json" | |
| --filter "${{ inputs.suite-filter || 'all' }}" | |
| ) | |
| if [ -n "${TEST_PATH:-}" ]; then | |
| args+=(--match "${TEST_PATH}") | |
| fi | |
| node scripts/nextjs-deploy-manifest.mjs "${args[@]}" | |
| # Classify every Next.js e2e test file by which router(s) its fixture | |
| # exercises. The classification is independent of the test results — | |
| # it describes the Next.js checkout itself — so we POST it to its own | |
| # endpoint here in the build job and don't pass it through the test / | |
| # report jobs. The /compatibility UI joins it onto each result row at | |
| # query time. | |
| - name: Classify Next.js test suites by router | |
| env: | |
| NEXTJS_DIR: ${{ github.workspace }}/next.js | |
| run: | | |
| # 1) Enumerate every *.test.{ts,tsx,js,jsx} under next.js/test/e2e/. | |
| # 2) Run the classifier (walks each fixture dir, decides router kind). | |
| node scripts/list-nextjs-e2e-suites.mjs "$NEXTJS_DIR" nextjs-all-suites.json | |
| node scripts/classify-nextjs-suites.mjs \ | |
| "$NEXTJS_DIR" \ | |
| nextjs-all-suites.json \ | |
| nextjs-suite-routers.json | |
| # Submit the classification to the worker. Decoupled from results | |
| # ingestion so a re-classify can happen without re-running tests, and | |
| # so a partial test run still publishes fresh classifications. | |
| # | |
| # Guardrails mirror the results-ingest step at the end of the report | |
| # job: only the full suite (suite-filter=all), only against main. | |
| # A filtered run still produces a complete classification (the | |
| # classifier walks the whole Next.js checkout regardless of which | |
| # tests will actually run), so the partial-data argument from the | |
| # results endpoint doesn't apply here — but we keep the same gates | |
| # for consistency and to avoid surprising the operator running a | |
| # one-off workflow_dispatch. | |
| - name: Submit suite classification to worker | |
| if: | | |
| (inputs.suite-filter == 'all' || inputs.suite-filter == '' || github.event_name == 'schedule') | |
| && (inputs.test-path == '' || github.event_name == 'schedule') | |
| && ( | |
| (github.event_name == 'schedule' && github.ref == 'refs/heads/main') | |
| || (github.event_name == 'workflow_dispatch' && (inputs.vinext-ref == 'main' || inputs.vinext-ref == '')) | |
| ) | |
| env: | |
| COMPAT_INGEST_URL: ${{ vars.COMPAT_INGEST_URL || 'https://vinext-web.vinext.workers.dev/api/compatibility' }} | |
| COMPAT_INGEST_SECRET: ${{ secrets.COMPAT_INGEST_SECRET }} | |
| run: | | |
| if [ -z "${COMPAT_INGEST_SECRET:-}" ]; then | |
| echo "COMPAT_INGEST_SECRET not configured; skipping classification submission." | |
| exit 0 | |
| fi | |
| # Convert the suite → router map (object) into the endpoint's | |
| # expected `{ classifiedAt, suites: [{ suite, router }, ...] }` shape. | |
| # Done with node to avoid a jq dependency and to keep the JSON | |
| # construction safely string-typed. | |
| node -e ' | |
| const fs = require("node:fs"); | |
| const map = JSON.parse(fs.readFileSync("nextjs-suite-routers.json", "utf8")); | |
| const body = { | |
| classifiedAt: Date.now(), | |
| suites: Object.entries(map).map(([suite, router]) => ({ suite, router })), | |
| }; | |
| fs.writeFileSync("nextjs-classify-payload.json", JSON.stringify(body)); | |
| console.log("Classify payload:", body.suites.length, "suites"); | |
| ' | |
| CLASSIFY_URL="${COMPAT_INGEST_URL%/api/compatibility}/api/compatibility/classify" | |
| echo "Submitting classifications to ${CLASSIFY_URL}" | |
| if ! curl --silent --show-error --fail-with-body \ | |
| -X POST "${CLASSIFY_URL}" \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Compat-Secret: ${COMPAT_INGEST_SECRET}" \ | |
| --data-binary @nextjs-classify-payload.json; then | |
| echo "::warning::Suite classification POST failed (non-fatal)." | |
| fi | |
| # Kept as an artifact even after submission so the classification can | |
| # be re-submitted manually (e.g. if the live POST hiccupped) or | |
| # inspected when debugging misclassifications. | |
| - name: Upload suite router classification | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: nextjs-suite-routers | |
| path: | | |
| nextjs-suite-routers.json | |
| nextjs-classify-payload.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Save build artifacts | |
| uses: actions/cache/save@v5 | |
| with: | |
| # The test job needs: vinext built dist + workspace metadata (for | |
| # e2e-deploy.sh dependency injection), the scripts, the prepared | |
| # Next.js checkout, Playwright browsers, and the filtered manifest. | |
| # next.js/ is under $GITHUB_WORKSPACE so it's included in ".". | |
| path: | | |
| . | |
| ~/.cache/ms-playwright | |
| nextjs-deploy-manifest.json | |
| key: nextjs-deploy-build-${{ github.sha }}-${{ github.run_id }} | |
| test: | |
| name: Deploy suite (${{ matrix.group }}) | |
| needs: build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| group: | |
| [ | |
| 1/16, | |
| 2/16, | |
| 3/16, | |
| 4/16, | |
| 5/16, | |
| 6/16, | |
| 7/16, | |
| 8/16, | |
| 9/16, | |
| 10/16, | |
| 11/16, | |
| 12/16, | |
| 13/16, | |
| 14/16, | |
| 15/16, | |
| 16/16, | |
| ] | |
| steps: | |
| - name: Restore build artifacts | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: | | |
| . | |
| ~/.cache/ms-playwright | |
| nextjs-deploy-manifest.json | |
| key: nextjs-deploy-build-${{ github.sha }}-${{ github.run_id }} | |
| - uses: ./.github/actions/setup | |
| with: | |
| node-version: "24" | |
| # Skip install — the workspace is already fully built in the cache | |
| run-install: false | |
| - name: Enable pnpm shim for Next.js scripts | |
| run: corepack enable pnpm | |
| - name: Ensure Playwright browsers | |
| working-directory: next.js | |
| run: corepack pnpm playwright install chromium chromium-headless-shell | |
| - name: Make scripts executable | |
| run: chmod +x scripts/e2e-deploy.sh scripts/e2e-logs.sh scripts/e2e-cleanup.sh | |
| - name: Run deploy shard | |
| working-directory: next.js | |
| env: | |
| CI: true | |
| NEXT_TEST_MODE: deploy | |
| NEXT_E2E_TEST_TIMEOUT: "240000" | |
| NEXT_EXTERNAL_TESTS_FILTERS: ${{ github.workspace }}/nextjs-deploy-manifest.json | |
| VINEXT_DIR: ${{ github.workspace }} | |
| ADAPTER_DIR: ${{ github.workspace }} | |
| # Required by the Next.js deploy test manifest — tests are keyed to | |
| # the turbopack variant, so this must be set even though vinext uses Vite. | |
| IS_TURBOPACK_TEST: "1" | |
| NEXT_TEST_JOB: "1" | |
| NEXT_TELEMETRY_DISABLED: "1" | |
| NEXT_TEST_DEPLOY_SCRIPT_PATH: ${{ github.workspace }}/scripts/e2e-deploy.sh | |
| NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH: ${{ github.workspace }}/scripts/e2e-logs.sh | |
| NEXT_TEST_CLEANUP_SCRIPT_PATH: ${{ github.workspace }}/scripts/e2e-cleanup.sh | |
| run: node run-tests.js --timings --retries 0 -g ${{ matrix.group }} -c ${{ inputs.test-concurrency || '2' }} --type e2e | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-${{ strategy.job-index }} | |
| path: next.js/test/**/*.results.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Upload deploy debug logs | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: deploy-debug-${{ strategy.job-index }} | |
| path: reports/nextjs-deploy-debug/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| report: | |
| name: Test report | |
| needs: test | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test-results-* | |
| path: results | |
| merge-multiple: true | |
| - name: Generate report | |
| id: report | |
| uses: actions/github-script@v7 | |
| env: | |
| # Resolve the GitHub Actions expressions to env vars once, instead | |
| # of splicing `${{ ... }}` into JS string literals further down. | |
| # The latter is fragile — a single quote in an input value would | |
| # break the generated JS — and not exploitable in practice but | |
| # cleaner this way. | |
| VINEXT_REF: ${{ inputs.vinext-ref || github.ref }} | |
| NEXT_REF: ${{ inputs.next-ref || 'v16.2.6' }} | |
| SUITE_FILTER: ${{ inputs.suite-filter || 'all' }} | |
| with: | |
| script: | | |
| const fs = require('node:fs'); | |
| const path = require('node:path'); | |
| function findResultFiles(dir) { | |
| const files = []; | |
| if (!fs.existsSync(dir)) return files; | |
| for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { | |
| const full = path.join(dir, entry.name); | |
| if (entry.isDirectory()) { | |
| files.push(...findResultFiles(full)); | |
| } else if (entry.name.endsWith('.results.json')) { | |
| files.push(full); | |
| } | |
| } | |
| return files; | |
| } | |
| const resultFiles = findResultFiles('results'); | |
| if (resultFiles.length === 0) { | |
| core.summary.addHeading('Next.js Deploy Suite', 2); | |
| core.summary.addRaw('No test result files found. All shards may have been skipped or produced no output.'); | |
| await core.summary.write(); | |
| return; | |
| } | |
| const passed = []; | |
| const failed = []; | |
| const skipped = []; | |
| // Per-test-file counts for the /compatibility page ingest. Router | |
| // classification is NOT attached here — it's submitted separately | |
| // by the build job to /api/compatibility/classify, and the UI | |
| // joins it onto each result row at query time. See the | |
| // "Submit suite classification" step earlier in this workflow. | |
| const fileCounts = new Map(); | |
| function bumpFile(suite, key) { | |
| let cur = fileCounts.get(suite); | |
| if (!cur) { | |
| cur = { suite, total: 0, passed: 0, failed: 0, skipped: 0 }; | |
| fileCounts.set(suite, cur); | |
| } | |
| cur.total++; | |
| cur[key]++; | |
| } | |
| // Derive a canonical Next.js test path from the result-file path. | |
| // | |
| // Next.js's run-tests.js does NOT populate testResults[].testFilePath | |
| // in the JSON, so we have to recover it from where the file lives in | |
| // the artifact tree. Each shard uploads files at their relative path | |
| // inside next.js/test/, e.g. `e2e/app-dir/foo/foo.test.ts.results.json`. | |
| // | |
| // The report job uses `merge-multiple: true`, which flattens the | |
| // contents of every artifact into `results/`. We also defensively | |
| // strip a leading `test-results-N/` segment in case that ever | |
| // changes and shards get nested under their artifact names. | |
| // Result: a canonical path of the form `test/e2e/app-dir/foo/foo.test.ts`. | |
| function deriveSuiteName(file) { | |
| let rel = path.relative('results', file); | |
| // Defensive: strip a leading `test-results-<n>/` shard prefix if | |
| // the download layout ever changes. | |
| rel = rel.replace(/^test-results-[^/]+\//, ''); | |
| const stripped = rel.replace(/\.results\.json$/, ''); | |
| if (stripped && stripped.includes('/')) { | |
| return `test/${stripped}`; | |
| } | |
| // Fallback: file sits at the top level of results/ (shouldn't | |
| // happen in practice). Use the basename so we don't break ingest. | |
| return path.basename(file, '.results.json'); | |
| } | |
| for (const file of resultFiles) { | |
| try { | |
| const data = JSON.parse(fs.readFileSync(file, 'utf8')); | |
| const suiteName = deriveSuiteName(file); | |
| for (const suite of data.testResults || []) { | |
| for (const tc of suite.assertionResults || []) { | |
| const testName = tc.ancestorTitles | |
| ? [...tc.ancestorTitles, tc.title].join(' > ') | |
| : tc.fullName || tc.title; | |
| if (tc.status === 'passed') { | |
| passed.push({ suite: suiteName, test: testName }); | |
| bumpFile(suiteName, 'passed'); | |
| } else if (tc.status === 'failed') { | |
| const msg = (tc.failureMessages || []).join('\n').slice(0, 500); | |
| failed.push({ suite: suiteName, test: testName, message: msg }); | |
| bumpFile(suiteName, 'failed'); | |
| } else { | |
| skipped.push({ suite: suiteName, test: testName }); | |
| bumpFile(suiteName, 'skipped'); | |
| } | |
| } | |
| } | |
| } catch (e) { | |
| core.warning(`Failed to parse ${file}: ${e.message}`); | |
| } | |
| } | |
| // Persist the per-file counts for the next step to submit to D1. | |
| fs.mkdirSync('report', { recursive: true }); | |
| fs.writeFileSync( | |
| 'report/compat-ingest.json', | |
| JSON.stringify( | |
| { | |
| kind: 'deploy', | |
| runKey: String(context.runId), | |
| vinextRef: process.env.VINEXT_REF, | |
| nextRef: process.env.NEXT_REF, | |
| commitSha: context.sha, | |
| files: Array.from(fileCounts.values()), | |
| }, | |
| null, | |
| 2, | |
| ) + '\n', | |
| ); | |
| const total = passed.length + failed.length + skipped.length; | |
| const passRate = total > 0 ? ((passed.length / total) * 100).toFixed(1) : '0.0'; | |
| const statsLine = `${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped (${total} total, ${passRate}% pass rate)`; | |
| // Top-level annotation visible on the workflow run page | |
| if (failed.length > 0) { | |
| core.warning(`Next.js Deploy Suite: ${statsLine}`, { title: 'Next.js Deploy Suite results' }); | |
| } else { | |
| core.notice(`Next.js Deploy Suite: ${statsLine}`, { title: 'Next.js Deploy Suite results' }); | |
| } | |
| // Job summary | |
| core.summary.addHeading('Next.js Deploy Suite', 2); | |
| core.summary.addRaw( | |
| `**${passed.length}** passed, **${failed.length}** failed, **${skipped.length}** skipped (${total} total, ${passRate}% pass rate)\n\n` | |
| ); | |
| if (failed.length > 0) { | |
| core.summary.addHeading('Failed tests', 3); | |
| const rows = [ | |
| [{data: 'Suite', header: true}, {data: 'Test', header: true}, {data: 'Error', header: true}], | |
| ...failed.slice(0, 100).map(f => [ | |
| f.suite, | |
| f.test, | |
| f.message ? `<details><summary>Details</summary>\n\n\`\`\`\n${f.message}\n\`\`\`\n</details>` : '' | |
| ]) | |
| ]; | |
| core.summary.addTable(rows); | |
| if (failed.length > 100) { | |
| core.summary.addRaw(`\n_...and ${failed.length - 100} more failures. Download the test-results artifacts for full details._\n`); | |
| } | |
| } | |
| if (passed.length > 0) { | |
| const suites = [...new Set(passed.map(p => p.suite))].sort(); | |
| core.summary.addHeading('Passed suites', 3); | |
| core.summary.addDetails( | |
| `${suites.length} suites (click to expand)`, | |
| suites.map(s => `- \`${s}\``).join('\n') | |
| ); | |
| } | |
| await core.summary.write(); | |
| // Also write a JSON report as an artifact | |
| const report = { | |
| timestamp: new Date().toISOString(), | |
| vinextRef: process.env.VINEXT_REF, | |
| nextRef: process.env.NEXT_REF, | |
| suiteFilter: process.env.SUITE_FILTER, | |
| summary: { total, passed: passed.length, failed: failed.length, skipped: skipped.length }, | |
| failed: failed.map(f => ({ suite: f.suite, test: f.test })), | |
| passed: passed.map(p => ({ suite: p.suite, test: p.test })), | |
| }; | |
| fs.mkdirSync('report', { recursive: true }); | |
| fs.writeFileSync('report/deploy-suite-report.json', JSON.stringify(report, null, 2) + '\n'); | |
| if (failed.length > 0) { | |
| core.setFailed(`${failed.length} test(s) failed`); | |
| } | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: deploy-suite-report | |
| # Include both files: the human-readable report and the | |
| # /compatibility ingest payload. Keeping the ingest payload | |
| # archived lets us manually re-POST a run if the live ingest | |
| # step fails (network blip, bad URL, expired secret, etc.) | |
| # without re-running the 40-minute suite. | |
| path: | | |
| report/deploy-suite-report.json | |
| report/compat-ingest.json | |
| retention-days: 90 | |
| # Submit per-file results to the /compatibility page's ingest endpoint. | |
| # Requires the COMPAT_INGEST_SECRET repo secret (matched against the | |
| # worker's COMPAT_INGEST_SECRET set via `wrangler secret put`). | |
| # No-op if the secret or ingest URL aren't configured. | |
| # | |
| # Guardrails: | |
| # - Only the full suite (suite-filter=all, no test-path). Filtered | |
| # runs (`app`/`pages` only, or a path-narrowed run) would publish a | |
| # misleading partial snapshot that drags the historical pass-rate | |
| # trend off-course. | |
| # - Only when running against main. Scheduled runs always test main | |
| # (github.ref == refs/heads/main). For manual workflow_dispatch we | |
| # check that `inputs.vinext-ref` resolves to main (its default). | |
| # Branch / PR runs are useful for spot-checks but should not pollute | |
| # the historical record. | |
| - name: Submit results to /compatibility ingest | |
| if: | | |
| always() | |
| && (inputs.suite-filter == 'all' || inputs.suite-filter == '' || github.event_name == 'schedule') | |
| && (inputs.test-path == '' || github.event_name == 'schedule') | |
| && ( | |
| (github.event_name == 'schedule' && github.ref == 'refs/heads/main') | |
| || (github.event_name == 'workflow_dispatch' && (inputs.vinext-ref == 'main' || inputs.vinext-ref == '')) | |
| ) | |
| && hashFiles('report/compat-ingest.json') != '' | |
| env: | |
| COMPAT_INGEST_URL: ${{ vars.COMPAT_INGEST_URL || 'https://vinext-web.vinext.workers.dev/api/compatibility' }} | |
| COMPAT_INGEST_SECRET: ${{ secrets.COMPAT_INGEST_SECRET }} | |
| run: | | |
| if [ -z "${COMPAT_INGEST_SECRET:-}" ]; then | |
| echo "COMPAT_INGEST_SECRET not configured; skipping ingest submission." | |
| exit 0 | |
| fi | |
| echo "Submitting compat results to ${COMPAT_INGEST_URL}" | |
| # --fail-with-body keeps stdout/stderr but exits non-zero on HTTP errors. | |
| # We still don't want the report job to fail if ingest hiccups, so trap | |
| # the exit and just log. | |
| if ! curl --silent --show-error --fail-with-body \ | |
| -X POST "${COMPAT_INGEST_URL}" \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Compat-Secret: ${COMPAT_INGEST_SECRET}" \ | |
| --data-binary @report/compat-ingest.json; then | |
| echo "::warning::Compatibility ingest POST failed (non-fatal)." | |
| fi |