From 311cfb17fd75e683bac3a5f00661e003ffc915ed Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Fri, 12 Jun 2026 00:59:42 -0700 Subject: [PATCH 1/7] [cueweb] Add Stuck Frames page (CueCommander parity) Add a /stuck-frames route listing frames in RUNNING state longer than a configurable threshold, with per-row Retry and Kill actions. - Threshold slider (1-48h, default 8h) persisted to localStorage; filtering is client-side so the slider is instant. - Server-side aggregation route /api/stuck-frames lists unfinished jobs (GetJobs) and fans out GetFrames per job filtered to RUNNING (FrameState 2), so the browser makes a single request. - Retry/Kill reuse retryFrames/killFrames; kill records the signed-in user in the reason. Polls every 30s. --- cueweb/app/api/stuck-frames/route.ts | 83 ++++++++++ cueweb/app/stuck-frames/page.tsx | 221 +++++++++++++++++++++++++++ cueweb/app/utils/get_utils.ts | 12 ++ 3 files changed, 316 insertions(+) create mode 100644 cueweb/app/api/stuck-frames/route.ts create mode 100644 cueweb/app/stuck-frames/page.tsx diff --git a/cueweb/app/api/stuck-frames/route.ts b/cueweb/app/api/stuck-frames/route.ts new file mode 100644 index 000000000..301c230be --- /dev/null +++ b/cueweb/app/api/stuck-frames/route.ts @@ -0,0 +1,83 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { fetchObjectFromRestGateway } from '@/app/utils/api_utils'; +import { NextRequest, NextResponse } from "next/server"; + +// Server-side aggregation for the Stuck Frames page. There is no single RPC +// that returns every running frame, so we list the unfinished jobs +// (GetJobs) and fan out a GetFrames call per job filtered to RUNNING +// (FrameState 2), then flatten. Doing the fan-out here keeps the browser to a +// single request and avoids leaking the N+1 to the client. The page applies +// the running-time threshold locally so the slider stays instant. +// +// RPCs: /job.JobInterface/GetJobs, /job.JobInterface/GetFrames. + +const RUNNING_STATE = 2; // FrameState.RUNNING (proto/src/job.proto) +const MAX_FRAMES_PER_JOB = 1000; + +export async function POST(_request: NextRequest) { + try { + // 1. All unfinished jobs. + const jobsResp = await fetchObjectFromRestGateway( + "/job.JobInterface/GetJobs", + "POST", + JSON.stringify({ r: { include_finished: false } }), + ); + const jobsJson = await jobsResp.json(); + if (jobsJson?.error) { + return NextResponse.json({ error: jobsJson.error }, { status: 500 }); + } + const jobs: any[] = jobsJson?.data?.jobs?.jobs ?? []; + + // 2. Running frames per job, in parallel. A single job's failure drops to + // an empty list rather than failing the whole page. + const perJob = await Promise.all( + jobs.map(async (job) => { + const body = JSON.stringify({ + job: { id: job.id, name: job.name }, + req: { + include_finished: false, + page: 1, + limit: MAX_FRAMES_PER_JOB, + states: { frame_states: [RUNNING_STATE] }, + }, + }); + try { + const framesResp = await fetchObjectFromRestGateway( + "/job.JobInterface/GetFrames", + "POST", + body, + ); + const framesJson = await framesResp.json(); + if (framesJson?.error) return []; + const frames: any[] = framesJson?.data?.frames?.frames ?? []; + // Defensive: keep only RUNNING even if the state filter was ignored, + // and stamp the parent job so the table can show / act on it. + return frames + .filter((f) => f.state === "RUNNING") + .map((f) => ({ ...f, jobId: job.id, jobName: job.name })); + } catch { + return []; + } + }), + ); + + return NextResponse.json({ data: perJob.flat() }, { status: 200 }); + } catch (error) { + return NextResponse.json({ error: (error as Error).message }, { status: 500 }); + } +} diff --git a/cueweb/app/stuck-frames/page.tsx b/cueweb/app/stuck-frames/page.tsx new file mode 100644 index 000000000..888ceb486 --- /dev/null +++ b/cueweb/app/stuck-frames/page.tsx @@ -0,0 +1,221 @@ +"use client"; + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as React from "react"; +import { useSession } from "next-auth/react"; + +import type { Frame } from "@/app/frames/frame-columns"; +import { StuckFrame, getStuckFrames } from "@/app/utils/get_utils"; +import { killFrames, retryFrames } from "@/app/utils/action_utils"; +import { handleError } from "@/app/utils/notify_utils"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; + +const REFRESH_MS = 30000; +const THRESHOLD_KEY = "cueweb.stuck-frames.thresholdHours"; +const DEFAULT_HOURS = 8; +const MIN_HOURS = 1; +const MAX_HOURS = 48; + +function fmtDuration(seconds: number): string { + if (!Number.isFinite(seconds) || seconds < 0) return "—"; + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + const s = Math.floor(seconds % 60); + return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`; +} + +// last_resource is "host/procid"; show just the host. +const hostOf = (lastResource: string) => (lastResource || "").split("/")[0] || "—"; + +export default function StuckFramesPage() { + const { data: session } = useSession(); + const username = session?.user?.name ?? session?.user?.email ?? "cueweb"; + + const [frames, setFrames] = React.useState(null); + const [thresholdHours, setThresholdHours] = React.useState(DEFAULT_HOURS); + const [now, setNow] = React.useState(() => Date.now() / 1000); + const [busyId, setBusyId] = React.useState(null); + + const load = React.useCallback(async (isCancelled?: () => boolean) => { + try { + const data = await getStuckFrames(); + if (isCancelled?.()) return; + setFrames(data); + setNow(Date.now() / 1000); + } catch (err) { + if (isCancelled?.()) return; + handleError(err, "Could not load stuck frames"); + setFrames((prev) => prev ?? []); + } + }, []); + + React.useEffect(() => { + // Restore the persisted threshold on mount (kept out of the initial state + // to avoid an SSR/client hydration mismatch). + const stored = window.localStorage.getItem(THRESHOLD_KEY); + if (stored) { + const n = Number(stored); + if (Number.isFinite(n) && n >= MIN_HOURS && n <= MAX_HOURS) setThresholdHours(n); + } + let cancelled = false; + const isCancelled = () => cancelled; + load(isCancelled); + const interval = setInterval(() => load(isCancelled), REFRESH_MS); + return () => { + cancelled = true; + clearInterval(interval); + }; + }, [load]); + + function changeThreshold(hours: number) { + setThresholdHours(hours); + window.localStorage.setItem(THRESHOLD_KEY, String(hours)); + } + + const runtimeOf = React.useCallback( + (f: StuckFrame) => (f.startTime ? now - f.startTime : 0), + [now], + ); + + const stuck = React.useMemo(() => { + if (!frames) return null; + const thresholdSeconds = thresholdHours * 3600; + return frames + .filter((f) => runtimeOf(f) > thresholdSeconds) + .sort((a, b) => runtimeOf(b) - runtimeOf(a)); + }, [frames, thresholdHours, runtimeOf]); + + // Strip the page-only jobId/jobName before sending the frame to a Cuebot RPC + // (they are not Frame proto fields). + function toFrame(sf: StuckFrame): Frame { + const { jobId: _jobId, jobName: _jobName, ...frame } = sf; + return frame as Frame; + } + + async function handleRetry(sf: StuckFrame) { + setBusyId(sf.id); + try { + await retryFrames([toFrame(sf)]); + await load(); + } finally { + setBusyId(null); + } + } + + async function handleKill(sf: StuckFrame) { + setBusyId(sf.id); + try { + await killFrames([toFrame(sf)], username, `Manual frame kill from CueWeb Stuck Frames by ${username}`); + await load(); + } finally { + setBusyId(null); + } + } + + return ( +
+

Stuck Frames

+ +
+ + {stuck ? ( + + {stuck.length} {stuck.length === 1 ? "frame" : "frames"} + + ) : null} +
+ + {stuck === null ? ( +
+ + + +
+ ) : stuck.length === 0 ? ( +

+ No frames have been running longer than {thresholdHours}{" "} + {thresholdHours === 1 ? "hour" : "hours"}. +

+ ) : ( +
+ + + + + + + + + + + + + {stuck.map((f) => ( + + + + + + + + + ))} + +
JobLayerFrameHostRuntimeActions
{f.jobName}{f.layerName}{f.number}{hostOf(f.lastResource)}{fmtDuration(runtimeOf(f))} +
+ + +
+
+
+ )} +
+ ); +} diff --git a/cueweb/app/utils/get_utils.ts b/cueweb/app/utils/get_utils.ts index 12e5f92ff..448fdad6a 100644 --- a/cueweb/app/utils/get_utils.ts +++ b/cueweb/app/utils/get_utils.ts @@ -187,6 +187,18 @@ export async function getFrames(body: string): Promise { return response ? response : []; } +// A running frame plus its parent job, for the Stuck Frames page. +export type StuckFrame = Frame & { jobId: string; jobName: string }; + +// Fetch every RUNNING frame across all unfinished jobs (server-aggregated via +// /api/stuck-frames). The Stuck Frames page applies the running-time threshold +// locally so the slider stays instant. +export async function getStuckFrames(): Promise { + const ENDPOINT = "/api/stuck-frames"; + const response = await accessGetApi(ENDPOINT, JSON.stringify({})); + return Array.isArray(response) ? response : []; +} + // Fetch a pending job based on the request body export async function getPendingJob(body: string): Promise { const ENDPOINT = "/api/job/getjob"; From e5502dea37a8b095722089c9168f5c7ff616379f Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Fri, 12 Jun 2026 01:25:19 -0700 Subject: [PATCH 2/7] [cueweb] Stuck Frames: full StuckFramePlugin parity (CueCommander) Replace the running-time MVP with CueGUI's StuckFramePlugin behavior: - Detection predicate (LLU / % stuck / avg-completion / runtime / >500s, %stuck<1.1) applied live client-side. - Multi-service filter bar (catch-all + per-service filters) with % Run Since LLU, Min LLU, % Avg Completion, Total Runtime, Exclude Keywords, Enable; CueGUI service defaults; persisted to localStorage. - Job-grouped table: Name, Comment, Frame, Host, LLU, Runtime, % Stuck, Average, Last Line (lazy rqlog tail). - Frame menu: Tail/View/Last Log, Retry/Eat/Kill, Log + Log-and-X, Frame Not Stuck, Add/Exclude Job, Core Up, View Host. Job menu: View Comments, Job Not Stuck, Add/Exclude Job, Core Up. Auto-refresh, Notification, Refresh, Clear. - Aggregation route attaches service/avgFrameSec/layer via GetLayers; add /api/stuck-frames/lastline and /api/layer/action/setmincores. --- .../app/api/layer/action/setmincores/route.ts | 45 ++ cueweb/app/api/stuck-frames/lastline/route.ts | 44 ++ cueweb/app/api/stuck-frames/route.ts | 115 ++-- cueweb/app/stuck-frames/page.tsx | 573 ++++++++++++++---- cueweb/app/utils/action_utils.ts | 7 + cueweb/app/utils/get_utils.ts | 34 +- cueweb/components/ui/stuck-frame-filters.tsx | 192 ++++++ 7 files changed, 852 insertions(+), 158 deletions(-) create mode 100644 cueweb/app/api/layer/action/setmincores/route.ts create mode 100644 cueweb/app/api/stuck-frames/lastline/route.ts create mode 100644 cueweb/components/ui/stuck-frame-filters.tsx diff --git a/cueweb/app/api/layer/action/setmincores/route.ts b/cueweb/app/api/layer/action/setmincores/route.ts new file mode 100644 index 000000000..5d398a100 --- /dev/null +++ b/cueweb/app/api/layer/action/setmincores/route.ts @@ -0,0 +1,45 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { handleRoute } from '@/app/utils/api_utils'; +import { NextRequest, NextResponse } from "next/server"; + +// Set a layer's minimum cores (CueGUI Stuck Frame "Core Up"). Request: +// { layer, cores }. RPC: /job.LayerInterface/SetMinCores. +export async function POST(request: NextRequest) { + const endpoint = "/job.LayerInterface/SetMinCores"; + const method = request.method; + if (method !== 'POST') { + return NextResponse.json({ error: 'Invalid method. Only POST is allowed.' }, { status: 405 }); + } + + let jsonBody: any; + try { + jsonBody = await request.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON in request body' }, { status: 400 }); + } + if (!jsonBody || typeof jsonBody !== 'object' || !jsonBody.layer || typeof jsonBody.cores !== 'number') { + return NextResponse.json({ error: 'Invalid request body: layer and numeric cores are required' }, { status: 400 }); + } + + const body = JSON.stringify(jsonBody); + const response = await handleRoute(method, endpoint, body, true); + const responseData = await response.json(); + + if (!response.ok) return NextResponse.json({ error: responseData.error }, { status: response.status }); + return NextResponse.json({ data: responseData.data }, { status: response.status }); +} diff --git a/cueweb/app/api/stuck-frames/lastline/route.ts b/cueweb/app/api/stuck-frames/lastline/route.ts new file mode 100644 index 000000000..c25cbc09c --- /dev/null +++ b/cueweb/app/api/stuck-frames/lastline/route.ts @@ -0,0 +1,44 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextRequest, NextResponse } from "next/server"; +import { execFile as execFileCallback } from "child_process"; +import { promisify } from "util"; + +const execFile = promisify(execFileCallback); + +// Returns the last non-empty line of a frame's .rqlog (the Stuck Frames +// "Last Line" column, mirroring CueGUI's getLastLine). Best-effort: if the log +// filesystem isn't mounted in this deployment, or the file is missing, it +// returns an empty line rather than erroring. execFile (no shell) + an rqlog +// path allowlist keep the caller-supplied path from being abused. +export async function GET(request: NextRequest) { + const path = request.nextUrl.searchParams.get("path"); + if (!path || !path.endsWith(".rqlog") || path.includes("..")) { + return NextResponse.json({ lastLine: "" }, { status: 200 }); + } + try { + // tail the file, then keep the last non-blank line. + const { stdout } = await execFile("tail", ["-n", "20", "--", path], { + timeout: 5000, + maxBuffer: 1024 * 1024, + }); + const lines = stdout.split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() !== ""); + return NextResponse.json({ lastLine: lines.length ? lines[lines.length - 1] : "" }, { status: 200 }); + } catch { + return NextResponse.json({ lastLine: "" }, { status: 200 }); + } +} diff --git a/cueweb/app/api/stuck-frames/route.ts b/cueweb/app/api/stuck-frames/route.ts index 301c230be..5690fe29a 100644 --- a/cueweb/app/api/stuck-frames/route.ts +++ b/cueweb/app/api/stuck-frames/route.ts @@ -17,62 +17,97 @@ import { fetchObjectFromRestGateway } from '@/app/utils/api_utils'; import { NextRequest, NextResponse } from "next/server"; -// Server-side aggregation for the Stuck Frames page. There is no single RPC -// that returns every running frame, so we list the unfinished jobs -// (GetJobs) and fan out a GetFrames call per job filtered to RUNNING -// (FrameState 2), then flatten. Doing the fan-out here keeps the browser to a -// single request and avoids leaking the N+1 to the client. The page applies -// the running-time threshold locally so the slider stays instant. +// Server-side data gathering for the Stuck Frames page. CueGUI's +// StuckFramePlugin walks every show's procs; we approximate by listing the +// unfinished jobs (GetJobs) and, per job, fetching its RUNNING frames +// (GetFrames, FrameState 2) and its layers (GetLayers, for the per-service +// average frame time). Each frame is stamped with its job, service and the +// layer's average frame time so the client can apply the full CueGUI +// stuck-detection predicate (LLU / % stuck / avg-completion / runtime) live +// against the user's per-service filter thresholds. // -// RPCs: /job.JobInterface/GetJobs, /job.JobInterface/GetFrames. +// RPCs: /job.JobInterface/GetJobs, /job.JobInterface/GetFrames, +// /job.JobInterface/GetLayers. const RUNNING_STATE = 2; // FrameState.RUNNING (proto/src/job.proto) const MAX_FRAMES_PER_JOB = 1000; +async function gatewayJson(endpoint: string, body: string): Promise { + try { + const resp = await fetchObjectFromRestGateway(endpoint, "POST", body); + const json = await resp.json(); + if (json?.error) return null; + return json?.data ?? null; + } catch { + return null; + } +} + export async function POST(_request: NextRequest) { try { - // 1. All unfinished jobs. - const jobsResp = await fetchObjectFromRestGateway( + const jobsData = await gatewayJson( "/job.JobInterface/GetJobs", - "POST", JSON.stringify({ r: { include_finished: false } }), ); - const jobsJson = await jobsResp.json(); - if (jobsJson?.error) { - return NextResponse.json({ error: jobsJson.error }, { status: 500 }); + if (jobsData === null) { + return NextResponse.json({ error: "Failed to list jobs" }, { status: 500 }); } - const jobs: any[] = jobsJson?.data?.jobs?.jobs ?? []; + const jobs: any[] = jobsData?.jobs?.jobs ?? []; - // 2. Running frames per job, in parallel. A single job's failure drops to - // an empty list rather than failing the whole page. const perJob = await Promise.all( jobs.map(async (job) => { - const body = JSON.stringify({ - job: { id: job.id, name: job.name }, - req: { - include_finished: false, - page: 1, - limit: MAX_FRAMES_PER_JOB, - states: { frame_states: [RUNNING_STATE] }, - }, - }); - try { - const framesResp = await fetchObjectFromRestGateway( + const [framesData, layersData] = await Promise.all([ + gatewayJson( "/job.JobInterface/GetFrames", - "POST", - body, - ); - const framesJson = await framesResp.json(); - if (framesJson?.error) return []; - const frames: any[] = framesJson?.data?.frames?.frames ?? []; - // Defensive: keep only RUNNING even if the state filter was ignored, - // and stamp the parent job so the table can show / act on it. - return frames - .filter((f) => f.state === "RUNNING") - .map((f) => ({ ...f, jobId: job.id, jobName: job.name })); - } catch { - return []; + JSON.stringify({ + job: { id: job.id, name: job.name }, + req: { + include_finished: false, + page: 1, + limit: MAX_FRAMES_PER_JOB, + states: { frame_states: [RUNNING_STATE] }, + }, + }), + ), + gatewayJson( + "/job.JobInterface/GetLayers", + JSON.stringify({ job: { id: job.id, name: job.name } }), + ), + ]); + + const layers: any[] = layersData?.layers?.layers ?? []; + // layerName -> details for attaching to each frame (service + average + // frame time for detection; id + minCores for the Core Up action). + const layerInfo = new Map< + string, + { id: string; service: string; avgFrameSec: number; minCores: number } + >(); + for (const layer of layers) { + layerInfo.set(layer.name, { + id: layer.id ?? "", + service: Array.isArray(layer.services) && layer.services.length ? layer.services[0] : "", + avgFrameSec: Number(layer.layerStats?.avgFrameSec ?? 0), + minCores: Number(layer.minCores ?? 0), + }); } + + const frames: any[] = framesData?.frames?.frames ?? []; + return frames + .filter((f) => f.state === "RUNNING") + .map((f) => { + const info = layerInfo.get(f.layerName); + return { + ...f, + jobId: job.id, + jobName: job.name, + jobLogDir: job.logDir ?? "", + jobHasComment: !!job.hasComment, + service: info?.service ?? "", + avgFrameSec: info?.avgFrameSec ?? 0, + layerId: info?.id ?? "", + layerMinCores: info?.minCores ?? 0, + }; + }); }), ); diff --git a/cueweb/app/stuck-frames/page.tsx b/cueweb/app/stuck-frames/page.tsx index 888ceb486..1aa0d5ed1 100644 --- a/cueweb/app/stuck-frames/page.tsx +++ b/cueweb/app/stuck-frames/page.tsx @@ -18,204 +18,549 @@ import * as React from "react"; import { useSession } from "next-auth/react"; +import { useRouter } from "next/navigation"; +import { MessageSquare } from "lucide-react"; import type { Frame } from "@/app/frames/frame-columns"; -import { StuckFrame, getStuckFrames } from "@/app/utils/get_utils"; -import { killFrames, retryFrames } from "@/app/utils/action_utils"; -import { handleError } from "@/app/utils/notify_utils"; +import { StuckFrame, getStuckFrames, getStuckFrameLastLine } from "@/app/utils/get_utils"; +import { eatFrames, killFrames, retryFrames, setLayerMinCores } from "@/app/utils/action_utils"; +import { handleError, toastSuccess } from "@/app/utils/notify_utils"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; import { Skeleton } from "@/components/ui/skeleton"; +import { + DEFAULT_FILTER, + StuckFrameFilters, + type StuckFilter, +} from "@/components/ui/stuck-frame-filters"; -const REFRESH_MS = 30000; -const THRESHOLD_KEY = "cueweb.stuck-frames.thresholdHours"; -const DEFAULT_HOURS = 8; -const MIN_HOURS = 1; -const MAX_HOURS = 48; +const AUTO_REFRESH_MS = 60000; +const FILTERS_KEY = "cueweb.stuck-frames.filters"; -function fmtDuration(seconds: number): string { - if (!Number.isFinite(seconds) || seconds < 0) return "—"; +// --- formatting ----------------------------------------------------------- +function fmtDur(seconds: number): string { + if (!Number.isFinite(seconds) || seconds <= 0) return ""; const h = Math.floor(seconds / 3600); const m = Math.floor((seconds % 3600) / 60); const s = Math.floor(seconds % 60); return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`; } +const hostOf = (lastResource: string) => (lastResource || "").split("/")[0] || ""; -// last_resource is "host/procid"; show just the host. -const hostOf = (lastResource: string) => (lastResource || "").split("/")[0] || "—"; +// --- detection (CueGUI StuckFramePlugin parity) --------------------------- +type Metrics = { runtime: number; llu: number; percentStuck: number; avg: number }; + +function metricsOf(f: StuckFrame, now: number): Metrics { + const runtime = f.startTime ? now - f.startTime : 0; + const llu = f.state === "RUNNING" && f.lluTime ? now - f.lluTime : 0; + const percentStuck = runtime > 0 ? llu / runtime : 0; + return { runtime, llu, percentStuck, avg: f.avgFrameSec }; +} + +// The catch-all filter (index 0) applies unless a later, service-specific +// filter matches the frame's service. +function pickFilter(f: StuckFrame, filters: StuckFilter[]): StuckFilter | undefined { + const specific = filters.find((flt, i) => i > 0 && flt.service && flt.service === f.service); + return specific ?? filters[0]; +} + +function isExcluded(f: StuckFrame, filter: StuckFilter): boolean { + const keywords = filter.regex.split(",").map((s) => s.trim()).filter(Boolean); + return keywords.some((kw) => { + try { + const re = new RegExp(kw, "i"); + return re.test(f.jobName) || re.test(f.layerName); + } catch { + const k = kw.toLowerCase(); + return f.jobName.toLowerCase().includes(k) || f.layerName.toLowerCase().includes(k); + } + }); +} + +// Mirrors CueGUI: lluTime > minLLU AND %stuck > threshold AND runtime > +// avg*avgComp% AND %stuck < 1.1 AND runtime > 500s. +function isStuck(f: StuckFrame, filter: StuckFilter | undefined, now: number): boolean { + if (!filter || !filter.enabled) return false; + if (isExcluded(f, filter)) return false; + const { runtime, llu, percentStuck, avg } = metricsOf(f, now); + return ( + llu > filter.minLlu * 60 && + percentStuck * 100 > filter.percentStuck && + runtime > (avg * filter.avgComp) / 100 && + percentStuck < 1.1 && + runtime > 500 + ); +} + +type MenuState = + | { kind: "frame"; x: number; y: number; frame: StuckFrame } + | { kind: "job"; x: number; y: number; jobName: string }; export default function StuckFramesPage() { + const router = useRouter(); const { data: session } = useSession(); const username = session?.user?.name ?? session?.user?.email ?? "cueweb"; - const [frames, setFrames] = React.useState(null); - const [thresholdHours, setThresholdHours] = React.useState(DEFAULT_HOURS); + const [raw, setRaw] = React.useState(null); const [now, setNow] = React.useState(() => Date.now() / 1000); + const [filters, setFilters] = React.useState([{ ...DEFAULT_FILTER }]); + const [autoRefresh, setAutoRefresh] = React.useState(false); + const [notify, setNotify] = React.useState(false); + const [loading, setLoading] = React.useState(false); + + // Client-side removals: "Frame/Job Not Stuck". + const [hiddenFrames, setHiddenFrames] = React.useState>(new Set()); + const [hiddenJobs, setHiddenJobs] = React.useState>(new Set()); + + const [lastLines, setLastLines] = React.useState>({}); + const [menu, setMenu] = React.useState(null); + const [coreUp, setCoreUp] = React.useState<{ targets: { id: string; name: string }[]; cores: string } | null>(null); const [busyId, setBusyId] = React.useState(null); + // Restore persisted filters on mount. + React.useEffect(() => { + const stored = window.localStorage.getItem(FILTERS_KEY); + if (stored) { + try { + const parsed = JSON.parse(stored); + if (Array.isArray(parsed) && parsed.length > 0) setFilters(parsed); + } catch { + /* ignore corrupt value */ + } + } + }, []); + + function persistFilters(next: StuckFilter[]) { + setFilters(next); + window.localStorage.setItem(FILTERS_KEY, JSON.stringify(next)); + } + const load = React.useCallback(async (isCancelled?: () => boolean) => { + setLoading(true); try { const data = await getStuckFrames(); if (isCancelled?.()) return; - setFrames(data); + setRaw(data); setNow(Date.now() / 1000); } catch (err) { if (isCancelled?.()) return; handleError(err, "Could not load stuck frames"); - setFrames((prev) => prev ?? []); + setRaw((prev) => prev ?? []); + } finally { + if (!isCancelled?.()) setLoading(false); } }, []); React.useEffect(() => { - // Restore the persisted threshold on mount (kept out of the initial state - // to avoid an SSR/client hydration mismatch). - const stored = window.localStorage.getItem(THRESHOLD_KEY); - if (stored) { - const n = Number(stored); - if (Number.isFinite(n) && n >= MIN_HOURS && n <= MAX_HOURS) setThresholdHours(n); - } let cancelled = false; - const isCancelled = () => cancelled; - load(isCancelled); - const interval = setInterval(() => load(isCancelled), REFRESH_MS); + load(() => cancelled); return () => { cancelled = true; - clearInterval(interval); }; }, [load]); - function changeThreshold(hours: number) { - setThresholdHours(hours); - window.localStorage.setItem(THRESHOLD_KEY, String(hours)); + // Auto-refresh. CueGUI refreshes ~every 30 min; a web monitor wants fresher + // data, so this polls every 60s while enabled. Fires a desktop notification + // on completion when armed and stuck frames are present. + React.useEffect(() => { + if (!autoRefresh) return; + let cancelled = false; + const id = setInterval(async () => { + await load(() => cancelled); + if (cancelled) return; + if (notify && typeof Notification !== "undefined" && Notification.permission === "granted") { + new Notification("CueWeb: stuck-frame scan complete"); + } + }, AUTO_REFRESH_MS); + return () => { + cancelled = true; + clearInterval(id); + }; + }, [autoRefresh, notify, load]); + + function toggleNotify(checked: boolean) { + setNotify(checked); + if (checked && typeof Notification !== "undefined" && Notification.permission === "default") { + Notification.requestPermission(); + } } - const runtimeOf = React.useCallback( - (f: StuckFrame) => (f.startTime ? now - f.startTime : 0), - [now], - ); + // Services present in the data, for the add-filter service dropdown. + const availableServices = React.useMemo(() => { + const set = new Set(); + (raw ?? []).forEach((f) => f.service && set.add(f.service)); + return Array.from(set).sort(); + }, [raw]); + + // Apply detection + client-side removals, group by job. + const groups = React.useMemo(() => { + if (!raw) return null; + const stuck = raw.filter( + (f) => + !hiddenFrames.has(f.id) && + !hiddenJobs.has(f.jobName) && + isStuck(f, pickFilter(f, filters), now), + ); + const byJob = new Map(); + for (const f of stuck) { + const arr = byJob.get(f.jobName) ?? []; + arr.push(f); + byJob.set(f.jobName, arr); + } + return Array.from(byJob.entries()) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([jobName, frames]) => ({ + jobName, + frames: frames.sort((a, b) => metricsOf(b, now).runtime - metricsOf(a, now).runtime), + })); + }, [raw, filters, now, hiddenFrames, hiddenJobs]); - const stuck = React.useMemo(() => { - if (!frames) return null; - const thresholdSeconds = thresholdHours * 3600; - return frames - .filter((f) => runtimeOf(f) > thresholdSeconds) - .sort((a, b) => runtimeOf(b) - runtimeOf(a)); - }, [frames, thresholdHours, runtimeOf]); + const totalStuck = groups?.reduce((n, g) => n + g.frames.length, 0) ?? 0; - // Strip the page-only jobId/jobName before sending the frame to a Cuebot RPC - // (they are not Frame proto fields). + // Lazily fetch the last log line for visible stuck frames. + React.useEffect(() => { + if (!groups) return; + const pending = groups + .flatMap((g) => g.frames) + .filter((f) => f.jobLogDir && lastLines[f.id] === undefined) + .slice(0, 50); // bound per pass + if (pending.length === 0) return; + let cancelled = false; + (async () => { + const entries = await Promise.all( + pending.map(async (f) => { + const logPath = `${f.jobLogDir}/${f.jobName}.${f.name}.rqlog`; + const line = await getStuckFrameLastLine(logPath); + return [f.id, line] as const; + }), + ); + if (cancelled) return; + setLastLines((prev) => { + const next = { ...prev }; + for (const [id, line] of entries) next[id] = line; + return next; + }); + })(); + return () => { + cancelled = true; + }; + }, [groups, lastLines]); + + // Close the context menu on any outside interaction. + React.useEffect(() => { + if (!menu) return; + const close = () => setMenu(null); + const onKey = (e: KeyboardEvent) => e.key === "Escape" && setMenu(null); + window.addEventListener("click", close); + window.addEventListener("scroll", close, true); + window.addEventListener("keydown", onKey); + return () => { + window.removeEventListener("click", close); + window.removeEventListener("scroll", close, true); + window.removeEventListener("keydown", onKey); + }; + }, [menu]); + + // --- helpers ------------------------------------------------------------- function toFrame(sf: StuckFrame): Frame { - const { jobId: _jobId, jobName: _jobName, ...frame } = sf; + const { + jobId: _a, jobName: _b, jobLogDir: _c, jobHasComment: _d, + service: _e, avgFrameSec: _f, layerId: _g, layerMinCores: _h, + ...frame + } = sf; return frame as Frame; } - async function handleRetry(sf: StuckFrame) { - setBusyId(sf.id); - try { - await retryFrames([toFrame(sf)]); - await load(); - } finally { - setBusyId(null); + function openLog(f: StuckFrame) { + const logDir = `${f.jobLogDir}/${f.jobName}.${f.name}.rqlog`; + const params = new URLSearchParams({ frameId: f.id, frameLogDir: logDir, username }); + window.open(`/frames/${encodeURIComponent(f.name)}?${params.toString()}`, "_blank", "noopener,noreferrer"); + } + + function exportLog(frames: StuckFrame[]) { + // Web adaptation of CueGUI's YAML "stuck_frames_db" file: a JSON download + // (the browser can't write to a fileshare). + const db: Record> = {}; + for (const f of frames) { + const { runtime, llu, avg } = metricsOf(f, now); + const byJob = db[f.jobName] ?? (db[f.jobName] = {}); + byJob[`${f.number}-${Math.floor(now)}`] = { + layer: f.layerName, + host: f.lastResource, + llu, + runtime, + average: avg, + log: lastLines[f.id] ?? "", + }; } + const blob = new Blob([JSON.stringify(db, null, 2)], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "stuck_frames.json"; + a.click(); + URL.revokeObjectURL(url); + toastSuccess(`Logged ${frames.length} stuck frame(s)`); } - async function handleKill(sf: StuckFrame) { - setBusyId(sf.id); + function hideFrame(f: StuckFrame) { + setHiddenFrames((prev) => new Set(prev).add(f.id)); + } + function hideJob(jobName: string) { + setHiddenJobs((prev) => new Set(prev).add(jobName)); + } + function addJobToExcludes(jobName: string) { + // Append the job name to the catch-all filter's exclude keywords. + persistFilters( + filters.map((flt, i) => + i === 0 + ? { ...flt, regex: flt.regex ? `${flt.regex}, ${jobName}` : jobName } + : flt, + ), + ); + toastSuccess(`Excluded ${jobName}`); + } + + async function act(f: StuckFrame, fn: () => Promise) { + setBusyId(f.id); + setMenu(null); try { - await killFrames([toFrame(sf)], username, `Manual frame kill from CueWeb Stuck Frames by ${username}`); + await fn(); + hideFrame(f); await load(); } finally { setBusyId(null); } } + const retry = (f: StuckFrame) => act(f, () => retryFrames([toFrame(f)])); + const eat = (f: StuckFrame) => act(f, () => eatFrames([toFrame(f)])); + const kill = (f: StuckFrame) => + act(f, () => killFrames([toFrame(f)], username, `Manual frame kill from CueWeb Stuck Frames by ${username}`)); + + function openCoreUpForFrame(f: StuckFrame) { + setMenu(null); + if (!f.layerId) return; + setCoreUp({ targets: [{ id: f.layerId, name: f.layerName }], cores: String(Math.max(1, f.layerMinCores || 1)) }); + } + function openCoreUpForJob(jobName: string) { + setMenu(null); + const frames = (raw ?? []).filter((f) => f.jobName === jobName && f.layerId); + const seen = new Map(); + frames.forEach((f) => seen.set(f.layerId, f.layerName)); + if (seen.size === 0) return; + setCoreUp({ targets: Array.from(seen.entries()).map(([id, name]) => ({ id, name })), cores: "1" }); + } + async function applyCoreUp() { + if (!coreUp) return; + const cores = Number(coreUp.cores); + if (!Number.isFinite(cores) || cores < 0) return; + await Promise.all(coreUp.targets.map((t) => setLayerMinCores(t, cores))); + setCoreUp(null); + await load(); + } + + // --- render -------------------------------------------------------------- + const menuItemCls = "block w-full rounded px-2 py-1.5 text-left hover:bg-accent disabled:opacity-50"; + return (
-

Stuck Frames

- -
- - {stuck ? ( - - {stuck.length} {stuck.length === 1 ? "frame" : "frames"} - - ) : null} +
+

Stuck Frames

+
+ + + + +
- {stuck === null ? ( +
+ +
+ + {groups === null ? (
- ) : stuck.length === 0 ? ( + ) : totalStuck === 0 ? (

- No frames have been running longer than {thresholdHours}{" "} - {thresholdHours === 1 ? "hour" : "hours"}. + No stuck frames detected with the current filters.

) : (
- - + + + - + + + - {stuck.map((f) => ( - - - - - - - { + e.preventDefault(); + setMenu({ kind: "job", x: e.clientX, y: e.clientY, jobName: g.jobName }); + }} + > + + + + {g.frames.map((f) => { + const m = metricsOf(f, now); + return ( + { + e.preventDefault(); + setMenu({ kind: "frame", x: e.clientX, y: e.clientY, frame: f }); + }} > - Retry - - - - - + + + + + + + + + + ); + })} + ))}
JobLayerName{/* comment icon col */} Frame HostLLU RuntimeActions% StuckAverageLast Line
{f.jobName}{f.layerName}{f.number}{hostOf(f.lastResource)}{fmtDuration(runtimeOf(f))} -
-
{g.jobName} + {g.frames[0]?.jobHasComment ? ( + + ) : null} + +
{f.layerName} + {f.number}{hostOf(f.lastResource)}{fmtDur(m.llu)}{fmtDur(m.runtime)}{(m.percentStuck * 100).toFixed(2)}{fmtDur(m.avg)} + {lastLines[f.id] ?? ""} + {busyId === f.id ? " …" : ""} +
)} + + {!loading && groups !== null ? ( +

+ {totalStuck} stuck frame(s) across {groups.length} job(s). +

+ ) : null} + + {/* Context menu */} + {menu ? ( +
e.stopPropagation()} + > + {menu.kind === "frame" ? ( + <> + + + {menu.frame.retryCount >= 1 ? ( + + ) : null} +
+ + + +
+ + + + +
+ + + +
+ + + + ) : ( + <> + +
+ + + +
+ + + )} +
+ ) : null} + + {/* Core Up dialog */} + !o && setCoreUp(null)}> + + + Core Up + +
+

+ Set minimum cores for {coreUp?.targets.length === 1 ? `layer "${coreUp.targets[0].name}"` : `${coreUp?.targets.length ?? 0} layer(s)`}. +

+ setCoreUp((c) => (c ? { ...c, cores: e.target.value } : c))} + aria-label="Minimum cores" + /> +
+ + + + + +
); } diff --git a/cueweb/app/utils/action_utils.ts b/cueweb/app/utils/action_utils.ts index 182b437d0..c291c0acf 100644 --- a/cueweb/app/utils/action_utils.ts +++ b/cueweb/app/utils/action_utils.ts @@ -163,6 +163,13 @@ export async function retryFrames(frames: Frame[]) { await performAction(endpoint, bodyAr, `Retried ${frames.length} frame(s)`); } +// Set a layer's minimum cores (CueGUI Stuck Frame "Core Up"). cores is a float +// core count. Returns success so callers can gate a refresh. +export async function setLayerMinCores(layer: { id: string; name?: string }, cores: number): Promise { + const endpoint = "/api/layer/action/setmincores"; + return performAction(endpoint, [JSON.stringify({ layer, cores })], `Set min cores to ${cores}`); +} + /**************************************/ // Unbook /**************************************/ diff --git a/cueweb/app/utils/get_utils.ts b/cueweb/app/utils/get_utils.ts index 448fdad6a..cac6e5b48 100644 --- a/cueweb/app/utils/get_utils.ts +++ b/cueweb/app/utils/get_utils.ts @@ -187,18 +187,44 @@ export async function getFrames(body: string): Promise { return response ? response : []; } -// A running frame plus its parent job, for the Stuck Frames page. -export type StuckFrame = Frame & { jobId: string; jobName: string }; +// A running frame plus the job/layer context the Stuck Frames page needs to +// apply CueGUI's per-service stuck-detection predicate (service + average +// frame time) and to act on the row (job, log dir, comment flag). +export type StuckFrame = Frame & { + jobId: string; + jobName: string; + jobLogDir: string; + jobHasComment: boolean; + service: string; + avgFrameSec: number; + layerId: string; + layerMinCores: number; +}; // Fetch every RUNNING frame across all unfinished jobs (server-aggregated via -// /api/stuck-frames). The Stuck Frames page applies the running-time threshold -// locally so the slider stays instant. +// /api/stuck-frames), each stamped with its service and average frame time. +// The Stuck Frames page applies the detection thresholds locally so the +// filters stay instant. export async function getStuckFrames(): Promise { const ENDPOINT = "/api/stuck-frames"; const response = await accessGetApi(ENDPOINT, JSON.stringify({})); return Array.isArray(response) ? response : []; } +// Best-effort fetch of a frame log's last line (the "Last Line" column). Empty +// when the log filesystem isn't reachable from the web server. +export async function getStuckFrameLastLine(logPath: string): Promise { + if (!logPath) return ""; + const base = process.env.NEXT_PUBLIC_URL ?? ""; + try { + const resp = await fetch(`${base}/api/stuck-frames/lastline?path=${encodeURIComponent(logPath)}`); + const json = await resp.json(); + return typeof json?.lastLine === "string" ? json.lastLine : ""; + } catch { + return ""; + } +} + // Fetch a pending job based on the request body export async function getPendingJob(body: string): Promise { const ENDPOINT = "/api/job/getjob"; diff --git a/cueweb/components/ui/stuck-frame-filters.tsx b/cueweb/components/ui/stuck-frame-filters.tsx new file mode 100644 index 000000000..04d7ac98f --- /dev/null +++ b/cueweb/components/ui/stuck-frame-filters.tsx @@ -0,0 +1,192 @@ +"use client"; + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as React from "react"; +import { Plus, X } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; + +// One detection filter (CueGUI StuckFrameBar). service === "" is the catch-all +// ("All" when it's the only filter, "All Other Types" when service filters +// exist). The four thresholds mirror CueGUI's spinboxes. +export type StuckFilter = { + service: string; + regex: string; // exclude keywords, comma-separated + percentStuck: number; // % of runtime since last log update + minLlu: number; // minutes + avgComp: number; // % of average completion time + runtime: number; // minutes + enabled: boolean; +}; + +// CueGUI defaults: [percentStuck, minLlu, avgComp, runtime]. +export const SERVICE_DEFAULTS: Record = { + preprocess: [1, 1, 115, 10], + nuke: [50, 5, 115, 10], + arnold: [50, 60, 115, 120], +}; + +export const DEFAULT_FILTER: StuckFilter = { + service: "", + regex: "", + percentStuck: 50, + minLlu: 30, + avgComp: 115, + runtime: 60, + enabled: true, +}; + +export function makeServiceFilter(service: string): StuckFilter { + const d = SERVICE_DEFAULTS[service]; + return d + ? { service, regex: "", percentStuck: d[0], minLlu: d[1], avgComp: d[2], runtime: d[3], enabled: true } + : { ...DEFAULT_FILTER, service }; +} + +const NUM = "h-8 w-20 text-right"; + +function NumberField({ + label, + suffix, + value, + disabled, + onChange, +}: { + label: string; + suffix: string; + value: number; + disabled: boolean; + onChange: (n: number) => void; +}) { + return ( + + ); +} + +export function StuckFrameFilters({ + filters, + onChange, + availableServices, +}: { + filters: StuckFilter[]; + onChange: (filters: StuckFilter[]) => void; + availableServices: string[]; +}) { + function update(index: number, patch: Partial) { + onChange(filters.map((f, i) => (i === index ? { ...f, ...patch } : f))); + } + function addFilter() { + // Default the new filter to the first available service not already used. + const used = new Set(filters.map((f) => f.service)); + const next = availableServices.find((s) => !used.has(s)) ?? ""; + onChange([...filters, makeServiceFilter(next)]); + } + function removeFilter(index: number) { + onChange(filters.filter((_, i) => i !== index)); + } + + const hasServiceFilters = filters.some((f, i) => i > 0); + + return ( +
+ {filters.map((f, i) => { + const isCatchAll = i === 0; + const disabled = !f.enabled; + return ( +
+
+ Layer Service + {isCatchAll ? ( + + {hasServiceFilters ? "All Other Types" : "All"} + + ) : ( + + )} +
+ + update(i, { percentStuck: n })} /> + update(i, { minLlu: n })} /> + update(i, { avgComp: n })} /> + update(i, { runtime: n })} /> + + + + + + {isCatchAll ? ( + + ) : ( + + )} +
+ ); + })} +
+ ); +} From e57ab7fdc205706a80473de31ec31e37feacdcb9 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 15 Jun 2026 13:06:08 -0700 Subject: [PATCH 3/7] [cueweb] Stuck Frames: bound per-job fan-out, paginate frames, surface load failures Address review feedback on the Stuck Frames endpoint and helper: - route.ts: cap the per-job GetFrames/GetLayers fan-out with a fixed-size worker pool (MAX_CONCURRENT_JOBS = 16) instead of Promise.all over every unfinished job, so a large farm no longer fires hundreds/thousands of concurrent gateway calls on each 30s poll. - route.ts: paginate GetFrames via getRunningFrames(), looping pages until a short page arrives (bounded by MAX_FRAME_PAGES) instead of fetching only page 1. Jobs with more than MAX_FRAMES_PER_JOB running frames are no longer silently truncated, so Retry/Kill stay available for every row. - get_utils.ts: getStuckFrames now throws on a non-array response (mirroring getHosts) instead of collapsing a failed fetch into []. A backend outage reaches the page's catch/error path rather than rendering the empty state. --- cueweb/app/api/stuck-frames/route.ts | 79 ++++++++++++++++++++++------ cueweb/app/utils/get_utils.ts | 5 +- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/cueweb/app/api/stuck-frames/route.ts b/cueweb/app/api/stuck-frames/route.ts index 5690fe29a..08f916057 100644 --- a/cueweb/app/api/stuck-frames/route.ts +++ b/cueweb/app/api/stuck-frames/route.ts @@ -31,6 +31,10 @@ import { NextRequest, NextResponse } from "next/server"; const RUNNING_STATE = 2; // FrameState.RUNNING (proto/src/job.proto) const MAX_FRAMES_PER_JOB = 1000; +// Cap the per-job GetFrames/GetLayers fan-out. Without a bound, a farm with +// thousands of unfinished jobs would fire that many concurrent gateway calls +// on every 30s poll; this keeps at most N jobs in flight at a time. +const MAX_CONCURRENT_JOBS = 16; async function gatewayJson(endpoint: string, body: string): Promise { try { @@ -43,6 +47,57 @@ async function gatewayJson(endpoint: string, body: string): Promise } } +// Map over items with a fixed-size worker pool, preserving input order. Plain +// promises (no extra dependency) so the route stays self-contained. +async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let cursor = 0; + async function worker() { + while (true) { + const index = cursor++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + } + const workerCount = Math.min(limit, items.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +// Safety bound on pagination so a malformed/never-shrinking response can't spin +// forever: MAX_FRAME_PAGES * MAX_FRAMES_PER_JOB frames per job. +const MAX_FRAME_PAGES = 50; + +// Fetch every RUNNING frame for a job, paging through GetFrames until a short +// page arrives. A single page caps at MAX_FRAMES_PER_JOB, so a job with more +// running frames than that would otherwise be silently truncated. +async function getRunningFrames(job: any): Promise { + const all: any[] = []; + for (let page = 1; page <= MAX_FRAME_PAGES; page++) { + const framesData = await gatewayJson( + "/job.JobInterface/GetFrames", + JSON.stringify({ + job: { id: job.id, name: job.name }, + req: { + include_finished: false, + page, + limit: MAX_FRAMES_PER_JOB, + states: { frame_states: [RUNNING_STATE] }, + }, + }), + ); + const batch: any[] = framesData?.frames?.frames ?? []; + all.push(...batch); + // A short (or empty/failed) page means we've reached the end. + if (batch.length < MAX_FRAMES_PER_JOB) break; + } + return all; +} + export async function POST(_request: NextRequest) { try { const jobsData = await gatewayJson( @@ -54,21 +109,12 @@ export async function POST(_request: NextRequest) { } const jobs: any[] = jobsData?.jobs?.jobs ?? []; - const perJob = await Promise.all( - jobs.map(async (job) => { - const [framesData, layersData] = await Promise.all([ - gatewayJson( - "/job.JobInterface/GetFrames", - JSON.stringify({ - job: { id: job.id, name: job.name }, - req: { - include_finished: false, - page: 1, - limit: MAX_FRAMES_PER_JOB, - states: { frame_states: [RUNNING_STATE] }, - }, - }), - ), + const perJob = await mapWithConcurrency( + jobs, + MAX_CONCURRENT_JOBS, + async (job) => { + const [frames, layersData] = await Promise.all([ + getRunningFrames(job), gatewayJson( "/job.JobInterface/GetLayers", JSON.stringify({ job: { id: job.id, name: job.name } }), @@ -91,7 +137,6 @@ export async function POST(_request: NextRequest) { }); } - const frames: any[] = framesData?.frames?.frames ?? []; return frames .filter((f) => f.state === "RUNNING") .map((f) => { @@ -108,7 +153,7 @@ export async function POST(_request: NextRequest) { layerMinCores: info?.minCores ?? 0, }; }); - }), + }, ); return NextResponse.json({ data: perJob.flat() }, { status: 200 }); diff --git a/cueweb/app/utils/get_utils.ts b/cueweb/app/utils/get_utils.ts index cac6e5b48..d07b63973 100644 --- a/cueweb/app/utils/get_utils.ts +++ b/cueweb/app/utils/get_utils.ts @@ -208,7 +208,10 @@ export type StuckFrame = Frame & { export async function getStuckFrames(): Promise { const ENDPOINT = "/api/stuck-frames"; const response = await accessGetApi(ENDPOINT, JSON.stringify({})); - return Array.isArray(response) ? response : []; + if (!Array.isArray(response)) { + throw new Error("Failed to load stuck frames from Cuebot."); + } + return response; } // Best-effort fetch of a frame log's last line (the "Last Line" column). Empty From 5870b618d74b9eed9433fa670031adc8fc317e75 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 15 Jun 2026 13:49:11 -0700 Subject: [PATCH 4/7] [cueweb] Stuck Frames: address review feedback - page: key job grouping, hide, Core Up and comment links by jobId instead of jobName, so two jobs sharing a name aren't merged or acted on together. - page / action_utils: retryFrames / eatFrames / killFrames now return a success boolean, and act() only hides the frame + reloads when the action succeeded (a failed Retry/Eat/Kill no longer removes the frame from view). - stuck-frame-filters: ignore non-finite numeric input so a NaN can't poison filter thresholds or persisted state; don't add an empty service filter when all services are taken (and disable the add button in that case). --- cueweb/app/stuck-frames/page.tsx | 56 +++++++++++--------- cueweb/app/utils/action_utils.ts | 12 ++--- cueweb/components/ui/stuck-frame-filters.tsx | 17 ++++-- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/cueweb/app/stuck-frames/page.tsx b/cueweb/app/stuck-frames/page.tsx index 1aa0d5ed1..d2a10361d 100644 --- a/cueweb/app/stuck-frames/page.tsx +++ b/cueweb/app/stuck-frames/page.tsx @@ -102,7 +102,7 @@ function isStuck(f: StuckFrame, filter: StuckFilter | undefined, now: number): b type MenuState = | { kind: "frame"; x: number; y: number; frame: StuckFrame } - | { kind: "job"; x: number; y: number; jobName: string }; + | { kind: "job"; x: number; y: number; jobId: string; jobName: string }; export default function StuckFramesPage() { const router = useRouter(); @@ -200,24 +200,26 @@ export default function StuckFramesPage() { return Array.from(set).sort(); }, [raw]); - // Apply detection + client-side removals, group by job. + // Apply detection + client-side removals, group by job. Identity is jobId, + // not jobName, so two jobs sharing a name aren't merged or acted on together. const groups = React.useMemo(() => { if (!raw) return null; const stuck = raw.filter( (f) => !hiddenFrames.has(f.id) && - !hiddenJobs.has(f.jobName) && + !hiddenJobs.has(f.jobId) && isStuck(f, pickFilter(f, filters), now), ); - const byJob = new Map(); + const byJob = new Map(); for (const f of stuck) { - const arr = byJob.get(f.jobName) ?? []; - arr.push(f); - byJob.set(f.jobName, arr); + const entry = byJob.get(f.jobId) ?? { jobName: f.jobName, frames: [] }; + entry.frames.push(f); + byJob.set(f.jobId, entry); } return Array.from(byJob.entries()) - .sort((a, b) => a[0].localeCompare(b[0])) - .map(([jobName, frames]) => ({ + .sort((a, b) => a[1].jobName.localeCompare(b[1].jobName)) + .map(([jobId, { jobName, frames }]) => ({ + jobId, jobName, frames: frames.sort((a, b) => metricsOf(b, now).runtime - metricsOf(a, now).runtime), })); @@ -314,8 +316,8 @@ export default function StuckFramesPage() { function hideFrame(f: StuckFrame) { setHiddenFrames((prev) => new Set(prev).add(f.id)); } - function hideJob(jobName: string) { - setHiddenJobs((prev) => new Set(prev).add(jobName)); + function hideJob(jobId: string) { + setHiddenJobs((prev) => new Set(prev).add(jobId)); } function addJobToExcludes(jobName: string) { // Append the job name to the catch-all filter's exclude keywords. @@ -329,13 +331,17 @@ export default function StuckFramesPage() { toastSuccess(`Excluded ${jobName}`); } - async function act(f: StuckFrame, fn: () => Promise) { + async function act(f: StuckFrame, fn: () => Promise) { setBusyId(f.id); setMenu(null); try { - await fn(); - hideFrame(f); - await load(); + // Only remove the frame from view when the backend action succeeded; + // performAction resolves false (without throwing) on failure. + const ok = await fn(); + if (ok) { + hideFrame(f); + await load(); + } } finally { setBusyId(null); } @@ -351,9 +357,9 @@ export default function StuckFramesPage() { if (!f.layerId) return; setCoreUp({ targets: [{ id: f.layerId, name: f.layerName }], cores: String(Math.max(1, f.layerMinCores || 1)) }); } - function openCoreUpForJob(jobName: string) { + function openCoreUpForJob(jobId: string) { setMenu(null); - const frames = (raw ?? []).filter((f) => f.jobName === jobName && f.layerId); + const frames = (raw ?? []).filter((f) => f.jobId === jobId && f.layerId); const seen = new Map(); frames.forEach((f) => seen.set(f.layerId, f.layerName)); if (seen.size === 0) return; @@ -425,12 +431,12 @@ export default function StuckFramesPage() { {groups.map((g) => ( - + { e.preventDefault(); - setMenu({ kind: "job", x: e.clientX, y: e.clientY, jobName: g.jobName }); + setMenu({ kind: "job", x: e.clientX, y: e.clientY, jobId: g.jobId, jobName: g.jobName }); }} > {g.jobName} @@ -440,7 +446,7 @@ export default function StuckFramesPage() { title="View comments" onClick={() => window.open( - `/jobs/${encodeURIComponent(g.jobName)}/comments?jobId=${encodeURIComponent(g.frames[0].jobId)}`, + `/jobs/${encodeURIComponent(g.jobName)}/comments?jobId=${encodeURIComponent(g.jobId)}`, "_blank", "noopener,noreferrer", ) @@ -517,20 +523,20 @@ export default function StuckFramesPage() {
- +
) : ( <> - +
- + - +
- + )}
diff --git a/cueweb/app/utils/action_utils.ts b/cueweb/app/utils/action_utils.ts index c291c0acf..8fffa5a26 100644 --- a/cueweb/app/utils/action_utils.ts +++ b/cueweb/app/utils/action_utils.ts @@ -72,14 +72,14 @@ export async function killLayers(layers: Layer[], username: string, reason: stri await performAction(endpoint, bodyAr, `Killed ${layers.length} layer(s)`); } -export async function killFrames(frames: Frame[], username: string, reason: string) { +export async function killFrames(frames: Frame[], username: string, reason: string): Promise { const endpoint = "/api/frame/action/kill"; const bodyAr = frames.map(frame => JSON.stringify({ frame, username, reason })); - await performAction(endpoint, bodyAr, `Killed ${frames.length} frame(s)`); + return performAction(endpoint, bodyAr, `Killed ${frames.length} frame(s)`); } @@ -104,12 +104,12 @@ export async function eatLayersFrames(layers: Layer[]) { await performAction(endpoint, bodyAr, `Ate ${layers.length} layer(s)`); } -export async function eatFrames(frames: Frame[]) { +export async function eatFrames(frames: Frame[]): Promise { const endpoint = "/api/frame/action/eat"; const bodyAr = frames.map(frame => JSON.stringify({ frame })); - await performAction(endpoint, bodyAr, `Ate ${frames.length} frame(s)`); + return performAction(endpoint, bodyAr, `Ate ${frames.length} frame(s)`); } @@ -155,12 +155,12 @@ export async function retryLayersDeadFrames(layers: Layer[]) { } } -export async function retryFrames(frames: Frame[]) { +export async function retryFrames(frames: Frame[]): Promise { const endpoint = "/api/frame/action/retry"; const bodyAr = frames.map(frame => JSON.stringify({ frame })); - await performAction(endpoint, bodyAr, `Retried ${frames.length} frame(s)`); + return performAction(endpoint, bodyAr, `Retried ${frames.length} frame(s)`); } // Set a layer's minimum cores (CueGUI Stuck Frame "Core Up"). cores is a float diff --git a/cueweb/components/ui/stuck-frame-filters.tsx b/cueweb/components/ui/stuck-frame-filters.tsx index 04d7ac98f..a05490cf0 100644 --- a/cueweb/components/ui/stuck-frame-filters.tsx +++ b/cueweb/components/ui/stuck-frame-filters.tsx @@ -84,7 +84,12 @@ function NumberField({ min={1} value={value} disabled={disabled} - onChange={(e) => onChange(Number(e.target.value))} + onChange={(e) => { + // Ignore transient/invalid input (empty field, partial entry) so a + // NaN can't poison the filter comparisons or persisted state. + const n = e.currentTarget.valueAsNumber; + if (Number.isFinite(n)) onChange(n); + }} className={NUM} aria-label={label} /> @@ -108,8 +113,10 @@ export function StuckFrameFilters({ } function addFilter() { // Default the new filter to the first available service not already used. - const used = new Set(filters.map((f) => f.service)); - const next = availableServices.find((s) => !used.has(s)) ?? ""; + // Bail out if every service is taken, rather than adding an empty row. + const used = new Set(filters.map((f) => f.service).filter(Boolean)); + const next = availableServices.find((s) => !used.has(s)); + if (!next) return; onChange([...filters, makeServiceFilter(next)]); } function removeFilter(index: number) { @@ -117,6 +124,8 @@ export function StuckFrameFilters({ } const hasServiceFilters = filters.some((f, i) => i > 0); + const usedServices = new Set(filters.map((f) => f.service).filter(Boolean)); + const canAddFilter = availableServices.some((s) => !usedServices.has(s)); return (
@@ -176,7 +185,7 @@ export function StuckFrameFilters({ {isCatchAll ? ( - ) : ( From f7de7fb453ba215d5fb53694b705eb71924ac52c Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 15 Jun 2026 14:06:51 -0700 Subject: [PATCH 5/7] [cueweb] Stuck Frames: address review feedback - page: auto-refresh notification now fires only when stuck frames are actually present. load() returns the fetched data so the interval applies the same detection + hidden filters and notifies with the count, instead of firing on every scan. - stuck-frame-filters: the service dropdown only offers the row's own service plus unused ones, preventing duplicate service-specific filters. - stuck-frame-filters: add aria-label to the icon-only add/remove filter buttons for screen-reader support. --- cueweb/app/stuck-frames/page.tsx | 31 +++++++++++++++----- cueweb/components/ui/stuck-frame-filters.tsx | 16 +++++----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/cueweb/app/stuck-frames/page.tsx b/cueweb/app/stuck-frames/page.tsx index d2a10361d..3b2faa562 100644 --- a/cueweb/app/stuck-frames/page.tsx +++ b/cueweb/app/stuck-frames/page.tsx @@ -143,17 +143,21 @@ export default function StuckFramesPage() { window.localStorage.setItem(FILTERS_KEY, JSON.stringify(next)); } - const load = React.useCallback(async (isCancelled?: () => boolean) => { + // Returns the loaded frames (null on cancel/error) so callers can act on the + // fresh data without waiting for the state/memo round-trip. + const load = React.useCallback(async (isCancelled?: () => boolean): Promise => { setLoading(true); try { const data = await getStuckFrames(); - if (isCancelled?.()) return; + if (isCancelled?.()) return null; setRaw(data); setNow(Date.now() / 1000); + return data; } catch (err) { - if (isCancelled?.()) return; + if (isCancelled?.()) return null; handleError(err, "Could not load stuck frames"); setRaw((prev) => prev ?? []); + return null; } finally { if (!isCancelled?.()) setLoading(false); } @@ -169,22 +173,33 @@ export default function StuckFramesPage() { // Auto-refresh. CueGUI refreshes ~every 30 min; a web monitor wants fresher // data, so this polls every 60s while enabled. Fires a desktop notification - // on completion when armed and stuck frames are present. + // on completion when armed and stuck frames are actually present. React.useEffect(() => { if (!autoRefresh) return; let cancelled = false; const id = setInterval(async () => { - await load(() => cancelled); - if (cancelled) return; + const data = await load(() => cancelled); + if (cancelled || !data) return; if (notify && typeof Notification !== "undefined" && Notification.permission === "granted") { - new Notification("CueWeb: stuck-frame scan complete"); + // Apply the same detection + hidden filters as the table so we only + // notify when a stuck frame would actually be shown. + const scanNow = Date.now() / 1000; + const stuckCount = data.filter( + (f) => + !hiddenFrames.has(f.id) && + !hiddenJobs.has(f.jobId) && + isStuck(f, pickFilter(f, filters), scanNow), + ).length; + if (stuckCount > 0) { + new Notification(`CueWeb: ${stuckCount} stuck frame(s) detected`); + } } }, AUTO_REFRESH_MS); return () => { cancelled = true; clearInterval(id); }; - }, [autoRefresh, notify, load]); + }, [autoRefresh, notify, load, filters, hiddenFrames, hiddenJobs]); function toggleNotify(checked: boolean) { setNotify(checked); diff --git a/cueweb/components/ui/stuck-frame-filters.tsx b/cueweb/components/ui/stuck-frame-filters.tsx index a05490cf0..648b5ba1c 100644 --- a/cueweb/components/ui/stuck-frame-filters.tsx +++ b/cueweb/components/ui/stuck-frame-filters.tsx @@ -151,11 +151,13 @@ export function StuckFrameFilters({ {f.service && !availableServices.includes(f.service) ? ( ) : null} - {availableServices.map((s) => ( - - ))} + {availableServices + .filter((s) => s === f.service || !usedServices.has(s)) + .map((s) => ( + + ))} )}
@@ -185,11 +187,11 @@ export function StuckFrameFilters({ {isCatchAll ? ( - ) : ( - )} From 0fc75ae706a2a8cf4c16b33ba251fc5760be1bc0 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 15 Jun 2026 15:46:04 -0700 Subject: [PATCH 6/7] [cueweb] Stuck Frames: address review feedback - lastline route: canonicalize the requested path with realpath (replacing the brittle ".." substring check), re-check the .rqlog extension on the real file, and restrict reads to an optional CUEWEB_LOG_ROOTS allow-list when set (unset = unrestricted, best-effort). Documented in .env.example. - setmincores route: reject non-finite (NaN/Infinity) and negative cores; keep fractional values allowed since the proto field is a float. - page: guard the auto-refresh interval with an in-flight flag so a slow backend can't pile up overlapping scans. - page: add an aria-label to the comments icon button for screen readers. --- cueweb/.env.example | 7 +++ .../app/api/layer/action/setmincores/route.ts | 13 +++- cueweb/app/api/stuck-frames/lastline/route.ts | 59 +++++++++++++++++-- cueweb/app/stuck-frames/page.tsx | 38 +++++++----- 4 files changed, 96 insertions(+), 21 deletions(-) diff --git a/cueweb/.env.example b/cueweb/.env.example index b4ff813e0..3f7e2e1c7 100644 --- a/cueweb/.env.example +++ b/cueweb/.env.example @@ -1,5 +1,12 @@ NEXT_PUBLIC_OPENCUE_ENDPOINT=http://your-rest-gateway-url.com +# Optional allow-list for the Stuck Frames "Last Line" reader +# (/api/stuck-frames/lastline), as a colon-separated list of absolute path +# prefixes. When set, only .rqlog files under one of these roots are read. +# When unset, reads are not restricted to a root (job log paths are +# site-specific). Set this to harden the deployment. +# CUEWEB_LOG_ROOTS=/mnt/logs:/shows + # Sentry values SENTRY_ENVIRONMENT='development' SENTRY_DSN = sentrydsn diff --git a/cueweb/app/api/layer/action/setmincores/route.ts b/cueweb/app/api/layer/action/setmincores/route.ts index 5d398a100..5d42012a6 100644 --- a/cueweb/app/api/layer/action/setmincores/route.ts +++ b/cueweb/app/api/layer/action/setmincores/route.ts @@ -32,8 +32,17 @@ export async function POST(request: NextRequest) { } catch { return NextResponse.json({ error: 'Invalid JSON in request body' }, { status: 400 }); } - if (!jsonBody || typeof jsonBody !== 'object' || !jsonBody.layer || typeof jsonBody.cores !== 'number') { - return NextResponse.json({ error: 'Invalid request body: layer and numeric cores are required' }, { status: 400 }); + // cores is a float proto field (fractional core counts are valid), so reject + // only non-finite (typeof NaN is "number") and negative values, not fractions. + if ( + !jsonBody || + typeof jsonBody !== 'object' || + !jsonBody.layer || + typeof jsonBody.cores !== 'number' || + !Number.isFinite(jsonBody.cores) || + jsonBody.cores < 0 + ) { + return NextResponse.json({ error: 'Invalid request body: layer and non-negative numeric cores are required' }, { status: 400 }); } const body = JSON.stringify(jsonBody); diff --git a/cueweb/app/api/stuck-frames/lastline/route.ts b/cueweb/app/api/stuck-frames/lastline/route.ts index c25cbc09c..bd31c1d45 100644 --- a/cueweb/app/api/stuck-frames/lastline/route.ts +++ b/cueweb/app/api/stuck-frames/lastline/route.ts @@ -17,22 +17,71 @@ import { NextRequest, NextResponse } from "next/server"; import { execFile as execFileCallback } from "child_process"; import { promisify } from "util"; +import { promises as fs } from "fs"; +import path from "path"; const execFile = promisify(execFileCallback); +// Optional per-site allow-list (colon-separated absolute prefixes). When set, +// only .rqlog files under one of these roots are read; when unset, reads aren't +// restricted to a root (job log paths are site-specific). +function allowedLogRoots(): string[] { + return (process.env.CUEWEB_LOG_ROOTS ?? "") + .split(":") + .map((r) => r.trim()) + .filter(Boolean); +} + // Returns the last non-empty line of a frame's .rqlog (the Stuck Frames // "Last Line" column, mirroring CueGUI's getLastLine). Best-effort: if the log // filesystem isn't mounted in this deployment, or the file is missing, it -// returns an empty line rather than erroring. execFile (no shell) + an rqlog -// path allowlist keep the caller-supplied path from being abused. +// returns an empty line rather than erroring. execFile (no shell) + canonical +// path validation (realpath, .rqlog extension, optional root allow-list) keep +// the caller-supplied path from being abused. export async function GET(request: NextRequest) { - const path = request.nextUrl.searchParams.get("path"); - if (!path || !path.endsWith(".rqlog") || path.includes("..")) { + const rawPath = request.nextUrl.searchParams.get("path"); + if (!rawPath || !rawPath.endsWith(".rqlog")) { + return NextResponse.json({ lastLine: "" }, { status: 200 }); + } + + // Canonicalize (follows symlinks) so the extension / root checks apply to the + // real file rather than a lexical path. A missing/unreadable file resolves to + // the best-effort empty response. + let target: string; + try { + target = await fs.realpath(path.resolve(rawPath)); + } catch { return NextResponse.json({ lastLine: "" }, { status: 200 }); } + if (!target.endsWith(".rqlog")) { + return NextResponse.json({ lastLine: "" }, { status: 200 }); + } + + const rawRoots = allowedLogRoots(); + if (rawRoots.length > 0) { + const roots = ( + await Promise.all( + rawRoots.map(async (r) => { + try { + return await fs.realpath(path.resolve(r)); + } catch { + return null; + } + }), + ) + ).filter((r): r is string => r !== null); + const inAllowedRoot = roots.some((root) => { + const rel = path.relative(root, target); + return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel)); + }); + if (!inAllowedRoot) { + return NextResponse.json({ lastLine: "" }, { status: 200 }); + } + } + try { // tail the file, then keep the last non-blank line. - const { stdout } = await execFile("tail", ["-n", "20", "--", path], { + const { stdout } = await execFile("tail", ["-n", "20", "--", target], { timeout: 5000, maxBuffer: 1024 * 1024, }); diff --git a/cueweb/app/stuck-frames/page.tsx b/cueweb/app/stuck-frames/page.tsx index 3b2faa562..2c9882aa1 100644 --- a/cueweb/app/stuck-frames/page.tsx +++ b/cueweb/app/stuck-frames/page.tsx @@ -177,22 +177,31 @@ export default function StuckFramesPage() { React.useEffect(() => { if (!autoRefresh) return; let cancelled = false; + // Skip a tick if the previous scan is still running, so a slow/degraded + // backend can't pile up overlapping, out-of-order refreshes. + let inFlight = false; const id = setInterval(async () => { - const data = await load(() => cancelled); - if (cancelled || !data) return; - if (notify && typeof Notification !== "undefined" && Notification.permission === "granted") { - // Apply the same detection + hidden filters as the table so we only - // notify when a stuck frame would actually be shown. - const scanNow = Date.now() / 1000; - const stuckCount = data.filter( - (f) => - !hiddenFrames.has(f.id) && - !hiddenJobs.has(f.jobId) && - isStuck(f, pickFilter(f, filters), scanNow), - ).length; - if (stuckCount > 0) { - new Notification(`CueWeb: ${stuckCount} stuck frame(s) detected`); + if (inFlight) return; + inFlight = true; + try { + const data = await load(() => cancelled); + if (cancelled || !data) return; + if (notify && typeof Notification !== "undefined" && Notification.permission === "granted") { + // Apply the same detection + hidden filters as the table so we only + // notify when a stuck frame would actually be shown. + const scanNow = Date.now() / 1000; + const stuckCount = data.filter( + (f) => + !hiddenFrames.has(f.id) && + !hiddenJobs.has(f.jobId) && + isStuck(f, pickFilter(f, filters), scanNow), + ).length; + if (stuckCount > 0) { + new Notification(`CueWeb: ${stuckCount} stuck frame(s) detected`); + } } + } finally { + inFlight = false; } }, AUTO_REFRESH_MS); return () => { @@ -459,6 +468,7 @@ export default function StuckFramesPage() { {g.frames[0]?.jobHasComment ? (
) : totalStuck === 0 ? ( -

- No stuck frames detected with the current filters. -

+ loadError ? ( +

{loadError}

+ ) : ( +

+ No stuck frames detected with the current filters. +

+ ) ) : (