// ===== Evals page ===== const { useState: eUseState, useEffect: eUseEffect, useMemo: eUseMemo, useRef: eUseRef } = React; function eRelTime(ts) { if (!ts) return "—"; const d = (Date.now() - ts) / 1000; if (d < 0) return "just now"; if (d < 60) return `${Math.floor(d)}s ago`; if (d < 3600) return `${Math.floor(d / 60)}m ago`; if (d < 86400) return `${Math.floor(d / 3600)}h ago`; return `${Math.floor(d / 86400)}d ago`; } function fmtCost(c) { return `$${Number(c || 0).toFixed(4)}`; } function evalStatusLabel(status) { return ({ pending: "Queued", running: "Running", complete: "Completed", failed: "Failed", cancelled: "Cancelled" })[status] || status || "Unknown"; } function evalStatusClass(status, passRate) { if (status === "failed" || status === "cancelled") return "err"; if (status === "pending" || status === "running") return "warn"; return passRate >= 0.9 ? "ok" : passRate >= 0.7 ? "warn" : "err"; } // ====== Datasets tab ====== function DatasetsTab({ datasets, refreshDatasets, agents, models, toolOptions, onRunStarted, loading }) { const [selectedId, setSelectedId] = eUseState(datasets[0]?.id || null); const [innerTab, setInnerTab] = eUseState("cases"); // cases | graders | settings const [selectedCaseId, setSelectedCaseId] = eUseState(null); const [addingCase, setAddingCase] = eUseState(false); const [runningDataset, setRunningDataset] = eUseState(null); eUseEffect(() => { if (datasets.length && !datasets.find((d) => d.id === selectedId)) { setSelectedId(datasets[0].id); } }, [datasets]); const ds = datasets.find((d) => d.id === selectedId); const createDs = async () => { const name = prompt("Dataset name?"); if (!name) return; try { const created = await window.api.createDataset({ name }); await refreshDatasets(); setSelectedId(created.dataset_id); } catch (err) { alert("Create failed: " + (err.message || err)); } }; const deleteDs = async () => { if (!ds || !confirm(`Delete dataset "${ds.name}"?`)) return; try { await window.api.deleteDataset(ds.id); await refreshDatasets(); } catch (err) { alert("Delete failed: " + (err.message || err)); } }; const deleteCase = async (caseId) => { if (!ds) return; try { await window.api.deleteDatasetCase(ds.id, caseId); await refreshDatasets(); } catch (err) { alert("Delete case failed: " + (err.message || err)); } }; const toggleGrader = async (graderId) => { if (!ds) return; const nextGraders = ds.graders.includes(graderId) ? ds.graders.filter((g) => g !== graderId) : [...ds.graders, graderId]; try { await window.api.updateDataset(ds.id, { graders: nextGraders }); await refreshDatasets(); } catch (err) { alert("Save graders failed: " + (err.message || err)); } }; if (!ds) { return (

Datasets

No datasets yet

Create one or add a case from Traces.

); } return (

Datasets

{datasets.length}
{datasets.map((d) => (
{ setSelectedId(d.id); setSelectedCaseId(null); }} >
{d.name}
{d.description &&
{d.description}
}
{d.cases.length} cases · {d.graders.length} graders
{d.tags?.length > 0 && (
{d.tags.map((t) => {t})}
)}
))}

{ds.name}

dataset · {ds.id} · created {new Date(ds.createdAt).toLocaleDateString()}
{ds.description &&
{ds.description}
}
Cases{ds.cases.length}
Graders{ds.graders.length}
From traces{ds.cases.filter((c) => c.source?.type === "trace").length}
Updated {ds.cases.length ? eRelTime(Math.max(...ds.cases.map((c) => c.addedAt))) : "—"}
{innerTab === "cases" && ( )}
{innerTab === "cases" && ( setAddingCase(true)} /> )} {innerTab === "graders" && (
{GRADER_LIBRARY.map((g) => { const on = ds.graders.includes(g.id); return (
{g.kind === "deterministic" ? "=" : g.kind === "llm" ? "L" : g.kind === "trajectory" ? "→" : "$"}
{g.name}
{g.desc}
toggleGrader(g.id)}>
); })}
)}
{addingCase && ( setAddingCase(false)} onAdded={() => refreshDatasets()} /> )} {runningDataset && ( setRunningDataset(null)} onStarted={onRunStarted} /> )}
); } function CasesTable({ dataset, selectedCaseId, onSelect, onDelete, onAddCase }) { if (dataset.cases.length === 0) { return (

No cases yet

Add a case manually, or open a run in{" "} Traces and click "Add to dataset".

); } const selected = dataset.cases.find((c) => c.id === selectedCaseId); return (
{dataset.cases.map((c, i) => { const active = c.id === selectedCaseId; return ( onSelect(active ? null : c.id)}> {active && ( )} ); })}
Input · expected Expected tools Source Added
#{i + 1} {c.input} {c.expected && → {c.expected}} {c.tools.length === 0 ? : c.tools.map((t, i) => ( {t} ))} {c.source?.type === "trace" ? ( e.stopPropagation()}> trace ) : ( manual )} {eRelTime(c.addedAt)}
Input
{c.input}
{c.expected && ( <>
Expected output
{c.expected}
)} {c.tags?.length > 0 && (
{c.tags.map((t) => #{t})}
)}
); } // ====== Runs tab ====== function RunsTab({ runs, datasets }) { const [selectedId, setSelectedId] = eUseState(runs[0]?.id || null); const [compareId, setCompareId] = eUseState(null); const selected = runs.find((r) => r.id === selectedId) || null; const compared = runs.find((r) => r.id === compareId) || null; if (runs.length === 0) { return (

Eval runs

No eval runs yet

Pick a dataset and click "Run eval".

); } return (
Run Pass rate Started Latency Cost
{runs.map((r) => { const summary = r.summary || {}; const status = r.status || "pending"; const expectedCaseCount = datasets.find((ds) => ds.id === r.datasetId)?.cases.length ?? r.cases.length; const pass = summary.passed_count ?? r.cases.filter((c) => c.pass).length; const passRate = summary.pass_rate ?? (r.cases.length ? pass / r.cases.length : 0); const dotCls = evalStatusClass(status, passRate); const p50Latency = summary.p50_latency_ms ?? 0; const totalCost = summary.total_cost_usd ?? r.cases.reduce((s, c) => s + (c.costUSD || 0), 0); return (
setSelectedId(r.id === selectedId ? null : r.id)} >
{r.datasetName} {r.datasetVersion} {evalStatusLabel(status)}
{r.agentName} · {r.model} · {r.cases.length}/{expectedCaseCount} cases
{r.error &&
{r.error}
}
{status === "pending" || status === "running" ? `${evalStatusLabel(status)} · ${r.cases.length} case(s)` : status === "failed" ? `Failed · ${summary.error_count || 0} case error(s)` : `${(passRate * 100).toFixed(0)}% · ${pass}/${expectedCaseCount} pass`}
{eRelTime(r.startedAt)}
p50 {fmtMs(p50Latency)}
{fmtCost(totalCost)}
); })} {selected && }
); } function RunDetail({ run, runs, datasets }) { const ds = datasets.find((d) => d.id === run.datasetId); const summary = run.summary || {}; const expectedCaseCount = ds?.cases.length ?? run.cases.length; const pass = summary.passed_count ?? run.cases.filter((c) => c.pass).length; const passRate = summary.pass_rate ?? (run.cases.length ? pass / run.cases.length : 0); const totalCost = summary.total_cost_usd ?? run.cases.reduce((s, c) => s + (c.costUSD || 0), 0); const status = run.status || "pending"; const otherRuns = runs.filter((r) => r.datasetId === run.datasetId && r.id !== run.id); const baseline = otherRuns[0] || null; const baselinePass = baseline ? baseline.cases.filter((c) => c.pass).length / baseline.cases.length : null; return (
Status {evalStatusLabel(status)}
Pass rate = 0.8 ? "var(--accent)" : passRate >= 0.5 ? "var(--orange)" : "#b91c1c" }}> {(passRate * 100).toFixed(0)}%
{baselinePass != null && (
vs previous = 0 ? "var(--accent)" : "#b91c1c" }}> {passRate - baselinePass >= 0 ? "+" : ""}{((passRate - baselinePass) * 100).toFixed(0)}%
)}
Cases {pass}/{expectedCaseCount}
p50 / p95 latency {fmtMs(summary.p50_latency_ms)} / {fmtMs(summary.p95_latency_ms)}
Total cost {fmtCost(totalCost)}
Input / output tokens {(summary.input_tokens || 0).toLocaleString()} / {(summary.output_tokens || 0).toLocaleString()}
Run id {run.id}
{run.error && (
{run.error}
)} {run.cases.length === 0 && ( )} {run.cases.map((rc) => { const dsCase = ds?.cases.find((c) => c.id === rc.caseId); const caseFailed = !!rc.error || rc.pass === false; const casePending = rc.pass == null && !rc.error; return ( ); })}
Result Case Grader scores Latency Cost
{status === "pending" || status === "running" ? "Waiting for the first case to start." : "No case result was recorded."}
{casePending ? "…" : caseFailed ? : } {casePending ? "running" : caseFailed ? "fail" : "pass"}
{dsCase?.input || rc.caseId}
{rc.error &&
{rc.error}
} {dsCase?.expected && (
→ {dsCase.expected}
)}
{Object.entries(rc.scores).map(([k, v]) => { const isPass = typeof v === "number" ? v >= (k.startsWith("llm") ? 3.5 : 1) : !!v; return ( {k}: {typeof v === "number" ? (k.startsWith("llm") ? v.toFixed(1) : v) : (v ? "✓" : "✗")} ); })} {fmtMs(rc.latencyMs)} {fmtCost(rc.costUSD)}
); } // ====== Main app ====== function EvalsApp() { const [tab, setTab] = eUseState("datasets"); // datasets | runs const [datasets, setDatasets] = eUseState([]); const [runs, setRuns] = eUseState([]); const [agents, setAgents] = eUseState([]); const [models, setModels] = eUseState([]); const [toolOptions, setToolOptions] = eUseState([]); const [loading, setLoading] = eUseState(true); const [loadError, setLoadError] = eUseState(null); // PR-S.17d · topbar 上 Users 入口 admin-gate const [isAdmin, setIsAdmin] = eUseState(false); eUseEffect(() => { window.api.authMe() .then((m) => setIsAdmin(!!m.is_admin)) .catch(() => {}); }, []); // Detail (cases) isn't in the list response, so we fan out one // getDataset() per row. Acceptable for current demo scale; future PR // can add `?include=cases` to the list endpoint. const refreshDatasets = async () => { const items = await window.api.listDatasets(); const detailed = await Promise.all(items.map( (ds) => window.api.getDataset(ds.dataset_id).catch(() => ({ ...ds, cases: [] })) )); setDatasets(detailed.map(window.decorateDataset)); }; const refreshRuns = async () => { const items = await window.api.listEvalRuns({ limit: 200 }); setRuns(items.map(window.decorateEvalRun)); }; const refreshRunOptions = async () => { const [agentItems, modelResponse, localToolItems, mcpItems] = await Promise.all([ window.api.listAgents(), window.api.listModels(), window.api.listLocalTools().catch(() => []), window.api.listMcpServers().catch(() => []), ]); setAgents(agentItems); setModels(modelResponse.items || []); const names = new Set((localToolItems || []).map((tool) => tool.name).filter(Boolean)); (mcpItems || []).forEach((server) => (server.tools || []).forEach((tool) => { const name = typeof tool === "string" ? tool : (tool?.name || tool?.id); if (name) names.add(name); })); setToolOptions([...names].sort().map((name) => ({ id: name, name, label: name }))); window.MODELS = modelResponse.items || []; window.MODEL_PROVIDERS = modelResponse.providers || []; }; eUseEffect(() => { (async () => { try { await Promise.all([refreshDatasets(), refreshRuns(), refreshRunOptions()]); } catch (err) { console.warn("evals: initial load failed", err); setLoadError(err.message || String(err)); } finally { setLoading(false); } })(); }, []); eUseEffect(() => { if (!runs.some((run) => run.status === "pending" || run.status === "running")) return undefined; const interval = window.setInterval(() => refreshRuns().catch(() => {}), 1000); return () => window.clearInterval(interval); }, [runs]); return (
baizhi / Evals
{loadError && (
⚠ Failed to load from backend: {loadError}. Datasets / Eval runs may be empty.
)} {tab === "datasets" ? { await refreshRuns(); setTab("runs"); }} loading={loading} /> : }
); } function navLinkStyle() { return { background: "transparent", border: 0, padding: "6px 10px", borderRadius: 6, color: "var(--text-muted)", fontSize: 13, textDecoration: "none", cursor: "pointer", }; } ReactDOM.createRoot(document.getElementById("root")).render();