oxedyne/fe2o3/fe2o3_austenite/tools/bench/bench_verdict.mjs
6.0 KiB, 1 run, executable
created by r1870400018:66220, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env node |
| 2 | // Reads the JSONL a bench driver wrote and prints the verdict tables, in markdown. |
| 3 | // |
| 4 | // bench_verdict.mjs typing --jsonl=FILE [--doc=NAME] |
| 5 | // typing_sessions.sh's sessions (D-20261001-08): each session's nearest-rank p50 and p95, the |
| 6 | // median of each across the valid sessions of an engine, T = max(1.25 x typst.ts, typst.ts + |
| 7 | // 16 ms) per percentile, and a pass when Austenite is at or under T for both. A round's last |
| 8 | // attempt is the one that counts; an invalid session is listed and left out of the medians. |
| 9 | // bench_verdict.mjs cost --jsonl=FILE [--doc=NAME] [--limit=1.5] |
| 10 | // cost_runs.sh's cold and unchanged runs: the median of each (doc, format, mode, engine) cell |
| 11 | // and its ratio to typst.ts, held against the limit (1.5). |
| 12 | import fs from 'node:fs'; |
| 13 | import { median } from './lib/stats.mjs'; |
| 14 | import { typingThreshold, floorBinds } from './lib/stats.mjs'; |
| 15 | |
| 16 | function parseArgs(argv) { |
| 17 | const out = { _: [] }; |
| 18 | for (const a of argv.slice(2)) { |
| 19 | const m = a.match(/^--([^=]+)=(.*)$/); |
| 20 | if (m) out[m[1]] = m[2]; |
| 21 | else out._.push(a); |
| 22 | } |
| 23 | return out; |
| 24 | } |
| 25 | |
| 26 | function readJsonl(file) { |
| 27 | return fs |
| 28 | .readFileSync(file, 'utf8') |
| 29 | .split('\n') |
| 30 | .filter((l) => l.trim()) |
| 31 | .map((l) => JSON.parse(l)); |
| 32 | } |
| 33 | |
| 34 | const ms = (s) => (s === null || s === undefined ? '-' : (s * 1000).toFixed(1)); |
| 35 | const f2 = (x) => (x === null || x === undefined ? '-' : x.toFixed(2)); |
| 36 | |
| 37 | function typing(rows, docFilter) { |
| 38 | const docs = [...new Set(rows.map((r) => r.doc))].filter((d) => !docFilter || d === docFilter); |
| 39 | const out = []; |
| 40 | for (const doc of docs) { |
| 41 | const mine = rows.filter((r) => r.doc === doc); |
| 42 | const sha = mine[0].doc_sha256; |
| 43 | const rounds = [...new Set(mine.map((r) => r.round))].sort((a, b) => a - b); |
| 44 | // The last attempt of each (engine, round). |
| 45 | const last = {}; |
| 46 | for (const r of mine) { |
| 47 | const k = `${r.engine}/${r.round}`; |
| 48 | if (!last[k] || r.attempt > last[k].attempt) last[k] = r; |
| 49 | } |
| 50 | const edits = mine[0].edits; |
| 51 | const valid = (r) => r && r.ok === true && Array.isArray(r.samples) && r.samples.length === edits; |
| 52 | out.push(`### ${doc} (sha256 \`${sha}\`)\n`); |
| 53 | out.push('| Session | Starts | Engine | p50 ms | p95 ms | open s | pages | PSI start / end | load1 start / end | attempt | note |'); |
| 54 | out.push('|---|---|---|---|---|---|---|---|---|---|---|'); |
| 55 | for (const rd of rounds) { |
| 56 | const first = rd % 2 === 0 ? 'austenite' : 'typstts'; |
| 57 | for (const eng of ['austenite', 'typstts']) { |
| 58 | const r = last[`${eng}/${rd}`]; |
| 59 | if (!r) continue; |
| 60 | const notes = []; |
| 61 | if (!valid(r)) notes.push(`INVALID: ${r.invalid || r.error || 'incomplete'}`); |
| 62 | if (r.flagged) notes.push('FLAGGED (PSI avg10 > 20)'); |
| 63 | out.push( |
| 64 | `| ${rd + 1} | ${first === eng ? 'first' : 'second'} | ${eng === 'austenite' ? 'Austenite' : 'typst.ts'} | ` + |
| 65 | `${valid(r) ? ms(r.p50_s) : '-'} | ${valid(r) ? ms(r.p95_s) : '-'} | ${f2(r.open_s)} | ${r.pages ?? '-'} | ` + |
| 66 | `${r.psi_before} / ${r.psi_after} | ${r.load_before} / ${r.load_after} | ${r.attempt} | ${notes.join('; ')} |` |
| 67 | ); |
| 68 | } |
| 69 | } |
| 70 | const cell = (eng, key) => { |
| 71 | const v = rounds.map((rd) => last[`${eng}/${rd}`]).filter(valid).map((r) => r[key]); |
| 72 | return { n: v.length, med: median(v) }; |
| 73 | }; |
| 74 | const a50 = cell('austenite', 'p50_s'), a95 = cell('austenite', 'p95_s'); |
| 75 | const t50 = cell('typstts', 'p50_s'), t95 = cell('typstts', 'p95_s'); |
| 76 | out.push(''); |
| 77 | out.push(`Valid sessions: Austenite ${a50.n}, typst.ts ${t50.n}. Median across sessions:\n`); |
| 78 | out.push('| Percentile | typst.ts ms | T ms | Austenite ms | A / typst.ts | A / T | 16 ms floor binds | Verdict |'); |
| 79 | out.push('|---|---|---|---|---|---|---|---|'); |
| 80 | const verdicts = []; |
| 81 | for (const [name, a, t] of [['p50', a50, t50], ['p95', a95, t95]]) { |
| 82 | if (a.med === null || t.med === null) { |
| 83 | out.push(`| ${name} | - | - | - | - | - | - | NO DATA |`); |
| 84 | verdicts.push(false); |
| 85 | continue; |
| 86 | } |
| 87 | const T = typingThreshold(t.med); |
| 88 | const pass = a.med <= T; |
| 89 | verdicts.push(pass); |
| 90 | out.push( |
| 91 | `| ${name} | ${ms(t.med)} | ${ms(T)} | ${ms(a.med)} | ${f2(a.med / t.med)} | ${f2(a.med / T)} | ${floorBinds(t.med) ? 'yes' : 'no'} | ${pass ? 'pass' : 'miss'} |` |
| 92 | ); |
| 93 | } |
| 94 | out.push(''); |
| 95 | out.push(`**${doc}: ${verdicts.length === 2 && verdicts.every(Boolean) ? 'PASS' : 'MISS'}** (pass needs both percentiles at or under T)\n`); |
| 96 | } |
| 97 | return out.join('\n'); |
| 98 | } |
| 99 | |
| 100 | function cost(rows, docFilter, limit) { |
| 101 | const docs = [...new Set(rows.map((r) => r.doc))].filter((d) => !docFilter || d === docFilter); |
| 102 | const out = []; |
| 103 | for (const doc of docs) { |
| 104 | const mine = rows.filter((r) => r.doc === doc); |
| 105 | out.push(`### ${doc} (sha256 \`${mine[0].doc_sha256}\`)\n`); |
| 106 | out.push(`| Format | Mode | Runs per engine | typst.ts median s | Austenite median s | Ratio | Limit | Verdict | Flagged runs |`); |
| 107 | out.push('|---|---|---|---|---|---|---|---|---|'); |
| 108 | const fmts = [...new Set(mine.map((r) => r.fmt))]; |
| 109 | for (const fmt of fmts) { |
| 110 | for (const mode of ['cold', 'unchanged']) { |
| 111 | const sel = mine.filter((r) => r.fmt === fmt && r.mode === mode); |
| 112 | if (!sel.length) continue; |
| 113 | const runs = (eng) => sel.filter((r) => r.engine === eng && r.ok === true).flatMap((r) => r.runs); |
| 114 | const a = runs('austenite'), t = runs('typstts'); |
| 115 | const flagged = sel.filter((r) => r.flagged).length; |
| 116 | if (!a.length || !t.length) { |
| 117 | out.push(`| ${fmt} | ${mode} | ${a.length}/${t.length} | - | - | - | - | NO DATA | ${flagged} |`); |
| 118 | continue; |
| 119 | } |
| 120 | const ratio = median(a) / median(t); |
| 121 | out.push( |
| 122 | `| ${fmt} | ${mode} | ${a.length}/${t.length} | ${median(t).toFixed(3)} | ${median(a).toFixed(3)} | ${ratio.toFixed(2)} | ${limit} | ${ratio <= limit ? 'pass' : 'miss'} | ${flagged} |` |
| 123 | ); |
| 124 | } |
| 125 | } |
| 126 | out.push(''); |
| 127 | } |
| 128 | return out.join('\n'); |
| 129 | } |
| 130 | |
| 131 | const args = parseArgs(process.argv); |
| 132 | const which = args._[0]; |
| 133 | if (!args.jsonl || !['typing', 'cost'].includes(which)) { |
| 134 | process.stderr.write('usage: bench_verdict.mjs typing|cost --jsonl=FILE [--doc=NAME] [--limit=1.5]\n'); |
| 135 | process.exit(2); |
| 136 | } |
| 137 | const rows = readJsonl(args.jsonl); |
| 138 | process.stdout.write((which === 'typing' ? typing(rows, args.doc) : cost(rows, args.doc, Number(args.limit || 1.5))) + '\n'); |