Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/bench_verdict.mjs

6.0 KiB, 1 run, executable

created by r1870400018:66220, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env node
2// Reads the JSONL a bench driver wrote and prints the verdict tables, in markdown.
3//
4// bench_verdict.mjs typing --jsonl=FILE [--doc=NAME]
5// typing_sessions.sh's sessions (D-20261001-08): each session's nearest-rank p50 and p95, the
6// median of each across the valid sessions of an engine, T = max(1.25 x typst.ts, typst.ts +
7// 16 ms) per percentile, and a pass when Austenite is at or under T for both. A round's last
8// attempt is the one that counts; an invalid session is listed and left out of the medians.
9// bench_verdict.mjs cost --jsonl=FILE [--doc=NAME] [--limit=1.5]
10// cost_runs.sh's cold and unchanged runs: the median of each (doc, format, mode, engine) cell
11// and its ratio to typst.ts, held against the limit (1.5).
12import fs from 'node:fs';
13import { median } from './lib/stats.mjs';
14import { typingThreshold, floorBinds } from './lib/stats.mjs';
15
16function parseArgs(argv) {
17 const out = { _: [] };
18 for (const a of argv.slice(2)) {
19 const m = a.match(/^--([^=]+)=(.*)$/);
20 if (m) out[m[1]] = m[2];
21 else out._.push(a);
22 }
23 return out;
24}
25
26function readJsonl(file) {
27 return fs
28 .readFileSync(file, 'utf8')
29 .split('\n')
30 .filter((l) => l.trim())
31 .map((l) => JSON.parse(l));
32}
33
34const ms = (s) => (s === null || s === undefined ? '-' : (s * 1000).toFixed(1));
35const f2 = (x) => (x === null || x === undefined ? '-' : x.toFixed(2));
36
37function typing(rows, docFilter) {
38 const docs = [...new Set(rows.map((r) => r.doc))].filter((d) => !docFilter || d === docFilter);
39 const out = [];
40 for (const doc of docs) {
41 const mine = rows.filter((r) => r.doc === doc);
42 const sha = mine[0].doc_sha256;
43 const rounds = [...new Set(mine.map((r) => r.round))].sort((a, b) => a - b);
44 // The last attempt of each (engine, round).
45 const last = {};
46 for (const r of mine) {
47 const k = `${r.engine}/${r.round}`;
48 if (!last[k] || r.attempt > last[k].attempt) last[k] = r;
49 }
50 const edits = mine[0].edits;
51 const valid = (r) => r && r.ok === true && Array.isArray(r.samples) && r.samples.length === edits;
52 out.push(`### ${doc} (sha256 \`${sha}\`)\n`);
53 out.push('| Session | Starts | Engine | p50 ms | p95 ms | open s | pages | PSI start / end | load1 start / end | attempt | note |');
54 out.push('|---|---|---|---|---|---|---|---|---|---|---|');
55 for (const rd of rounds) {
56 const first = rd % 2 === 0 ? 'austenite' : 'typstts';
57 for (const eng of ['austenite', 'typstts']) {
58 const r = last[`${eng}/${rd}`];
59 if (!r) continue;
60 const notes = [];
61 if (!valid(r)) notes.push(`INVALID: ${r.invalid || r.error || 'incomplete'}`);
62 if (r.flagged) notes.push('FLAGGED (PSI avg10 > 20)');
63 out.push(
64 `| ${rd + 1} | ${first === eng ? 'first' : 'second'} | ${eng === 'austenite' ? 'Austenite' : 'typst.ts'} | ` +
65 `${valid(r) ? ms(r.p50_s) : '-'} | ${valid(r) ? ms(r.p95_s) : '-'} | ${f2(r.open_s)} | ${r.pages ?? '-'} | ` +
66 `${r.psi_before} / ${r.psi_after} | ${r.load_before} / ${r.load_after} | ${r.attempt} | ${notes.join('; ')} |`
67 );
68 }
69 }
70 const cell = (eng, key) => {
71 const v = rounds.map((rd) => last[`${eng}/${rd}`]).filter(valid).map((r) => r[key]);
72 return { n: v.length, med: median(v) };
73 };
74 const a50 = cell('austenite', 'p50_s'), a95 = cell('austenite', 'p95_s');
75 const t50 = cell('typstts', 'p50_s'), t95 = cell('typstts', 'p95_s');
76 out.push('');
77 out.push(`Valid sessions: Austenite ${a50.n}, typst.ts ${t50.n}. Median across sessions:\n`);
78 out.push('| Percentile | typst.ts ms | T ms | Austenite ms | A / typst.ts | A / T | 16 ms floor binds | Verdict |');
79 out.push('|---|---|---|---|---|---|---|---|');
80 const verdicts = [];
81 for (const [name, a, t] of [['p50', a50, t50], ['p95', a95, t95]]) {
82 if (a.med === null || t.med === null) {
83 out.push(`| ${name} | - | - | - | - | - | - | NO DATA |`);
84 verdicts.push(false);
85 continue;
86 }
87 const T = typingThreshold(t.med);
88 const pass = a.med <= T;
89 verdicts.push(pass);
90 out.push(
91 `| ${name} | ${ms(t.med)} | ${ms(T)} | ${ms(a.med)} | ${f2(a.med / t.med)} | ${f2(a.med / T)} | ${floorBinds(t.med) ? 'yes' : 'no'} | ${pass ? 'pass' : 'miss'} |`
92 );
93 }
94 out.push('');
95 out.push(`**${doc}: ${verdicts.length === 2 && verdicts.every(Boolean) ? 'PASS' : 'MISS'}** (pass needs both percentiles at or under T)\n`);
96 }
97 return out.join('\n');
98}
99
100function cost(rows, docFilter, limit) {
101 const docs = [...new Set(rows.map((r) => r.doc))].filter((d) => !docFilter || d === docFilter);
102 const out = [];
103 for (const doc of docs) {
104 const mine = rows.filter((r) => r.doc === doc);
105 out.push(`### ${doc} (sha256 \`${mine[0].doc_sha256}\`)\n`);
106 out.push(`| Format | Mode | Runs per engine | typst.ts median s | Austenite median s | Ratio | Limit | Verdict | Flagged runs |`);
107 out.push('|---|---|---|---|---|---|---|---|---|');
108 const fmts = [...new Set(mine.map((r) => r.fmt))];
109 for (const fmt of fmts) {
110 for (const mode of ['cold', 'unchanged']) {
111 const sel = mine.filter((r) => r.fmt === fmt && r.mode === mode);
112 if (!sel.length) continue;
113 const runs = (eng) => sel.filter((r) => r.engine === eng && r.ok === true).flatMap((r) => r.runs);
114 const a = runs('austenite'), t = runs('typstts');
115 const flagged = sel.filter((r) => r.flagged).length;
116 if (!a.length || !t.length) {
117 out.push(`| ${fmt} | ${mode} | ${a.length}/${t.length} | - | - | - | - | NO DATA | ${flagged} |`);
118 continue;
119 }
120 const ratio = median(a) / median(t);
121 out.push(
122 `| ${fmt} | ${mode} | ${a.length}/${t.length} | ${median(t).toFixed(3)} | ${median(a).toFixed(3)} | ${ratio.toFixed(2)} | ${limit} | ${ratio <= limit ? 'pass' : 'miss'} | ${flagged} |`
123 );
124 }
125 }
126 out.push('');
127 }
128 return out.join('\n');
129}
130
131const args = parseArgs(process.argv);
132const which = args._[0];
133if (!args.jsonl || !['typing', 'cost'].includes(which)) {
134 process.stderr.write('usage: bench_verdict.mjs typing|cost --jsonl=FILE [--doc=NAME] [--limit=1.5]\n');
135 process.exit(2);
136}
137const rows = readJsonl(args.jsonl);
138process.stdout.write((which === 'typing' ? typing(rows, args.doc) : cost(rows, args.doc, Number(args.limit || 1.5))) + '\n');