Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/aggregate.py

7.9 KiB, 1 run, executable

created by r1870400018:60932, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2"""Aggregates the S0 bench harness's raw JSONL runs into one JSON report and
3one markdown summary.
4
5A run flagged `flagged_under_load` (the host's 1-minute load average was above
6`BENCH_LOAD_THRESHOLD`, default 2, before or after that run) is never folded
7into a median silently. By default the whole report refuses to print numbers
8when ANY run is flagged: `--force` overrides that and prints them anyway,
9loudly labelled as taken under load. This is the harness's own gate on the
10plan's rule: "A run whose load average is above 2 is flagged and repeated,
11never averaged in silently."
12
13Usage: aggregate.py --in DIR [--in DIR ...] --out-json FILE --out-md FILE
14 [--force] [--pages N]
15"""
16import argparse
17import json
18import statistics as st
19import sys
20from collections import defaultdict
21from pathlib import Path
22
23
24def load_jsonl(path):
25 rows = []
26 if not path.exists():
27 return rows
28 for line in path.read_text().splitlines():
29 line = line.strip()
30 if not line:
31 continue
32 try:
33 rows.append(json.loads(line))
34 except json.JSONDecodeError as e:
35 print(f"aggregate: skipping malformed line in {path}: {e}", file=sys.stderr)
36 return rows
37
38
39def median_min_max(values):
40 if not values:
41 return None
42 return {"median": st.median(values), "min": min(values), "max": max(values), "n": len(values)}
43
44
45def summarise_wall(rows, key_fields, wall_field="wall_s"):
46 """Groups rows by key_fields, returns {key: median_min_max} over wall_field."""
47 groups = defaultdict(list)
48 for r in rows:
49 if r.get("exit_code", 0) != 0 and wall_field == "wall_s":
50 continue # a failed run has no meaningful wall time
51 key = tuple(r.get(f) for f in key_fields)
52 v = r.get(wall_field)
53 if v is not None:
54 groups[key].append(v)
55 return {k: median_min_max(v) for k, v in groups.items()}
56
57
58def main():
59 ap = argparse.ArgumentParser()
60 ap.add_argument("--in", dest="in_dirs", action="append", required=True)
61 ap.add_argument("--out-json", required=True)
62 ap.add_argument("--out-md", required=True)
63 ap.add_argument("--force", action="store_true",
64 help="print numbers even though some runs were flagged under load")
65 ap.add_argument("--label", default="", help="one line noting the run's circumstances")
66 args = ap.parse_args()
67
68 native, wasm, edits = [], [], []
69 for d in args.in_dirs:
70 d = Path(d)
71 native += load_jsonl(d / "native_runs.jsonl")
72 wasm += load_jsonl(d / "wasm_runs.jsonl")
73 edits += load_jsonl(d / "edit_latency_runs.jsonl")
74
75 all_rows = native + wasm + edits
76 flagged = [r for r in all_rows if r.get("flagged_under_load")]
77 under_load = bool(flagged)
78
79 report = {
80 "label": args.label,
81 "under_load": under_load,
82 "flagged_run_count": len(flagged),
83 "total_run_count": len(all_rows),
84 "forced": args.force,
85 "native": {},
86 "wasm_compile": {},
87 "edit_latency": {},
88 }
89
90 if under_load and not args.force:
91 Path(args.out_json).write_text(json.dumps(report, indent=2) + "\n")
92 Path(args.out_md).write_text(
93 "# S0 bench report -- REFUSED\n\n"
94 f"{len(flagged)} of {len(all_rows)} runs were made while the host's 1-minute load "
95 "average was above the threshold. This harness refuses to report numbers taken under "
96 "load, per the S0 protocol, unless run with `--force`.\n\n"
97 "Re-run when the host is idle, or pass `--force` to see the (labelled, unreliable) "
98 "numbers anyway.\n"
99 )
100 print("REFUSED: runs were made under load; re-run idle or pass --force", file=sys.stderr)
101 return
102
103 # Native: doc x engine(jobs)
104 native_summary = summarise_wall(native, ["doc", "engine"])
105 rss_summary = summarise_wall(native, ["doc", "engine"], wall_field="rss_kb")
106 for (doc, engine), stat in native_summary.items():
107 report["native"].setdefault(doc, {})[engine] = {
108 "wall_s": stat,
109 "rss_kb": rss_summary.get((doc, engine)),
110 }
111 # Ratio austenite-native / typst-j16 median, per doc.
112 for doc, engines in report["native"].items():
113 a = engines.get("austenite-native", {}).get("wall_s")
114 t16 = engines.get("typst-j16", {}).get("wall_s")
115 if a and t16 and t16["median"]:
116 report["native"][doc]["ratio_austenite_over_typst_j16"] = round(a["median"] / t16["median"], 3)
117
118 # Wasm compile: doc x engine(wasm-austenite/wasm-typstts). Each row holds one
119 # process invocation's `runs` array (every warm-up-then-measured call inside
120 # that one capped process), not a single `wall_s` -- unlike the native rows,
121 # where `run_timed` writes one row per measured run.
122 wasm_by_key = defaultdict(list)
123 wasm_rss_rows = defaultdict(list)
124 for r in wasm:
125 if r.get("exit_code", 0) != 0 or not r.get("ok", True):
126 continue
127 key = (r.get("doc"), r.get("engine"))
128 wasm_by_key[key].extend(r.get("runs") or [])
129 if r.get("rss_kb") is not None:
130 wasm_rss_rows[key].append(r["rss_kb"])
131 for key, samples in wasm_by_key.items():
132 doc, engine = key
133 report["wasm_compile"].setdefault(doc, {})[engine] = {
134 "wall_s": median_min_max(samples),
135 "rss_kb": median_min_max(wasm_rss_rows.get(key, [])),
136 }
137 for doc, engines in report["wasm_compile"].items():
138 a = engines.get("wasm-austenite", {}).get("wall_s")
139 t = engines.get("wasm-typstts", {}).get("wall_s")
140 if a and t and t["median"]:
141 report["wasm_compile"][doc]["ratio_austenite_over_typstts"] = round(a["median"] / t["median"], 3)
142
143 # Edit latency: p50_s/p95_s come pre-computed from edit_latency.mjs, one row per (doc, engine).
144 for r in edits:
145 doc, engine = r.get("doc"), r.get("engine")
146 if doc is None:
147 continue
148 report["edit_latency"].setdefault(doc, {})[engine] = {
149 "p50_s": r.get("p50_s"), "p95_s": r.get("p95_s"),
150 "edits": r.get("edits"), "rss_kb": r.get("rss_kb"),
151 "ok": r.get("ok"),
152 }
153
154 Path(args.out_json).write_text(json.dumps(report, indent=2) + "\n")
155
156 md = []
157 md.append("# S0 bench report")
158 if args.label:
159 md.append(f"\n{args.label}\n")
160 if under_load:
161 md.append(
162 f"\n**TAKEN UNDER LOAD** -- {len(flagged)}/{len(all_rows)} runs were flagged "
163 "(1-minute load average above threshold before or after the run). These numbers "
164 "are NOT a valid baseline; re-run when the host is idle.\n"
165 )
166 md.append("\n## Native (typst -j16 / -j1 vs austenite)\n")
167 md.append("| Doc | Engine | wall median (s) | min | max | n | peak RSS median (kB) |")
168 md.append("|---|---|---|---|---|---|---|")
169 for doc, engines in sorted(report["native"].items()):
170 for engine, v in sorted(engines.items()):
171 if engine.startswith("ratio_"):
172 continue
173 w = v["wall_s"]
174 rss = v.get("rss_kb") or {}
175 md.append(f"| {doc} | {engine} | {w['median']:.3f} | {w['min']:.3f} | {w['max']:.3f} | "
176 f"{w['n']} | {rss.get('median', '-')} |")
177 ratio = engines.get("ratio_austenite_over_typst_j16")
178 if ratio is not None:
179 md.append(f"| {doc} | **ratio austenite/typst-j16** | {ratio} | | | | |")
180
181 md.append("\n## Wasm compile (Austenite wasm vs typst.ts wasm, node)\n")
182 md.append("| Doc | Engine | wall median (s) | min | max | n | peak RSS median (kB) |")
183 md.append("|---|---|---|---|---|---|---|")
184 for doc, engines in sorted(report["wasm_compile"].items()):
185 for engine, v in sorted(engines.items()):
186 if engine.startswith("ratio_"):
187 continue
188 w = v["wall_s"]
189 rss = v.get("rss_kb") or {}
190 md.append(f"| {doc} | {engine} | {w['median']:.3f} | {w['min']:.3f} | {w['max']:.3f} | "
191 f"{w['n']} | {rss.get('median', '-')} |")
192 ratio = engines.get("ratio_austenite_over_typstts")
193 if ratio is not None:
194 md.append(f"| {doc} | **ratio austenite/typst.ts** | {ratio} | | | | |")
195
196 md.append("\n## Edit latency (one-character edit to recompiled page)\n")
197 md.append("| Doc | Engine | p50 (s) | p95 (s) | edits | peak RSS (kB) | ok |")
198 md.append("|---|---|---|---|---|---|---|")
199 def fmt4(x):
200 return f"{x:.4f}" if isinstance(x, (int, float)) else "-"
201
202 for doc, engines in sorted(report["edit_latency"].items()):
203 for engine, v in sorted(engines.items()):
204 md.append(f"| {doc} | {engine} | {fmt4(v['p50_s'])} | {fmt4(v['p95_s'])} | "
205 f"{v['edits']} | {v.get('rss_kb', '-')} | {v['ok']} |")
206
207 Path(args.out_md).write_text("\n".join(md) + "\n")
208 print(f"wrote {args.out_json} and {args.out_md}", file=sys.stderr)
209
210
211if __name__ == "__main__":
212 main()