Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/report.py

9.4 KiB, 1 run, executable

created by r1870400018:72517, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2"""Summarises the sweep's JSONL (sweep.sh) as markdown: robustness, speed ratios, the worst documents, the
3stage ranking and the warm edit. A later file's record for an id replaces an earlier one, so a rerun file
4goes last.
5
6Usage: report.py [--rerun-ids FILE] [--fail-tsv FILE] <sweep.jsonl> [rerun.jsonl]...
7 --rerun-ids writes the ids to measure again with RUNS=4: the slowest 10 by ratio, every document over 3x,
8 and every document with a run at a load above 6.
9 --fail-tsv writes every Austenite failure: id, tier, status, typst status, message.
10"""
11
12import json
13import statistics
14import sys
15
16STAGES = ("load", "eval", "realise", "layout", "emit")
17DOCS = ("universe", "syn", "own", "tree") # the headline set; fixtures are reported apart
18
19
20def pct(xs, p):
21 xs = sorted(xs)
22 if not xs:
23 return float("nan")
24 k = (len(xs) - 1) * p
25 f = int(k)
26 c = min(f + 1, len(xs) - 1)
27 return xs[f] + (xs[c] - xs[f]) * (k - f)
28
29
30def load_max(r):
31 m = 0.0
32 for e in ("aus", "typ"):
33 for run in (r.get(e) or {}).get("runs", []):
34 try:
35 m = max(m, float(run.get("load") or 0))
36 except ValueError:
37 pass
38 return m
39
40
41def excess(r):
42 a = (r["aus"].get("stages") or {})
43 t = (r["typ"].get("stages") or {})
44 if not a or not t:
45 return None
46 return {s: a.get(s, 0.0) - t.get(s, 0.0) for s in STAGES}
47
48
49def dominant(r):
50 x = excess(r)
51 if x:
52 s = max(x, key=x.get)
53 return s, x[s]
54 a = r["aus"].get("stages") or {}
55 if a:
56 s = max(STAGES, key=lambda k: a.get(k, 0.0))
57 return s + " (share)", a.get(s, 0.0)
58 return "-", 0.0
59
60
61def ratio_stats(rs):
62 xs = [r["ratio"] for r in rs]
63 if not xs:
64 return "n=0"
65 w = max(rs, key=lambda r: r["ratio"])
66 return (f"n={len(xs)}, median {statistics.median(xs):.2f}x, p90 {pct(xs, 0.9):.2f}x, "
67 f"worst {w['ratio']:.2f}x ({w['id']}), geomean {statistics.geometric_mean(xs):.2f}x")
68
69
70def main():
71 args = sys.argv[1:]
72 rerun_out = fail_out = None
73 while args and args[0].startswith("--"):
74 flag = args.pop(0)
75 if flag == "--rerun-ids":
76 rerun_out = args.pop(0)
77 elif flag == "--fail-tsv":
78 fail_out = args.pop(0)
79 recs = {}
80 for path in args:
81 for line in open(path):
82 try:
83 r = json.loads(line)
84 except ValueError:
85 continue
86 recs[r["id"]] = r
87 allr = list(recs.values())
88 staged = [r for r in allr if "aus" in r]
89 print(f"# Sweep report: {len(allr)} records ({len(allr) - len(staged)} could not be staged)\n")
90
91 # Robustness.
92 print("## Robustness\n")
93 print("| tier | docs | aus ok | aus error | panic | timeout | killed/OOM | typst ok | both ok | pages differ |")
94 print("|---|---|---|---|---|---|---|---|---|---|")
95 tiers = sorted({r["tier"] for r in allr})
96 for tier in tiers:
97 rs = [r for r in staged if r["tier"] == tier]
98 st = lambda r: r["aus"]["status"]
99 cnt = lambda s: sum(1 for r in rs if st(r) == s)
100 both = [r for r in rs if st(r) == "ok" and r["typ"]["status"] == "ok"]
101 diff = sum(1 for r in both if r.get("pages") and r.get("typst_pages") and r["pages"] != r["typst_pages"])
102 nstage = sum(1 for r in allr if r["tier"] == tier and "aus" not in r)
103 sig = sum(1 for r in rs if st(r).startswith("signal"))
104 print(f"| {tier} | {len(rs)}{f' (+{nstage} unstaged)' if nstage else ''} | {cnt('ok')} | {cnt('error')} | "
105 f"{cnt('panic') + sig} | {cnt('timeout')} | {cnt('killed')} | "
106 f"{sum(1 for r in rs if r['typ']['status'] == 'ok')} | {len(both)} | {diff} |")
107 crash = [r for r in staged if r["aus"]["status"] in ("panic", "timeout", "killed") or r["aus"]["status"].startswith("signal")]
108 print(f"\n### Every Austenite crash, hang or kill ({len(crash)})\n")
109 for r in sorted(crash, key=lambda r: (r["aus"]["status"], r["id"])):
110 print(f"- `{r['id']}` ({r['tier']}): **{r['aus']['status']}**, typst {r['typ']['status']}: {r['aus'].get('msg', '')}")
111 wfail = [r for r in staged if r.get("warm", {}).get("status") == "fail"]
112 print(f"\n### Warm-edit failures ({len(wfail)})\n")
113 for r in wfail[:40]:
114 print(f"- `{r['id']}`: rc {r['warm'].get('rc')}: {r['warm'].get('msg', '')[:200]}")
115 # Errors where typst succeeds, grouped by message head.
116 refused = [r for r in staged if r["aus"]["status"] == "error" and r["typ"]["status"] == "ok"]
117 groups = {}
118 for r in refused:
119 m = r["aus"].get("msg", "")
120 key = m.split(" | ")[-1][:110]
121 groups.setdefault(key, []).append(r["id"])
122 print(f"\n### Austenite refuses, typst compiles ({len(refused)}), top messages\n")
123 for k, ids in sorted(groups.items(), key=lambda kv: -len(kv[1]))[:25]:
124 print(f"- {len(ids)} x `{k}` (e.g. {', '.join(ids[:3])})")
125 fx = [r for r in staged if r["tier"] == "fixture"]
126 agree = sum(1 for r in fx if (r["aus"]["status"] == "ok") == (r["typ"]["status"] == "ok"))
127 print(f"\nFixtures: Austenite and typst agree on accept/reject for {agree} of {len(fx)}.")
128
129 # Speed.
130 print("\n## Speed (cold, Austenite wall / typst -j1 wall, median of runs)\n")
131 both = [r for r in staged if r.get("ratio")]
132 docs = [r for r in both if r["tier"] in DOCS]
133 print(f"- **Documents (universe, syn, own, tree): {ratio_stats(docs)}**")
134 big = [r for r in docs if r["typ"]["wall"] >= 0.5]
135 print(f"- Documents where typst takes >= 0.5 s: {ratio_stats(big)}")
136 for tier in tiers:
137 print(f"- {tier}: {ratio_stats([r for r in both if r['tier'] == tier])}")
138 tot_a = sum(r["aus"]["wall"] for r in docs)
139 tot_t = sum(r["typ"]["wall"] for r in docs)
140 print(f"- Summed wall over documents: Austenite {tot_a:.1f} s, typst {tot_t:.1f} s ({tot_a / max(tot_t, 1e-9):.2f}x)")
141 over = [r for r in docs if r["ratio"] > 3]
142 print(f"- Documents over 3x: {len(over)}; over 1.5x: {sum(1 for r in docs if r['ratio'] > 1.5)}")
143
144 print("\n### The 20 worst documents by ratio\n")
145 print("| doc | tier | pages | typst s | Austenite s | ratio | passes | dominant stage (excess s) | warm s / passes / view s | load max |")
146 print("|---|---|---|---|---|---|---|---|---|---|")
147 worst = sorted(docs, key=lambda r: -r["ratio"])[:20]
148 for r in worst:
149 dom, ex = dominant(r)
150 w = r.get("warm") or {}
151 ws = f"{w.get('compile_s', '-')} / {w.get('passes', '-')} / {w.get('save_to_view_s', '-')}" if "mode" in w else "-"
152 st = r["aus"].get("stages") or {}
153 print(f"| `{r['id']}` | {r['tier']} | {r.get('pages', '-')} | {r['typ']['wall']:.3f} | {r['aus']['wall']:.3f} | "
154 f"{r['ratio']:.2f}x | {st.get('passes', '-')} | {dom} ({ex:+.3f}) | {ws} | {load_max(r):.1f} |")
155
156 print("\n### The 10 slowest documents by Austenite wall\n")
157 for r in sorted(docs, key=lambda r: -r["aus"]["wall"])[:10]:
158 dom, ex = dominant(r)
159 print(f"- `{r['id']}`: {r['aus']['wall']:.2f} s against {r['typ']['wall']:.2f} s ({r['ratio']:.2f}x), "
160 f"{r.get('pages', '-')} pages, dominant {dom} ({ex:+.2f} s)")
161
162 # Stage ranking.
163 print("\n## Stages ranked by total excess over typst (documents with both stage splits)\n")
164 tot = dict.fromkeys(STAGES, 0.0)
165 pos = dict.fromkeys(STAGES, 0.0)
166 dom_n = dict.fromkeys(STAGES, 0)
167 a_tot = dict.fromkeys(STAGES, 0.0)
168 t_tot = dict.fromkeys(STAGES, 0.0)
169 n = 0
170 for r in docs:
171 x = excess(r)
172 if not x:
173 continue
174 n += 1
175 for s in STAGES:
176 tot[s] += x[s]
177 pos[s] += max(x[s], 0.0)
178 a_tot[s] += r["aus"]["stages"].get(s, 0.0)
179 t_tot[s] += r["typ"]["stages"].get(s, 0.0)
180 if r["ratio"] > 1:
181 dom_n[max(x, key=x.get)] += 1
182 print(f"n={n}. Excess = Austenite stage time less typst's, summed over documents.\n")
183 print("| stage | Austenite s | typst s | net excess s | positive excess s | dominant in (docs slower than typst) |")
184 print("|---|---|---|---|---|---|")
185 for s in sorted(STAGES, key=lambda s: -pos[s]):
186 print(f"| {s} | {a_tot[s]:.1f} | {t_tot[s]:.1f} | {tot[s]:+.1f} | {pos[s]:.1f} | {dom_n[s]} |")
187 flow = sum((r["aus"].get("stages") or {}).get("flow", 0.0) for r in docs if excess(r))
188 print(f"\nOf Austenite's layout, flow (which holds line breaking) is {flow:.1f} s.")
189
190 # Warm edit.
191 ws = [r for r in docs if "mode" in (r.get("warm") or {})]
192 if ws:
193 cs = [r["warm"]["compile_s"] for r in ws]
194 vs = [r["warm"]["save_to_view_s"] for r in ws]
195 passes = {}
196 for r in ws:
197 passes[r["warm"]["passes"]] = passes.get(r["warm"]["passes"], 0) + 1
198 cold = sum(1 for r in ws if r["warm"]["mode"] != "warm")
199 print(f"\n## Warm one-letter edit (n={len(ws)}; {cold} ran cold)\n")
200 print(f"- compile: median {statistics.median(cs):.3f} s, p90 {pct(cs, 0.9):.3f} s, worst {max(cs):.3f} s")
201 print(f"- save-to-view: median {statistics.median(vs):.3f} s, p90 {pct(vs, 0.9):.3f} s, worst {max(vs):.3f} s "
202 f"({max(ws, key=lambda r: r['warm']['save_to_view_s'])['id']})")
203 print(f"- passes: {dict(sorted(passes.items(), key=lambda kv: (kv[0] is None, kv[0] or 0)))}")
204 for r in sorted(ws, key=lambda r: -r["warm"]["save_to_view_s"])[:5]:
205 print(f" - `{r['id']}`: view {r['warm']['save_to_view_s']} s, compile {r['warm']['compile_s']} s, "
206 f"{r['warm']['passes']} passes, cold {r['aus']['wall']:.2f} s")
207
208 hi = [r for r in both if load_max(r) > 6]
209 print(f"\nRuns measured at a load above 6: {len(hi)} documents.")
210
211 if rerun_out:
212 ids = {r["id"] for r in sorted(docs, key=lambda r: -r["ratio"])[:10]}
213 ids |= {r["id"] for r in docs if r["ratio"] > 3}
214 ids |= {r["id"] for r in hi if r["tier"] in DOCS}
215 with open(rerun_out, "w") as f:
216 for i in sorted(ids):
217 f.write(i + "\n")
218 if fail_out:
219 with open(fail_out, "w") as f:
220 for r in staged:
221 if r["aus"]["status"] != "ok":
222 f.write("\t".join([r["id"], r["tier"], r["aus"]["status"], r["typ"]["status"],
223 r["aus"].get("msg", "").replace("\t", " ")]) + "\n")
224 for r in allr:
225 if "aus" not in r:
226 f.write("\t".join([r["id"], r["tier"], "unstaged", "-", r.get("stage_err", "")]) + "\n")
227
228
229if __name__ == "__main__":
230 main()