Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/report_parity.py

3.0 KiB, 1 run

created by r1870400018:72520, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2# Summarises parity.py's JSONL: counts and shares by tier and verdict, the aus_fails causes ranked by the
3# number of documents each breaks, and a sample of differs rows. Usage: report_parity.py PARITY.jsonl [N].
4import collections
5import json
6import sys
7
8rows = {}
9for l in open(sys.argv[1]):
10 try:
11 r = json.loads(l)
12 except ValueError:
13 continue
14 rows[r["id"]] = r
15top = int(sys.argv[2]) if len(sys.argv) > 2 else 25
16V = ["equivalent", "near", "differs", "aus_fails", "typst_fails", "harness_error"]
17
18
19def table(sel, name):
20 c = collections.Counter(r["verdict"] for r in sel)
21 base = sum(c[v] for v in V[:4])
22 comp = c["equivalent"] + c["near"] + c["differs"]
23 print("\n### %s (%d documents; %d where typst compiles)\n" % (name, len(sel), base))
24 print("| Measure | Count | Share of typst-compiling |")
25 print("|---|---:|---:|")
26 pct = lambda n: "%.1f%%" % (100.0 * n / base) if base else "-"
27 print("| Austenite compiles | %d | %s |" % (comp, pct(comp)))
28 for v in V[:4]:
29 print("| %s | %d | %s |" % (v, c[v], pct(c[v])))
30 print("| typst_fails (apart) | %d | - |" % c["typst_fails"])
31 if c["harness_error"]:
32 print("| harness_error (apart) | %d | - |" % c["harness_error"])
33
34
35allr = list(rows.values())
36table(allr, "All tiers")
37for t in ("universe", "fixture", "tree"):
38 sel = [r for r in allr if r["tier"] == t]
39 if sel:
40 table(sel, "Tier " + t)
41
42print("\n### aus_fails causes, ranked by documents broken\n")
43print("| Rank | Cause | All | Universe | Where (top sites) |")
44print("|---:|---|---:|---:|---|")
45cz = collections.defaultdict(list)
46for r in allr:
47 if r["verdict"] == "aus_fails":
48 cz[r.get("cause") or "?"].append(r)
49for i, (k, rs) in enumerate(sorted(cz.items(), key=lambda kv: -len(kv[1]))[:top], 1):
50 u = sum(1 for r in rs if r["tier"] == "universe")
51 sites = collections.Counter(r.get("cause_site") or "?" for r in rs).most_common(3)
52 where = ", ".join("%s %d" % s for s in sites)
53 print("| %d | `%s` | %d | %d | %s |" % (i, k.replace("|", "/"), len(rs), u, where))
54
55print("\n### differs: what differs\n")
56d = [r for r in allr if r["verdict"] == "differs"]
57pg = sum(1 for r in d if r.get("typst_pages") != r.get("aus_pages"))
58tx = sum(1 for r in d if r.get("text_diff"))
59print("%d rows: %d with a different page count, %d with text lines differing.\n" % (len(d), pg, tx))
60print("| Document | Pages typst/aus | Text lines differing / typst lines | Worst raster | First text difference (typst / aus) |")
61print("|---|---|---|---:|---|")
62step = max(1, len(d) // 10)
63def at_split(a, b):
64 # The two lines from just before where they first part.
65 i = 0
66 while i < min(len(a), len(b)) and a[i] == b[i]:
67 i += 1
68 s = max(0, i - 25)
69 return a[s:s + 60], b[s:s + 60]
70
71
72for r in d[::step][:10]:
73 tf = dict(r.get("text_first") or {})
74 tf["typst"], tf["aus"] = at_split(tf.get("typst") or "", tf.get("aus") or "")
75 print("| %s | %s/%s | %s/%s | %s | %s / %s |" % (r["id"], r.get("typst_pages"), r.get("aus_pages"),
76 r.get("text_diff"), r.get("text_lines"), r.get("raster_worst"),
77 tf["typst"].replace("|", "/"), tf["aus"].replace("|", "/")))