Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/parity.py

14.2 KiB, 1 run

created by r1870400018:72510, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2# Render parity over the whole corpus (FIRE-3, 2026-10-11): every document in real_corpus.txt is copied once
3# to a work directory and compiled from that one copy by Austenite's evaluator (`--eval`) and by typst
4# 0.15.1, each under its own memory scope and wall-time cap. The two PDFs are compared by page count, by
5# `pdftotext -layout` text (runs of spaces collapsed, then the count of differing lines) and by raster (every
6# page up to MAXPAGES at DPI, `compare -metric AE -fuzz 10%`, the worst page's fraction). One JSONL row per
7# document. A crash, hang or out-of-memory is recorded and the run moves on. A document already in the JSONL
8# is skipped, so a killed run is simply started again.
9#
10# Verdicts: equivalent (same pages, no text line differs, worst raster <= RASTER_EQ), near (same pages, text
11# equal, raster above it), differs, aus_fails, typst_fails (counted apart from the shares). `cause` keys an
12# aus_fails row by the construct its diagnostic names, so failures can be ranked (report_parity.py).
13#
14# Run it detached as one job:
15# systemd-run --user --scope --quiet -p MemoryMax=4G --slice=claude-rc.slice --unit=<name> \
16# python3 -I tools/bench/corpus/parity.py
17#
18# Environment (defaults): W (~/.cache/austenite/claude-rc-7/parity), BIN (~/.local/bin/austenite), TYPST
19# (~/bin/typst), LIST (real_corpus.txt beside this script), OUT ($W/parity.jsonl), TMO (120 s per engine),
20# MEM (3G per engine), TIERS ("universe fixture tree"), ONLY (a file of ids, one per line), LIMIT (0: all),
21# DPI (50), MAXPAGES (20), RASTER_EQ (0.001).
22import difflib
23import json
24import os
25import re
26import shutil
27import subprocess
28import sys
29import time
30import unicodedata
31
32HERE = os.path.dirname(os.path.abspath(__file__))
33CRATE = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
34REPO = os.path.dirname(CRATE)
35HOME = os.path.expanduser("~")
36W = os.environ.get("W", os.path.join(HOME, ".cache/austenite/claude-rc-7/parity"))
37BIN = os.environ.get("BIN", os.path.join(HOME, ".local/bin/austenite"))
38TYPST = os.environ.get("TYPST", os.path.join(HOME, "bin/typst"))
39LIST = os.environ.get("LIST", os.path.join(HERE, "real_corpus.txt"))
40OUT = os.environ.get("OUT", os.path.join(W, "parity.jsonl"))
41TMO = int(os.environ.get("TMO", "120"))
42MEM = os.environ.get("MEM", "3G")
43TIERS = os.environ.get("TIERS", "universe fixture tree").split()
44ONLY = os.environ.get("ONLY", "")
45LIMIT = int(os.environ.get("LIMIT", "0"))
46DPI = os.environ.get("DPI", "50")
47MAXP = int(os.environ.get("MAXPAGES", "20"))
48RASTER_EQ = float(os.environ.get("RASTER_EQ", "0.001"))
49os.environ.setdefault("TYPST_PACKAGE_CACHE_PATH", os.path.join(W, "pkgs"))
50
51LOG = os.path.join(W, "parity.log")
52
53
54def log(msg):
55 with open(LOG, "a") as f:
56 f.write("%s %s\n" % (time.strftime("%T"), msg))
57
58
59def capped(cmd, cwd, out_prefix):
60 # One engine run under its own memory scope and wall-time cap; returns (exit code, seconds).
61 t0 = time.time()
62 with open(out_prefix + ".out", "w") as o, open(out_prefix + ".err", "w") as e:
63 rc = subprocess.call(
64 ["systemd-run", "--user", "--scope", "--quiet", "-p", "MemoryMax=" + MEM, "-p", "MemorySwapMax=0",
65 "--slice=claude-rc.slice", "timeout", "-k", "5", str(TMO)] + cmd,
66 cwd=cwd, stdout=o, stderr=e)
67 return rc, round(time.time() - t0, 2)
68
69
70def read(p, limit=None):
71 try:
72 with open(p, errors="replace") as f:
73 return f.read() if limit is None else f.read(limit)
74 except OSError:
75 return ""
76
77
78ANSI = re.compile(r"\x1b\[[0-9;]*m")
79
80
81def first_error(prefix, rc):
82 # The first line that names an error, else the last non-empty line, of stderr then stdout.
83 if rc == 124 or rc == 137:
84 return "timeout after %d s" % TMO if rc == 124 else "killed (memory cap %s or timeout)" % MEM
85 text = ANSI.sub("", read(prefix + ".err") + "\n" + read(prefix + ".out"))
86 lines = [l.strip() for l in text.splitlines() if l.strip()]
87 for l in lines:
88 if re.search(r"\berror\b|panicked|Error|ERR", l):
89 return l[:400]
90 return (lines[-1] if lines else "")[:400]
91
92
93def pages(pdf):
94 try:
95 info = subprocess.run(["pdfinfo", pdf], capture_output=True, text=True, timeout=60).stdout
96 except (subprocess.TimeoutExpired, OSError):
97 return None
98 m = re.search(r"^Pages:\s+(\d+)", info, re.M)
99 return int(m.group(1)) if m else None
100
101
102SPACES = re.compile(r" +")
103
104
105def text_lines(pdf):
106 try:
107 t = subprocess.run(["pdftotext", "-layout", pdf, "-"], capture_output=True, text=True,
108 errors="replace", timeout=120).stdout
109 except (subprocess.TimeoutExpired, OSError):
110 return None
111 # A page break is a line break; runs of spaces collapse; trailing space is not a difference.
112 # Invisible format characters (word joiner, zero-width space, soft hyphen; Unicode category Cf) draw
113 # nothing, so one engine emitting them and the other not is not a text difference.
114 t = "".join(c for c in t.replace("\f", "\n") if unicodedata.category(c) != "Cf")
115 return [SPACES.sub(" ", l).strip() for l in t.split("\n")]
116
117
118def text_diff(a, b):
119 if a is None or b is None:
120 return None
121 n = 0
122 for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
123 if tag != "equal":
124 n += max(i2 - i1, j2 - j1)
125 return n
126
127
128def raster(pdf, pfx, npages):
129 subprocess.run(["pdftoppm", "-r", DPI, "-png", "-l", str(min(npages, MAXP)), pdf, pfx],
130 capture_output=True, timeout=300)
131
132
133def page_pngs(d, pfx):
134 out = {}
135 for f in os.listdir(d):
136 m = re.match(re.escape(pfx) + r"-0*(\d+)\.png$", f)
137 if m:
138 out[int(m.group(1))] = os.path.join(d, f)
139 return out
140
141
142def png_size(p):
143 with open(p, "rb") as f:
144 h = f.read(24)
145 return int.from_bytes(h[16:20], "big"), int.from_bytes(h[20:24], "big")
146
147
148def page_diff(a, b):
149 # Fraction of pixels differing beyond 10% fuzz; 1.0 when the page sizes differ.
150 wa, ha = png_size(a)
151 wb, hb = png_size(b)
152 if (wa, ha) != (wb, hb):
153 return 1.0
154 r = subprocess.run(["compare", "-metric", "AE", "-fuzz", "10%", a, b, "null:"],
155 capture_output=True, text=True, timeout=120)
156 m = re.match(r"\s*([0-9.e+]+)", r.stderr)
157 if not m:
158 return 1.0
159 return float(m.group(1)) / float(wa * ha)
160
161
162# Cause keys. The evaluator's diagnostic names the construct; the key is the construct, not the site.
163CAUSES = [
164 (r"bibliography style `?[^`]*`? is not supported", "unsupported: bibliography style (CSL)"),
165 (r"type `?([\w.-]+)`? has no method `?([\w.-]+)", "no method: {}.{}"),
166 (r"`?([\w.-]+)`? does not contain field `?([\w.-]+)", "no field: {}.{}"),
167 (r"(expected [^,]+(?:, [^,]+)*,? found [\w.-]+)", "type mismatch: {}"),
168 (r"unknown variable:? `?([\w.-]+)", "unknown variable: {}"),
169 (r"unknown function:? `?([\w.-]+)", "unknown function: {}"),
170 (r"unknown field:? `?([\w.-]+)`?", "unknown field: {}"),
171 (r"(?:type|value) `?([\w.-]+)`? (?:has|does not have) (?:no )?(?:field|method) `?([\w.-]+)", "no field: {}.{}"),
172 (r"(?:no|unknown) (?:field|method) `?([\w.-]+)`? (?:on|of|for) `?([\w.-]+)", "no field: {1}.{0}"),
173 (r"unsupported element:? `?([\w.-]+)", "unsupported element: {}"),
174 (r"element `?([\w.-]+)`? (?:is )?(?:not supported|unsupported)", "unsupported element: {}"),
175 (r"unknown (?:argument|parameter):? `?([\w.-]+)", "unknown argument: {}"),
176 (r"unexpected argument:? `?([\w.-]+)", "unexpected argument: {}"),
177 (r"package `?(@[\w/.:-]+)`?", "package: {}"),
178 (r"(?:cannot|could not|failed to) (?:read|open|find|load) (?:file )?`?([^`:\s]+)", "file: {}"),
179]
180
181
182def norm_cause(err, rc):
183 if rc == 124:
184 return "timeout"
185 if rc == 137:
186 return "killed (memory or timeout)"
187 if "panicked" in err:
188 m = re.search(r"panicked at ([^:]+)", err)
189 return "panic: %s" % (m.group(1) if m else "?")
190 for pat, fmt in CAUSES:
191 m = re.search(pat, err, re.I)
192 if m:
193 g = m.groups()
194 name = fmt.format(*g)
195 if name.startswith("file: "):
196 name = "file: " + os.path.splitext(name[6:])[1]
197 return name
198 # The error class: the message with its names, numbers and quotes dropped.
199 e = re.sub(r"^.*?error:?\s*", "", err, flags=re.I)
200 e = re.sub(r"`[^`]*`|\"[^\"]*\"|'[^']*'", "_", e)
201 e = re.sub(r"\d+", "N", e)
202 e = re.sub(r"\s+at\s+\S+$", "", e)
203 return ("class: " + e.strip())[:120]
204
205
206SITE = re.compile(r"(@[\w-]+/[\w.-]+:[\d.]+)/|(?:^|\s)(/\S+?\.typ):\d+")
207
208
209def cause_site(err):
210 # Where the failing construct sits: a package (name:version) or "document" for the template's own files.
211 m = SITE.search(err or "")
212 if not m:
213 return None
214 return m.group(1) if m.group(1) else "document"
215
216
217def stage(tier, ident, spec, work):
218 # Copies the document's tree into work/src; returns (root, source, fonts) or raises RuntimeError.
219 src_dir = os.path.join(work, "src")
220 if tier == "universe":
221 cache = os.path.join(W, "src", "universe", ident.replace(":", "-"))
222 if not os.path.isfile(os.path.join(cache, ".staged")):
223 shutil.rmtree(cache, ignore_errors=True)
224 os.makedirs(os.path.dirname(cache), exist_ok=True)
225 r = subprocess.run([TYPST, "init", "@preview/" + ident, cache], capture_output=True, text=True,
226 timeout=180)
227 if r.returncode != 0:
228 shutil.rmtree(cache, ignore_errors=True)
229 raise RuntimeError("typst init: " + " ".join(r.stderr.split())[:300])
230 open(os.path.join(cache, ".staged"), "w").close()
231 shutil.copytree(cache, src_dir, symlinks=True, ignore=shutil.ignore_patterns(".staged"))
232 source = os.path.join(src_dir, spec)
233 else:
234 path = os.path.join(REPO, spec)
235 shutil.copytree(os.path.dirname(path), src_dir, symlinks=True, ignore=shutil.ignore_patterns("*.pdf"))
236 source = os.path.join(src_dir, os.path.basename(path))
237 if not os.path.isfile(source):
238 raise RuntimeError("no source " + spec)
239 fonts = None
240 for dp, _, fs in os.walk(src_dir):
241 if any(f.lower().endswith((".ttf", ".otf", ".ttc")) for f in fs):
242 fonts = src_dir
243 break
244 return src_dir, source, fonts
245
246
247def measure(tier, ident, spec, extra):
248 row = {"id": ident, "tier": tier, "spec": spec, "extra": extra}
249 work = os.path.join(W, "run", "cur")
250 shutil.rmtree(work, ignore_errors=True)
251 os.makedirs(work)
252 try:
253 root, source, fonts = stage(tier, ident, spec, work)
254 except (RuntimeError, OSError, subprocess.TimeoutExpired) as e:
255 row.update({"typst_ok": False, "typst_rc": None, "typst_err": "stage: " + str(e)[:300],
256 "aus_ok": None, "verdict": "typst_fails", "stage_fail": True})
257 return row
258 fa = ["--font-path", fonts] if fonts else []
259 tpdf = os.path.join(work, "t.pdf")
260 adir = os.path.join(work, "ao")
261 # typst first: it fetches the template's imports into the package cache Austenite then reads.
262 trc, tsec = capped([TYPST, "compile", "--ignore-system-fonts", "--root", root] + fa + [source, tpdf],
263 os.path.dirname(source), os.path.join(work, "t"))
264 arc, asec = capped([BIN, "--eval", "--root", root] + fa + [source, adir], os.path.dirname(source),
265 os.path.join(work, "a"))
266 apdf = os.path.join(adir, "document.pdf")
267 if not os.path.isfile(apdf):
268 cands = [f for f in os.listdir(adir) if f.endswith(".pdf")] if os.path.isdir(adir) else []
269 apdf = os.path.join(adir, cands[0]) if cands else apdf
270 t_ok = trc == 0 and os.path.isfile(tpdf)
271 a_ok = arc == 0 and os.path.isfile(apdf)
272 row.update({
273 "typst_ok": t_ok, "typst_rc": trc, "typst_s": tsec, "typst_err": None if t_ok else first_error(os.path.join(work, "t"), trc),
274 "aus_ok": a_ok, "aus_rc": arc, "aus_s": asec, "aus_err": None if a_ok else first_error(os.path.join(work, "a"), arc),
275 })
276 if not t_ok:
277 row["verdict"] = "typst_fails"
278 return row
279 row["typst_pages"] = pages(tpdf)
280 if not a_ok:
281 row["verdict"] = "aus_fails"
282 row["cause"] = norm_cause(row["aus_err"], arc)
283 row["cause_site"] = cause_site(row["aus_err"])
284 return row
285 row["aus_pages"] = pages(apdf)
286 tl = text_lines(tpdf)
287 al = text_lines(apdf)
288 row["text_lines"] = len(tl) if tl is not None else None
289 row["text_diff"] = text_diff(tl, al)
290 if tl is not None and al is not None and row["text_diff"]:
291 # The first differing pair, for the report's sample.
292 for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, tl, al, autojunk=False).get_opcodes():
293 if tag != "equal":
294 row["text_first"] = {"line": i1 + 1, "typst": " | ".join(tl[i1:i2][:2])[:200],
295 "aus": " | ".join(al[j1:j2][:2])[:200]}
296 break
297 worst, worst_p, npages = None, None, min(row["typst_pages"] or 0, row["aus_pages"] or 0)
298 if npages:
299 raster(tpdf, os.path.join(work, "t"), npages)
300 raster(apdf, os.path.join(work, "a"), npages)
301 tp = page_pngs(work, "t")
302 ap = page_pngs(work, "a")
303 for p in sorted(tp):
304 if p in ap:
305 try:
306 d = page_diff(tp[p], ap[p])
307 except (OSError, subprocess.TimeoutExpired):
308 d = 1.0
309 if worst is None or d > worst:
310 worst, worst_p = d, p
311 row["raster_worst"] = None if worst is None else round(worst, 6)
312 row["raster_page"] = worst_p
313 same = row["typst_pages"] == row["aus_pages"] and row["typst_pages"] is not None
314 if same and row["text_diff"] == 0 and worst is not None and worst <= RASTER_EQ:
315 row["verdict"] = "equivalent"
316 elif same and row["text_diff"] == 0:
317 row["verdict"] = "near"
318 else:
319 row["verdict"] = "differs"
320 if row["verdict"] == "differs" and worst_p is not None:
321 keep = os.path.join(W, "keep", re.sub(r"[^\w.-]", "_", ident))
322 os.makedirs(keep, exist_ok=True)
323 for pfx, pp in (("t", tp), ("a", ap)):
324 shutil.copy(pp[worst_p], os.path.join(keep, "%s-%d.png" % (pfx, worst_p)))
325 return row
326
327
328def main():
329 os.makedirs(W, exist_ok=True)
330 os.makedirs(os.environ["TYPST_PACKAGE_CACHE_PATH"], exist_ok=True)
331 done = set()
332 if os.path.isfile(OUT):
333 for l in open(OUT):
334 try:
335 done.add(json.loads(l)["id"])
336 except (ValueError, KeyError):
337 pass
338 want = None
339 if ONLY:
340 want = set(x.strip() for x in open(ONLY) if x.strip())
341 tv = subprocess.run([TYPST, "--version"], capture_output=True, text=True).stdout.strip()
342 log("parity start: list %s, out %s, bin %s, %s" % (LIST, OUT, BIN, tv))
343 n = 0
344 for line in open(LIST):
345 f = line.rstrip("\n").split("\t")
346 if not f[0] or f[0].startswith("#") or len(f) < 3 or f[0] not in TIERS:
347 continue
348 tier, ident, spec = f[0], f[1], f[2]
349 extra = f[3] if len(f) > 3 else ""
350 if want is not None and ident not in want:
351 continue
352 if ident in done:
353 continue
354 if LIMIT and n >= LIMIT:
355 break
356 n += 1
357 t0 = time.time()
358 try:
359 row = measure(tier, ident, spec, extra)
360 except Exception as e:
361 row = {"id": ident, "tier": tier, "spec": spec, "verdict": "harness_error", "harness_err": repr(e)[:300]}
362 with open(OUT, "a") as o:
363 o.write(json.dumps(row) + "\n")
364 log("%d %s %s %s %.1fs" % (n, tier, ident, row.get("verdict"), time.time() - t0))
365 shutil.rmtree(os.path.join(W, "run", "cur"), ignore_errors=True)
366 log("parity end: %d documents" % n)
367
368
369if __name__ == "__main__":
370 main()