Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/parity_diff.py

2.4 KiB, 1 run

created by r1870400018:73737, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2# Compares two parity.py JSONL runs row by row: regressions (a row equivalent or near in the base that is
3# now worse, equivalent to near included), gains, verdict moves and the compile, equivalent and near shares per tier.
4# Usage: parity_diff.py BASE.jsonl NEW.jsonl [--moves]. Exits 1 on any regression.
5import collections
6import json
7import sys
8
9RANK = {"equivalent": 0, "near": 1, "differs": 2, "aus_fails": 3}
10KEEP = ("equivalent", "near")
11
12
13def load(path):
14 rows = {}
15 for l in open(path):
16 try:
17 r = json.loads(l)
18 except ValueError:
19 continue
20 rows[r["id"]] = r
21 return rows
22
23
24def shares(rows, tier):
25 c = collections.Counter(r["verdict"] for r in rows.values() if r["tier"] == tier)
26 base = sum(c[v] for v in RANK)
27 comp = c["equivalent"] + c["near"] + c["differs"]
28 return base, comp, c["equivalent"], c["near"]
29
30
31def pct(n, d):
32 return "%.1f%%" % (100.0 * n / d) if d else "-"
33
34
35def main(argv):
36 if len(argv) < 3:
37 print("usage: parity_diff.py BASE.jsonl NEW.jsonl [--moves]")
38 return 2
39 a = load(argv[1])
40 b = load(argv[2])
41 moves = "--moves" in argv
42 tiers = sorted(set(r["tier"] for r in a.values()) | set(r["tier"] for r in b.values()))
43 print("| Tier | Measure | Base | New | Delta |")
44 print("|---|---|---:|---:|---:|")
45 for t in tiers:
46 ba, ca, ea, na = shares(a, t)
47 bb, cb, eb, nb = shares(b, t)
48 if not (ba or bb):
49 continue
50 for name, x, y in (("compile", ca, cb), ("equivalent", ea, eb), ("near", na, nb)):
51 print("| %s | %s | %d/%d %s | %d/%d %s | %+d |" % (t, name, x, ba, pct(x, ba), y, bb, pct(y, bb), y - x))
52 regs = []
53 gains = []
54 mv = collections.Counter()
55 for k in sorted(set(a) & set(b)):
56 va = a[k]["verdict"]
57 vb = b[k]["verdict"]
58 if va == vb:
59 continue
60 mv[(va, vb)] += 1
61 if va in KEEP and RANK.get(vb, 9) > RANK[va]:
62 regs.append((k, va, vb))
63 elif vb in KEEP and va not in KEEP:
64 gains.append((k, va, vb))
65 if moves:
66 print("move %s: %s -> %s" % (k, va, vb))
67 print("\nVerdict moves:")
68 for (va, vb), n in sorted(mv.items(), key=lambda x: -x[1]):
69 print(" %s -> %s: %d" % (va, vb, n))
70 print("\nGains into equivalent or near: %d" % len(gains))
71 for k, va, vb in gains:
72 print(" + %s: %s -> %s" % (k, va, vb))
73 print("Regressions out of equivalent or near: %d" % len(regs))
74 for k, va, vb in regs:
75 print(" - %s: %s -> %s" % (k, va, vb))
76 only = (set(a) ^ set(b))
77 if only:
78 print("Rows in one run only: %d" % len(only))
79 return 1 if regs else 0
80
81
82if __name__ == "__main__":
83 sys.exit(main(sys.argv))