Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/tools/csl_fetch.py

7.3 KiB, 1 run

created by r1870400018:73164, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2"""Rebuild fe2o3_text's CSL archive, `data/csl/`, as the CSL XML hayagriva 0.10.1 archived.
3
4hayagriva embeds its 91 styles and 64 locales as CBOR in citationberg's serde schema. This script
5reads each record's `<updated>` stamp and recovers, from the CSL styles and locales repositories,
6the XML revision that carries it: walking a file's history backwards from hayagriva 0.10.1's
7publication, it takes the first revision whose `<updated>` equals the record's. Where none does,
8it takes the latest revision on or before the record's date and marks the row `nearest` in the
9index. `alphanumeric` is hayagriva's own style and is copied from the crate.
10
11Std-only. Usage:
12
13 python3 -I tools/csl_fetch.py HAYAGRIVA_CRATE_DIR STYLES_REPO LOCALES_REPO OUT_DATA_DIR OUT_RS
14
15where the repositories are clones (a `--filter=blob:none` clone is enough) of
16https://github.com/citation-style-language/styles and .../locales, OUT_DATA_DIR is
17`fe2o3_text/data/csl` and OUT_RS is `fe2o3_text/src/csl/archive_index.rs`.
18"""
19
20import os
21import re
22import subprocess
23import sys
24
25sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
26import cbor # noqa: E402
27
28CUTOFF = "2026-06-14T10:08:30Z" # hayagriva 0.10.1 on crates.io
29HAYAGRIVA = "hayagriva 0.10.1"
30
31
32def git(repo, *args, binary=False):
33 out = subprocess.run(["git", "-C", repo] + list(args), check=True, capture_output=True)
34 return out.stdout if binary else out.stdout.decode("utf-8")
35
36
37def updated_of_xml(text):
38 m = re.search(r"<updated>\s*([^<\s]+)\s*</updated>", text)
39 return m.group(1) if m else ""
40
41
42def updated_of_record(rec):
43 info = rec.get("info") or {}
44 up = info.get("updated")
45 if isinstance(up, dict):
46 up = up.get("$text") or up.get("$value")
47 return up or ""
48
49
50def same_stamp(a, b):
51 # citationberg normalises the timestamp; compare its date and time, not its spelling.
52 def norm(s):
53 s = s.strip().replace("Z", "+00:00")
54 return s[:19]
55 return norm(a) == norm(b)
56
57
58def recover(repo, path, stamp):
59 revs = git(repo, "log", "--format=%H %cI", "--before=" + CUTOFF, "--", path).split("\n")
60 revs = [r.split(" ") for r in revs if r.strip()]
61 if not revs:
62 raise SystemExit("no history for %s in %s" % (path, repo))
63 fallback = None
64 for sha, date in revs:
65 text = git(repo, "show", "%s:%s" % (sha, path))
66 up = updated_of_xml(text)
67 if same_stamp(up, stamp):
68 return sha, text, "exact"
69 if fallback is None and up[:10] <= stamp[:10]:
70 fallback = (sha, text, "nearest")
71 if up[:10] < stamp[:10] and fallback is not None:
72 break
73 if fallback is None:
74 raise SystemExit("no revision of %s at or before %s" % (path, stamp))
75 return fallback
76
77
78def parse_archive_rs(src):
79 # Variant to CBOR file, and the names hayagriva accepts for each variant.
80 files = dict(re.findall(r"Self::(\w+) => include_bytes!\(\"\.\./\.\./archive/styles/([\w.-]+)\.cbor\"\)", src))
81 body = src[src.index("pub fn by_name"):src.index("pub fn by_id")]
82 names = {}
83 for name, var in re.findall(r"\"([^\"]+)\" => Some\(Self::(\w+)\)", body):
84 names.setdefault(var, []).append(name)
85 order = re.findall(r"Self::(\w+) => include_bytes!", src)
86 locales = re.findall(r"include_bytes!\(\"\.\./\.\./archive/locales/([\w-]+)\.cbor\"\)", src)
87 return order, files, names, locales
88
89
90def rust_str(s):
91 return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
92
93
94def main(argv):
95 if len(argv) != 6:
96 raise SystemExit(__doc__)
97 crate, styles_repo, locales_repo, out_dir, out_rs = argv[1:]
98 src = open(os.path.join(crate, "src/csl/archive.rs"), encoding="utf-8").read()
99 order, files, names, locales = parse_archive_rs(src)
100 os.makedirs(os.path.join(out_dir, "styles"), exist_ok=True)
101 os.makedirs(os.path.join(out_dir, "locales"), exist_ok=True)
102 style_rows = []
103 for var in order:
104 stem = files[var]
105 rec = cbor.decode(open(os.path.join(crate, "archive/styles", stem + ".cbor"), "rb").read())
106 stamp = updated_of_record(rec)
107 cid = rec["info"]["id"]
108 if stem == "alphanumeric":
109 text = open(os.path.join(crate, "styles/alphanumeric.csl"), encoding="utf-8").read()
110 sha, how = HAYAGRIVA, "crate"
111 else:
112 base = cid.rsplit("/", 1)[-1]
113 sha, text, how = recover(styles_repo, base + ".csl", stamp)
114 with open(os.path.join(out_dir, "styles", stem + ".csl"), "w", encoding="utf-8") as f:
115 f.write(text)
116 style_rows.append((stem, cid, names.get(var, []), stamp, sha, how))
117 print("style %-60s %s %s" % (stem, how, sha[:10]), file=sys.stderr)
118 locale_rows = []
119 newest = ""
120 for code in locales:
121 rec = cbor.decode(open(os.path.join(crate, "archive/locales", code + ".cbor"), "rb").read())
122 stamp = updated_of_record(rec)
123 path = "locales-%s.xml" % code
124 sha, text, how = recover(locales_repo, path, stamp)
125 date = git(locales_repo, "log", "-1", "--format=%cI", sha).strip()
126 newest = max(newest, date)
127 with open(os.path.join(out_dir, "locales", code + ".xml"), "w", encoding="utf-8") as f:
128 f.write(text)
129 locale_rows.append((code, stamp, sha, how))
130 print("locale %-10s %s %s" % (code, how, sha[:10]), file=sys.stderr)
131 # locales.json as it stood with the newest archived locale.
132 jsha = git(locales_repo, "log", "-1", "--format=%H", "--before=" + newest, "--", "locales.json").strip()
133 with open(os.path.join(out_dir, "locales.json"), "w", encoding="utf-8") as f:
134 f.write(git(locales_repo, "show", "%s:locales.json" % jsha))
135 with open(os.path.join(out_dir, "index.tsv"), "w", encoding="utf-8") as f:
136 f.write("# kind\tfile\tcsl id or code\tnames\tupdated\tsource revision\tmatch\n")
137 for stem, cid, ns, stamp, sha, how in style_rows:
138 f.write("style\tstyles/%s.csl\t%s\t%s\t%s\t%s\t%s\n" % (stem, cid, ",".join(ns), stamp, sha, how))
139 for code, stamp, sha, how in locale_rows:
140 f.write("locale\tlocales/%s.xml\t%s\t\t%s\t%s\t%s\n" % (code, code, stamp, sha, how))
141 f.write("json\tlocales.json\t\t\t\t%s\texact\n" % jsha)
142 write_rs(out_rs, style_rows, locale_rows)
143
144
145def write_rs(path, style_rows, locale_rows):
146 out = []
147 out.append("// Generated by tools/csl_fetch.py from hayagriva 0.10.1's archive. Do not edit by hand.\n")
148 out.append("// Columns: file stem, CSL id, the names Typst accepts, `<updated>` as hayagriva archived it.\n")
149 out.append("pub(super) const STYLES: &[(&str, &str, &[&str], &str)] = &[\n")
150 for stem, cid, ns, stamp, _sha, _how in style_rows:
151 out.append("\t(%s, %s, &[%s], %s),\n" % (rust_str(stem), rust_str(cid), ", ".join(rust_str(n) for n in ns), rust_str(stamp)))
152 out.append("];\n\n")
153 out.append("pub(super) const LOCALES: &[&str] = &[\n")
154 for code, _stamp, _sha, _how in locale_rows:
155 out.append("\t%s,\n" % rust_str(code))
156 out.append("];\n\n")
157 out.append("#[cfg(feature = \"csl-archive\")]\n")
158 out.append("pub(super) fn style_xml(stem: &str) -> Option<&'static str> {\n\tSome(match stem {\n")
159 for stem, *_ in style_rows:
160 out.append("\t\t%s\t=> include_str!(\"../../data/csl/styles/%s.csl\"),\n" % (rust_str(stem), stem))
161 out.append("\t\t_ => return None,\n\t})\n}\n\n")
162 out.append("#[cfg(feature = \"csl-archive\")]\n")
163 out.append("pub(super) fn locale_xml(code: &str) -> Option<&'static str> {\n\tSome(match code {\n")
164 for code, *_ in locale_rows:
165 out.append("\t\t%s\t=> include_str!(\"../../data/csl/locales/%s.xml\"),\n" % (rust_str(code), code))
166 out.append("\t\t_ => return None,\n\t})\n}\n\n")
167 out.append("#[cfg(feature = \"csl-archive\")]\n")
168 out.append("pub(super) const LOCALES_JSON: &str = include_str!(\"../../data/csl/locales.json\");\n")
169 with open(path, "w", encoding="utf-8") as f:
170 f.write("".join(out))
171
172
173if __name__ == "__main__":
174 main(sys.argv)