oxedyne/fe2o3/fe2o3_text/tools/csl_fetch.py
7.3 KiB, 1 run
created by r1870400018:73164, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | """Rebuild fe2o3_text's CSL archive, `data/csl/`, as the CSL XML hayagriva 0.10.1 archived. |
| 3 | |
| 4 | hayagriva embeds its 91 styles and 64 locales as CBOR in citationberg's serde schema. This script |
| 5 | reads each record's `<updated>` stamp and recovers, from the CSL styles and locales repositories, |
| 6 | the XML revision that carries it: walking a file's history backwards from hayagriva 0.10.1's |
| 7 | publication, it takes the first revision whose `<updated>` equals the record's. Where none does, |
| 8 | it takes the latest revision on or before the record's date and marks the row `nearest` in the |
| 9 | index. `alphanumeric` is hayagriva's own style and is copied from the crate. |
| 10 | |
| 11 | Std-only. Usage: |
| 12 | |
| 13 | python3 -I tools/csl_fetch.py HAYAGRIVA_CRATE_DIR STYLES_REPO LOCALES_REPO OUT_DATA_DIR OUT_RS |
| 14 | |
| 15 | where the repositories are clones (a `--filter=blob:none` clone is enough) of |
| 16 | https://github.com/citation-style-language/styles and .../locales, OUT_DATA_DIR is |
| 17 | `fe2o3_text/data/csl` and OUT_RS is `fe2o3_text/src/csl/archive_index.rs`. |
| 18 | """ |
| 19 | |
| 20 | import os |
| 21 | import re |
| 22 | import subprocess |
| 23 | import sys |
| 24 | |
| 25 | sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| 26 | import cbor # noqa: E402 |
| 27 | |
| 28 | CUTOFF = "2026-06-14T10:08:30Z" # hayagriva 0.10.1 on crates.io |
| 29 | HAYAGRIVA = "hayagriva 0.10.1" |
| 30 | |
| 31 | |
| 32 | def git(repo, *args, binary=False): |
| 33 | out = subprocess.run(["git", "-C", repo] + list(args), check=True, capture_output=True) |
| 34 | return out.stdout if binary else out.stdout.decode("utf-8") |
| 35 | |
| 36 | |
| 37 | def updated_of_xml(text): |
| 38 | m = re.search(r"<updated>\s*([^<\s]+)\s*</updated>", text) |
| 39 | return m.group(1) if m else "" |
| 40 | |
| 41 | |
| 42 | def updated_of_record(rec): |
| 43 | info = rec.get("info") or {} |
| 44 | up = info.get("updated") |
| 45 | if isinstance(up, dict): |
| 46 | up = up.get("$text") or up.get("$value") |
| 47 | return up or "" |
| 48 | |
| 49 | |
| 50 | def same_stamp(a, b): |
| 51 | # citationberg normalises the timestamp; compare its date and time, not its spelling. |
| 52 | def norm(s): |
| 53 | s = s.strip().replace("Z", "+00:00") |
| 54 | return s[:19] |
| 55 | return norm(a) == norm(b) |
| 56 | |
| 57 | |
| 58 | def recover(repo, path, stamp): |
| 59 | revs = git(repo, "log", "--format=%H %cI", "--before=" + CUTOFF, "--", path).split("\n") |
| 60 | revs = [r.split(" ") for r in revs if r.strip()] |
| 61 | if not revs: |
| 62 | raise SystemExit("no history for %s in %s" % (path, repo)) |
| 63 | fallback = None |
| 64 | for sha, date in revs: |
| 65 | text = git(repo, "show", "%s:%s" % (sha, path)) |
| 66 | up = updated_of_xml(text) |
| 67 | if same_stamp(up, stamp): |
| 68 | return sha, text, "exact" |
| 69 | if fallback is None and up[:10] <= stamp[:10]: |
| 70 | fallback = (sha, text, "nearest") |
| 71 | if up[:10] < stamp[:10] and fallback is not None: |
| 72 | break |
| 73 | if fallback is None: |
| 74 | raise SystemExit("no revision of %s at or before %s" % (path, stamp)) |
| 75 | return fallback |
| 76 | |
| 77 | |
| 78 | def parse_archive_rs(src): |
| 79 | # Variant to CBOR file, and the names hayagriva accepts for each variant. |
| 80 | files = dict(re.findall(r"Self::(\w+) => include_bytes!\(\"\.\./\.\./archive/styles/([\w.-]+)\.cbor\"\)", src)) |
| 81 | body = src[src.index("pub fn by_name"):src.index("pub fn by_id")] |
| 82 | names = {} |
| 83 | for name, var in re.findall(r"\"([^\"]+)\" => Some\(Self::(\w+)\)", body): |
| 84 | names.setdefault(var, []).append(name) |
| 85 | order = re.findall(r"Self::(\w+) => include_bytes!", src) |
| 86 | locales = re.findall(r"include_bytes!\(\"\.\./\.\./archive/locales/([\w-]+)\.cbor\"\)", src) |
| 87 | return order, files, names, locales |
| 88 | |
| 89 | |
| 90 | def rust_str(s): |
| 91 | return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"' |
| 92 | |
| 93 | |
| 94 | def main(argv): |
| 95 | if len(argv) != 6: |
| 96 | raise SystemExit(__doc__) |
| 97 | crate, styles_repo, locales_repo, out_dir, out_rs = argv[1:] |
| 98 | src = open(os.path.join(crate, "src/csl/archive.rs"), encoding="utf-8").read() |
| 99 | order, files, names, locales = parse_archive_rs(src) |
| 100 | os.makedirs(os.path.join(out_dir, "styles"), exist_ok=True) |
| 101 | os.makedirs(os.path.join(out_dir, "locales"), exist_ok=True) |
| 102 | style_rows = [] |
| 103 | for var in order: |
| 104 | stem = files[var] |
| 105 | rec = cbor.decode(open(os.path.join(crate, "archive/styles", stem + ".cbor"), "rb").read()) |
| 106 | stamp = updated_of_record(rec) |
| 107 | cid = rec["info"]["id"] |
| 108 | if stem == "alphanumeric": |
| 109 | text = open(os.path.join(crate, "styles/alphanumeric.csl"), encoding="utf-8").read() |
| 110 | sha, how = HAYAGRIVA, "crate" |
| 111 | else: |
| 112 | base = cid.rsplit("/", 1)[-1] |
| 113 | sha, text, how = recover(styles_repo, base + ".csl", stamp) |
| 114 | with open(os.path.join(out_dir, "styles", stem + ".csl"), "w", encoding="utf-8") as f: |
| 115 | f.write(text) |
| 116 | style_rows.append((stem, cid, names.get(var, []), stamp, sha, how)) |
| 117 | print("style %-60s %s %s" % (stem, how, sha[:10]), file=sys.stderr) |
| 118 | locale_rows = [] |
| 119 | newest = "" |
| 120 | for code in locales: |
| 121 | rec = cbor.decode(open(os.path.join(crate, "archive/locales", code + ".cbor"), "rb").read()) |
| 122 | stamp = updated_of_record(rec) |
| 123 | path = "locales-%s.xml" % code |
| 124 | sha, text, how = recover(locales_repo, path, stamp) |
| 125 | date = git(locales_repo, "log", "-1", "--format=%cI", sha).strip() |
| 126 | newest = max(newest, date) |
| 127 | with open(os.path.join(out_dir, "locales", code + ".xml"), "w", encoding="utf-8") as f: |
| 128 | f.write(text) |
| 129 | locale_rows.append((code, stamp, sha, how)) |
| 130 | print("locale %-10s %s %s" % (code, how, sha[:10]), file=sys.stderr) |
| 131 | # locales.json as it stood with the newest archived locale. |
| 132 | jsha = git(locales_repo, "log", "-1", "--format=%H", "--before=" + newest, "--", "locales.json").strip() |
| 133 | with open(os.path.join(out_dir, "locales.json"), "w", encoding="utf-8") as f: |
| 134 | f.write(git(locales_repo, "show", "%s:locales.json" % jsha)) |
| 135 | with open(os.path.join(out_dir, "index.tsv"), "w", encoding="utf-8") as f: |
| 136 | f.write("# kind\tfile\tcsl id or code\tnames\tupdated\tsource revision\tmatch\n") |
| 137 | for stem, cid, ns, stamp, sha, how in style_rows: |
| 138 | f.write("style\tstyles/%s.csl\t%s\t%s\t%s\t%s\t%s\n" % (stem, cid, ",".join(ns), stamp, sha, how)) |
| 139 | for code, stamp, sha, how in locale_rows: |
| 140 | f.write("locale\tlocales/%s.xml\t%s\t\t%s\t%s\t%s\n" % (code, code, stamp, sha, how)) |
| 141 | f.write("json\tlocales.json\t\t\t\t%s\texact\n" % jsha) |
| 142 | write_rs(out_rs, style_rows, locale_rows) |
| 143 | |
| 144 | |
| 145 | def write_rs(path, style_rows, locale_rows): |
| 146 | out = [] |
| 147 | out.append("// Generated by tools/csl_fetch.py from hayagriva 0.10.1's archive. Do not edit by hand.\n") |
| 148 | out.append("// Columns: file stem, CSL id, the names Typst accepts, `<updated>` as hayagriva archived it.\n") |
| 149 | out.append("pub(super) const STYLES: &[(&str, &str, &[&str], &str)] = &[\n") |
| 150 | for stem, cid, ns, stamp, _sha, _how in style_rows: |
| 151 | out.append("\t(%s, %s, &[%s], %s),\n" % (rust_str(stem), rust_str(cid), ", ".join(rust_str(n) for n in ns), rust_str(stamp))) |
| 152 | out.append("];\n\n") |
| 153 | out.append("pub(super) const LOCALES: &[&str] = &[\n") |
| 154 | for code, _stamp, _sha, _how in locale_rows: |
| 155 | out.append("\t%s,\n" % rust_str(code)) |
| 156 | out.append("];\n\n") |
| 157 | out.append("#[cfg(feature = \"csl-archive\")]\n") |
| 158 | out.append("pub(super) fn style_xml(stem: &str) -> Option<&'static str> {\n\tSome(match stem {\n") |
| 159 | for stem, *_ in style_rows: |
| 160 | out.append("\t\t%s\t=> include_str!(\"../../data/csl/styles/%s.csl\"),\n" % (rust_str(stem), stem)) |
| 161 | out.append("\t\t_ => return None,\n\t})\n}\n\n") |
| 162 | out.append("#[cfg(feature = \"csl-archive\")]\n") |
| 163 | out.append("pub(super) fn locale_xml(code: &str) -> Option<&'static str> {\n\tSome(match code {\n") |
| 164 | for code, *_ in locale_rows: |
| 165 | out.append("\t\t%s\t=> include_str!(\"../../data/csl/locales/%s.xml\"),\n" % (rust_str(code), code)) |
| 166 | out.append("\t\t_ => return None,\n\t})\n}\n\n") |
| 167 | out.append("#[cfg(feature = \"csl-archive\")]\n") |
| 168 | out.append("pub(super) const LOCALES_JSON: &str = include_str!(\"../../data/csl/locales.json\");\n") |
| 169 | with open(path, "w", encoding="utf-8") as f: |
| 170 | f.write("".join(out)) |
| 171 | |
| 172 | |
| 173 | if __name__ == "__main__": |
| 174 | main(sys.argv) |