Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/lib/packages.mjs

12.7 KiB, 1 run

created by r1870400018:66239, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Typst packages for the typst.ts leg, supplied as Daimond supplies them (`src/wasm/typst.rs`: `Spec`,
2// `imports`, `repoint`, `prepare`, `packages`, `parse_pack`). The scanners are ports of its own, so what
3// is gathered here, what is followed into a package's dependencies and what is re-pointed cannot disagree.
4//
5// typst.ts has no network and no package registry, so Daimond carries packages beside the page and
6// re-points every `#import "@preview/name:1.2.3"` at a one-line module it writes at
7// `/_pkg/<ns>/<name>/<version>/<name>.typ`, which star-imports the package's entrypoint. A package is
8// looked for in a directory laid out in either of two ways:
9//
10// <dir>/<ns>/<name>/<version>/... Typst's own package cache; the .typ files, `typst.toml` and the
11// licence are taken, as Daimond's refresh.sh takes them
12// <dir>/<ns>/<name>/<version>.pack Daimond's vendored packs: an ASCII header, one blank line, the
13// file bodies end to end (`www/assets/typst/packs`)
14//
15// A package is `{ spec, pack }`: `spec` is `{ ns, name, version }` and `pack` is `{ entry, files }`, with
16// `files` as `[[package-relative path, bytes]]`. That is the shape a host hands a compiler that takes
17// packages from outside.
18
19import fs from 'node:fs';
20import path from 'node:path';
21
22export const PKG_ROOT = '/_pkg';
23
24// A refusal carries a closed `code`, so a caller that must not print the message can print the code.
25export function refuse(code, message) {
26 const e = new Error(message);
27 e.code = code;
28 return e;
29}
30
31const ext = (name) => {
32 const i = name.lastIndexOf('.');
33 return i < 0 ? '' : name.slice(i + 1).toLowerCase();
34};
35
36// ── A spec ──────────────────────────────────────────────────────────────────
37
38const isWord = (s) => /^[A-Za-z0-9_-]+$/.test(s);
39const isSemver = (s) => /^[0-9]+\.[0-9]+\.[0-9]+$/.test(s);
40
41/// A spec from the text of a quoted literal, `@preview/cetz:0.3.4`, or null when it is not one.
42export function parseSpec(lit) {
43 if (!lit.startsWith('@')) return null;
44 const rest = lit.slice(1);
45 const slash = rest.indexOf('/');
46 if (slash < 0) return null;
47 const ns = rest.slice(0, slash);
48 const tail = rest.slice(slash + 1);
49 const colon = tail.indexOf(':');
50 if (colon < 0) return null;
51 const name = tail.slice(0, colon);
52 const version = tail.slice(colon + 1);
53 if (!isWord(ns) || !isWord(name) || !isSemver(version)) return null;
54 return { ns, name, version };
55}
56
57export const specText = (s) => `@${s.ns}/${s.name}:${s.version}`;
58export const specDir = (s) => `${PKG_ROOT}/${s.ns}/${s.name}/${s.version}`;
59// Named for the package, so that `#import "@preview/cetz:0.3.4"` goes on binding `cetz`.
60export const specShim = (s) => `${specDir(s)}/${s.name}.typ`;
61
62// ── Reading a source for what it names ──────────────────────────────────────
63
64/// Every double-quoted literal in `src` as `[start, end, raw text]`; `start` is the opening quote and `end`
65/// one past the closing one.
66export function quoted(src) {
67 const out = [];
68 const n = src.length;
69 let i = 0;
70 while (i < n) {
71 if (src[i] !== '"') {
72 i++;
73 continue;
74 }
75 const start = i;
76 i++;
77 const from = i;
78 let closed = false;
79 while (i < n) {
80 if (src[i] === '"') {
81 closed = true;
82 break;
83 }
84 if (src[i] === '\\' && i + 1 < n) {
85 i += 2;
86 continue;
87 }
88 if (src[i] === '\n') break;
89 i++;
90 }
91 if (closed) {
92 out.push([start, i + 1, src.slice(from, i)]);
93 i++;
94 } else {
95 i = start + 1;
96 }
97 }
98 return out;
99}
100
101/// Every raw block and raw span in `src`, as `[start, end]`: a run of N backticks opens a region that the
102/// next run of exactly N closes.
103export function rawSpans(src) {
104 const out = [];
105 const n = src.length;
106 let i = 0;
107 while (i < n) {
108 if (src[i] !== '`') {
109 i++;
110 continue;
111 }
112 const open = i;
113 let run = 0;
114 while (i < n && src[i] === '`') {
115 run++;
116 i++;
117 }
118 let j = i;
119 let end = null;
120 while (j < n) {
121 if (src[j] !== '`') {
122 j++;
123 continue;
124 }
125 let m = 0;
126 while (j < n && src[j] === '`') {
127 m++;
128 j++;
129 }
130 if (m === run) {
131 end = j;
132 break;
133 }
134 }
135 if (end === null) break;
136 out.push([open, end]);
137 i = end;
138 }
139 return out;
140}
141
142const within = (spans, at) => spans.some(([a, b]) => at >= a && at < b);
143
144/// Is the literal beginning at `at` what an `#import` or `#include` is being given? A quotation of an import
145/// in prose is not one.
146function isImportOperand(src, at) {
147 let i = at;
148 while (i > 0 && /\s/.test(src[i - 1])) i--;
149 for (const kw of ['import', 'include']) {
150 const n = kw.length;
151 if (i < n || src.slice(i - n, i) !== kw) continue;
152 if (i === n) return true;
153 // A longer identifier that merely ends in "import" is not the keyword.
154 if (!/[A-Za-z0-9_-]/.test(src[i - n - 1])) return true;
155 }
156 return false;
157}
158
159/// Every package `src` imports, as `{ start, spec }`; a literal inside a raw block or span is not an import.
160export function imports(src) {
161 if (!src.includes('@')) return [];
162 const raws = rawSpans(src);
163 const out = [];
164 for (const [start, , raw] of quoted(src)) {
165 const spec = parseSpec(raw);
166 if (!spec) continue;
167 if (!isImportOperand(src, start)) continue;
168 if (within(raws, start)) continue;
169 out.push({ start, spec });
170 }
171 return out;
172}
173
174/// `.` and `..` collapsed in a slash path; null when it climbs above the top.
175export function norm(p) {
176 const out = [];
177 for (const seg of p.split('/')) {
178 if (seg === '' || seg === '.') continue;
179 if (seg === '..') {
180 if (out.pop() === undefined) return null;
181 continue;
182 }
183 out.push(seg);
184 }
185 return out.join('/');
186}
187
188/// Re-points every package and package-root reference in `text`.
189///
190/// # Arguments
191/// * `inside` - When `text` is a package's own file: `{ dir, names }`, the shadow directory the package sits
192/// at and every path it holds. Null for a project's own source.
193export function repoint(text, inside = null) {
194 const imps = imports(text);
195 const raws = rawSpans(text);
196 let out = '';
197 let last = 0;
198 for (const [start, end, raw] of quoted(text)) {
199 let rep = null;
200 const imp = imps.find((i) => i.start === start);
201 if (imp) {
202 rep = specShim(imp.spec);
203 } else if (inside && raw.startsWith('/') && !within(raws, start)) {
204 // Inside a package a leading "/" means the package root, not the project's. Re-pointed only when it
205 // names a file the pack holds, so a string of prose that begins with a slash is left alone.
206 const p = norm(raw);
207 if (p !== null && inside.names.includes(p)) rep = `${inside.dir}/pkg/${p}`;
208 }
209 if (rep !== null) {
210 out += text.slice(last, start) + '"' + rep + '"';
211 last = end;
212 }
213 }
214 return out + text.slice(last);
215}
216
217const lossy = new TextDecoder('utf-8');
218
219/// A package laid out for the compiler, read for what it imports in turn.
220///
221/// Returns `{ sources, assets, deps }`: shadow `.typ` files, re-pointed, with the one-line module that
222/// carries the binding; shadow files that are not sources (the manifest, the licence); and the specs the
223/// package imports, each with the file of its own that names it.
224export function prepare(spec, pack) {
225 const dir = specDir(spec);
226 const names = pack.files.map(([p]) => p);
227 const sources = [];
228 const assets = [];
229 const deps = [];
230 for (const [rel, data] of pack.files) {
231 const shadow = `${dir}/pkg/${rel}`;
232 if (ext(rel) !== 'typ') {
233 assets.push([shadow, data]);
234 continue;
235 }
236 const text = lossy.decode(data);
237 for (const { spec: dep } of imports(text)) {
238 if (specText(dep) === specText(spec) || deps.some(([d]) => specText(d) === specText(dep))) continue;
239 deps.push([dep, rel]);
240 }
241 sources.push([shadow, repoint(text, { dir, names })]);
242 }
243 // The binding: a star import re-exports whatever the entrypoint exports.
244 sources.push([specShim(spec), `#import "pkg/${pack.entry}": *\n`]);
245 return { sources, assets, deps };
246}
247
248// ── Finding a package ───────────────────────────────────────────────────────
249
250const isDir = (p) => {
251 try {
252 return fs.statSync(p).isDirectory();
253 } catch {
254 return false;
255 }
256};
257const isFile = (p) => {
258 try {
259 return fs.statSync(p).isFile();
260 } catch {
261 return false;
262 }
263};
264
265// What of a package directory is taken, as Daimond's refresh.sh takes it.
266const keep = (rel) => {
267 const b = rel.slice(rel.lastIndexOf('/') + 1);
268 return ext(rel) === 'typ' || b === 'typst.toml' || b === 'LICENSE' || b.startsWith('LICENSE.');
269};
270
271function readDirPackage(d) {
272 const files = [];
273 const walk = (dir, prefix) => {
274 for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
275 const rel = prefix ? `${prefix}/${e.name}` : e.name;
276 const p = path.join(dir, e.name);
277 if (e.isDirectory() || (e.isSymbolicLink() && isDir(p))) walk(p, rel);
278 else if (keep(rel) && isFile(p)) files.push([rel, fs.readFileSync(p)]);
279 }
280 };
281 walk(d, '');
282 files.sort((a, b) => (Buffer.from(a[0]).compare(Buffer.from(b[0]))));
283 const toml = files.find(([rel]) => rel === 'typst.toml');
284 const m = toml && /^[ \t]*entrypoint[ \t]*=[ \t]*"(.*)".*$/m.exec(Buffer.from(toml[1]).toString('utf8'));
285 if (!m) throw refuse('bad-package', 'a package names no entrypoint');
286 const entry = m[1].replace(/^\/+/, '');
287 if (!files.some(([rel]) => rel === entry)) throw refuse('bad-package', 'the entrypoint is not among the files');
288 return { entry, files };
289}
290
291function readPack(bytes, spec) {
292 let split = -1;
293 for (let i = 0; i + 1 < bytes.length; i++) {
294 if (bytes[i] === 0x0a && bytes[i + 1] === 0x0a) {
295 split = i;
296 break;
297 }
298 }
299 if (split < 0) throw refuse('bad-package', 'a pack has no header');
300 const lines = Buffer.from(bytes.subarray(0, split)).toString('utf8').split('\n');
301 if (lines[0] !== 'DAIMOND TYPST PACK 1') throw refuse('bad-package', 'not a pack this reader knows');
302 let entry = '';
303 const got = { ns: '', name: '', version: '' };
304 const names = [];
305 for (const line of lines.slice(1)) {
306 const sp = line.indexOf(' ');
307 if (sp < 0) continue;
308 const key = line.slice(0, sp);
309 const rest = line.slice(sp + 1).trim();
310 if (key === 'namespace') got.ns = rest;
311 else if (key === 'name') got.name = rest;
312 else if (key === 'version') got.version = rest;
313 else if (key === 'entrypoint') entry = rest.replace(/^\/+/, '');
314 else if (key === 'file') {
315 const s2 = rest.indexOf(' ');
316 if (s2 < 0) continue;
317 const len = Number(rest.slice(0, s2));
318 if (!Number.isInteger(len) || len < 0) continue;
319 names.push([rest.slice(s2 + 1), len]);
320 }
321 }
322 if (specText(got) !== specText(spec)) throw refuse('bad-package', 'a pack holds another package');
323 if (!entry) throw refuse('bad-package', 'a pack names no entrypoint');
324 const body = bytes.subarray(split + 2);
325 const total = names.reduce((n, [, len]) => n + len, 0);
326 if (total !== body.length) throw refuse('bad-package', 'a pack header does not account for its bytes');
327 const files = [];
328 let at = 0;
329 for (const [p, len] of names) {
330 files.push([p, body.subarray(at, at + len)]);
331 at += len;
332 }
333 if (!files.some(([p]) => p === entry)) throw refuse('bad-package', 'the entrypoint is not among the files');
334 return { entry, files };
335}
336
337/// The package `spec` names under `dir`, or null when `dir` holds none.
338export function readPackage(dir, spec) {
339 const d = path.join(dir, spec.ns, spec.name, spec.version);
340 if (isDir(d)) return readDirPackage(d);
341 const f = `${d}.pack`;
342 if (isFile(f)) return readPack(fs.readFileSync(f), spec);
343 return null;
344}
345
346/// The transitive closure of the packages the project's `.typ` sources import, looked for under `dir`
347/// (null looks nowhere, so every spec is missing).
348///
349/// The project's own imports are taken in the order it makes them, and each package found is read for the
350/// imports it makes, which go on the end of the same queue; a dependency is never assumed from a list.
351/// Returns `{ found: [{ spec, pack }], missing: [spec], specs: [{ spec, found }] }`, `specs` in queue order.
352///
353/// # Arguments
354/// * `sources` - `[[shadow path, text]]`; only those ending `.typ` are read.
355export function closure(sources, dir) {
356 const want = [];
357 const has = (s) => want.some((w) => specText(w) === specText(s));
358 for (const [p, text] of sources) {
359 if (ext(p) !== 'typ') continue;
360 for (const { spec } of imports(text)) if (!has(spec)) want.push(spec);
361 }
362 const found = [];
363 const missing = [];
364 const specs = [];
365 for (let at = 0; at < want.length; at++) {
366 const spec = want[at];
367 const pack = dir ? readPackage(dir, spec) : null;
368 if (!pack) {
369 missing.push(spec);
370 specs.push({ spec, found: false });
371 continue;
372 }
373 specs.push({ spec, found: true });
374 found.push({ spec, pack });
375 for (const [dep] of prepare(spec, pack).deps) if (!has(dep)) want.push(dep);
376 }
377 return { found, missing, specs };
378}