Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/lib/gather.mjs

8.1 KiB, 1 run

created by r1870400018:66237, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Walks a project root into the shape Daimond's gatherer (`src/wasm/typst.rs`, `gather`) hands the
2// compiler: `{ root, main, sources, assets, fonts }`, with every path as the compiler's shadow
3// filesystem names it, `/` and then the path under the root. It takes paths as arguments and knows no
4// project: the same code gathers a three-file synthetic and a book.
5//
6// How a file is classified, by extension, after the directory it is in:
7// - under a font directory: a font when it is .ttf, .otf, .ttc or .otc, and not gathered otherwise;
8// - a `.typ` file, or text data (.bib .csv .json .yaml .yml .toml .xml .txt) that is valid UTF-8: a source;
9// - anything else: an asset, as bytes.
10// Left out, as Daimond leaves them out: a `.pdf` (typst.ts cannot place one, and a project directory
11// usually holds the last build), and any name that begins with a dot (`.git`, `.stversions`, ...).
12//
13// Given `--packages <dir>`, it also follows the project's `#import "@ns/name:x.y.z"` into that directory
14// and returns the packages it finds, with their dependencies (see `packages.mjs` for the two layouts the
15// directory may have and for how Daimond supplies them). Specs not found are counted, never fatal.
16//
17// Run directly, it prints counts and nothing else, so it is safe to point at a project whose text
18// must not appear in a log:
19//
20// node gather.mjs <root> <main> [--font-dir <dir>]... [--exclude <dir>]... [--packages <dir>]
21
22import fs from 'node:fs';
23import path from 'node:path';
24import { pathToFileURL } from 'node:url';
25
26import { closure, refuse } from './packages.mjs';
27
28export const TEXT_EXTS = ['typ', 'bib', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'txt'];
29export const FONT_EXTS = ['ttf', 'otf', 'ttc', 'otc'];
30export const SKIP_EXTS = ['pdf'];
31export const FONT_DIRS = ['assets/fonts', 'fonts']; // Daimond's, relative to the root
32export const MAX_BYTES = 512 * 1024 * 1024; // refused above this, never cut short
33
34// The stat of a path, or null for a link that points nowhere.
35const statOf = (p) => {
36 try {
37 return fs.statSync(p);
38 } catch {
39 return null;
40 }
41};
42
43const ext = (name) => {
44 const i = name.lastIndexOf('.');
45 return i < 0 ? '' : name.slice(i + 1).toLowerCase();
46};
47
48// The path under `root` as a shadow path, or null when `p` is not under it.
49function shadow(root, p) {
50 const rel = path.relative(root, p);
51 if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) return null;
52 return '/' + rel.split(path.sep).join('/');
53}
54
55/// Gathers the project under `root` whose entry file is `main`.
56///
57/// # Arguments
58/// * `main` - The entry `.typ` file, under `root`, relative to it or absolute.
59/// * `fontDirs` - Directories searched for fonts, relative to `root` or absolute; those that do not
60/// exist are skipped. Defaults to Daimond's `assets/fonts` and `fonts`.
61/// * `exclude` - Directories under `root` left out of the walk, relative to `root` or absolute.
62/// * `packages` - A directory of packages to resolve the project's package imports against, or null.
63export function gather(root, main, { fontDirs = FONT_DIRS, exclude = [], maxBytes = MAX_BYTES, packages: pkgDir = null } = {}) {
64 let rootAbs;
65 try {
66 rootAbs = fs.realpathSync(path.resolve(root));
67 } catch {
68 throw refuse('no-root', 'the root does not exist');
69 }
70 if (!fs.statSync(rootAbs).isDirectory()) throw refuse('no-root', 'the root is not a directory');
71 const mainAbs = path.resolve(rootAbs, main);
72 const mainShadow = shadow(rootAbs, mainAbs);
73 if (!mainShadow || ext(mainAbs) !== 'typ' || !fs.existsSync(mainAbs)) {
74 throw refuse('no-main', 'the main file is not a .typ file under the root');
75 }
76 if (pkgDir !== null && !fs.existsSync(pkgDir)) throw refuse('no-packages', 'the package directory does not exist');
77 const abs = (d) => path.resolve(rootAbs, d);
78 const fontRoots = fontDirs.map(abs).filter((d) => fs.existsSync(d) && fs.statSync(d).isDirectory());
79 const skip = exclude.map(abs);
80 const under = (p, dir) => p === dir || p.startsWith(dir + path.sep);
81
82 const sources = [];
83 const assets = [];
84 const fonts = [];
85 const seenFont = new Set(); // by leaf name, as Daimond does: one face reached twice is one font
86 const ancestry = new Set(); // real paths of the directories being walked, so a link back up is not followed
87 let bytes = 0;
88
89 const take = (n) => {
90 bytes += n;
91 if (bytes > maxBytes) throw refuse('too-large', 'the project is past the gather limit');
92 };
93
94 // Font directories are walked first and in full, wherever they are; the main walk then steps over them.
95 const walkFonts = (dir) => {
96 for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) {
97 if (e.name.startsWith('.')) continue;
98 const p = path.join(dir, e.name);
99 const st = statOf(p);
100 if (!st) continue;
101 if (st.isDirectory()) {
102 walkFonts(p);
103 } else if (FONT_EXTS.includes(ext(e.name))) {
104 const leaf = e.name.toLowerCase();
105 if (seenFont.has(leaf)) continue;
106 seenFont.add(leaf);
107 const buf = fs.readFileSync(p);
108 take(buf.length);
109 fonts.push([e.name, buf]);
110 }
111 }
112 };
113 for (const d of fontRoots) walkFonts(d);
114
115 const decoder = new TextDecoder('utf-8', { fatal: true });
116 const walk = (dir) => {
117 const real = fs.realpathSync(dir);
118 if (ancestry.has(real)) return; // a link back into the walk
119 ancestry.add(real);
120 for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) {
121 if (e.name.startsWith('.')) continue;
122 const p = path.join(dir, e.name);
123 if (skip.some((s) => under(p, s)) || fontRoots.some((f) => under(p, f))) continue;
124 const st = statOf(p);
125 if (!st) continue;
126 if (st.isDirectory()) {
127 walk(p);
128 continue;
129 }
130 const x = ext(e.name);
131 if (SKIP_EXTS.includes(x)) continue;
132 const sp = shadow(rootAbs, p);
133 if (!sp) continue;
134 const buf = fs.readFileSync(p);
135 take(buf.length);
136 if (TEXT_EXTS.includes(x)) {
137 try {
138 sources.push([sp, decoder.decode(buf)]);
139 continue;
140 } catch {
141 // Not UTF-8, so not text to the compiler either: it goes in as bytes.
142 }
143 }
144 assets.push([sp, buf]);
145 }
146 ancestry.delete(real);
147 };
148 walk(rootAbs);
149
150 // The packages the sources import, and theirs in turn. A spec the directory does not hold is counted as
151 // missing; with no directory every spec is.
152 const pk = closure(sources, pkgDir);
153
154 return {
155 root: rootAbs,
156 main: mainShadow,
157 sources,
158 assets,
159 fonts,
160 packages: pk.found,
161 specs: pk.specs,
162 stats: {
163 sources: sources.length,
164 assets: assets.length,
165 fonts: fonts.length,
166 packages: pk.specs.length,
167 missing: pk.missing.length,
168 bytes,
169 },
170 };
171}
172
173// ── Command line ─────────────────────────────────────────────────────────────
174
175// Parses `<root> <main> [--font-dir d]... [--exclude d]... [--packages d]`; null on any other shape.
176export function parseArgs(argv) {
177 const pos = [];
178 const fontDirs = [];
179 const exclude = [];
180 let packages = null;
181 for (let i = 0; i < argv.length; i++) {
182 const a = argv[i];
183 if (a === '--font-dir' || a === '--exclude') {
184 if (i + 1 >= argv.length) return null;
185 (a === '--font-dir' ? fontDirs : exclude).push(argv[++i]);
186 } else if (a === '--packages') {
187 if (i + 1 >= argv.length) return null;
188 packages = argv[++i];
189 } else if (a.startsWith('--')) {
190 return null;
191 } else {
192 pos.push(a);
193 }
194 }
195 if (pos.length !== 2) return null;
196 return { root: pos[0], main: pos[1], fontDirs: fontDirs.length ? fontDirs : FONT_DIRS, exclude, packages };
197}
198
199if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
200 const a = parseArgs(process.argv.slice(2));
201 if (!a) {
202 console.log('gather: error bad-args');
203 process.exit(2);
204 }
205 try {
206 const g = gather(a.root, a.main, { fontDirs: a.fontDirs, exclude: a.exclude, packages: a.packages });
207 const s = g.stats;
208 console.log(`gather: sources ${s.sources} assets ${s.assets} fonts ${s.fonts} packages ${s.packages} missing ${s.missing} bytes ${s.bytes}`);
209 } catch (e) {
210 // Only the closed code: a message could name a file of the project.
211 console.log(`gather: error ${e && e.code ? e.code : 'failed'}`);
212 process.exit(1);
213 }
214}