oxedyne/fe2o3/fe2o3_austenite/tools/bench/lib/gather.mjs
8.1 KiB, 1 run
created by r1870400018:66237, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // Walks a project root into the shape Daimond's gatherer (`src/wasm/typst.rs`, `gather`) hands the |
| 2 | // compiler: `{ root, main, sources, assets, fonts }`, with every path as the compiler's shadow |
| 3 | // filesystem names it, `/` and then the path under the root. It takes paths as arguments and knows no |
| 4 | // project: the same code gathers a three-file synthetic and a book. |
| 5 | // |
| 6 | // How a file is classified, by extension, after the directory it is in: |
| 7 | // - under a font directory: a font when it is .ttf, .otf, .ttc or .otc, and not gathered otherwise; |
| 8 | // - a `.typ` file, or text data (.bib .csv .json .yaml .yml .toml .xml .txt) that is valid UTF-8: a source; |
| 9 | // - anything else: an asset, as bytes. |
| 10 | // Left out, as Daimond leaves them out: a `.pdf` (typst.ts cannot place one, and a project directory |
| 11 | // usually holds the last build), and any name that begins with a dot (`.git`, `.stversions`, ...). |
| 12 | // |
| 13 | // Given `--packages <dir>`, it also follows the project's `#import "@ns/name:x.y.z"` into that directory |
| 14 | // and returns the packages it finds, with their dependencies (see `packages.mjs` for the two layouts the |
| 15 | // directory may have and for how Daimond supplies them). Specs not found are counted, never fatal. |
| 16 | // |
| 17 | // Run directly, it prints counts and nothing else, so it is safe to point at a project whose text |
| 18 | // must not appear in a log: |
| 19 | // |
| 20 | // node gather.mjs <root> <main> [--font-dir <dir>]... [--exclude <dir>]... [--packages <dir>] |
| 21 | |
| 22 | import fs from 'node:fs'; |
| 23 | import path from 'node:path'; |
| 24 | import { pathToFileURL } from 'node:url'; |
| 25 | |
| 26 | import { closure, refuse } from './packages.mjs'; |
| 27 | |
| 28 | export const TEXT_EXTS = ['typ', 'bib', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'txt']; |
| 29 | export const FONT_EXTS = ['ttf', 'otf', 'ttc', 'otc']; |
| 30 | export const SKIP_EXTS = ['pdf']; |
| 31 | export const FONT_DIRS = ['assets/fonts', 'fonts']; // Daimond's, relative to the root |
| 32 | export const MAX_BYTES = 512 * 1024 * 1024; // refused above this, never cut short |
| 33 | |
| 34 | // The stat of a path, or null for a link that points nowhere. |
| 35 | const statOf = (p) => { |
| 36 | try { |
| 37 | return fs.statSync(p); |
| 38 | } catch { |
| 39 | return null; |
| 40 | } |
| 41 | }; |
| 42 | |
| 43 | const ext = (name) => { |
| 44 | const i = name.lastIndexOf('.'); |
| 45 | return i < 0 ? '' : name.slice(i + 1).toLowerCase(); |
| 46 | }; |
| 47 | |
| 48 | // The path under `root` as a shadow path, or null when `p` is not under it. |
| 49 | function shadow(root, p) { |
| 50 | const rel = path.relative(root, p); |
| 51 | if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) return null; |
| 52 | return '/' + rel.split(path.sep).join('/'); |
| 53 | } |
| 54 | |
| 55 | /// Gathers the project under `root` whose entry file is `main`. |
| 56 | /// |
| 57 | /// # Arguments |
| 58 | /// * `main` - The entry `.typ` file, under `root`, relative to it or absolute. |
| 59 | /// * `fontDirs` - Directories searched for fonts, relative to `root` or absolute; those that do not |
| 60 | /// exist are skipped. Defaults to Daimond's `assets/fonts` and `fonts`. |
| 61 | /// * `exclude` - Directories under `root` left out of the walk, relative to `root` or absolute. |
| 62 | /// * `packages` - A directory of packages to resolve the project's package imports against, or null. |
| 63 | export function gather(root, main, { fontDirs = FONT_DIRS, exclude = [], maxBytes = MAX_BYTES, packages: pkgDir = null } = {}) { |
| 64 | let rootAbs; |
| 65 | try { |
| 66 | rootAbs = fs.realpathSync(path.resolve(root)); |
| 67 | } catch { |
| 68 | throw refuse('no-root', 'the root does not exist'); |
| 69 | } |
| 70 | if (!fs.statSync(rootAbs).isDirectory()) throw refuse('no-root', 'the root is not a directory'); |
| 71 | const mainAbs = path.resolve(rootAbs, main); |
| 72 | const mainShadow = shadow(rootAbs, mainAbs); |
| 73 | if (!mainShadow || ext(mainAbs) !== 'typ' || !fs.existsSync(mainAbs)) { |
| 74 | throw refuse('no-main', 'the main file is not a .typ file under the root'); |
| 75 | } |
| 76 | if (pkgDir !== null && !fs.existsSync(pkgDir)) throw refuse('no-packages', 'the package directory does not exist'); |
| 77 | const abs = (d) => path.resolve(rootAbs, d); |
| 78 | const fontRoots = fontDirs.map(abs).filter((d) => fs.existsSync(d) && fs.statSync(d).isDirectory()); |
| 79 | const skip = exclude.map(abs); |
| 80 | const under = (p, dir) => p === dir || p.startsWith(dir + path.sep); |
| 81 | |
| 82 | const sources = []; |
| 83 | const assets = []; |
| 84 | const fonts = []; |
| 85 | const seenFont = new Set(); // by leaf name, as Daimond does: one face reached twice is one font |
| 86 | const ancestry = new Set(); // real paths of the directories being walked, so a link back up is not followed |
| 87 | let bytes = 0; |
| 88 | |
| 89 | const take = (n) => { |
| 90 | bytes += n; |
| 91 | if (bytes > maxBytes) throw refuse('too-large', 'the project is past the gather limit'); |
| 92 | }; |
| 93 | |
| 94 | // Font directories are walked first and in full, wherever they are; the main walk then steps over them. |
| 95 | const walkFonts = (dir) => { |
| 96 | for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) { |
| 97 | if (e.name.startsWith('.')) continue; |
| 98 | const p = path.join(dir, e.name); |
| 99 | const st = statOf(p); |
| 100 | if (!st) continue; |
| 101 | if (st.isDirectory()) { |
| 102 | walkFonts(p); |
| 103 | } else if (FONT_EXTS.includes(ext(e.name))) { |
| 104 | const leaf = e.name.toLowerCase(); |
| 105 | if (seenFont.has(leaf)) continue; |
| 106 | seenFont.add(leaf); |
| 107 | const buf = fs.readFileSync(p); |
| 108 | take(buf.length); |
| 109 | fonts.push([e.name, buf]); |
| 110 | } |
| 111 | } |
| 112 | }; |
| 113 | for (const d of fontRoots) walkFonts(d); |
| 114 | |
| 115 | const decoder = new TextDecoder('utf-8', { fatal: true }); |
| 116 | const walk = (dir) => { |
| 117 | const real = fs.realpathSync(dir); |
| 118 | if (ancestry.has(real)) return; // a link back into the walk |
| 119 | ancestry.add(real); |
| 120 | for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) { |
| 121 | if (e.name.startsWith('.')) continue; |
| 122 | const p = path.join(dir, e.name); |
| 123 | if (skip.some((s) => under(p, s)) || fontRoots.some((f) => under(p, f))) continue; |
| 124 | const st = statOf(p); |
| 125 | if (!st) continue; |
| 126 | if (st.isDirectory()) { |
| 127 | walk(p); |
| 128 | continue; |
| 129 | } |
| 130 | const x = ext(e.name); |
| 131 | if (SKIP_EXTS.includes(x)) continue; |
| 132 | const sp = shadow(rootAbs, p); |
| 133 | if (!sp) continue; |
| 134 | const buf = fs.readFileSync(p); |
| 135 | take(buf.length); |
| 136 | if (TEXT_EXTS.includes(x)) { |
| 137 | try { |
| 138 | sources.push([sp, decoder.decode(buf)]); |
| 139 | continue; |
| 140 | } catch { |
| 141 | // Not UTF-8, so not text to the compiler either: it goes in as bytes. |
| 142 | } |
| 143 | } |
| 144 | assets.push([sp, buf]); |
| 145 | } |
| 146 | ancestry.delete(real); |
| 147 | }; |
| 148 | walk(rootAbs); |
| 149 | |
| 150 | // The packages the sources import, and theirs in turn. A spec the directory does not hold is counted as |
| 151 | // missing; with no directory every spec is. |
| 152 | const pk = closure(sources, pkgDir); |
| 153 | |
| 154 | return { |
| 155 | root: rootAbs, |
| 156 | main: mainShadow, |
| 157 | sources, |
| 158 | assets, |
| 159 | fonts, |
| 160 | packages: pk.found, |
| 161 | specs: pk.specs, |
| 162 | stats: { |
| 163 | sources: sources.length, |
| 164 | assets: assets.length, |
| 165 | fonts: fonts.length, |
| 166 | packages: pk.specs.length, |
| 167 | missing: pk.missing.length, |
| 168 | bytes, |
| 169 | }, |
| 170 | }; |
| 171 | } |
| 172 | |
| 173 | // ── Command line ───────────────────────────────────────────────────────────── |
| 174 | |
| 175 | // Parses `<root> <main> [--font-dir d]... [--exclude d]... [--packages d]`; null on any other shape. |
| 176 | export function parseArgs(argv) { |
| 177 | const pos = []; |
| 178 | const fontDirs = []; |
| 179 | const exclude = []; |
| 180 | let packages = null; |
| 181 | for (let i = 0; i < argv.length; i++) { |
| 182 | const a = argv[i]; |
| 183 | if (a === '--font-dir' || a === '--exclude') { |
| 184 | if (i + 1 >= argv.length) return null; |
| 185 | (a === '--font-dir' ? fontDirs : exclude).push(argv[++i]); |
| 186 | } else if (a === '--packages') { |
| 187 | if (i + 1 >= argv.length) return null; |
| 188 | packages = argv[++i]; |
| 189 | } else if (a.startsWith('--')) { |
| 190 | return null; |
| 191 | } else { |
| 192 | pos.push(a); |
| 193 | } |
| 194 | } |
| 195 | if (pos.length !== 2) return null; |
| 196 | return { root: pos[0], main: pos[1], fontDirs: fontDirs.length ? fontDirs : FONT_DIRS, exclude, packages }; |
| 197 | } |
| 198 | |
| 199 | if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { |
| 200 | const a = parseArgs(process.argv.slice(2)); |
| 201 | if (!a) { |
| 202 | console.log('gather: error bad-args'); |
| 203 | process.exit(2); |
| 204 | } |
| 205 | try { |
| 206 | const g = gather(a.root, a.main, { fontDirs: a.fontDirs, exclude: a.exclude, packages: a.packages }); |
| 207 | const s = g.stats; |
| 208 | console.log(`gather: sources ${s.sources} assets ${s.assets} fonts ${s.fonts} packages ${s.packages} missing ${s.missing} bytes ${s.bytes}`); |
| 209 | } catch (e) { |
| 210 | // Only the closed code: a message could name a file of the project. |
| 211 | console.log(`gather: error ${e && e.code ? e.code : 'failed'}`); |
| 212 | process.exit(1); |
| 213 | } |
| 214 | } |