oxedyne/fe2o3/fe2o3_austenite/tools/bench/lib/packages.mjs
12.7 KiB, 1 run
created by r1870400018:66239, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // Typst packages for the typst.ts leg, supplied as Daimond supplies them (`src/wasm/typst.rs`: `Spec`, |
| 2 | // `imports`, `repoint`, `prepare`, `packages`, `parse_pack`). The scanners are ports of its own, so what |
| 3 | // is gathered here, what is followed into a package's dependencies and what is re-pointed cannot disagree. |
| 4 | // |
| 5 | // typst.ts has no network and no package registry, so Daimond carries packages beside the page and |
| 6 | // re-points every `#import "@preview/name:1.2.3"` at a one-line module it writes at |
| 7 | // `/_pkg/<ns>/<name>/<version>/<name>.typ`, which star-imports the package's entrypoint. A package is |
| 8 | // looked for in a directory laid out in either of two ways: |
| 9 | // |
| 10 | // <dir>/<ns>/<name>/<version>/... Typst's own package cache; the .typ files, `typst.toml` and the |
| 11 | // licence are taken, as Daimond's refresh.sh takes them |
| 12 | // <dir>/<ns>/<name>/<version>.pack Daimond's vendored packs: an ASCII header, one blank line, the |
| 13 | // file bodies end to end (`www/assets/typst/packs`) |
| 14 | // |
| 15 | // A package is `{ spec, pack }`: `spec` is `{ ns, name, version }` and `pack` is `{ entry, files }`, with |
| 16 | // `files` as `[[package-relative path, bytes]]`. That is the shape a host hands a compiler that takes |
| 17 | // packages from outside. |
| 18 | |
| 19 | import fs from 'node:fs'; |
| 20 | import path from 'node:path'; |
| 21 | |
| 22 | export const PKG_ROOT = '/_pkg'; |
| 23 | |
| 24 | // A refusal carries a closed `code`, so a caller that must not print the message can print the code. |
| 25 | export function refuse(code, message) { |
| 26 | const e = new Error(message); |
| 27 | e.code = code; |
| 28 | return e; |
| 29 | } |
| 30 | |
| 31 | const ext = (name) => { |
| 32 | const i = name.lastIndexOf('.'); |
| 33 | return i < 0 ? '' : name.slice(i + 1).toLowerCase(); |
| 34 | }; |
| 35 | |
| 36 | // ── A spec ────────────────────────────────────────────────────────────────── |
| 37 | |
| 38 | const isWord = (s) => /^[A-Za-z0-9_-]+$/.test(s); |
| 39 | const isSemver = (s) => /^[0-9]+\.[0-9]+\.[0-9]+$/.test(s); |
| 40 | |
| 41 | /// A spec from the text of a quoted literal, `@preview/cetz:0.3.4`, or null when it is not one. |
| 42 | export function parseSpec(lit) { |
| 43 | if (!lit.startsWith('@')) return null; |
| 44 | const rest = lit.slice(1); |
| 45 | const slash = rest.indexOf('/'); |
| 46 | if (slash < 0) return null; |
| 47 | const ns = rest.slice(0, slash); |
| 48 | const tail = rest.slice(slash + 1); |
| 49 | const colon = tail.indexOf(':'); |
| 50 | if (colon < 0) return null; |
| 51 | const name = tail.slice(0, colon); |
| 52 | const version = tail.slice(colon + 1); |
| 53 | if (!isWord(ns) || !isWord(name) || !isSemver(version)) return null; |
| 54 | return { ns, name, version }; |
| 55 | } |
| 56 | |
| 57 | export const specText = (s) => `@${s.ns}/${s.name}:${s.version}`; |
| 58 | export const specDir = (s) => `${PKG_ROOT}/${s.ns}/${s.name}/${s.version}`; |
| 59 | // Named for the package, so that `#import "@preview/cetz:0.3.4"` goes on binding `cetz`. |
| 60 | export const specShim = (s) => `${specDir(s)}/${s.name}.typ`; |
| 61 | |
| 62 | // ── Reading a source for what it names ────────────────────────────────────── |
| 63 | |
| 64 | /// Every double-quoted literal in `src` as `[start, end, raw text]`; `start` is the opening quote and `end` |
| 65 | /// one past the closing one. |
| 66 | export function quoted(src) { |
| 67 | const out = []; |
| 68 | const n = src.length; |
| 69 | let i = 0; |
| 70 | while (i < n) { |
| 71 | if (src[i] !== '"') { |
| 72 | i++; |
| 73 | continue; |
| 74 | } |
| 75 | const start = i; |
| 76 | i++; |
| 77 | const from = i; |
| 78 | let closed = false; |
| 79 | while (i < n) { |
| 80 | if (src[i] === '"') { |
| 81 | closed = true; |
| 82 | break; |
| 83 | } |
| 84 | if (src[i] === '\\' && i + 1 < n) { |
| 85 | i += 2; |
| 86 | continue; |
| 87 | } |
| 88 | if (src[i] === '\n') break; |
| 89 | i++; |
| 90 | } |
| 91 | if (closed) { |
| 92 | out.push([start, i + 1, src.slice(from, i)]); |
| 93 | i++; |
| 94 | } else { |
| 95 | i = start + 1; |
| 96 | } |
| 97 | } |
| 98 | return out; |
| 99 | } |
| 100 | |
| 101 | /// Every raw block and raw span in `src`, as `[start, end]`: a run of N backticks opens a region that the |
| 102 | /// next run of exactly N closes. |
| 103 | export function rawSpans(src) { |
| 104 | const out = []; |
| 105 | const n = src.length; |
| 106 | let i = 0; |
| 107 | while (i < n) { |
| 108 | if (src[i] !== '`') { |
| 109 | i++; |
| 110 | continue; |
| 111 | } |
| 112 | const open = i; |
| 113 | let run = 0; |
| 114 | while (i < n && src[i] === '`') { |
| 115 | run++; |
| 116 | i++; |
| 117 | } |
| 118 | let j = i; |
| 119 | let end = null; |
| 120 | while (j < n) { |
| 121 | if (src[j] !== '`') { |
| 122 | j++; |
| 123 | continue; |
| 124 | } |
| 125 | let m = 0; |
| 126 | while (j < n && src[j] === '`') { |
| 127 | m++; |
| 128 | j++; |
| 129 | } |
| 130 | if (m === run) { |
| 131 | end = j; |
| 132 | break; |
| 133 | } |
| 134 | } |
| 135 | if (end === null) break; |
| 136 | out.push([open, end]); |
| 137 | i = end; |
| 138 | } |
| 139 | return out; |
| 140 | } |
| 141 | |
| 142 | const within = (spans, at) => spans.some(([a, b]) => at >= a && at < b); |
| 143 | |
| 144 | /// Is the literal beginning at `at` what an `#import` or `#include` is being given? A quotation of an import |
| 145 | /// in prose is not one. |
| 146 | function isImportOperand(src, at) { |
| 147 | let i = at; |
| 148 | while (i > 0 && /\s/.test(src[i - 1])) i--; |
| 149 | for (const kw of ['import', 'include']) { |
| 150 | const n = kw.length; |
| 151 | if (i < n || src.slice(i - n, i) !== kw) continue; |
| 152 | if (i === n) return true; |
| 153 | // A longer identifier that merely ends in "import" is not the keyword. |
| 154 | if (!/[A-Za-z0-9_-]/.test(src[i - n - 1])) return true; |
| 155 | } |
| 156 | return false; |
| 157 | } |
| 158 | |
| 159 | /// Every package `src` imports, as `{ start, spec }`; a literal inside a raw block or span is not an import. |
| 160 | export function imports(src) { |
| 161 | if (!src.includes('@')) return []; |
| 162 | const raws = rawSpans(src); |
| 163 | const out = []; |
| 164 | for (const [start, , raw] of quoted(src)) { |
| 165 | const spec = parseSpec(raw); |
| 166 | if (!spec) continue; |
| 167 | if (!isImportOperand(src, start)) continue; |
| 168 | if (within(raws, start)) continue; |
| 169 | out.push({ start, spec }); |
| 170 | } |
| 171 | return out; |
| 172 | } |
| 173 | |
| 174 | /// `.` and `..` collapsed in a slash path; null when it climbs above the top. |
| 175 | export function norm(p) { |
| 176 | const out = []; |
| 177 | for (const seg of p.split('/')) { |
| 178 | if (seg === '' || seg === '.') continue; |
| 179 | if (seg === '..') { |
| 180 | if (out.pop() === undefined) return null; |
| 181 | continue; |
| 182 | } |
| 183 | out.push(seg); |
| 184 | } |
| 185 | return out.join('/'); |
| 186 | } |
| 187 | |
| 188 | /// Re-points every package and package-root reference in `text`. |
| 189 | /// |
| 190 | /// # Arguments |
| 191 | /// * `inside` - When `text` is a package's own file: `{ dir, names }`, the shadow directory the package sits |
| 192 | /// at and every path it holds. Null for a project's own source. |
| 193 | export function repoint(text, inside = null) { |
| 194 | const imps = imports(text); |
| 195 | const raws = rawSpans(text); |
| 196 | let out = ''; |
| 197 | let last = 0; |
| 198 | for (const [start, end, raw] of quoted(text)) { |
| 199 | let rep = null; |
| 200 | const imp = imps.find((i) => i.start === start); |
| 201 | if (imp) { |
| 202 | rep = specShim(imp.spec); |
| 203 | } else if (inside && raw.startsWith('/') && !within(raws, start)) { |
| 204 | // Inside a package a leading "/" means the package root, not the project's. Re-pointed only when it |
| 205 | // names a file the pack holds, so a string of prose that begins with a slash is left alone. |
| 206 | const p = norm(raw); |
| 207 | if (p !== null && inside.names.includes(p)) rep = `${inside.dir}/pkg/${p}`; |
| 208 | } |
| 209 | if (rep !== null) { |
| 210 | out += text.slice(last, start) + '"' + rep + '"'; |
| 211 | last = end; |
| 212 | } |
| 213 | } |
| 214 | return out + text.slice(last); |
| 215 | } |
| 216 | |
| 217 | const lossy = new TextDecoder('utf-8'); |
| 218 | |
| 219 | /// A package laid out for the compiler, read for what it imports in turn. |
| 220 | /// |
| 221 | /// Returns `{ sources, assets, deps }`: shadow `.typ` files, re-pointed, with the one-line module that |
| 222 | /// carries the binding; shadow files that are not sources (the manifest, the licence); and the specs the |
| 223 | /// package imports, each with the file of its own that names it. |
| 224 | export function prepare(spec, pack) { |
| 225 | const dir = specDir(spec); |
| 226 | const names = pack.files.map(([p]) => p); |
| 227 | const sources = []; |
| 228 | const assets = []; |
| 229 | const deps = []; |
| 230 | for (const [rel, data] of pack.files) { |
| 231 | const shadow = `${dir}/pkg/${rel}`; |
| 232 | if (ext(rel) !== 'typ') { |
| 233 | assets.push([shadow, data]); |
| 234 | continue; |
| 235 | } |
| 236 | const text = lossy.decode(data); |
| 237 | for (const { spec: dep } of imports(text)) { |
| 238 | if (specText(dep) === specText(spec) || deps.some(([d]) => specText(d) === specText(dep))) continue; |
| 239 | deps.push([dep, rel]); |
| 240 | } |
| 241 | sources.push([shadow, repoint(text, { dir, names })]); |
| 242 | } |
| 243 | // The binding: a star import re-exports whatever the entrypoint exports. |
| 244 | sources.push([specShim(spec), `#import "pkg/${pack.entry}": *\n`]); |
| 245 | return { sources, assets, deps }; |
| 246 | } |
| 247 | |
| 248 | // ── Finding a package ─────────────────────────────────────────────────────── |
| 249 | |
| 250 | const isDir = (p) => { |
| 251 | try { |
| 252 | return fs.statSync(p).isDirectory(); |
| 253 | } catch { |
| 254 | return false; |
| 255 | } |
| 256 | }; |
| 257 | const isFile = (p) => { |
| 258 | try { |
| 259 | return fs.statSync(p).isFile(); |
| 260 | } catch { |
| 261 | return false; |
| 262 | } |
| 263 | }; |
| 264 | |
| 265 | // What of a package directory is taken, as Daimond's refresh.sh takes it. |
| 266 | const keep = (rel) => { |
| 267 | const b = rel.slice(rel.lastIndexOf('/') + 1); |
| 268 | return ext(rel) === 'typ' || b === 'typst.toml' || b === 'LICENSE' || b.startsWith('LICENSE.'); |
| 269 | }; |
| 270 | |
| 271 | function readDirPackage(d) { |
| 272 | const files = []; |
| 273 | const walk = (dir, prefix) => { |
| 274 | for (const e of fs.readdirSync(dir, { withFileTypes: true })) { |
| 275 | const rel = prefix ? `${prefix}/${e.name}` : e.name; |
| 276 | const p = path.join(dir, e.name); |
| 277 | if (e.isDirectory() || (e.isSymbolicLink() && isDir(p))) walk(p, rel); |
| 278 | else if (keep(rel) && isFile(p)) files.push([rel, fs.readFileSync(p)]); |
| 279 | } |
| 280 | }; |
| 281 | walk(d, ''); |
| 282 | files.sort((a, b) => (Buffer.from(a[0]).compare(Buffer.from(b[0])))); |
| 283 | const toml = files.find(([rel]) => rel === 'typst.toml'); |
| 284 | const m = toml && /^[ \t]*entrypoint[ \t]*=[ \t]*"(.*)".*$/m.exec(Buffer.from(toml[1]).toString('utf8')); |
| 285 | if (!m) throw refuse('bad-package', 'a package names no entrypoint'); |
| 286 | const entry = m[1].replace(/^\/+/, ''); |
| 287 | if (!files.some(([rel]) => rel === entry)) throw refuse('bad-package', 'the entrypoint is not among the files'); |
| 288 | return { entry, files }; |
| 289 | } |
| 290 | |
| 291 | function readPack(bytes, spec) { |
| 292 | let split = -1; |
| 293 | for (let i = 0; i + 1 < bytes.length; i++) { |
| 294 | if (bytes[i] === 0x0a && bytes[i + 1] === 0x0a) { |
| 295 | split = i; |
| 296 | break; |
| 297 | } |
| 298 | } |
| 299 | if (split < 0) throw refuse('bad-package', 'a pack has no header'); |
| 300 | const lines = Buffer.from(bytes.subarray(0, split)).toString('utf8').split('\n'); |
| 301 | if (lines[0] !== 'DAIMOND TYPST PACK 1') throw refuse('bad-package', 'not a pack this reader knows'); |
| 302 | let entry = ''; |
| 303 | const got = { ns: '', name: '', version: '' }; |
| 304 | const names = []; |
| 305 | for (const line of lines.slice(1)) { |
| 306 | const sp = line.indexOf(' '); |
| 307 | if (sp < 0) continue; |
| 308 | const key = line.slice(0, sp); |
| 309 | const rest = line.slice(sp + 1).trim(); |
| 310 | if (key === 'namespace') got.ns = rest; |
| 311 | else if (key === 'name') got.name = rest; |
| 312 | else if (key === 'version') got.version = rest; |
| 313 | else if (key === 'entrypoint') entry = rest.replace(/^\/+/, ''); |
| 314 | else if (key === 'file') { |
| 315 | const s2 = rest.indexOf(' '); |
| 316 | if (s2 < 0) continue; |
| 317 | const len = Number(rest.slice(0, s2)); |
| 318 | if (!Number.isInteger(len) || len < 0) continue; |
| 319 | names.push([rest.slice(s2 + 1), len]); |
| 320 | } |
| 321 | } |
| 322 | if (specText(got) !== specText(spec)) throw refuse('bad-package', 'a pack holds another package'); |
| 323 | if (!entry) throw refuse('bad-package', 'a pack names no entrypoint'); |
| 324 | const body = bytes.subarray(split + 2); |
| 325 | const total = names.reduce((n, [, len]) => n + len, 0); |
| 326 | if (total !== body.length) throw refuse('bad-package', 'a pack header does not account for its bytes'); |
| 327 | const files = []; |
| 328 | let at = 0; |
| 329 | for (const [p, len] of names) { |
| 330 | files.push([p, body.subarray(at, at + len)]); |
| 331 | at += len; |
| 332 | } |
| 333 | if (!files.some(([p]) => p === entry)) throw refuse('bad-package', 'the entrypoint is not among the files'); |
| 334 | return { entry, files }; |
| 335 | } |
| 336 | |
| 337 | /// The package `spec` names under `dir`, or null when `dir` holds none. |
| 338 | export function readPackage(dir, spec) { |
| 339 | const d = path.join(dir, spec.ns, spec.name, spec.version); |
| 340 | if (isDir(d)) return readDirPackage(d); |
| 341 | const f = `${d}.pack`; |
| 342 | if (isFile(f)) return readPack(fs.readFileSync(f), spec); |
| 343 | return null; |
| 344 | } |
| 345 | |
| 346 | /// The transitive closure of the packages the project's `.typ` sources import, looked for under `dir` |
| 347 | /// (null looks nowhere, so every spec is missing). |
| 348 | /// |
| 349 | /// The project's own imports are taken in the order it makes them, and each package found is read for the |
| 350 | /// imports it makes, which go on the end of the same queue; a dependency is never assumed from a list. |
| 351 | /// Returns `{ found: [{ spec, pack }], missing: [spec], specs: [{ spec, found }] }`, `specs` in queue order. |
| 352 | /// |
| 353 | /// # Arguments |
| 354 | /// * `sources` - `[[shadow path, text]]`; only those ending `.typ` are read. |
| 355 | export function closure(sources, dir) { |
| 356 | const want = []; |
| 357 | const has = (s) => want.some((w) => specText(w) === specText(s)); |
| 358 | for (const [p, text] of sources) { |
| 359 | if (ext(p) !== 'typ') continue; |
| 360 | for (const { spec } of imports(text)) if (!has(spec)) want.push(spec); |
| 361 | } |
| 362 | const found = []; |
| 363 | const missing = []; |
| 364 | const specs = []; |
| 365 | for (let at = 0; at < want.length; at++) { |
| 366 | const spec = want[at]; |
| 367 | const pack = dir ? readPackage(dir, spec) : null; |
| 368 | if (!pack) { |
| 369 | missing.push(spec); |
| 370 | specs.push({ spec, found: false }); |
| 371 | continue; |
| 372 | } |
| 373 | specs.push({ spec, found: true }); |
| 374 | found.push({ spec, pack }); |
| 375 | for (const [dep] of prepare(spec, pack).deps) if (!has(dep)) want.push(dep); |
| 376 | } |
| 377 | return { found, missing, specs }; |
| 378 | } |