oxedyne/fe2o3/fe2o3_net/lens/js/reader.mjs
42.0 KiB, 1 run
created by r1870400018:65602, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // reader.mjs -- the node half of the debug lens: pull a mirror, keep an archive, answer questions. |
| 2 | // |
| 3 | // Extracted from Daimond's `dev/lens.mjs` (2026-10-02) so that Daimond and Oxegen share one reader. |
| 4 | // What the app alone knows is a parameter of `createReader`: where the home directory is, how the |
| 5 | // mirror is fetched, which names and strings are secret, what a snapshot or a telemetry bundle |
| 6 | // holds. The views (`turns`, `errors`, `status` and the rest) stay in the app; they read the archive |
| 7 | // this file keeps and use the helpers it exports. |
| 8 | // |
| 9 | // THE LINE. `fe2o3_net::lens::Sink` writes one object a line, `{"bt":<recv ms>,"device":"..", |
| 10 | // "ts":<client ms>,"tag":"..","data":".."}`, with `data` a string (an event's JSON as text, so a |
| 11 | // clip never makes an invalid line). Daimond's gateway still writes the text block the line was |
| 12 | // taken from (`===== <recv ms> account=<a> device=<d> rows=<n> =====` and ` <ts> <tag> <data>` |
| 13 | // under it); this reader takes either, and both reach the same archive records. |
| 14 | // |
| 15 | // THE FILES. A key's live file is `<key>.ndjson`. A rotation closes it as `<key>.ndjson.<stamp>`, |
| 16 | // `<stamp>` being `YYYYMMDDTHHMMSSZ` and `-n` added on a taken stamp. A rotated file is closed for |
| 17 | // good, so once it has been read it is skipped until its size changes. |
| 18 | // |
| 19 | // THE ARCHIVE, kept under `<home>/archive/`: |
| 20 | // |
| 21 | // <device>.events.ndjson every `ev` row as {src:'ev',device,bt,ts,kind,ev}, every other row as |
| 22 | // {src:'row',device,bt,ts,tag,data} |
| 23 | // <device>.telemetry.ndjson one record per assembled `ds telemetry` set |
| 24 | // <device>.snapshots.ndjson one index record per `ds snapshot` set, its body in `snapshots/` |
| 25 | // |
| 26 | // DEDUPE. An event is filed once by its envelope `(d, n)`, whichever file or retry delivers it. A |
| 27 | // chunk of a `ds` set is filed once by its index. A line is told from a line already read by its |
| 28 | // receive time and a hash of its text, per stream (one key's files), so a torn tail that is |
| 29 | // completed on the next pull loses nothing and files nothing twice. |
| 30 | // |
| 31 | // SCRUBBING AGAIN. Every record passes the caller's redactor (by name and by string test, where |
| 32 | // given) and then the content scrubber on its way to disk, because the writer's own guard is the |
| 33 | // first line and a client bug or an older client is what the second line is for. |
| 34 | // |
| 35 | // OUTPUT IS THE BUDGET. A digest is read by a language model at every pickup, so `digest` and |
| 36 | // `capBytes` hold it to 2 KB inside the function, and say what they withheld. |
| 37 | import { execFileSync } from 'node:child_process'; |
| 38 | import crypto from 'node:crypto'; |
| 39 | import fs from 'node:fs'; |
| 40 | import path from 'node:path'; |
| 41 | import { createScrubber } from './redact.js'; |
| 42 | |
| 43 | // ── Arguments ──────────────────────────────────────────────────────── |
| 44 | |
| 45 | export function parseArgs(argv) { |
| 46 | const a = { cmd: argv[0] || 'digest', flags: {}, bool: {} }; |
| 47 | for (let i = 1; i < argv.length; i++) { |
| 48 | const t = argv[i]; |
| 49 | if (!t.startsWith('--')) continue; |
| 50 | const k = t.slice(2); |
| 51 | const next = argv[i + 1]; |
| 52 | if (next !== undefined && !next.startsWith('--')) { a.flags[k] = next; i++; } |
| 53 | else a.bool[k] = true; |
| 54 | } |
| 55 | return a; |
| 56 | } |
| 57 | |
| 58 | /// Turn `--since` into an absolute millisecond bound. Bare numbers are seconds, |
| 59 | /// so `--since 90` and `--since 90s` agree. |
| 60 | export function sinceMs(v, dflt) { |
| 61 | if (v === undefined || v === null || v === '') return dflt; |
| 62 | const s = String(v).trim(); |
| 63 | if (s === 'all') return 0; |
| 64 | if (s === 'today') { const d = new Date(); d.setHours(0, 0, 0, 0); return d.getTime(); } |
| 65 | const m = s.match(/^(\d+(?:\.\d+)?)\s*([smhdw]?)$/i); |
| 66 | if (m) { |
| 67 | const mult = { '': 1000, s: 1000, m: 60000, h: 3600000, d: 86400000, w: 604800000 }; |
| 68 | return Date.now() - Number(m[1]) * mult[m[2].toLowerCase()]; |
| 69 | } |
| 70 | const t = Date.parse(s); |
| 71 | return Number.isFinite(t) ? t : dflt; |
| 72 | } |
| 73 | |
| 74 | // ── Small file helpers ─────────────────────────────────────────────── |
| 75 | |
| 76 | export function ensureDir(d) { fs.mkdirSync(d, { recursive: true }); } |
| 77 | export function safeName(s) { return String(s || 'unknown').replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 64); } |
| 78 | |
| 79 | export function readJson(f, dflt) { |
| 80 | try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch (e) { return dflt; } |
| 81 | } |
| 82 | |
| 83 | export function writeJson(f, obj) { |
| 84 | ensureDir(path.dirname(f)); |
| 85 | // Through a temporary file, so a pull killed mid-write (the timer's unit |
| 86 | // stopping, an OOM) cannot leave a truncated state that loses the archive's |
| 87 | // whole ingestion history. |
| 88 | const tmp = f + '.tmp'; |
| 89 | fs.writeFileSync(tmp, JSON.stringify(obj)); |
| 90 | fs.renameSync(tmp, f); |
| 91 | } |
| 92 | |
| 93 | export function readNdjson(f) { |
| 94 | let txt = ''; |
| 95 | try { txt = fs.readFileSync(f, 'utf8'); } catch (e) { return []; } |
| 96 | const out = []; |
| 97 | for (const line of txt.split('\n')) { |
| 98 | if (!line) continue; |
| 99 | // A line half-written by a killed pull is skipped rather than fatal: the |
| 100 | // archive is append-only, so the loss is one record at the tail. |
| 101 | try { out.push(JSON.parse(line)); } catch (e) { /* torn tail */ } |
| 102 | } |
| 103 | return out; |
| 104 | } |
| 105 | |
| 106 | // ── Integer range sets ─────────────────────────────────────────────── |
| 107 | // |
| 108 | // Chunk indices and event sequence numbers both arrive almost in order with |
| 109 | // occasional redelivery, so a list of [lo,hi] ranges collapses to one or two |
| 110 | // entries and keeps state.json small where a plain set of every number seen |
| 111 | // would reach megabytes within a day. |
| 112 | |
| 113 | export function rangeHas(rs, n) { |
| 114 | for (const r of rs) if (n >= r[0] && n <= r[1]) return true; |
| 115 | return false; |
| 116 | } |
| 117 | |
| 118 | export function rangeAdd(rs, n) { |
| 119 | if (rangeHas(rs, n)) return false; |
| 120 | rs.push([n, n]); |
| 121 | rs.sort((a, b) => a[0] - b[0]); |
| 122 | for (let i = rs.length - 1; i > 0; i--) { |
| 123 | if (rs[i][0] <= rs[i - 1][1] + 1) { |
| 124 | rs[i - 1][1] = Math.max(rs[i - 1][1], rs[i][1]); |
| 125 | rs.splice(i, 1); |
| 126 | } |
| 127 | } |
| 128 | return true; |
| 129 | } |
| 130 | |
| 131 | export function rangeCount(rs) { |
| 132 | let n = 0; |
| 133 | for (const r of rs) n += r[1] - r[0] + 1; |
| 134 | return n; |
| 135 | } |
| 136 | |
| 137 | /// The numbers in 1..n that the ranges do not cover, as a short human string. |
| 138 | export function rangeMissing(rs, n, cap = 6) { |
| 139 | const gaps = []; |
| 140 | let next = 1; |
| 141 | for (const r of rs) { |
| 142 | if (r[0] > next) gaps.push([next, r[0] - 1]); |
| 143 | next = Math.max(next, r[1] + 1); |
| 144 | } |
| 145 | if (next <= n) gaps.push([next, n]); |
| 146 | const shown = gaps.slice(0, cap).map(g => g[0] === g[1] ? String(g[0]) : g[0] + '-' + g[1]); |
| 147 | if (gaps.length > cap) shown.push('+' + (gaps.length - cap) + ' more'); |
| 148 | return shown.join(','); |
| 149 | } |
| 150 | |
| 151 | // ── Parsing what the sink wrote ────────────────────────────────────── |
| 152 | |
| 153 | const HEADER_RE = /^===== (\d+) account=(\S+) device=(\S+) rows=(\d+) =====$/; |
| 154 | const CHUNK_RE = /^ds (telemetry|snapshot) (\S+) (\d+)\/(\d+)$/; |
| 155 | const EV_RE = /^ev (\S+)$/; |
| 156 | |
| 157 | /// One line of a sink file as `{bt, device, ts, tag, data}`, or null for anything that is not one: a |
| 158 | /// torn tail, a blank, a value that is not an object, a line with no tag or no receive time, or a |
| 159 | /// `data` that is not a string (T5a always writes a string). |
| 160 | export function parseLine(text) { |
| 161 | const raw = String(text).trim(); |
| 162 | if (!raw) return null; |
| 163 | let o; |
| 164 | try { o = JSON.parse(raw); } catch (e) { return null; } |
| 165 | if (!o || typeof o !== 'object' || Array.isArray(o)) return null; |
| 166 | if (!Number.isFinite(o.bt) || typeof o.tag !== 'string') return null; |
| 167 | if (o.data !== undefined && typeof o.data !== 'string') return null; |
| 168 | return { |
| 169 | bt: o.bt, |
| 170 | device: typeof o.device === 'string' ? o.device : '', |
| 171 | ts: Number.isFinite(o.ts) ? o.ts : o.bt, |
| 172 | tag: o.tag, |
| 173 | data: o.data === undefined ? '' : o.data, |
| 174 | }; |
| 175 | } |
| 176 | |
| 177 | /// Every line of a file that parses, each with its trimmed text as `raw`, and how many non-blank lines |
| 178 | /// did not. A file that is not there reads as empty. |
| 179 | export function readLines(file) { |
| 180 | let txt = ''; |
| 181 | try { txt = fs.readFileSync(file, 'utf8'); } catch (e) { return { rows: [], bad: 0 }; } |
| 182 | const rows = []; |
| 183 | let bad = 0; |
| 184 | for (const line of txt.split('\n')) { |
| 185 | if (!line.trim()) continue; |
| 186 | const r = parseLine(line); |
| 187 | if (r) { r.raw = line.trim(); rows.push(r); } else bad++; |
| 188 | } |
| 189 | return { rows, bad }; |
| 190 | } |
| 191 | |
| 192 | /// Group consecutive rows of one receive time and device, which is one post, into the blocks the |
| 193 | /// ingest works on. |
| 194 | export function lineBlocks(rows) { |
| 195 | const out = []; |
| 196 | let cur = null; |
| 197 | for (const r of rows) { |
| 198 | if (!cur || cur.bt !== r.bt || cur.device !== r.device) { |
| 199 | cur = { bt: r.bt, device: r.device, rows: [] }; |
| 200 | out.push(cur); |
| 201 | } |
| 202 | cur.rows.push({ ts: r.ts, tag: r.tag, data: r.data, raw: r.raw }); |
| 203 | } |
| 204 | return out; |
| 205 | } |
| 206 | |
| 207 | /// Which family a tag belongs to: `ev <kind>`, `ds <telemetry|snapshot> <id> i/N`, or any other row. |
| 208 | export function tagFamily(tag) { |
| 209 | const c = CHUNK_RE.exec(tag); |
| 210 | if (c) return { fam: 'ds', kind: c[1], id: c[2], i: Number(c[3]), n: Number(c[4]) }; |
| 211 | const e = EV_RE.exec(tag); |
| 212 | if (e) return { fam: 'ev', kind: e[1] }; |
| 213 | return { fam: 'row' }; |
| 214 | } |
| 215 | |
| 216 | /// Walk one text-block file (Daimond's gateway format), calling `onBlock({bt, account, device, |
| 217 | /// rows})` in file order. Rows are `{ts, tag, data}`; a row the gateway marked malformed comes |
| 218 | /// through with tag `(malformed row)` rather than being dropped, since the client having sent |
| 219 | /// something unexpected is itself a finding. Tag and data are separated by the FIRST double space, |
| 220 | /// because a tag may hold single spaces and data may hold double ones. |
| 221 | export function eachBlock(file, onBlock) { |
| 222 | let txt = ''; |
| 223 | try { txt = fs.readFileSync(file, 'utf8'); } catch (e) { return; } |
| 224 | let cur = null; |
| 225 | for (const line of txt.split('\n')) { |
| 226 | const h = HEADER_RE.exec(line); |
| 227 | if (h) { |
| 228 | if (cur) onBlock(cur); |
| 229 | cur = { bt: Number(h[1]), account: h[2], device: h[3], claimed: Number(h[4]), rows: [] }; |
| 230 | continue; |
| 231 | } |
| 232 | if (!cur || !line.startsWith(' ')) continue; |
| 233 | const rest = line.slice(2); |
| 234 | const m = /^(\d+) /.exec(rest); |
| 235 | if (!m) { cur.rows.push({ ts: cur.bt, tag: rest.trim(), data: '' }); continue; } |
| 236 | const after = rest.slice(m[0].length); |
| 237 | const cut = after.indexOf(' '); |
| 238 | cur.rows.push({ |
| 239 | ts: Number(m[1]), |
| 240 | tag: cut < 0 ? after : after.slice(0, cut), |
| 241 | data: cut < 0 ? '' : after.slice(cut + 2), |
| 242 | }); |
| 243 | } |
| 244 | if (cur) onBlock(cur); |
| 245 | } |
| 246 | |
| 247 | const ROT_RE = /^(.+)\.ndjson\.(\d{8}T\d{6}Z)(?:-(\d+))?$/; |
| 248 | |
| 249 | /// The lens files under a mirror directory, oldest content first: every rotated file before every |
| 250 | /// live one, so a stream's receive times only rise, and within a stream by stamp. Directories are |
| 251 | /// walked (a sink keeps `client/` and `peer/` apart). `fmt` is `lines` for `<key>.ndjson[.<stamp>]` |
| 252 | /// and `text` for Daimond's `<name>.log` and `<name>.log.1`; `sealed` marks a stamped file, which |
| 253 | /// is closed. A pre-stamp `<key>.ndjson.1` is kept and read first, as the oldest. |
| 254 | export function mirrorFiles(dir) { |
| 255 | const out = []; |
| 256 | const walk = (d, rel, depth) => { |
| 257 | let ents = []; |
| 258 | try { ents = fs.readdirSync(d, { withFileTypes: true }); } catch (e) { return; } |
| 259 | for (const e of ents) { |
| 260 | const p = path.join(d, e.name), r = rel ? rel + '/' + e.name : e.name; |
| 261 | if (e.isDirectory()) { if (depth < 4) walk(p, r, depth + 1); continue; } |
| 262 | let m, f = null; |
| 263 | const base = (n) => (rel ? rel + '/' : '') + n; |
| 264 | if ((m = ROT_RE.exec(e.name))) f = { fmt: 'lines', rank: 0, stem: base(m[1]), stamp: m[2], n: Number(m[3] || 0), sealed: true }; |
| 265 | else if ((m = /^(.+)\.ndjson\.1$/.exec(e.name))) f = { fmt: 'lines', rank: 0, stem: base(m[1]), stamp: '', n: 0, sealed: false }; |
| 266 | else if ((m = /^(.+)\.ndjson$/.exec(e.name))) f = { fmt: 'lines', rank: 1, stem: base(m[1]), stamp: '', n: 0, sealed: false }; |
| 267 | else if ((m = /^(.+)\.log\.1$/.exec(e.name))) f = { fmt: 'text', rank: 0, stem: base(m[1]), stamp: '', n: 0, sealed: false }; |
| 268 | else if ((m = /^(.+)\.log$/.exec(e.name))) f = { fmt: 'text', rank: 1, stem: base(m[1]), stamp: '', n: 0, sealed: false }; |
| 269 | if (!f) continue; |
| 270 | let size = 0; |
| 271 | try { size = fs.statSync(p).size; } catch (er) { continue; } |
| 272 | out.push(Object.assign(f, { file: p, rel: r, size })); |
| 273 | } |
| 274 | }; |
| 275 | walk(dir, '', 0); |
| 276 | out.sort((a, b) => a.rank - b.rank || (a.stem < b.stem ? -1 : a.stem > b.stem ? 1 : 0) |
| 277 | || (a.stamp < b.stamp ? -1 : a.stamp > b.stamp ? 1 : 0) || a.n - b.n || (a.rel < b.rel ? -1 : 1)); |
| 278 | return out; |
| 279 | } |
| 280 | |
| 281 | // ── Formatting ─────────────────────────────────────────────────────── |
| 282 | |
| 283 | export function num(n) { |
| 284 | n = Number(n) || 0; |
| 285 | if (n >= 1e9) return (n / 1e9).toFixed(2) + 'G'; |
| 286 | if (n >= 1e6) return (n / 1e6).toFixed(2) + 'M'; |
| 287 | if (n >= 1e3) return Math.round(n / 1e3) + 'k'; |
| 288 | return String(Math.round(n)); |
| 289 | } |
| 290 | |
| 291 | export function hm(ts) { |
| 292 | if (!ts) return '--:--:--'; |
| 293 | const d = new Date(ts); |
| 294 | const p = (n) => String(n).padStart(2, '0'); |
| 295 | return p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds()); |
| 296 | } |
| 297 | |
| 298 | /// A `--since` bound as a label. Time alone for a window inside the day, date |
| 299 | /// and time beyond it, so `--since 7d` does not report itself as "00:11:56". |
| 300 | export function sinceLabel(ts) { |
| 301 | if (!ts) return 'the start of the archive'; |
| 302 | if (Date.now() - ts < 20 * 3600000) return hm(ts); |
| 303 | return new Date(ts).toISOString().slice(0, 16).replace('T', ' ') + 'Z'; |
| 304 | } |
| 305 | |
| 306 | /// A wall-clock label in UTC, for anything a reader might compare against a |
| 307 | /// gateway log or another machine. |
| 308 | export function utcLabel(ts) { |
| 309 | if (!ts) return '?'; |
| 310 | return new Date(ts).toISOString().slice(0, 16).replace('T', ' ') + 'Z'; |
| 311 | } |
| 312 | |
| 313 | /// A coverage stamp, in UTC: the clock alone inside the day, the date in front |
| 314 | /// of it beyond that, because "14:41Z" three days ago is a trap. |
| 315 | export function covLabel(ts) { |
| 316 | if (!ts) return '?'; |
| 317 | const d = new Date(ts); |
| 318 | const p = (n) => String(n).padStart(2, '0'); |
| 319 | const hhmm = p(d.getUTCHours()) + ':' + p(d.getUTCMinutes()) + 'Z'; |
| 320 | return (Date.now() - ts < 20 * 3600000) ? hhmm : d.toISOString().slice(0, 10) + ' ' + hhmm; |
| 321 | } |
| 322 | |
| 323 | /// How a window should describe itself once coverage is taken into account. |
| 324 | export function windowLabel(since, cov) { |
| 325 | if (cov && since < cov) return utcLabel(cov) + ' (archive start)'; |
| 326 | return sinceLabel(since); |
| 327 | } |
| 328 | |
| 329 | /// How long before `at` (now, unless given) a time was. |
| 330 | export function ago(ts, at) { |
| 331 | if (!ts) return 'never'; |
| 332 | const s = Math.max(0, Math.round(((at == null ? Date.now() : at) - ts) / 1000)); |
| 333 | if (s < 90) return s + 's'; |
| 334 | if (s < 5400) return Math.round(s / 60) + 'm'; |
| 335 | if (s < 172800) return Math.round(s / 3600) + 'h'; |
| 336 | return Math.round(s / 86400) + 'd'; |
| 337 | } |
| 338 | |
| 339 | export function dev7(d) { return String(d || '?').slice(0, 7); } |
| 340 | |
| 341 | /// Match two device identifiers loosely. A command-line `--device`, this |
| 342 | /// archive's directory name, and the full id a snapshot's roster or event |
| 343 | /// carries are three different lengths for the SAME device, so a plain |
| 344 | /// `startsWith` has to run both directions. |
| 345 | export function idsMatch(a, b) { |
| 346 | a = String(a); b = String(b); |
| 347 | return a.startsWith(b.slice(0, 16)) || b.startsWith(a.slice(0, 16)); |
| 348 | } |
| 349 | |
| 350 | // ── Output size is the budget ──────────────────────────────────────── |
| 351 | |
| 352 | /// Print at most `cap` lines, saying how many were withheld rather than going |
| 353 | /// quiet -- a truncated answer that does not admit it is a wrong answer. |
| 354 | export function emit(lines, cap) { |
| 355 | if (!cap || lines.length <= cap) { console.log(lines.join('\n')); return; } |
| 356 | const keep = lines.slice(0, cap - 1); |
| 357 | keep.push(`… ${lines.length - cap + 1} more line(s); raise --limit or use --json`); |
| 358 | console.log(keep.join('\n')); |
| 359 | } |
| 360 | |
| 361 | // The longest prefix of `s` within `n` bytes of UTF-8, never splitting a character. |
| 362 | function clipBytes(s, n) { |
| 363 | if (n <= 0) return ''; |
| 364 | if (Buffer.byteLength(s) <= n) return s; |
| 365 | let end = Math.min(s.length, n); |
| 366 | while (end > 0 && Buffer.byteLength(s.slice(0, end)) > n) end--; |
| 367 | const c = s.charCodeAt(end - 1); |
| 368 | if (end > 0 && c >= 0xD800 && c <= 0xDBFF) end--; |
| 369 | return s.slice(0, end); |
| 370 | } |
| 371 | |
| 372 | /// Join lines into text of at most `max` bytes. The first line stays, and so do the last two; |
| 373 | /// the lines between go from the back of the middle first, and a notice after the first line |
| 374 | /// says how many went. A single line too long for the cap is clipped on a character boundary, |
| 375 | /// so the result is never over `max`, whatever it is given. |
| 376 | export function capBytes(lines, max) { |
| 377 | const cap = Math.max(0, Math.floor(Number(max) || 0)); |
| 378 | const keep = lines.map(String); |
| 379 | const bytes = keep.map((l) => Buffer.byteLength(l)); |
| 380 | const total = () => bytes.reduce((a, b) => a + b, 0) + Math.max(0, keep.length - 1); |
| 381 | if (total() <= cap) return keep.join('\n'); |
| 382 | let dropped = 0, size = total(); |
| 383 | const notice = () => '… ' + dropped + ' line(s) withheld by the ' + cap + '-byte cap'; |
| 384 | while (keep.length > 2 && size + (dropped ? Buffer.byteLength(notice()) + 1 : 0) > cap) { |
| 385 | const i = Math.max(1, keep.length - 3); |
| 386 | size -= bytes[i] + 1; |
| 387 | keep.splice(i, 1); bytes.splice(i, 1); |
| 388 | dropped++; |
| 389 | } |
| 390 | if (dropped) keep.splice(Math.min(1, keep.length), 0, notice()); |
| 391 | return clipBytes(keep.join('\n'), cap); |
| 392 | } |
| 393 | |
| 394 | // ── Questions over archive records ─────────────────────────────────── |
| 395 | |
| 396 | const label = (r) => r.src === 'ev' ? r.kind : r.tag; |
| 397 | |
| 398 | function bodyOf(r) { |
| 399 | if (r.src === 'ev') return r.ev; |
| 400 | const d = r.data; |
| 401 | if (typeof d !== 'string' || d.charAt(0) !== '{') return null; |
| 402 | try { return JSON.parse(d); } catch (e) { return null; } |
| 403 | } |
| 404 | |
| 405 | /// Keep the records that fall in a window, name a device, carry a kind (an `ev` kind or a row's tag) |
| 406 | /// and hold a pattern (a string or a RegExp, tried over the label and the body). `wantDevice`, when |
| 407 | /// given, replaces the id-prefix test for the device. |
| 408 | export function filterRows(rows, o) { |
| 409 | o = o || {}; |
| 410 | const re = o.grep == null ? null : (o.grep instanceof RegExp ? o.grep : new RegExp(String(o.grep), 'i')); |
| 411 | const dev = o.device == null || o.device === '' ? null |
| 412 | : (o.wantDevice ? (d) => o.wantDevice(d, o.device) |
| 413 | : (d) => String(d).startsWith(String(o.device)) || String(o.device).startsWith(String(d))); |
| 414 | return rows.filter((r) => { |
| 415 | if (o.since && (r.ts || r.bt || 0) < o.since) return false; |
| 416 | if (dev && !dev(r.device)) return false; |
| 417 | if (o.kind && label(r) !== o.kind) return false; |
| 418 | if (re) { |
| 419 | const body = r.src === 'ev' ? JSON.stringify(r.ev) : (r.data || ''); |
| 420 | re.lastIndex = 0; |
| 421 | if (!re.test(label(r) + ' ' + body)) return false; |
| 422 | } |
| 423 | return true; |
| 424 | }); |
| 425 | } |
| 426 | |
| 427 | /// Archive records as timeline entries, `{bt, ts, device, src, label, run, conn, sid, rec}`, ordered |
| 428 | /// by the time the sink received them and then by the client's own clock. The ids come from the |
| 429 | /// event's payload, or from the row's data when that is a JSON object; `fields` renames them. |
| 430 | export function timeline(records, o) { |
| 431 | const f = Object.assign({ run: 'run', conn: 'conn', sid: 'sid' }, o && o.fields); |
| 432 | const out = records.map((r, i) => { |
| 433 | const body = bodyOf(r); |
| 434 | const id = (k) => (body && typeof body === 'object' && body[f[k]] != null) ? body[f[k]] : null; |
| 435 | return { bt: r.bt || 0, ts: r.ts || 0, device: r.device, src: r.src, label: label(r), |
| 436 | run: id('run'), conn: id('conn'), sid: id('sid'), rec: r, i }; |
| 437 | }); |
| 438 | out.sort((a, b) => a.bt - b.bt || a.ts - b.ts || a.i - b.i); |
| 439 | return out.map((e) => { delete e.i; return e; }); |
| 440 | } |
| 441 | |
| 442 | /// The entries of one page load, one connection or one process, from `timeline`'s output. A page |
| 443 | /// load (`sid`) takes its own rows and every row of the connections it carried; a connection |
| 444 | /// (`conn`, with `run` to say which process, since a connection number restarts) takes its rows and |
| 445 | /// the page loads that rode it; a `run` alone takes that process's rows. A connection is told from |
| 446 | /// another of the same number by its run, so a restarted peer's conn 7 is not this one. |
| 447 | export function session(entries, key) { |
| 448 | key = key || {}; |
| 449 | const same = (a, b) => a != null && String(a) === String(b); |
| 450 | const cid = (e) => (e.run == null ? '' : String(e.run)) + '|' + e.conn; |
| 451 | if (key.run != null && key.sid == null && key.conn == null) return entries.filter((e) => same(e.run, key.run)); |
| 452 | let seeds = []; |
| 453 | if (key.sid != null) seeds = entries.filter((e) => same(e.sid, key.sid)); |
| 454 | else if (key.conn != null) seeds = entries.filter((e) => same(e.conn, key.conn) && (key.run == null || same(e.run, key.run))); |
| 455 | if (!seeds.length) return []; |
| 456 | const sids = new Set(seeds.filter((e) => e.sid != null).map((e) => String(e.sid))); |
| 457 | const conns = new Set(seeds.filter((e) => e.conn != null).map(cid)); |
| 458 | return entries.filter((e) => (e.sid != null && sids.has(String(e.sid))) || (e.conn != null && conns.has(cid(e)))); |
| 459 | } |
| 460 | |
| 461 | export const DIGEST_MAX = 2048; |
| 462 | // A label that reads as something going wrong. |
| 463 | const BAD_RE = /(?:^|[._ -])(?:error|fail|fault|drop|refus|reject|exception|throttl)/i; |
| 464 | |
| 465 | /// What a pickup needs in at most 2 KB: when, how many devices, how stale each is, what the window |
| 466 | /// held, the commonest labels, and the worst errors. `extra` (an array, or a function from the |
| 467 | /// windowed records to one) adds the app's own lines after the device block. The cap is enforced here |
| 468 | /// and a caller may lower it with `max` and never raise it. |
| 469 | export function digest(records, o) { |
| 470 | o = o || {}; |
| 471 | const now = o.now == null ? Date.now() : o.now; |
| 472 | const since = o.since || 0; |
| 473 | const max = Math.min(DIGEST_MAX, Number.isFinite(o.max) && o.max > 0 ? Math.floor(o.max) : DIGEST_MAX); |
| 474 | const all = records || []; |
| 475 | let first = 0; |
| 476 | for (const r of all) { const t = r.ts || r.bt || 0; if (t && (!first || t < first)) first = t; } |
| 477 | const recs = all.filter((r) => (r.ts || r.bt || 0) >= since); |
| 478 | const by = new Map(), tags = new Map(), errs = new Map(); |
| 479 | let bad = 0; |
| 480 | for (const r of recs) { |
| 481 | const t = r.ts || r.bt || 0, lab = label(r); |
| 482 | const d = by.get(r.device) || { n: 0, last: 0, e: 0 }; |
| 483 | d.n++; d.last = Math.max(d.last, t); |
| 484 | tags.set(lab, (tags.get(lab) || 0) + 1); |
| 485 | const body = r.src === 'ev' ? r.ev : null; |
| 486 | if (BAD_RE.test(lab) || (body && body.lvl === 'error')) { |
| 487 | d.e++; bad++; |
| 488 | let msg = body ? (body.msg ?? body.m ?? body.err ?? '') : (r.data || ''); |
| 489 | if (typeof msg !== 'string') msg = JSON.stringify(msg); |
| 490 | const key = lab + '|' + msg.replace(/\d+/g, 'N'); |
| 491 | const e = errs.get(key) || { lab, msg, n: 0, last: 0 }; |
| 492 | e.n++; e.last = Math.max(e.last, t); |
| 493 | errs.set(key, e); |
| 494 | } |
| 495 | by.set(r.device, d); |
| 496 | } |
| 497 | const lines = [(o.title || 'lens') + ' — ' + utcLabel(now) + ' ' + by.size + ' device(s) archive since ' + utcLabel(first)]; |
| 498 | for (const [dv, d] of [...by].sort((a, b) => b[1].last - a[1].last).slice(0, 4)) { |
| 499 | lines.push(dev7(dv) + ' ' + ago(d.last, now) + ' ago ' + d.n + ' row(s)' + (d.e ? ' ' + d.e + ' err' : '')); |
| 500 | } |
| 501 | const extra = typeof o.extra === 'function' ? o.extra(recs) : (o.extra || []); |
| 502 | for (const l of extra || []) lines.push(String(l)); |
| 503 | lines.push('window: ' + recs.length + ' row(s), ' + bad + ' error-like'); |
| 504 | const top = [...tags].sort((a, b) => b[1] - a[1]).slice(0, 5); |
| 505 | if (top.length) lines.push('top: ' + top.map(([k, n]) => 'x' + n + ' ' + k).join(' ')); |
| 506 | for (const e of [...errs.values()].sort((a, b) => b.n - a.n || b.last - a.last).slice(0, 3)) { |
| 507 | lines.push(' x' + e.n + ' ' + e.lab + ': ' + e.msg.slice(0, 88)); |
| 508 | } |
| 509 | return capBytes(lines, max); |
| 510 | } |
| 511 | |
| 512 | // ── The reader: a home directory, a mirror, an archive ─────────────── |
| 513 | |
| 514 | /// A reader over `cfg.home`. All of it optional but `home`: |
| 515 | /// |
| 516 | /// mirror the mirror's directory name under home (default `mirror`); |
| 517 | /// pull how the mirror is fetched: an argv array, an array of them (run in order, no shell), or |
| 518 | /// a function; a failure is reported in the stat and never stops the ingest; |
| 519 | /// scrub a scrubber `{text, deep}` (default `createScrubber()`), run on every record filed; |
| 520 | /// redact a redactor from `createRedactor`, run first by name and by string test; |
| 521 | /// prepare `bundle -> bundle`, a snapshot's second-line redaction before it is filed; |
| 522 | /// tick `bundle -> fields`, what a telemetry tick keeps of its bundle (default all of it); |
| 523 | /// index `bundle -> fields`, facts lifted into a snapshot's index record; |
| 524 | /// nameFor `device -> name`, so `--device` can name a roster entry as well as an id prefix; |
| 525 | /// recentMs, pendingTtlMs, now the dedupe window, how long an unfinished set waits, the clock. |
| 526 | export function createReader(cfg) { |
| 527 | cfg = cfg || {}; |
| 528 | if (!cfg.home) throw new Error('createReader needs a home directory'); |
| 529 | const dirs = { |
| 530 | home: cfg.home, |
| 531 | mirror: path.join(cfg.home, cfg.mirror || 'mirror'), |
| 532 | archive: path.join(cfg.home, 'archive'), |
| 533 | snaps: path.join(cfg.home, 'archive', 'snapshots'), |
| 534 | pending: path.join(cfg.home, 'pending'), |
| 535 | state: path.join(cfg.home, 'state.json'), |
| 536 | }; |
| 537 | const now = cfg.now || (() => Date.now()); |
| 538 | const recentMs = cfg.recentMs || 10 * 60 * 1000; |
| 539 | const ttlMs = cfg.pendingTtlMs || 6 * 60 * 60 * 1000; |
| 540 | const scrub = cfg.scrub || createScrubber(); |
| 541 | const redact = cfg.redact || null; |
| 542 | const prepare = cfg.prepare || ((b) => b); |
| 543 | const tick = cfg.tick || ((b) => b); |
| 544 | const index = cfg.index || (() => ({})); |
| 545 | const nameFor = cfg.nameFor || (() => ''); |
| 546 | |
| 547 | const partsFile = (device, kind, id) => path.join(dirs.pending, safeName(device), safeName(kind) + '_' + safeName(id) + '.parts'); |
| 548 | const eventsFile = (device) => path.join(dirs.archive, safeName(device) + '.events.ndjson'); |
| 549 | const telFile = (device) => path.join(dirs.archive, safeName(device) + '.telemetry.ndjson'); |
| 550 | const snapIndexFile = (device) => path.join(dirs.archive, safeName(device) + '.snapshots.ndjson'); |
| 551 | const snapBodyFile = (device, id) => path.join(dirs.snaps, safeName(device) + '-' + safeName(id) + '.json'); |
| 552 | |
| 553 | /// Append one archive record. THE INGEST SEAM: every event, telemetry tick and |
| 554 | /// snapshot-index row the archive holds is written here and nowhere else, so the |
| 555 | /// content scrubber runs here -- once, on the way to disk. |
| 556 | function appendLine(f, obj) { |
| 557 | ensureDir(path.dirname(f)); |
| 558 | fs.appendFileSync(f, JSON.stringify(scrub.deep(obj)) + '\n'); |
| 559 | } |
| 560 | |
| 561 | // ── State ──────────────────────────────────────────────────────── |
| 562 | |
| 563 | function loadState() { |
| 564 | const s = readJson(dirs.state, null); |
| 565 | if (s && s.v === 1) return s; |
| 566 | return { v: 1, devices: {} }; |
| 567 | } |
| 568 | |
| 569 | function devState(st, device) { |
| 570 | if (!st.devices[device]) { |
| 571 | st.devices[device] = { |
| 572 | maxBt: 0, // newest text-block receive time ingested for this device |
| 573 | minBt: 0, // earliest receive time of any kind, which is where coverage starts |
| 574 | top: 0, // newest receive time of any kind, which ages the closed sets |
| 575 | recent: {}, // text-block times within recentMs of maxBt, exactly |
| 576 | ev: {}, // envelope device id -> seen sequence ranges |
| 577 | pending: {}, // "<kind>:<id>" -> chunk set being assembled |
| 578 | done: {}, // "<kind>:<id>" -> when the set was closed |
| 579 | }; |
| 580 | } |
| 581 | return st.devices[device]; |
| 582 | } |
| 583 | |
| 584 | // One stream is one key's files, live and rotated: the unit whose receive times only rise. |
| 585 | function streamState(st, stem) { |
| 586 | if (!st.streams) st.streams = {}; |
| 587 | if (!st.streams[stem]) st.streams[stem] = { maxBt: 0, lines: {} }; |
| 588 | return st.streams[stem]; |
| 589 | } |
| 590 | |
| 591 | /// Has this text block already been ingested? Blocks arrive in receive order within a |
| 592 | /// stream, so anything well behind the watermark is certainly seen; near it, the exact |
| 593 | /// key decides, which is what makes two posts in one millisecond safe. |
| 594 | function blockSeen(ds, bt) { |
| 595 | if (bt < ds.maxBt - recentMs) return true; |
| 596 | return !!ds.recent[String(bt)]; |
| 597 | } |
| 598 | |
| 599 | function blockMark(ds, bt) { |
| 600 | ds.recent[String(bt)] = 1; |
| 601 | if (bt > ds.maxBt) ds.maxBt = bt; |
| 602 | if (bt > ds.top) ds.top = bt; |
| 603 | if (!ds.minBt || bt < ds.minBt) ds.minBt = bt; |
| 604 | const floor = ds.maxBt - recentMs; |
| 605 | for (const k of Object.keys(ds.recent)) if (Number(k) < floor) delete ds.recent[k]; |
| 606 | } |
| 607 | |
| 608 | // A line is told by its receive time and a hash of its text. `k` counts the times the same text |
| 609 | // has come up in this file, so a row that was genuinely posted twice is kept twice and a file |
| 610 | // read again adds nothing. |
| 611 | const lineHash = (raw) => crypto.createHash('sha1').update(raw).digest('hex').slice(0, 16); |
| 612 | |
| 613 | function lineSeen(ss, bt, h, k) { |
| 614 | if (bt < ss.maxBt - recentMs) return true; |
| 615 | const m = ss.lines[String(bt)]; |
| 616 | return !!m && (m[h] || 0) >= k; |
| 617 | } |
| 618 | |
| 619 | function lineMark(ss, bt, h, k) { |
| 620 | const m = ss.lines[String(bt)] || (ss.lines[String(bt)] = {}); |
| 621 | m[h] = Math.max(m[h] || 0, k); |
| 622 | if (bt > ss.maxBt) ss.maxBt = bt; |
| 623 | const floor = ss.maxBt - recentMs; |
| 624 | for (const t of Object.keys(ss.lines)) if (Number(t) < floor) delete ss.lines[t]; |
| 625 | } |
| 626 | |
| 627 | /// Mark a chunk set closed, and forget closures older than a day so the marker |
| 628 | /// map cannot grow for the life of the archive. |
| 629 | function closeSet(ds, key, bt) { |
| 630 | if (!ds.done) ds.done = {}; |
| 631 | ds.done[key] = bt || now(); |
| 632 | const floor = (ds.top || ds.maxBt || now()) - 86400000; |
| 633 | for (const k of Object.keys(ds.done)) if (ds.done[k] < floor) delete ds.done[k]; |
| 634 | } |
| 635 | |
| 636 | // ── Ingest ─────────────────────────────────────────────────────── |
| 637 | |
| 638 | /// Concatenate a completed chunk set, decode it, and file it. A set that will not decode is |
| 639 | /// recorded as a broken set rather than thrown: one corrupt bundle must not stop a pull. |
| 640 | function materialise(device, p, stat) { |
| 641 | const pf = partsFile(device, p.kind, p.id); |
| 642 | const parts = new Map(); |
| 643 | let txt = ''; |
| 644 | try { txt = fs.readFileSync(pf, 'utf8'); } catch (e) { /* nothing on disk */ } |
| 645 | for (const line of txt.split('\n')) { |
| 646 | if (!line) continue; |
| 647 | const tab = line.indexOf('\t'); |
| 648 | if (tab < 0) continue; |
| 649 | const i = Number(line.slice(0, tab)); |
| 650 | // Last writer wins, which for a duplicated chunk is the same bytes. |
| 651 | if (Number.isFinite(i)) parts.set(i, line.slice(tab + 1)); |
| 652 | } |
| 653 | const ordered = []; |
| 654 | for (let i = 1; i <= p.n; i++) ordered.push(parts.get(i) || ''); |
| 655 | let bundle = null, error = null; |
| 656 | try { bundle = JSON.parse(Buffer.from(ordered.join(''), 'base64').toString('utf8')); } |
| 657 | catch (e) { error = String(e && e.message || e).slice(0, 120); } |
| 658 | if (bundle && redact) bundle = redact.value(bundle); |
| 659 | |
| 660 | if (p.kind === 'telemetry') { |
| 661 | if (bundle) { |
| 662 | appendLine(telFile(device), Object.assign({}, tick(bundle), { |
| 663 | id: p.id, device, bt: p.lastBt, ts: bundle.ts || p.lastTs, |
| 664 | })); |
| 665 | stat.telemetry++; |
| 666 | } else { |
| 667 | appendLine(telFile(device), { id: p.id, device, bt: p.lastBt, ts: p.lastTs, broken: error }); |
| 668 | stat.broken++; |
| 669 | } |
| 670 | } else { |
| 671 | const rec = { |
| 672 | id: p.id, device, bt: p.lastBt, ts: bundle ? (bundle.ts || p.lastTs) : p.lastTs, |
| 673 | n: p.n, have: rangeCount(p.have), complete: rangeCount(p.have) === p.n, |
| 674 | missing: rangeCount(p.have) === p.n ? '' : rangeMissing(p.have, p.n), |
| 675 | }; |
| 676 | if (bundle) { |
| 677 | // The app's own second line first (a camelCase name the client missed), then the |
| 678 | // content scrubber over the whole body, which is the only thing filed here that |
| 679 | // does not pass through `appendLine`. |
| 680 | bundle = scrub.deep(prepare(bundle)); |
| 681 | ensureDir(dirs.snaps); |
| 682 | fs.writeFileSync(snapBodyFile(device, p.id), JSON.stringify(bundle)); |
| 683 | rec.body = path.basename(snapBodyFile(device, p.id)); |
| 684 | rec.iso = bundle.iso || null; |
| 685 | // A few facts are lifted into the index so a status never has to open a |
| 686 | // megabyte of snapshot to answer a one-line question. |
| 687 | Object.assign(rec, index(bundle)); |
| 688 | stat.snapshots++; |
| 689 | } else { |
| 690 | rec.broken = error; |
| 691 | stat.broken++; |
| 692 | } |
| 693 | appendLine(snapIndexFile(device), rec); |
| 694 | } |
| 695 | try { fs.unlinkSync(pf); } catch (e) { /* already gone */ } |
| 696 | return bundle ? 'ok' : 'broken'; |
| 697 | } |
| 698 | |
| 699 | /// Record a chunk set that can never complete, and stop holding its parts open. |
| 700 | /// The index keeps the gap so a reader can say what was lost rather than going |
| 701 | /// quiet about a bundle it never saw. |
| 702 | function expire(device, p, stat) { |
| 703 | if (p.kind === 'telemetry') { |
| 704 | appendLine(telFile(device), { |
| 705 | id: p.id, device, bt: p.lastBt, ts: p.lastTs, |
| 706 | gap: true, n: p.n, have: rangeCount(p.have), missing: rangeMissing(p.have, p.n), |
| 707 | }); |
| 708 | } else { |
| 709 | appendLine(snapIndexFile(device), { |
| 710 | id: p.id, device, bt: p.lastBt, ts: p.lastTs, |
| 711 | n: p.n, have: rangeCount(p.have), complete: false, |
| 712 | missing: rangeMissing(p.have, p.n), gap: true, |
| 713 | }); |
| 714 | } |
| 715 | try { fs.renameSync(partsFile(device, p.kind, p.id), partsFile(device, p.kind, p.id) + '.stale'); } |
| 716 | catch (e) { /* nothing to keep */ } |
| 717 | stat.gapped++; |
| 718 | } |
| 719 | |
| 720 | function ingestRow(ds, device, bt, row, stat) { |
| 721 | const t = tagFamily(row.tag); |
| 722 | if (t.fam === 'ds') { |
| 723 | const key = t.kind + ':' + t.id; |
| 724 | // A chunk of a set that is already filed. The client retries a post whose |
| 725 | // reply it lost, so this arrives after the set has completed -- and without |
| 726 | // this line it would open the set again and leave a phantom half-assembled |
| 727 | // bundle behind that nothing ever finishes. |
| 728 | if (!ds.done) ds.done = {}; |
| 729 | if (ds.done[key]) { stat.dupChunks++; return; } |
| 730 | let p = ds.pending[key]; |
| 731 | if (!p) p = ds.pending[key] = { kind: t.kind, id: t.id, n: t.n, have: [], firstTs: row.ts, lastTs: row.ts, lastBt: bt }; |
| 732 | p.n = Math.max(p.n, t.n); |
| 733 | p.lastTs = row.ts; |
| 734 | p.lastBt = bt; |
| 735 | // A duplicated chunk index is written once. The client can resend a post |
| 736 | // the sink accepted but whose reply was lost, so this is ordinary. |
| 737 | if (rangeAdd(p.have, t.i)) { |
| 738 | const pf = partsFile(device, t.kind, t.id); |
| 739 | ensureDir(path.dirname(pf)); |
| 740 | fs.appendFileSync(pf, t.i + '\t' + row.data + '\n'); |
| 741 | stat.chunks++; |
| 742 | } else { |
| 743 | stat.dupChunks++; |
| 744 | } |
| 745 | if (rangeCount(p.have) >= p.n) { |
| 746 | materialise(device, p, stat); |
| 747 | delete ds.pending[key]; |
| 748 | closeSet(ds, key, bt); |
| 749 | } |
| 750 | return; |
| 751 | } |
| 752 | |
| 753 | if (t.fam === 'ev') { |
| 754 | let obj = null; |
| 755 | try { obj = JSON.parse(row.data); } catch (e) { obj = null; } |
| 756 | if (obj && typeof obj === 'object') { |
| 757 | // The envelope's own device id is the dedupe axis, not the filename: |
| 758 | // `(d,n)` is what the client makes unique, and a device that has been |
| 759 | // re-keyed writes to the same file under a new `d`. |
| 760 | const d = String(obj.d || device); |
| 761 | const n = Number(obj.n); |
| 762 | const rs = ds.ev[d] || (ds.ev[d] = []); |
| 763 | if (Number.isFinite(n) && !rangeAdd(rs, n)) { stat.dupEvents++; return; } |
| 764 | appendLine(eventsFile(device), { |
| 765 | src: 'ev', device, bt, ts: obj.t || row.ts, kind: t.kind, ev: redact ? redact.value(obj) : obj, |
| 766 | }); |
| 767 | stat.events++; |
| 768 | return; |
| 769 | } |
| 770 | // An `ev` row whose data will not parse is still a fact about the client. |
| 771 | appendLine(eventsFile(device), { |
| 772 | src: 'row', device, bt, ts: row.ts, tag: row.tag, data: guardText(row.data), malformed: true, |
| 773 | }); |
| 774 | stat.rows++; |
| 775 | return; |
| 776 | } |
| 777 | |
| 778 | // Everything else -- election traces, hand-off latency, diagnostics -- is kept verbatim as a |
| 779 | // generic row, so a search can reach it without anyone having taught this file what the |
| 780 | // tag means. |
| 781 | appendLine(eventsFile(device), { src: 'row', device, bt, ts: row.ts, tag: row.tag, data: guardText(row.data) }); |
| 782 | stat.rows++; |
| 783 | } |
| 784 | |
| 785 | // A row's text, replaced whole when the caller's string test calls it secret. |
| 786 | function guardText(s) { |
| 787 | if (!redact) return s; |
| 788 | const r = redact.text(s); |
| 789 | return r == null ? s : r; |
| 790 | } |
| 791 | |
| 792 | function ingestLines(f, st, stat) { |
| 793 | const { rows, bad } = readLines(f.file); |
| 794 | stat.bad += bad; |
| 795 | const ss = streamState(st, f.stem); |
| 796 | const counts = new Map(); |
| 797 | for (const blk of lineBlocks(rows)) { |
| 798 | const ds = devState(st, blk.device); |
| 799 | const fresh = []; |
| 800 | for (const r of blk.rows) { |
| 801 | const h = lineHash(r.raw), ck = blk.bt + '|' + h; |
| 802 | const k = (counts.get(ck) || 0) + 1; |
| 803 | counts.set(ck, k); |
| 804 | if (lineSeen(ss, blk.bt, h, k)) continue; |
| 805 | lineMark(ss, blk.bt, h, k); |
| 806 | fresh.push(r); |
| 807 | } |
| 808 | if (!fresh.length) { stat.skipped++; continue; } |
| 809 | stat.blocks++; |
| 810 | if (!ds.minBt || blk.bt < ds.minBt) ds.minBt = blk.bt; |
| 811 | if (blk.bt > ds.top) ds.top = blk.bt; |
| 812 | for (const r of fresh) ingestRow(ds, blk.device, blk.bt, r, stat); |
| 813 | } |
| 814 | } |
| 815 | |
| 816 | // The caller's command, or commands, as one word for the stat. |
| 817 | function runPull() { |
| 818 | const p = cfg.pull; |
| 819 | if (!p || (Array.isArray(p) && !p.length)) return 'skipped'; |
| 820 | const first = (e) => String(e && e.message || e).split('\n')[0].slice(0, 80); |
| 821 | if (typeof p === 'function') { |
| 822 | try { p(dirs); return 'ok'; } catch (e) { return 'failed: ' + first(e); } |
| 823 | } |
| 824 | let failed = null; |
| 825 | for (const argv of (Array.isArray(p[0]) ? p : [p])) { |
| 826 | // An argv, never a shell string, so a host name or a path is never parsed. |
| 827 | try { execFileSync(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'pipe'], timeout: cfg.pullTimeoutMs || 300000 }); } |
| 828 | catch (e) { failed = failed || first(e); } |
| 829 | } |
| 830 | return failed ? 'failed: ' + failed : 'ok'; |
| 831 | } |
| 832 | |
| 833 | /// Fetch the mirror by the caller's command (unless `noPull`), then file what it holds that the |
| 834 | /// archive has not seen. Idempotent. A failed fetch still ingests whatever the mirror has, which |
| 835 | /// is the point of keeping a local archive at all. |
| 836 | function pull(o) { |
| 837 | o = o || {}; |
| 838 | ensureDir(dirs.mirror); ensureDir(dirs.archive); |
| 839 | const stat = { |
| 840 | blocks: 0, skipped: 0, rows: 0, events: 0, dupEvents: 0, chunks: 0, dupChunks: 0, |
| 841 | telemetry: 0, snapshots: 0, gapped: 0, broken: 0, bad: 0, sealed: 0, pull: 'skipped', |
| 842 | }; |
| 843 | if (!o.noPull) stat.pull = runPull(); |
| 844 | const st = loadState(); |
| 845 | if (!st.sealed) st.sealed = {}; |
| 846 | for (const f of mirrorFiles(dirs.mirror)) { |
| 847 | if (f.sealed && st.sealed[f.rel] === f.size) { stat.sealed++; continue; } |
| 848 | if (f.fmt === 'text') { |
| 849 | eachBlock(f.file, (b) => { |
| 850 | const ds = devState(st, b.device); |
| 851 | if (blockSeen(ds, b.bt)) { stat.skipped++; return; } |
| 852 | blockMark(ds, b.bt); |
| 853 | stat.blocks++; |
| 854 | for (const row of b.rows) ingestRow(ds, b.device, b.bt, row, stat); |
| 855 | }); |
| 856 | } else { |
| 857 | ingestLines(f, st, stat); |
| 858 | } |
| 859 | if (f.sealed) st.sealed[f.rel] = f.size; |
| 860 | } |
| 861 | // Chunk sets whose missing rows rotated away at the source are closed off, so the |
| 862 | // pending area cannot grow without bound and the gap is on the record. |
| 863 | const at = now(); |
| 864 | for (const device of Object.keys(st.devices)) { |
| 865 | const ds = st.devices[device]; |
| 866 | for (const key of Object.keys(ds.pending)) { |
| 867 | const p = ds.pending[key]; |
| 868 | if (at - (p.lastBt || p.lastTs || 0) > ttlMs) { |
| 869 | expire(device, p, stat); |
| 870 | delete ds.pending[key]; |
| 871 | closeSet(ds, key, p.lastBt || at); |
| 872 | } |
| 873 | } |
| 874 | } |
| 875 | writeJson(dirs.state, st); |
| 876 | return stat; |
| 877 | } |
| 878 | |
| 879 | // ── Reading the archive ────────────────────────────────────────── |
| 880 | |
| 881 | function archiveDevices() { |
| 882 | let names = []; |
| 883 | try { names = fs.readdirSync(dirs.archive); } catch (e) { return []; } |
| 884 | const set = new Set(); |
| 885 | for (const n of names) { |
| 886 | const m = /^(.+)\.(events|telemetry|snapshots)\.ndjson$/.exec(n); |
| 887 | if (m) set.add(m[1]); |
| 888 | } |
| 889 | return [...set].sort(); |
| 890 | } |
| 891 | |
| 892 | function wantDevice(device, filter) { |
| 893 | if (!filter) return true; |
| 894 | const f = String(filter); |
| 895 | if (String(device).startsWith(f) || f.startsWith(String(device))) return true; |
| 896 | // `--device` naming the roster instead of the id, matched case-insensitively against |
| 897 | // whatever this archive has ever called that device. The id-prefix check above runs |
| 898 | // first and is unaffected: this only fires for a filter that was never a prefix. |
| 899 | const nm = nameFor(device); |
| 900 | return !!nm && nm.toLowerCase().includes(f.toLowerCase()); |
| 901 | } |
| 902 | |
| 903 | /// When this archive's record begins, PER DEVICE and PER SOURCE. The two are not the same number |
| 904 | /// and conflating them lies: a device's oldest row may be an election trace from three days ago |
| 905 | /// while its oldest telemetry tick, the only source that can see a turn or a cost, is from this |
| 906 | /// afternoon. |
| 907 | /// |
| 908 | /// `ticks` earliest telemetry tick, and earliest `ev` event; |
| 909 | /// `rows` earliest row of any kind, which is what a search can reach and nothing more. |
| 910 | function coverage(filter) { |
| 911 | const st = loadState(); |
| 912 | const byDevice = {}; |
| 913 | let ticks = 0, rows = 0; |
| 914 | const lower = (a, b) => (!a || (b && b < a)) ? b : a; |
| 915 | for (const d of archiveDevices()) { |
| 916 | if (!wantDevice(d, filter)) continue; |
| 917 | const rec = { device: d, ticks: 0, rows: 0 }; |
| 918 | // Each archive file is append-ordered, so its first record is its earliest; only the |
| 919 | // events file has to be walked, because it mixes `ev` events with generic rows. |
| 920 | const t0 = readNdjson(telFile(d))[0]; |
| 921 | if (t0) rec.ticks = t0.ts || t0.bt || 0; |
| 922 | for (const r of readNdjson(eventsFile(d))) { |
| 923 | const ts = r.ts || r.bt || 0; |
| 924 | if (!ts) continue; |
| 925 | rec.rows = lower(rec.rows, ts); |
| 926 | if (r.src === 'ev') rec.ticks = lower(rec.ticks, ts); |
| 927 | } |
| 928 | const s0 = readNdjson(snapIndexFile(d))[0]; |
| 929 | if (s0) rec.rows = lower(rec.rows, s0.ts || s0.bt || 0); |
| 930 | if (st.devices && st.devices[d] && st.devices[d].minBt) rec.rows = lower(rec.rows, st.devices[d].minBt); |
| 931 | rec.rows = lower(rec.rows, rec.ticks); |
| 932 | byDevice[d] = rec; |
| 933 | ticks = lower(ticks, rec.ticks); |
| 934 | rows = lower(rows, rec.rows); |
| 935 | } |
| 936 | return { ticks: ticks || 0, rows: rows || 0, byDevice }; |
| 937 | } |
| 938 | |
| 939 | function loadFrom(fileOf, filter, since) { |
| 940 | const out = []; |
| 941 | for (const d of archiveDevices()) { |
| 942 | if (!wantDevice(d, filter)) continue; |
| 943 | for (const r of readNdjson(fileOf(d))) { |
| 944 | if (since && (r.ts || r.bt || 0) < since) continue; |
| 945 | out.push(r); |
| 946 | } |
| 947 | } |
| 948 | out.sort((a, b) => (a.ts || a.bt || 0) - (b.ts || b.bt || 0)); |
| 949 | return out; |
| 950 | } |
| 951 | const loadTelemetry = (filter, since) => loadFrom(telFile, filter, since); |
| 952 | const loadEvents = (filter, since) => loadFrom(eventsFile, filter, since); |
| 953 | const loadSnapshotIndex = (filter) => loadFrom(snapIndexFile, filter, 0); |
| 954 | |
| 955 | /// Chunk sets part-assembled and still waiting for their remaining rows. They are a live fact, |
| 956 | /// not a fault: a snapshot is thousands of rows and drains over minutes. |
| 957 | function pendingSets(filter) { |
| 958 | const st = loadState(); |
| 959 | const out = []; |
| 960 | for (const device of Object.keys(st.devices || {})) { |
| 961 | if (!wantDevice(device, filter)) continue; |
| 962 | const pend = st.devices[device].pending || {}; |
| 963 | for (const key of Object.keys(pend)) { |
| 964 | const p = pend[key]; |
| 965 | out.push({ device, kind: p.kind, id: p.id, n: p.n, have: rangeCount(p.have || []), lastBt: p.lastBt }); |
| 966 | } |
| 967 | } |
| 968 | out.sort((a, b) => (b.lastBt || 0) - (a.lastBt || 0)); |
| 969 | return out; |
| 970 | } |
| 971 | |
| 972 | /// The digest of what the archive holds for a device and a window: `digest` over its events. |
| 973 | function digestOf(o) { |
| 974 | o = o || {}; |
| 975 | const recs = loadEvents(o.device, o.since || 0); |
| 976 | return digest(recs, Object.assign({ now: now() }, o)); |
| 977 | } |
| 978 | |
| 979 | return { |
| 980 | dirs, appendLine, loadState, pull, archiveDevices, wantDevice, coverage, |
| 981 | loadTelemetry, loadEvents, loadSnapshotIndex, pendingSets, digest: digestOf, |
| 982 | eventsFile, telFile, snapIndexFile, snapBodyFile, partsFile, |
| 983 | }; |
| 984 | } |