Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/lens/js/reader.mjs

42.0 KiB, 1 run

created by r1870400018:65602, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// reader.mjs -- the node half of the debug lens: pull a mirror, keep an archive, answer questions.
2//
3// Extracted from Daimond's `dev/lens.mjs` (2026-10-02) so that Daimond and Oxegen share one reader.
4// What the app alone knows is a parameter of `createReader`: where the home directory is, how the
5// mirror is fetched, which names and strings are secret, what a snapshot or a telemetry bundle
6// holds. The views (`turns`, `errors`, `status` and the rest) stay in the app; they read the archive
7// this file keeps and use the helpers it exports.
8//
9// THE LINE. `fe2o3_net::lens::Sink` writes one object a line, `{"bt":<recv ms>,"device":"..",
10// "ts":<client ms>,"tag":"..","data":".."}`, with `data` a string (an event's JSON as text, so a
11// clip never makes an invalid line). Daimond's gateway still writes the text block the line was
12// taken from (`===== <recv ms> account=<a> device=<d> rows=<n> =====` and ` <ts> <tag> <data>`
13// under it); this reader takes either, and both reach the same archive records.
14//
15// THE FILES. A key's live file is `<key>.ndjson`. A rotation closes it as `<key>.ndjson.<stamp>`,
16// `<stamp>` being `YYYYMMDDTHHMMSSZ` and `-n` added on a taken stamp. A rotated file is closed for
17// good, so once it has been read it is skipped until its size changes.
18//
19// THE ARCHIVE, kept under `<home>/archive/`:
20//
21// <device>.events.ndjson every `ev` row as {src:'ev',device,bt,ts,kind,ev}, every other row as
22// {src:'row',device,bt,ts,tag,data}
23// <device>.telemetry.ndjson one record per assembled `ds telemetry` set
24// <device>.snapshots.ndjson one index record per `ds snapshot` set, its body in `snapshots/`
25//
26// DEDUPE. An event is filed once by its envelope `(d, n)`, whichever file or retry delivers it. A
27// chunk of a `ds` set is filed once by its index. A line is told from a line already read by its
28// receive time and a hash of its text, per stream (one key's files), so a torn tail that is
29// completed on the next pull loses nothing and files nothing twice.
30//
31// SCRUBBING AGAIN. Every record passes the caller's redactor (by name and by string test, where
32// given) and then the content scrubber on its way to disk, because the writer's own guard is the
33// first line and a client bug or an older client is what the second line is for.
34//
35// OUTPUT IS THE BUDGET. A digest is read by a language model at every pickup, so `digest` and
36// `capBytes` hold it to 2 KB inside the function, and say what they withheld.
37import { execFileSync } from 'node:child_process';
38import crypto from 'node:crypto';
39import fs from 'node:fs';
40import path from 'node:path';
41import { createScrubber } from './redact.js';
42
43// ── Arguments ────────────────────────────────────────────────────────
44
45export function parseArgs(argv) {
46 const a = { cmd: argv[0] || 'digest', flags: {}, bool: {} };
47 for (let i = 1; i < argv.length; i++) {
48 const t = argv[i];
49 if (!t.startsWith('--')) continue;
50 const k = t.slice(2);
51 const next = argv[i + 1];
52 if (next !== undefined && !next.startsWith('--')) { a.flags[k] = next; i++; }
53 else a.bool[k] = true;
54 }
55 return a;
56}
57
58/// Turn `--since` into an absolute millisecond bound. Bare numbers are seconds,
59/// so `--since 90` and `--since 90s` agree.
60export function sinceMs(v, dflt) {
61 if (v === undefined || v === null || v === '') return dflt;
62 const s = String(v).trim();
63 if (s === 'all') return 0;
64 if (s === 'today') { const d = new Date(); d.setHours(0, 0, 0, 0); return d.getTime(); }
65 const m = s.match(/^(\d+(?:\.\d+)?)\s*([smhdw]?)$/i);
66 if (m) {
67 const mult = { '': 1000, s: 1000, m: 60000, h: 3600000, d: 86400000, w: 604800000 };
68 return Date.now() - Number(m[1]) * mult[m[2].toLowerCase()];
69 }
70 const t = Date.parse(s);
71 return Number.isFinite(t) ? t : dflt;
72}
73
74// ── Small file helpers ───────────────────────────────────────────────
75
76export function ensureDir(d) { fs.mkdirSync(d, { recursive: true }); }
77export function safeName(s) { return String(s || 'unknown').replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 64); }
78
79export function readJson(f, dflt) {
80 try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch (e) { return dflt; }
81}
82
83export function writeJson(f, obj) {
84 ensureDir(path.dirname(f));
85 // Through a temporary file, so a pull killed mid-write (the timer's unit
86 // stopping, an OOM) cannot leave a truncated state that loses the archive's
87 // whole ingestion history.
88 const tmp = f + '.tmp';
89 fs.writeFileSync(tmp, JSON.stringify(obj));
90 fs.renameSync(tmp, f);
91}
92
93export function readNdjson(f) {
94 let txt = '';
95 try { txt = fs.readFileSync(f, 'utf8'); } catch (e) { return []; }
96 const out = [];
97 for (const line of txt.split('\n')) {
98 if (!line) continue;
99 // A line half-written by a killed pull is skipped rather than fatal: the
100 // archive is append-only, so the loss is one record at the tail.
101 try { out.push(JSON.parse(line)); } catch (e) { /* torn tail */ }
102 }
103 return out;
104}
105
106// ── Integer range sets ───────────────────────────────────────────────
107//
108// Chunk indices and event sequence numbers both arrive almost in order with
109// occasional redelivery, so a list of [lo,hi] ranges collapses to one or two
110// entries and keeps state.json small where a plain set of every number seen
111// would reach megabytes within a day.
112
113export function rangeHas(rs, n) {
114 for (const r of rs) if (n >= r[0] && n <= r[1]) return true;
115 return false;
116}
117
118export function rangeAdd(rs, n) {
119 if (rangeHas(rs, n)) return false;
120 rs.push([n, n]);
121 rs.sort((a, b) => a[0] - b[0]);
122 for (let i = rs.length - 1; i > 0; i--) {
123 if (rs[i][0] <= rs[i - 1][1] + 1) {
124 rs[i - 1][1] = Math.max(rs[i - 1][1], rs[i][1]);
125 rs.splice(i, 1);
126 }
127 }
128 return true;
129}
130
131export function rangeCount(rs) {
132 let n = 0;
133 for (const r of rs) n += r[1] - r[0] + 1;
134 return n;
135}
136
137/// The numbers in 1..n that the ranges do not cover, as a short human string.
138export function rangeMissing(rs, n, cap = 6) {
139 const gaps = [];
140 let next = 1;
141 for (const r of rs) {
142 if (r[0] > next) gaps.push([next, r[0] - 1]);
143 next = Math.max(next, r[1] + 1);
144 }
145 if (next <= n) gaps.push([next, n]);
146 const shown = gaps.slice(0, cap).map(g => g[0] === g[1] ? String(g[0]) : g[0] + '-' + g[1]);
147 if (gaps.length > cap) shown.push('+' + (gaps.length - cap) + ' more');
148 return shown.join(',');
149}
150
151// ── Parsing what the sink wrote ──────────────────────────────────────
152
153const HEADER_RE = /^===== (\d+) account=(\S+) device=(\S+) rows=(\d+) =====$/;
154const CHUNK_RE = /^ds (telemetry|snapshot) (\S+) (\d+)\/(\d+)$/;
155const EV_RE = /^ev (\S+)$/;
156
157/// One line of a sink file as `{bt, device, ts, tag, data}`, or null for anything that is not one: a
158/// torn tail, a blank, a value that is not an object, a line with no tag or no receive time, or a
159/// `data` that is not a string (T5a always writes a string).
160export function parseLine(text) {
161 const raw = String(text).trim();
162 if (!raw) return null;
163 let o;
164 try { o = JSON.parse(raw); } catch (e) { return null; }
165 if (!o || typeof o !== 'object' || Array.isArray(o)) return null;
166 if (!Number.isFinite(o.bt) || typeof o.tag !== 'string') return null;
167 if (o.data !== undefined && typeof o.data !== 'string') return null;
168 return {
169 bt: o.bt,
170 device: typeof o.device === 'string' ? o.device : '',
171 ts: Number.isFinite(o.ts) ? o.ts : o.bt,
172 tag: o.tag,
173 data: o.data === undefined ? '' : o.data,
174 };
175}
176
177/// Every line of a file that parses, each with its trimmed text as `raw`, and how many non-blank lines
178/// did not. A file that is not there reads as empty.
179export function readLines(file) {
180 let txt = '';
181 try { txt = fs.readFileSync(file, 'utf8'); } catch (e) { return { rows: [], bad: 0 }; }
182 const rows = [];
183 let bad = 0;
184 for (const line of txt.split('\n')) {
185 if (!line.trim()) continue;
186 const r = parseLine(line);
187 if (r) { r.raw = line.trim(); rows.push(r); } else bad++;
188 }
189 return { rows, bad };
190}
191
192/// Group consecutive rows of one receive time and device, which is one post, into the blocks the
193/// ingest works on.
194export function lineBlocks(rows) {
195 const out = [];
196 let cur = null;
197 for (const r of rows) {
198 if (!cur || cur.bt !== r.bt || cur.device !== r.device) {
199 cur = { bt: r.bt, device: r.device, rows: [] };
200 out.push(cur);
201 }
202 cur.rows.push({ ts: r.ts, tag: r.tag, data: r.data, raw: r.raw });
203 }
204 return out;
205}
206
207/// Which family a tag belongs to: `ev <kind>`, `ds <telemetry|snapshot> <id> i/N`, or any other row.
208export function tagFamily(tag) {
209 const c = CHUNK_RE.exec(tag);
210 if (c) return { fam: 'ds', kind: c[1], id: c[2], i: Number(c[3]), n: Number(c[4]) };
211 const e = EV_RE.exec(tag);
212 if (e) return { fam: 'ev', kind: e[1] };
213 return { fam: 'row' };
214}
215
216/// Walk one text-block file (Daimond's gateway format), calling `onBlock({bt, account, device,
217/// rows})` in file order. Rows are `{ts, tag, data}`; a row the gateway marked malformed comes
218/// through with tag `(malformed row)` rather than being dropped, since the client having sent
219/// something unexpected is itself a finding. Tag and data are separated by the FIRST double space,
220/// because a tag may hold single spaces and data may hold double ones.
221export function eachBlock(file, onBlock) {
222 let txt = '';
223 try { txt = fs.readFileSync(file, 'utf8'); } catch (e) { return; }
224 let cur = null;
225 for (const line of txt.split('\n')) {
226 const h = HEADER_RE.exec(line);
227 if (h) {
228 if (cur) onBlock(cur);
229 cur = { bt: Number(h[1]), account: h[2], device: h[3], claimed: Number(h[4]), rows: [] };
230 continue;
231 }
232 if (!cur || !line.startsWith(' ')) continue;
233 const rest = line.slice(2);
234 const m = /^(\d+) /.exec(rest);
235 if (!m) { cur.rows.push({ ts: cur.bt, tag: rest.trim(), data: '' }); continue; }
236 const after = rest.slice(m[0].length);
237 const cut = after.indexOf(' ');
238 cur.rows.push({
239 ts: Number(m[1]),
240 tag: cut < 0 ? after : after.slice(0, cut),
241 data: cut < 0 ? '' : after.slice(cut + 2),
242 });
243 }
244 if (cur) onBlock(cur);
245}
246
247const ROT_RE = /^(.+)\.ndjson\.(\d{8}T\d{6}Z)(?:-(\d+))?$/;
248
249/// The lens files under a mirror directory, oldest content first: every rotated file before every
250/// live one, so a stream's receive times only rise, and within a stream by stamp. Directories are
251/// walked (a sink keeps `client/` and `peer/` apart). `fmt` is `lines` for `<key>.ndjson[.<stamp>]`
252/// and `text` for Daimond's `<name>.log` and `<name>.log.1`; `sealed` marks a stamped file, which
253/// is closed. A pre-stamp `<key>.ndjson.1` is kept and read first, as the oldest.
254export function mirrorFiles(dir) {
255 const out = [];
256 const walk = (d, rel, depth) => {
257 let ents = [];
258 try { ents = fs.readdirSync(d, { withFileTypes: true }); } catch (e) { return; }
259 for (const e of ents) {
260 const p = path.join(d, e.name), r = rel ? rel + '/' + e.name : e.name;
261 if (e.isDirectory()) { if (depth < 4) walk(p, r, depth + 1); continue; }
262 let m, f = null;
263 const base = (n) => (rel ? rel + '/' : '') + n;
264 if ((m = ROT_RE.exec(e.name))) f = { fmt: 'lines', rank: 0, stem: base(m[1]), stamp: m[2], n: Number(m[3] || 0), sealed: true };
265 else if ((m = /^(.+)\.ndjson\.1$/.exec(e.name))) f = { fmt: 'lines', rank: 0, stem: base(m[1]), stamp: '', n: 0, sealed: false };
266 else if ((m = /^(.+)\.ndjson$/.exec(e.name))) f = { fmt: 'lines', rank: 1, stem: base(m[1]), stamp: '', n: 0, sealed: false };
267 else if ((m = /^(.+)\.log\.1$/.exec(e.name))) f = { fmt: 'text', rank: 0, stem: base(m[1]), stamp: '', n: 0, sealed: false };
268 else if ((m = /^(.+)\.log$/.exec(e.name))) f = { fmt: 'text', rank: 1, stem: base(m[1]), stamp: '', n: 0, sealed: false };
269 if (!f) continue;
270 let size = 0;
271 try { size = fs.statSync(p).size; } catch (er) { continue; }
272 out.push(Object.assign(f, { file: p, rel: r, size }));
273 }
274 };
275 walk(dir, '', 0);
276 out.sort((a, b) => a.rank - b.rank || (a.stem < b.stem ? -1 : a.stem > b.stem ? 1 : 0)
277 || (a.stamp < b.stamp ? -1 : a.stamp > b.stamp ? 1 : 0) || a.n - b.n || (a.rel < b.rel ? -1 : 1));
278 return out;
279}
280
281// ── Formatting ───────────────────────────────────────────────────────
282
283export function num(n) {
284 n = Number(n) || 0;
285 if (n >= 1e9) return (n / 1e9).toFixed(2) + 'G';
286 if (n >= 1e6) return (n / 1e6).toFixed(2) + 'M';
287 if (n >= 1e3) return Math.round(n / 1e3) + 'k';
288 return String(Math.round(n));
289}
290
291export function hm(ts) {
292 if (!ts) return '--:--:--';
293 const d = new Date(ts);
294 const p = (n) => String(n).padStart(2, '0');
295 return p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
296}
297
298/// A `--since` bound as a label. Time alone for a window inside the day, date
299/// and time beyond it, so `--since 7d` does not report itself as "00:11:56".
300export function sinceLabel(ts) {
301 if (!ts) return 'the start of the archive';
302 if (Date.now() - ts < 20 * 3600000) return hm(ts);
303 return new Date(ts).toISOString().slice(0, 16).replace('T', ' ') + 'Z';
304}
305
306/// A wall-clock label in UTC, for anything a reader might compare against a
307/// gateway log or another machine.
308export function utcLabel(ts) {
309 if (!ts) return '?';
310 return new Date(ts).toISOString().slice(0, 16).replace('T', ' ') + 'Z';
311}
312
313/// A coverage stamp, in UTC: the clock alone inside the day, the date in front
314/// of it beyond that, because "14:41Z" three days ago is a trap.
315export function covLabel(ts) {
316 if (!ts) return '?';
317 const d = new Date(ts);
318 const p = (n) => String(n).padStart(2, '0');
319 const hhmm = p(d.getUTCHours()) + ':' + p(d.getUTCMinutes()) + 'Z';
320 return (Date.now() - ts < 20 * 3600000) ? hhmm : d.toISOString().slice(0, 10) + ' ' + hhmm;
321}
322
323/// How a window should describe itself once coverage is taken into account.
324export function windowLabel(since, cov) {
325 if (cov && since < cov) return utcLabel(cov) + ' (archive start)';
326 return sinceLabel(since);
327}
328
329/// How long before `at` (now, unless given) a time was.
330export function ago(ts, at) {
331 if (!ts) return 'never';
332 const s = Math.max(0, Math.round(((at == null ? Date.now() : at) - ts) / 1000));
333 if (s < 90) return s + 's';
334 if (s < 5400) return Math.round(s / 60) + 'm';
335 if (s < 172800) return Math.round(s / 3600) + 'h';
336 return Math.round(s / 86400) + 'd';
337}
338
339export function dev7(d) { return String(d || '?').slice(0, 7); }
340
341/// Match two device identifiers loosely. A command-line `--device`, this
342/// archive's directory name, and the full id a snapshot's roster or event
343/// carries are three different lengths for the SAME device, so a plain
344/// `startsWith` has to run both directions.
345export function idsMatch(a, b) {
346 a = String(a); b = String(b);
347 return a.startsWith(b.slice(0, 16)) || b.startsWith(a.slice(0, 16));
348}
349
350// ── Output size is the budget ────────────────────────────────────────
351
352/// Print at most `cap` lines, saying how many were withheld rather than going
353/// quiet -- a truncated answer that does not admit it is a wrong answer.
354export function emit(lines, cap) {
355 if (!cap || lines.length <= cap) { console.log(lines.join('\n')); return; }
356 const keep = lines.slice(0, cap - 1);
357 keep.push(`… ${lines.length - cap + 1} more line(s); raise --limit or use --json`);
358 console.log(keep.join('\n'));
359}
360
361// The longest prefix of `s` within `n` bytes of UTF-8, never splitting a character.
362function clipBytes(s, n) {
363 if (n <= 0) return '';
364 if (Buffer.byteLength(s) <= n) return s;
365 let end = Math.min(s.length, n);
366 while (end > 0 && Buffer.byteLength(s.slice(0, end)) > n) end--;
367 const c = s.charCodeAt(end - 1);
368 if (end > 0 && c >= 0xD800 && c <= 0xDBFF) end--;
369 return s.slice(0, end);
370}
371
372/// Join lines into text of at most `max` bytes. The first line stays, and so do the last two;
373/// the lines between go from the back of the middle first, and a notice after the first line
374/// says how many went. A single line too long for the cap is clipped on a character boundary,
375/// so the result is never over `max`, whatever it is given.
376export function capBytes(lines, max) {
377 const cap = Math.max(0, Math.floor(Number(max) || 0));
378 const keep = lines.map(String);
379 const bytes = keep.map((l) => Buffer.byteLength(l));
380 const total = () => bytes.reduce((a, b) => a + b, 0) + Math.max(0, keep.length - 1);
381 if (total() <= cap) return keep.join('\n');
382 let dropped = 0, size = total();
383 const notice = () => '… ' + dropped + ' line(s) withheld by the ' + cap + '-byte cap';
384 while (keep.length > 2 && size + (dropped ? Buffer.byteLength(notice()) + 1 : 0) > cap) {
385 const i = Math.max(1, keep.length - 3);
386 size -= bytes[i] + 1;
387 keep.splice(i, 1); bytes.splice(i, 1);
388 dropped++;
389 }
390 if (dropped) keep.splice(Math.min(1, keep.length), 0, notice());
391 return clipBytes(keep.join('\n'), cap);
392}
393
394// ── Questions over archive records ───────────────────────────────────
395
396const label = (r) => r.src === 'ev' ? r.kind : r.tag;
397
398function bodyOf(r) {
399 if (r.src === 'ev') return r.ev;
400 const d = r.data;
401 if (typeof d !== 'string' || d.charAt(0) !== '{') return null;
402 try { return JSON.parse(d); } catch (e) { return null; }
403}
404
405/// Keep the records that fall in a window, name a device, carry a kind (an `ev` kind or a row's tag)
406/// and hold a pattern (a string or a RegExp, tried over the label and the body). `wantDevice`, when
407/// given, replaces the id-prefix test for the device.
408export function filterRows(rows, o) {
409 o = o || {};
410 const re = o.grep == null ? null : (o.grep instanceof RegExp ? o.grep : new RegExp(String(o.grep), 'i'));
411 const dev = o.device == null || o.device === '' ? null
412 : (o.wantDevice ? (d) => o.wantDevice(d, o.device)
413 : (d) => String(d).startsWith(String(o.device)) || String(o.device).startsWith(String(d)));
414 return rows.filter((r) => {
415 if (o.since && (r.ts || r.bt || 0) < o.since) return false;
416 if (dev && !dev(r.device)) return false;
417 if (o.kind && label(r) !== o.kind) return false;
418 if (re) {
419 const body = r.src === 'ev' ? JSON.stringify(r.ev) : (r.data || '');
420 re.lastIndex = 0;
421 if (!re.test(label(r) + ' ' + body)) return false;
422 }
423 return true;
424 });
425}
426
427/// Archive records as timeline entries, `{bt, ts, device, src, label, run, conn, sid, rec}`, ordered
428/// by the time the sink received them and then by the client's own clock. The ids come from the
429/// event's payload, or from the row's data when that is a JSON object; `fields` renames them.
430export function timeline(records, o) {
431 const f = Object.assign({ run: 'run', conn: 'conn', sid: 'sid' }, o && o.fields);
432 const out = records.map((r, i) => {
433 const body = bodyOf(r);
434 const id = (k) => (body && typeof body === 'object' && body[f[k]] != null) ? body[f[k]] : null;
435 return { bt: r.bt || 0, ts: r.ts || 0, device: r.device, src: r.src, label: label(r),
436 run: id('run'), conn: id('conn'), sid: id('sid'), rec: r, i };
437 });
438 out.sort((a, b) => a.bt - b.bt || a.ts - b.ts || a.i - b.i);
439 return out.map((e) => { delete e.i; return e; });
440}
441
442/// The entries of one page load, one connection or one process, from `timeline`'s output. A page
443/// load (`sid`) takes its own rows and every row of the connections it carried; a connection
444/// (`conn`, with `run` to say which process, since a connection number restarts) takes its rows and
445/// the page loads that rode it; a `run` alone takes that process's rows. A connection is told from
446/// another of the same number by its run, so a restarted peer's conn 7 is not this one.
447export function session(entries, key) {
448 key = key || {};
449 const same = (a, b) => a != null && String(a) === String(b);
450 const cid = (e) => (e.run == null ? '' : String(e.run)) + '|' + e.conn;
451 if (key.run != null && key.sid == null && key.conn == null) return entries.filter((e) => same(e.run, key.run));
452 let seeds = [];
453 if (key.sid != null) seeds = entries.filter((e) => same(e.sid, key.sid));
454 else if (key.conn != null) seeds = entries.filter((e) => same(e.conn, key.conn) && (key.run == null || same(e.run, key.run)));
455 if (!seeds.length) return [];
456 const sids = new Set(seeds.filter((e) => e.sid != null).map((e) => String(e.sid)));
457 const conns = new Set(seeds.filter((e) => e.conn != null).map(cid));
458 return entries.filter((e) => (e.sid != null && sids.has(String(e.sid))) || (e.conn != null && conns.has(cid(e))));
459}
460
461export const DIGEST_MAX = 2048;
462// A label that reads as something going wrong.
463const BAD_RE = /(?:^|[._ -])(?:error|fail|fault|drop|refus|reject|exception|throttl)/i;
464
465/// What a pickup needs in at most 2 KB: when, how many devices, how stale each is, what the window
466/// held, the commonest labels, and the worst errors. `extra` (an array, or a function from the
467/// windowed records to one) adds the app's own lines after the device block. The cap is enforced here
468/// and a caller may lower it with `max` and never raise it.
469export function digest(records, o) {
470 o = o || {};
471 const now = o.now == null ? Date.now() : o.now;
472 const since = o.since || 0;
473 const max = Math.min(DIGEST_MAX, Number.isFinite(o.max) && o.max > 0 ? Math.floor(o.max) : DIGEST_MAX);
474 const all = records || [];
475 let first = 0;
476 for (const r of all) { const t = r.ts || r.bt || 0; if (t && (!first || t < first)) first = t; }
477 const recs = all.filter((r) => (r.ts || r.bt || 0) >= since);
478 const by = new Map(), tags = new Map(), errs = new Map();
479 let bad = 0;
480 for (const r of recs) {
481 const t = r.ts || r.bt || 0, lab = label(r);
482 const d = by.get(r.device) || { n: 0, last: 0, e: 0 };
483 d.n++; d.last = Math.max(d.last, t);
484 tags.set(lab, (tags.get(lab) || 0) + 1);
485 const body = r.src === 'ev' ? r.ev : null;
486 if (BAD_RE.test(lab) || (body && body.lvl === 'error')) {
487 d.e++; bad++;
488 let msg = body ? (body.msg ?? body.m ?? body.err ?? '') : (r.data || '');
489 if (typeof msg !== 'string') msg = JSON.stringify(msg);
490 const key = lab + '|' + msg.replace(/\d+/g, 'N');
491 const e = errs.get(key) || { lab, msg, n: 0, last: 0 };
492 e.n++; e.last = Math.max(e.last, t);
493 errs.set(key, e);
494 }
495 by.set(r.device, d);
496 }
497 const lines = [(o.title || 'lens') + ' — ' + utcLabel(now) + ' ' + by.size + ' device(s) archive since ' + utcLabel(first)];
498 for (const [dv, d] of [...by].sort((a, b) => b[1].last - a[1].last).slice(0, 4)) {
499 lines.push(dev7(dv) + ' ' + ago(d.last, now) + ' ago ' + d.n + ' row(s)' + (d.e ? ' ' + d.e + ' err' : ''));
500 }
501 const extra = typeof o.extra === 'function' ? o.extra(recs) : (o.extra || []);
502 for (const l of extra || []) lines.push(String(l));
503 lines.push('window: ' + recs.length + ' row(s), ' + bad + ' error-like');
504 const top = [...tags].sort((a, b) => b[1] - a[1]).slice(0, 5);
505 if (top.length) lines.push('top: ' + top.map(([k, n]) => 'x' + n + ' ' + k).join(' '));
506 for (const e of [...errs.values()].sort((a, b) => b.n - a.n || b.last - a.last).slice(0, 3)) {
507 lines.push(' x' + e.n + ' ' + e.lab + ': ' + e.msg.slice(0, 88));
508 }
509 return capBytes(lines, max);
510}
511
512// ── The reader: a home directory, a mirror, an archive ───────────────
513
514/// A reader over `cfg.home`. All of it optional but `home`:
515///
516/// mirror the mirror's directory name under home (default `mirror`);
517/// pull how the mirror is fetched: an argv array, an array of them (run in order, no shell), or
518/// a function; a failure is reported in the stat and never stops the ingest;
519/// scrub a scrubber `{text, deep}` (default `createScrubber()`), run on every record filed;
520/// redact a redactor from `createRedactor`, run first by name and by string test;
521/// prepare `bundle -> bundle`, a snapshot's second-line redaction before it is filed;
522/// tick `bundle -> fields`, what a telemetry tick keeps of its bundle (default all of it);
523/// index `bundle -> fields`, facts lifted into a snapshot's index record;
524/// nameFor `device -> name`, so `--device` can name a roster entry as well as an id prefix;
525/// recentMs, pendingTtlMs, now the dedupe window, how long an unfinished set waits, the clock.
526export function createReader(cfg) {
527 cfg = cfg || {};
528 if (!cfg.home) throw new Error('createReader needs a home directory');
529 const dirs = {
530 home: cfg.home,
531 mirror: path.join(cfg.home, cfg.mirror || 'mirror'),
532 archive: path.join(cfg.home, 'archive'),
533 snaps: path.join(cfg.home, 'archive', 'snapshots'),
534 pending: path.join(cfg.home, 'pending'),
535 state: path.join(cfg.home, 'state.json'),
536 };
537 const now = cfg.now || (() => Date.now());
538 const recentMs = cfg.recentMs || 10 * 60 * 1000;
539 const ttlMs = cfg.pendingTtlMs || 6 * 60 * 60 * 1000;
540 const scrub = cfg.scrub || createScrubber();
541 const redact = cfg.redact || null;
542 const prepare = cfg.prepare || ((b) => b);
543 const tick = cfg.tick || ((b) => b);
544 const index = cfg.index || (() => ({}));
545 const nameFor = cfg.nameFor || (() => '');
546
547 const partsFile = (device, kind, id) => path.join(dirs.pending, safeName(device), safeName(kind) + '_' + safeName(id) + '.parts');
548 const eventsFile = (device) => path.join(dirs.archive, safeName(device) + '.events.ndjson');
549 const telFile = (device) => path.join(dirs.archive, safeName(device) + '.telemetry.ndjson');
550 const snapIndexFile = (device) => path.join(dirs.archive, safeName(device) + '.snapshots.ndjson');
551 const snapBodyFile = (device, id) => path.join(dirs.snaps, safeName(device) + '-' + safeName(id) + '.json');
552
553 /// Append one archive record. THE INGEST SEAM: every event, telemetry tick and
554 /// snapshot-index row the archive holds is written here and nowhere else, so the
555 /// content scrubber runs here -- once, on the way to disk.
556 function appendLine(f, obj) {
557 ensureDir(path.dirname(f));
558 fs.appendFileSync(f, JSON.stringify(scrub.deep(obj)) + '\n');
559 }
560
561 // ── State ────────────────────────────────────────────────────────
562
563 function loadState() {
564 const s = readJson(dirs.state, null);
565 if (s && s.v === 1) return s;
566 return { v: 1, devices: {} };
567 }
568
569 function devState(st, device) {
570 if (!st.devices[device]) {
571 st.devices[device] = {
572 maxBt: 0, // newest text-block receive time ingested for this device
573 minBt: 0, // earliest receive time of any kind, which is where coverage starts
574 top: 0, // newest receive time of any kind, which ages the closed sets
575 recent: {}, // text-block times within recentMs of maxBt, exactly
576 ev: {}, // envelope device id -> seen sequence ranges
577 pending: {}, // "<kind>:<id>" -> chunk set being assembled
578 done: {}, // "<kind>:<id>" -> when the set was closed
579 };
580 }
581 return st.devices[device];
582 }
583
584 // One stream is one key's files, live and rotated: the unit whose receive times only rise.
585 function streamState(st, stem) {
586 if (!st.streams) st.streams = {};
587 if (!st.streams[stem]) st.streams[stem] = { maxBt: 0, lines: {} };
588 return st.streams[stem];
589 }
590
591 /// Has this text block already been ingested? Blocks arrive in receive order within a
592 /// stream, so anything well behind the watermark is certainly seen; near it, the exact
593 /// key decides, which is what makes two posts in one millisecond safe.
594 function blockSeen(ds, bt) {
595 if (bt < ds.maxBt - recentMs) return true;
596 return !!ds.recent[String(bt)];
597 }
598
599 function blockMark(ds, bt) {
600 ds.recent[String(bt)] = 1;
601 if (bt > ds.maxBt) ds.maxBt = bt;
602 if (bt > ds.top) ds.top = bt;
603 if (!ds.minBt || bt < ds.minBt) ds.minBt = bt;
604 const floor = ds.maxBt - recentMs;
605 for (const k of Object.keys(ds.recent)) if (Number(k) < floor) delete ds.recent[k];
606 }
607
608 // A line is told by its receive time and a hash of its text. `k` counts the times the same text
609 // has come up in this file, so a row that was genuinely posted twice is kept twice and a file
610 // read again adds nothing.
611 const lineHash = (raw) => crypto.createHash('sha1').update(raw).digest('hex').slice(0, 16);
612
613 function lineSeen(ss, bt, h, k) {
614 if (bt < ss.maxBt - recentMs) return true;
615 const m = ss.lines[String(bt)];
616 return !!m && (m[h] || 0) >= k;
617 }
618
619 function lineMark(ss, bt, h, k) {
620 const m = ss.lines[String(bt)] || (ss.lines[String(bt)] = {});
621 m[h] = Math.max(m[h] || 0, k);
622 if (bt > ss.maxBt) ss.maxBt = bt;
623 const floor = ss.maxBt - recentMs;
624 for (const t of Object.keys(ss.lines)) if (Number(t) < floor) delete ss.lines[t];
625 }
626
627 /// Mark a chunk set closed, and forget closures older than a day so the marker
628 /// map cannot grow for the life of the archive.
629 function closeSet(ds, key, bt) {
630 if (!ds.done) ds.done = {};
631 ds.done[key] = bt || now();
632 const floor = (ds.top || ds.maxBt || now()) - 86400000;
633 for (const k of Object.keys(ds.done)) if (ds.done[k] < floor) delete ds.done[k];
634 }
635
636 // ── Ingest ───────────────────────────────────────────────────────
637
638 /// Concatenate a completed chunk set, decode it, and file it. A set that will not decode is
639 /// recorded as a broken set rather than thrown: one corrupt bundle must not stop a pull.
640 function materialise(device, p, stat) {
641 const pf = partsFile(device, p.kind, p.id);
642 const parts = new Map();
643 let txt = '';
644 try { txt = fs.readFileSync(pf, 'utf8'); } catch (e) { /* nothing on disk */ }
645 for (const line of txt.split('\n')) {
646 if (!line) continue;
647 const tab = line.indexOf('\t');
648 if (tab < 0) continue;
649 const i = Number(line.slice(0, tab));
650 // Last writer wins, which for a duplicated chunk is the same bytes.
651 if (Number.isFinite(i)) parts.set(i, line.slice(tab + 1));
652 }
653 const ordered = [];
654 for (let i = 1; i <= p.n; i++) ordered.push(parts.get(i) || '');
655 let bundle = null, error = null;
656 try { bundle = JSON.parse(Buffer.from(ordered.join(''), 'base64').toString('utf8')); }
657 catch (e) { error = String(e && e.message || e).slice(0, 120); }
658 if (bundle && redact) bundle = redact.value(bundle);
659
660 if (p.kind === 'telemetry') {
661 if (bundle) {
662 appendLine(telFile(device), Object.assign({}, tick(bundle), {
663 id: p.id, device, bt: p.lastBt, ts: bundle.ts || p.lastTs,
664 }));
665 stat.telemetry++;
666 } else {
667 appendLine(telFile(device), { id: p.id, device, bt: p.lastBt, ts: p.lastTs, broken: error });
668 stat.broken++;
669 }
670 } else {
671 const rec = {
672 id: p.id, device, bt: p.lastBt, ts: bundle ? (bundle.ts || p.lastTs) : p.lastTs,
673 n: p.n, have: rangeCount(p.have), complete: rangeCount(p.have) === p.n,
674 missing: rangeCount(p.have) === p.n ? '' : rangeMissing(p.have, p.n),
675 };
676 if (bundle) {
677 // The app's own second line first (a camelCase name the client missed), then the
678 // content scrubber over the whole body, which is the only thing filed here that
679 // does not pass through `appendLine`.
680 bundle = scrub.deep(prepare(bundle));
681 ensureDir(dirs.snaps);
682 fs.writeFileSync(snapBodyFile(device, p.id), JSON.stringify(bundle));
683 rec.body = path.basename(snapBodyFile(device, p.id));
684 rec.iso = bundle.iso || null;
685 // A few facts are lifted into the index so a status never has to open a
686 // megabyte of snapshot to answer a one-line question.
687 Object.assign(rec, index(bundle));
688 stat.snapshots++;
689 } else {
690 rec.broken = error;
691 stat.broken++;
692 }
693 appendLine(snapIndexFile(device), rec);
694 }
695 try { fs.unlinkSync(pf); } catch (e) { /* already gone */ }
696 return bundle ? 'ok' : 'broken';
697 }
698
699 /// Record a chunk set that can never complete, and stop holding its parts open.
700 /// The index keeps the gap so a reader can say what was lost rather than going
701 /// quiet about a bundle it never saw.
702 function expire(device, p, stat) {
703 if (p.kind === 'telemetry') {
704 appendLine(telFile(device), {
705 id: p.id, device, bt: p.lastBt, ts: p.lastTs,
706 gap: true, n: p.n, have: rangeCount(p.have), missing: rangeMissing(p.have, p.n),
707 });
708 } else {
709 appendLine(snapIndexFile(device), {
710 id: p.id, device, bt: p.lastBt, ts: p.lastTs,
711 n: p.n, have: rangeCount(p.have), complete: false,
712 missing: rangeMissing(p.have, p.n), gap: true,
713 });
714 }
715 try { fs.renameSync(partsFile(device, p.kind, p.id), partsFile(device, p.kind, p.id) + '.stale'); }
716 catch (e) { /* nothing to keep */ }
717 stat.gapped++;
718 }
719
720 function ingestRow(ds, device, bt, row, stat) {
721 const t = tagFamily(row.tag);
722 if (t.fam === 'ds') {
723 const key = t.kind + ':' + t.id;
724 // A chunk of a set that is already filed. The client retries a post whose
725 // reply it lost, so this arrives after the set has completed -- and without
726 // this line it would open the set again and leave a phantom half-assembled
727 // bundle behind that nothing ever finishes.
728 if (!ds.done) ds.done = {};
729 if (ds.done[key]) { stat.dupChunks++; return; }
730 let p = ds.pending[key];
731 if (!p) p = ds.pending[key] = { kind: t.kind, id: t.id, n: t.n, have: [], firstTs: row.ts, lastTs: row.ts, lastBt: bt };
732 p.n = Math.max(p.n, t.n);
733 p.lastTs = row.ts;
734 p.lastBt = bt;
735 // A duplicated chunk index is written once. The client can resend a post
736 // the sink accepted but whose reply was lost, so this is ordinary.
737 if (rangeAdd(p.have, t.i)) {
738 const pf = partsFile(device, t.kind, t.id);
739 ensureDir(path.dirname(pf));
740 fs.appendFileSync(pf, t.i + '\t' + row.data + '\n');
741 stat.chunks++;
742 } else {
743 stat.dupChunks++;
744 }
745 if (rangeCount(p.have) >= p.n) {
746 materialise(device, p, stat);
747 delete ds.pending[key];
748 closeSet(ds, key, bt);
749 }
750 return;
751 }
752
753 if (t.fam === 'ev') {
754 let obj = null;
755 try { obj = JSON.parse(row.data); } catch (e) { obj = null; }
756 if (obj && typeof obj === 'object') {
757 // The envelope's own device id is the dedupe axis, not the filename:
758 // `(d,n)` is what the client makes unique, and a device that has been
759 // re-keyed writes to the same file under a new `d`.
760 const d = String(obj.d || device);
761 const n = Number(obj.n);
762 const rs = ds.ev[d] || (ds.ev[d] = []);
763 if (Number.isFinite(n) && !rangeAdd(rs, n)) { stat.dupEvents++; return; }
764 appendLine(eventsFile(device), {
765 src: 'ev', device, bt, ts: obj.t || row.ts, kind: t.kind, ev: redact ? redact.value(obj) : obj,
766 });
767 stat.events++;
768 return;
769 }
770 // An `ev` row whose data will not parse is still a fact about the client.
771 appendLine(eventsFile(device), {
772 src: 'row', device, bt, ts: row.ts, tag: row.tag, data: guardText(row.data), malformed: true,
773 });
774 stat.rows++;
775 return;
776 }
777
778 // Everything else -- election traces, hand-off latency, diagnostics -- is kept verbatim as a
779 // generic row, so a search can reach it without anyone having taught this file what the
780 // tag means.
781 appendLine(eventsFile(device), { src: 'row', device, bt, ts: row.ts, tag: row.tag, data: guardText(row.data) });
782 stat.rows++;
783 }
784
785 // A row's text, replaced whole when the caller's string test calls it secret.
786 function guardText(s) {
787 if (!redact) return s;
788 const r = redact.text(s);
789 return r == null ? s : r;
790 }
791
792 function ingestLines(f, st, stat) {
793 const { rows, bad } = readLines(f.file);
794 stat.bad += bad;
795 const ss = streamState(st, f.stem);
796 const counts = new Map();
797 for (const blk of lineBlocks(rows)) {
798 const ds = devState(st, blk.device);
799 const fresh = [];
800 for (const r of blk.rows) {
801 const h = lineHash(r.raw), ck = blk.bt + '|' + h;
802 const k = (counts.get(ck) || 0) + 1;
803 counts.set(ck, k);
804 if (lineSeen(ss, blk.bt, h, k)) continue;
805 lineMark(ss, blk.bt, h, k);
806 fresh.push(r);
807 }
808 if (!fresh.length) { stat.skipped++; continue; }
809 stat.blocks++;
810 if (!ds.minBt || blk.bt < ds.minBt) ds.minBt = blk.bt;
811 if (blk.bt > ds.top) ds.top = blk.bt;
812 for (const r of fresh) ingestRow(ds, blk.device, blk.bt, r, stat);
813 }
814 }
815
816 // The caller's command, or commands, as one word for the stat.
817 function runPull() {
818 const p = cfg.pull;
819 if (!p || (Array.isArray(p) && !p.length)) return 'skipped';
820 const first = (e) => String(e && e.message || e).split('\n')[0].slice(0, 80);
821 if (typeof p === 'function') {
822 try { p(dirs); return 'ok'; } catch (e) { return 'failed: ' + first(e); }
823 }
824 let failed = null;
825 for (const argv of (Array.isArray(p[0]) ? p : [p])) {
826 // An argv, never a shell string, so a host name or a path is never parsed.
827 try { execFileSync(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'pipe'], timeout: cfg.pullTimeoutMs || 300000 }); }
828 catch (e) { failed = failed || first(e); }
829 }
830 return failed ? 'failed: ' + failed : 'ok';
831 }
832
833 /// Fetch the mirror by the caller's command (unless `noPull`), then file what it holds that the
834 /// archive has not seen. Idempotent. A failed fetch still ingests whatever the mirror has, which
835 /// is the point of keeping a local archive at all.
836 function pull(o) {
837 o = o || {};
838 ensureDir(dirs.mirror); ensureDir(dirs.archive);
839 const stat = {
840 blocks: 0, skipped: 0, rows: 0, events: 0, dupEvents: 0, chunks: 0, dupChunks: 0,
841 telemetry: 0, snapshots: 0, gapped: 0, broken: 0, bad: 0, sealed: 0, pull: 'skipped',
842 };
843 if (!o.noPull) stat.pull = runPull();
844 const st = loadState();
845 if (!st.sealed) st.sealed = {};
846 for (const f of mirrorFiles(dirs.mirror)) {
847 if (f.sealed && st.sealed[f.rel] === f.size) { stat.sealed++; continue; }
848 if (f.fmt === 'text') {
849 eachBlock(f.file, (b) => {
850 const ds = devState(st, b.device);
851 if (blockSeen(ds, b.bt)) { stat.skipped++; return; }
852 blockMark(ds, b.bt);
853 stat.blocks++;
854 for (const row of b.rows) ingestRow(ds, b.device, b.bt, row, stat);
855 });
856 } else {
857 ingestLines(f, st, stat);
858 }
859 if (f.sealed) st.sealed[f.rel] = f.size;
860 }
861 // Chunk sets whose missing rows rotated away at the source are closed off, so the
862 // pending area cannot grow without bound and the gap is on the record.
863 const at = now();
864 for (const device of Object.keys(st.devices)) {
865 const ds = st.devices[device];
866 for (const key of Object.keys(ds.pending)) {
867 const p = ds.pending[key];
868 if (at - (p.lastBt || p.lastTs || 0) > ttlMs) {
869 expire(device, p, stat);
870 delete ds.pending[key];
871 closeSet(ds, key, p.lastBt || at);
872 }
873 }
874 }
875 writeJson(dirs.state, st);
876 return stat;
877 }
878
879 // ── Reading the archive ──────────────────────────────────────────
880
881 function archiveDevices() {
882 let names = [];
883 try { names = fs.readdirSync(dirs.archive); } catch (e) { return []; }
884 const set = new Set();
885 for (const n of names) {
886 const m = /^(.+)\.(events|telemetry|snapshots)\.ndjson$/.exec(n);
887 if (m) set.add(m[1]);
888 }
889 return [...set].sort();
890 }
891
892 function wantDevice(device, filter) {
893 if (!filter) return true;
894 const f = String(filter);
895 if (String(device).startsWith(f) || f.startsWith(String(device))) return true;
896 // `--device` naming the roster instead of the id, matched case-insensitively against
897 // whatever this archive has ever called that device. The id-prefix check above runs
898 // first and is unaffected: this only fires for a filter that was never a prefix.
899 const nm = nameFor(device);
900 return !!nm && nm.toLowerCase().includes(f.toLowerCase());
901 }
902
903 /// When this archive's record begins, PER DEVICE and PER SOURCE. The two are not the same number
904 /// and conflating them lies: a device's oldest row may be an election trace from three days ago
905 /// while its oldest telemetry tick, the only source that can see a turn or a cost, is from this
906 /// afternoon.
907 ///
908 /// `ticks` earliest telemetry tick, and earliest `ev` event;
909 /// `rows` earliest row of any kind, which is what a search can reach and nothing more.
910 function coverage(filter) {
911 const st = loadState();
912 const byDevice = {};
913 let ticks = 0, rows = 0;
914 const lower = (a, b) => (!a || (b && b < a)) ? b : a;
915 for (const d of archiveDevices()) {
916 if (!wantDevice(d, filter)) continue;
917 const rec = { device: d, ticks: 0, rows: 0 };
918 // Each archive file is append-ordered, so its first record is its earliest; only the
919 // events file has to be walked, because it mixes `ev` events with generic rows.
920 const t0 = readNdjson(telFile(d))[0];
921 if (t0) rec.ticks = t0.ts || t0.bt || 0;
922 for (const r of readNdjson(eventsFile(d))) {
923 const ts = r.ts || r.bt || 0;
924 if (!ts) continue;
925 rec.rows = lower(rec.rows, ts);
926 if (r.src === 'ev') rec.ticks = lower(rec.ticks, ts);
927 }
928 const s0 = readNdjson(snapIndexFile(d))[0];
929 if (s0) rec.rows = lower(rec.rows, s0.ts || s0.bt || 0);
930 if (st.devices && st.devices[d] && st.devices[d].minBt) rec.rows = lower(rec.rows, st.devices[d].minBt);
931 rec.rows = lower(rec.rows, rec.ticks);
932 byDevice[d] = rec;
933 ticks = lower(ticks, rec.ticks);
934 rows = lower(rows, rec.rows);
935 }
936 return { ticks: ticks || 0, rows: rows || 0, byDevice };
937 }
938
939 function loadFrom(fileOf, filter, since) {
940 const out = [];
941 for (const d of archiveDevices()) {
942 if (!wantDevice(d, filter)) continue;
943 for (const r of readNdjson(fileOf(d))) {
944 if (since && (r.ts || r.bt || 0) < since) continue;
945 out.push(r);
946 }
947 }
948 out.sort((a, b) => (a.ts || a.bt || 0) - (b.ts || b.bt || 0));
949 return out;
950 }
951 const loadTelemetry = (filter, since) => loadFrom(telFile, filter, since);
952 const loadEvents = (filter, since) => loadFrom(eventsFile, filter, since);
953 const loadSnapshotIndex = (filter) => loadFrom(snapIndexFile, filter, 0);
954
955 /// Chunk sets part-assembled and still waiting for their remaining rows. They are a live fact,
956 /// not a fault: a snapshot is thousands of rows and drains over minutes.
957 function pendingSets(filter) {
958 const st = loadState();
959 const out = [];
960 for (const device of Object.keys(st.devices || {})) {
961 if (!wantDevice(device, filter)) continue;
962 const pend = st.devices[device].pending || {};
963 for (const key of Object.keys(pend)) {
964 const p = pend[key];
965 out.push({ device, kind: p.kind, id: p.id, n: p.n, have: rangeCount(p.have || []), lastBt: p.lastBt });
966 }
967 }
968 out.sort((a, b) => (b.lastBt || 0) - (a.lastBt || 0));
969 return out;
970 }
971
972 /// The digest of what the archive holds for a device and a window: `digest` over its events.
973 function digestOf(o) {
974 o = o || {};
975 const recs = loadEvents(o.device, o.since || 0);
976 return digest(recs, Object.assign({ now: now() }, o));
977 }
978
979 return {
980 dirs, appendLine, loadState, pull, archiveDevices, wantDevice, coverage,
981 loadTelemetry, loadEvents, loadSnapshotIndex, pendingSets, digest: digestOf,
982 eventsFile, telFile, snapIndexFile, snapBodyFile, partsFile,
983 };
984}