oxedyne/fe2o3/fe2o3_net/lens/js/test/adapter/lens_daimond.mjs
62.0 KiB, 1 run
created by r1870400018:65608, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | /* lens_daimond.mjs -- GENERATED by make_lens_adapter.mjs. Daimond's dev/lens.mjs as it would |
| 2 | * stand after adopting fe2o3_net/lens/js/reader.mjs: its own views over the extracted reader core. |
| 3 | * Run Daimond's test against it with |
| 4 | * LENS_UNDER_TEST=<this file> node <daimond>/dev/lens.test.mjs |
| 5 | */ |
| 6 | // lens.mjs — the reader half of Daimond's full-visibility feed. |
| 7 | // |
| 8 | // USAGE |
| 9 | // |
| 10 | // node dev/lens.mjs pull [--no-rsync] fetch + archive (idempotent) |
| 11 | // node dev/lens.mjs status [--device D] [--since today] one block per device |
| 12 | // node dev/lens.mjs turns [--since 24h] [--device D] one line per turn |
| 13 | // node dev/lens.mjs errors [--since 1h] [--grep RE] deduplicated, worst first |
| 14 | // node dev/lens.mjs console [--since 1h] [--lvl warn|all] what the tab printed |
| 15 | // node dev/lens.mjs events [--kind K] [--since 1h] [--grep RE] |
| 16 | // node dev/lens.mjs snapshot [--latest|--id ID] [--device D] [--out FILE] |
| 17 | // node dev/lens.mjs watch [--since 5m] [--once] [--no-pull] |
| 18 | // node dev/lens.mjs digest <= 2 KB, for /pickup |
| 19 | // |
| 20 | // Common: --json (machine form), --limit N, --since 30s|5m|2h|7d|today|<ISO>, |
| 21 | // --device (an id prefix, OR a roster name substring: "argonaut", |
| 22 | // "gilgamesh", "iOS" -- whatever this archive has ever called it). |
| 23 | // Every device is printed as its id BESIDE that name, because the id |
| 24 | // alone is how two sessions in one day mis-assigned a7b34e2 |
| 25 | // (gilgamesh) as argonaut (96a1474 is the real argonaut). |
| 26 | // |
| 27 | // `console` is the browser console as the device saw it -- every level, one row |
| 28 | // per distinct line with `x` saying how many times it repeated inside its minute. |
| 29 | // It defaults to warn and error, because that is what a reader is looking for; |
| 30 | // `--lvl all` adds log, info and debug. The same warn/error lines also join |
| 31 | // `errors`, so a question asked there does not have to be asked twice. |
| 32 | // |
| 33 | // `errors` HIDES ABORTED REQUESTS by default. A reload kills every call in |
| 34 | // flight and each arrives as a status 0 that means nothing was wrong; the client |
| 35 | // marks those `aborted`, and a status 0 within five seconds of the same device's |
| 36 | // next `boot` is read the same way even when it did not -- an unannounced reload |
| 37 | // (the updater's own, a crash-and-restart, iOS re-instantiating a backgrounded |
| 38 | // tab) kills the same calls with nothing set. `--aborted` shows them. |
| 39 | // |
| 40 | // A `fetch.fail` GROUPS BY PATH AND STATUS, not by its flattened message. Every |
| 41 | // other kind dedupes on a message with the digits turned into `N`, which read |
| 42 | // `/api/sync 409`, `413` and `502` as one line with a count -- three faults with |
| 43 | // three remedies, reported as one. |
| 44 | // |
| 45 | // WHERE THE DATA LIVES |
| 46 | // |
| 47 | // jarrah:~/usr/daimond-gateway/debug-traces/ the gateway's own files, rotated |
| 48 | // at 2 MB with only two kept |
| 49 | // ~/.cache/daimond-lens/traces/ the rsync mirror |
| 50 | // ~/.cache/daimond-lens/archive/ the durable archive (NDJSON) |
| 51 | // ~/.cache/daimond-lens/state.json what has been ingested |
| 52 | // |
| 53 | // Rotation on jarrah destroys a device's third-oldest file, so `pull` runs on a |
| 54 | // five-minute timer (dev/lens-pull.timer) and appends only what it has not seen. |
| 55 | // The archive is the authority for every question; nothing re-reads jarrah. |
| 56 | // |
| 57 | // THE ON-THE-WIRE FORMAT (gateway/src/handlers/debug_trace.rs, render_block) |
| 58 | // |
| 59 | // ===== <recv ms> account=<a> device=<d> rows=<n> ===== |
| 60 | // <client ms> <tag> <data> |
| 61 | // |
| 62 | // Tag and data are separated by the FIRST double space, because a tag may hold |
| 63 | // single spaces (`ds telemetry <id> 3/12`, `election peer`) and data may hold |
| 64 | // double ones. Three row families matter: |
| 65 | // |
| 66 | // ds telemetry <id> i/N base64 JSON chunk of a per-tick telemetry bundle |
| 67 | // ds snapshot <id> i/N base64 JSON chunk of a periodic full snapshot |
| 68 | // ev <kind> one JSON event, envelope {v,d,n,b,t,...} |
| 69 | // |
| 70 | // `ev console` is one of those kinds and needs no special ingestion: it is filed |
| 71 | // like every other event, on the same `(d,n)` dedupe. Its payload is |
| 72 | // `{lvl, msg, src, x}` -- level, the message clipped to 200 characters, the |
| 73 | // `file:line` it came from where the client could sample a stack, and the repeat |
| 74 | // count inside its dedupe window. |
| 75 | // |
| 76 | // `ev` rows do not exist in the logs yet — the client half is a parallel lane — |
| 77 | // so every question prefers them and falls back to what telemetry already |
| 78 | // carries: the raw ledger delta (one entry per TURN, `{t,m,p,c,ca,u,r,e}`), the |
| 79 | // per-model RUNNING TOTALS (`stats.models`, differenced tick to tick where the |
| 80 | // ledger delta was lost), the live context/worker numbers (`stats.live`), the |
| 81 | // breadcrumb trail (`{t,a,w,d}`) and the diagnostics ring (`{ts,a,tag,data}`). |
| 82 | // |
| 83 | // COVERAGE IS STATED, NOT ASSUMED, AND IT IS NOT ONE NUMBER. jarrah keeps two |
| 84 | // 2 MB files per device, so whatever rotated away before the first pull is gone |
| 85 | // for good — and each device's record starts at a different moment, from a |
| 86 | // different source. An election trace from three days ago is coverage for |
| 87 | // `events` and none at all for a turn or a cost, so the header quotes the |
| 88 | // earliest TELEMETRY tick (or `ev` event) and names the row coverage separately |
| 89 | // only when it differs. Every device line carries its own first tick, and a |
| 90 | // device whose record starts after the window says so in place of a count. A |
| 91 | // zero over a window the archive cannot see is not a quiet day. |
| 92 | // |
| 93 | // OUTPUT SIZE IS THE BUDGET. `status` and `digest` are read by a language model |
| 94 | // at every pickup, so they are capped (40 lines / 2 KB) and the cap is enforced |
| 95 | // here rather than trusted to the caller. A change that grows them is a |
| 96 | // regression, not a feature. |
| 97 | |
| 98 | import { execFileSync } from 'node:child_process'; |
| 99 | import fs from 'node:fs'; |
| 100 | import os from 'node:os'; |
| 101 | import path from 'node:path'; |
| 102 | |
| 103 | // ── Where everything lives ─────────────────────────────────────────── |
| 104 | |
| 105 | const HOME = process.env.DAIMOND_LENS_HOME |
| 106 | || path.join(os.homedir(), '.cache', 'daimond-lens'); |
| 107 | // Set (even to empty) by the verifier, which has no jarrah and no network; an |
| 108 | // `||` default would read an empty string as unset and try to reach the host. |
| 109 | const REMOTE = process.env.DAIMOND_LENS_REMOTE !== undefined |
| 110 | ? process.env.DAIMOND_LENS_REMOTE |
| 111 | : 'jarrah:~/usr/daimond-gateway/debug-traces/'; |
| 112 | const TRACES = path.join(HOME, 'traces'); |
| 113 | const ARCHIVE = path.join(HOME, 'archive'); |
| 114 | const SNAPS = path.join(ARCHIVE, 'snapshots'); |
| 115 | const PENDING = path.join(HOME, 'pending'); |
| 116 | const STATEF = path.join(HOME, 'state.json'); |
| 117 | |
| 118 | // A block whose receive time is more than this behind the newest one seen for |
| 119 | // that device cannot be new: the gateway appends in receive order and we read |
| 120 | // `.log.1` before `.log`. Inside the window we consult an exact key set, so two |
| 121 | // blocks sharing a millisecond are still told apart. |
| 122 | const RECENT_MS = 10 * 60 * 1000; |
| 123 | // A chunk set that has not gained a part in this long will never complete -- |
| 124 | // the missing rows were rotated away on jarrah before a pull reached them. |
| 125 | const PENDING_TTL_MS = 6 * 60 * 60 * 1000; |
| 126 | const STATUS_MAX_LINES = 40; |
| 127 | const DIGEST_MAX_BYTES = 2048; |
| 128 | // How close a status 0 has to sit in front of a device's next boot to be read as |
| 129 | // a request the reload took rather than a gateway that failed. Generous enough to |
| 130 | // cover a tab that takes its time coming up, short enough that an ordinary |
| 131 | // network failure minutes before an unrelated reload is still a fault. |
| 132 | const RELOAD_GRACE_MS = 5000; |
| 133 | |
| 134 | // ── The core's reader, handed Daimond's seams ──────────────────────── |
| 135 | import { createRedactor, createScrubber } from '../../redact.js'; |
| 136 | import { |
| 137 | ago, covLabel, createReader, dev7, emit, ensureDir, idsMatch, num, parseArgs, rangeAdd, rangeCount, |
| 138 | rangeHas, rangeMissing, readJson, readNdjson, safeName, sinceLabel, sinceMs, utcLabel, windowLabel, writeJson, hm, |
| 139 | } from '../../reader.mjs'; |
| 140 | |
| 141 | const R = createReader({ |
| 142 | home: HOME, |
| 143 | mirror: 'traces', |
| 144 | pull: REMOTE ? [['rsync', '-az', REMOTE, TRACES + '/']] : null, |
| 145 | // The gateway's own session cookie, and the fields whose values are ids this reader navigates by. |
| 146 | scrub: createScrubber({ |
| 147 | pairs: [{ k: 'cookie', g: 2, re: /(daimond_gw_sess=)([^;,\s"'\[\]]{4,})/g }], |
| 148 | idKeys: { holder: 1, self: 1, nominated: 1, body: 1 }, |
| 149 | }), |
| 150 | prepare: (b) => { if (b.config) b.config = createRedactor({ loose: true }).value(b.config); return b; }, |
| 151 | tick: (b) => ({ iso: b.iso || null, stats: b.stats || null, ledger: b.ledger || [], trail: b.trail || [], diag: b.diag || [] }), |
| 152 | index: (b) => summariseSnapshot(b), |
| 153 | nameFor: (d) => nameFor(d), |
| 154 | recentMs: RECENT_MS, pendingTtlMs: PENDING_TTL_MS, |
| 155 | }); |
| 156 | const { appendLine, archiveDevices, coverage, eventsFile, loadEvents, loadSnapshotIndex, loadState, loadTelemetry, |
| 157 | pendingSets, snapBodyFile, snapIndexFile, telFile, wantDevice } = R; |
| 158 | |
| 159 | |
| 160 | // ── Arguments ──────────────────────────────────────────────────────── |
| 161 | |
| 162 | |
| 163 | const ARGS = parseArgs(process.argv.slice(2)); |
| 164 | const JSONOUT = !!ARGS.bool.json; |
| 165 | |
| 166 | |
| 167 | // ── Small file helpers ─────────────────────────────────────────────── |
| 168 | |
| 169 | |
| 170 | |
| 171 | |
| 172 | |
| 173 | |
| 174 | // ── Integer range sets ─────────────────────────────────────────────── |
| 175 | // |
| 176 | // Chunk indices and event sequence numbers both arrive almost in order with |
| 177 | // occasional redelivery, so a list of [lo,hi] ranges collapses to one or two |
| 178 | // entries and keeps state.json small where a plain set of every number seen |
| 179 | // would reach megabytes within a day. |
| 180 | |
| 181 | |
| 182 | |
| 183 | |
| 184 | |
| 185 | // ── Parsing a trace file ───────────────────────────────────────────── |
| 186 | |
| 187 | |
| 188 | |
| 189 | |
| 190 | // ── State ──────────────────────────────────────────────────────────── |
| 191 | |
| 192 | |
| 193 | |
| 194 | |
| 195 | |
| 196 | |
| 197 | // ── Ingest ─────────────────────────────────────────────────────────── |
| 198 | |
| 199 | |
| 200 | |
| 201 | /// The shortest form of a roster entry's `name` a reader still recognises: the |
| 202 | /// word after " on " (`"Chrome on argonaut"` -> `argonaut`, `"Safari on iOS"` -> |
| 203 | /// `iOS`), which is the machine or platform a reader actually asks about -- |
| 204 | /// two sessions today read the bare id instead and mis-assigned a7b34e2 to |
| 205 | /// argonaut when it was gilgamesh. Falls back to the whole name where there is |
| 206 | /// no " on " to split on. |
| 207 | function shortRosterName(name) { |
| 208 | const s = String(name || '').trim(); |
| 209 | const m = /\bon\s+(\S+)\s*$/i.exec(s); |
| 210 | return (m ? m[1] : s).replace(/[.,;]+$/, ''); |
| 211 | } |
| 212 | |
| 213 | /// The handful of facts `status` and `digest` read out of a snapshot, so the |
| 214 | /// body file stays closed for every ordinary question. |
| 215 | function summariseSnapshot(bundle) { |
| 216 | const out = { builds: {}, names: {}, chats: 0, roster: 0 }; |
| 217 | const reg = bundle && bundle.roster; |
| 218 | if (reg && typeof reg === 'object') { |
| 219 | for (const id of Object.keys(reg)) { |
| 220 | const r = reg[id] || {}; |
| 221 | out.roster++; |
| 222 | if (r.build) out.builds[id] = String(r.build).slice(0, 32); |
| 223 | if (r.name) out.names[id] = shortRosterName(r.name); |
| 224 | } |
| 225 | } |
| 226 | if (Array.isArray(bundle && bundle.tokenStats)) out.chats = bundle.tokenStats.length; |
| 227 | if (bundle && bundle.election && bundle.election.self) out.self = String(bundle.election.self); |
| 228 | return out; |
| 229 | } |
| 230 | |
| 231 | |
| 232 | |
| 233 | |
| 234 | |
| 235 | function cmdPull(args) { |
| 236 | const stat = R.pull({ noPull: !!args.bool['no-rsync'] }); |
| 237 | stat.rsync = stat.pull; |
| 238 | if (JSONOUT) { console.log(JSON.stringify(stat)); return 0; } |
| 239 | console.log(`pull: rsync ${stat.rsync}; ${stat.blocks} new block(s), ${stat.skipped} seen`); |
| 240 | console.log(` rows ${stat.rows} events ${stat.events} (+${stat.dupEvents} dup)` |
| 241 | + ` chunks ${stat.chunks} (+${stat.dupChunks} dup)`); |
| 242 | console.log(` telemetry sets ${stat.telemetry} snapshots ${stat.snapshots}` |
| 243 | + ` gapped ${stat.gapped} broken ${stat.broken}`); |
| 244 | return 0; |
| 245 | } |
| 246 | |
| 247 | // ── Reading the archive ────────────────────────────────────────────── |
| 248 | |
| 249 | |
| 250 | |
| 251 | |
| 252 | |
| 253 | |
| 254 | |
| 255 | |
| 256 | |
| 257 | |
| 258 | // ── The derived model ──────────────────────────────────────────────── |
| 259 | // |
| 260 | // Every question is answered from `ev` rows where they exist and from telemetry |
| 261 | // where they do not. The two paths produce the same record shapes, so the |
| 262 | // commands below never branch on which half of the feed is shipped. |
| 263 | |
| 264 | /// One record per TURN, from whichever of three sources can see it. Each record |
| 265 | /// carries a `turns` COUNT, because the third source can only say how many turns |
| 266 | /// an interval held, not what each one did. |
| 267 | /// |
| 268 | /// `ev` an `ev turn.end` event — everything known, one turn. |
| 269 | /// `ledger` a cost-ledger entry — one turn, with its own tokens and USD |
| 270 | /// (`p` and `ca` are that turn's cumulative prompt and cached |
| 271 | /// tokens, `u` its USD, `r` set when the provider priced it). |
| 272 | /// `cumulative` the difference between two ticks of `stats.models[]`, which is |
| 273 | /// a running total per model and survives the loss of the ledger |
| 274 | /// deltas that rotation eats. It knows the count, the spend and |
| 275 | /// the tokens for an interval, and nothing about outcome. |
| 276 | /// |
| 277 | /// The third is used only where the first two are silent for that device, that |
| 278 | /// model and that interval, so a turn is never counted twice. |
| 279 | /// |
| 280 | /// `turn.end`'s `out` and `ended`'s `how` are two different vocabularies for the |
| 281 | /// same question (daimond.js, two call sites, two histories) -- `cap` against |
| 282 | /// `round_limit`. Collapsed to one here so a reader never has to know both. |
| 283 | function normalizeOutcome(w) { |
| 284 | return w === 'cap' ? 'round_limit' : w; |
| 285 | } |
| 286 | |
| 287 | function turnRecords(filter, since) { |
| 288 | const out = []; |
| 289 | // Loaded ONCE and reused below for the `turn.end`, `ended` and `round`/`fold` |
| 290 | // passes -- previously three separate reads of the same NDJSON. Still |
| 291 | // unbounded (`0`, not `since`): a marathon daimon turn can straddle the |
| 292 | // window, and its early `round`s must still reach `maxPromptByTurn` even |
| 293 | // when the turn itself closes inside `since`. Only the RETURNED rows are |
| 294 | // filtered to the window, at the very end, as before. |
| 295 | const events = loadEvents(filter, 0); |
| 296 | // `turn.end`'s own fields, short for the 360-byte cap (daimond.js `dsEvent('turn.end', …)`): |
| 297 | // `p`/`c`/`ca` prompt/completion/cached, `usd`, `r` rounds, `out` outcome. The long names |
| 298 | // (`prompt`/`rounds`/`outcome`/…) are read too, so a fixture or a future caller using either |
| 299 | // shape still works -- but the short ones are what the app actually sends. Collected rather |
| 300 | // than pushed straight to `out`, so the ledger join below can match against them first. |
| 301 | const endedTurns = new Set(); // turn ids already covered by a `turn.end` row |
| 302 | const turnEndRows = []; |
| 303 | for (const e of events) { |
| 304 | if (e.src !== 'ev' || e.kind !== 'turn.end') continue; |
| 305 | const p = e.ev || {}; |
| 306 | if (p.turn) endedTurns.add(String(p.turn)); |
| 307 | turnEndRows.push({ |
| 308 | ts: e.ts, device: e.device, model: p.model || p.m || '?', turns: 1, |
| 309 | turn: p.turn || null, |
| 310 | rounds: p.rounds ?? p.r ?? null, prompt: p.prompt ?? p.p ?? 0, |
| 311 | completion: p.completion ?? p.c ?? 0, |
| 312 | cached: p.cached ?? p.ca ?? 0, usd: p.usd ?? p.cost ?? 0, |
| 313 | // `prov`, short like `turn.start`'s own field -- absent on a build before |
| 314 | // this fix, so a `?` model there is what says the id join still has to |
| 315 | // borrow the ledger's. |
| 316 | provider: p.provider || p.prov || '', |
| 317 | outcome: normalizeOutcome(p.outcome || p.out || 'done'), reported: !!p.reported, src: 'ev', |
| 318 | }); |
| 319 | } |
| 320 | // A daimon turn (`round`/`fold`/`tool` with no chat-shaped close) still ends with |
| 321 | // its OWN `turn.end` -- the 2026-09-13 archive confirms it for a7b34e2 (gilgamesh) -- |
| 322 | // so this only ever fires for a turn id that genuinely named no `turn.end` at all. |
| 323 | // Tokens/spend are left unknown (`null`) rather than guessed, since neither the |
| 324 | // ledger nor the cumulative totals below are keyed by turn id. |
| 325 | for (const e of events) { |
| 326 | if (e.src !== 'ev' || e.kind !== 'ended') continue; |
| 327 | const p = e.ev || {}; |
| 328 | if (p.turn && endedTurns.has(String(p.turn))) continue; |
| 329 | out.push({ |
| 330 | ts: e.ts, device: e.device, model: '?', turns: 1, turn: p.turn || null, |
| 331 | rounds: p.rounds ?? null, prompt: null, completion: null, cached: null, usd: null, |
| 332 | outcome: p.how || 'done', reported: false, src: 'ended', |
| 333 | }); |
| 334 | if (p.turn) endedTurns.add(String(p.turn)); |
| 335 | } |
| 336 | // Per-round tells, joined onto turns by `turn` id: the HIGHEST per-round |
| 337 | // `last_prompt_tokens` the turn reached (the figure that actually tracks |
| 338 | // prompt-carry across a marathon turn's rounds) and how many times it |
| 339 | // folded. Both are throttled at the source -- a `round` only every 5th |
| 340 | // round, plus the one it ended on -- so `maxPrompt` can miss a spike |
| 341 | // between samples; it is still the best count an unthrottled re-read |
| 342 | // would give for the same budget of events. |
| 343 | const maxPromptByTurn = new Map(), foldsByTurn = new Map(); |
| 344 | // The LAST `round_meta`/`stalled` seen for a turn: the provider's own trace facts for |
| 345 | // whichever round they were most recently true of. Proposal 15, 2026-09-15 -- a round |
| 346 | // that answered nothing had no id, no `finish_reason` and no upstream provider name |
| 347 | // anywhere this tool could read, so a stall or a cut could not be traced back to the |
| 348 | // provider afterwards. Last rather than first: the round that mattered is the one the |
| 349 | // turn actually ended on. |
| 350 | const lastRoundByTurn = new Map(); |
| 351 | for (const e of events) { |
| 352 | if (e.src !== 'ev') continue; |
| 353 | const p = e.ev || {}; |
| 354 | if (!p.turn) continue; |
| 355 | const key = String(p.turn); |
| 356 | if (e.kind === 'round') { |
| 357 | const ctx = Number(p.ctx) || 0; |
| 358 | if (ctx > (maxPromptByTurn.get(key) || 0)) maxPromptByTurn.set(key, ctx); |
| 359 | } else if (e.kind === 'fold') { |
| 360 | foldsByTurn.set(key, (foldsByTurn.get(key) || 0) + 1); |
| 361 | } else if (e.kind === 'round_meta' || e.kind === 'stalled') { |
| 362 | lastRoundByTurn.set(key, { |
| 363 | r: p.r ?? null, gen: p.gen || '', finish: p.finish || '', |
| 364 | nfinish: p.nfinish || '', provider: p.provider || '', |
| 365 | stalled: e.kind === 'stalled', |
| 366 | }); |
| 367 | } |
| 368 | } |
| 369 | // The ledger delta is resent whole after a reload (the client's high-water |
| 370 | // mark is per page life), so the same turn arrives on many ticks. It also |
| 371 | // SYNCS TO EVERY DEVICE -- the same account-wide ledger, held in full by each |
| 372 | // device's own telemetry snapshot -- so keying the dedupe by `t.device` as |
| 373 | // well as the entry's own fingerprint made one turn into one row PER DEVICE |
| 374 | // it happened to sync to: "24h $28.39 / 23 turns" read as ~6 turns, ~$7. The |
| 375 | // entry's own (t, m, p) already names the turn; the device that happened to |
| 376 | // report it first is not part of what makes it unique. |
| 377 | const led = new Set(); |
| 378 | const ledgerRows = []; |
| 379 | for (const t of loadTelemetry(filter, 0)) { |
| 380 | for (const e of (t.ledger || [])) { |
| 381 | if (!e || typeof e !== 'object') continue; |
| 382 | const key = e.t + '|' + (e.m || '') + '|' + (e.p || 0); |
| 383 | if (led.has(key)) continue; |
| 384 | led.add(key); |
| 385 | ledgerRows.push({ |
| 386 | ts: e.t || t.ts, device: t.device, model: e.m || '?', turns: 1, |
| 387 | rounds: null, prompt: e.p || 0, completion: e.c || 0, cached: e.ca || 0, |
| 388 | usd: e.u || 0, outcome: e.e ? 'error' : 'done', reported: !!e.r, |
| 389 | provider: e.pv || '', src: 'ledger', |
| 390 | // The turn id, when this entry was written by a build that stamps one |
| 391 | // (`www/js/ledger.js` `record()`'s `tid`). Absent on every entry written |
| 392 | // before that build, and on the few paths (a whole-chat fold) that spend |
| 393 | // against no single turn at all. |
| 394 | tid: e.tid ? String(e.tid) : null, |
| 395 | }); |
| 396 | } |
| 397 | } |
| 398 | // A daimon turn posts BOTH the ledger tick and its own `turn.end` -- the same |
| 399 | // turn under two channels, which is why a `turns`/`status` total once counted |
| 400 | // its spend twice (`glm-5.3 r- rep $0.87` alongside `? r24 est $0.87`, summing |
| 401 | // $1.74). The two used to be joined by device + a five-second clock skew + |
| 402 | // equal prompt tokens, which is a GUESS: the ledger SYNCS TO EVERY DEVICE (see |
| 403 | // above), so `lr.device` can be whichever device's telemetry copy happened to |
| 404 | // carry the entry, not the device that ran the turn -- and the owner saw the |
| 405 | // same turn twice, once correctly and once as `? r33 … est` on a device that |
| 406 | // never ran it. Fixed by joining on the turn's OWN identity first: `turn.end` |
| 407 | // has always carried `p.turn`, and the ledger entry now carries the same id as |
| 408 | // `tid` (`www/js/ledger.js` `record()`, threaded through `recordSpend` from |
| 409 | // the chat and daimon-steer call sites). An id match is authoritative -- |
| 410 | // the merged row takes `turn.end`'s device (the one that actually ran it), |
| 411 | // turn id, rounds, model and provider outright, keeping only the ledger's |
| 412 | // price and `reported` flag, which `turn.end` does not carry. |
| 413 | const teByTid = new Map(); |
| 414 | for (const te of turnEndRows) if (te.turn) teByTid.set(String(te.turn), te); |
| 415 | const usedTurnEnd = new Set(), idMatchedLedger = new Set(); |
| 416 | for (const lr of ledgerRows) { |
| 417 | if (!lr.tid) continue; |
| 418 | const te = teByTid.get(lr.tid); |
| 419 | if (!te || usedTurnEnd.has(te)) continue; |
| 420 | usedTurnEnd.add(te); |
| 421 | idMatchedLedger.add(lr); |
| 422 | out.push(Object.assign({}, lr, { |
| 423 | device: te.device, |
| 424 | turn: te.turn, |
| 425 | rounds: te.rounds, |
| 426 | model: te.model && te.model !== '?' ? te.model : lr.model, |
| 427 | provider: te.provider || lr.provider, |
| 428 | outcome: te.outcome || lr.outcome, |
| 429 | })); |
| 430 | } |
| 431 | // The old heuristic survives ONLY for a ledger entry with no `tid` -- one |
| 432 | // written before this build. It is a guess, so a row it produces is marked |
| 433 | // `est` even where the ledger itself said a provider billed it (`r`): an |
| 434 | // unproven join must not borrow the confidence a proven one earned. A |
| 435 | // ledger entry that DOES carry a `tid` but matched no `turn.end` (the event |
| 436 | // was lost to rotation, say) is never handed to this guess -- it stands on |
| 437 | // its own rather than risk pairing with an unrelated turn. |
| 438 | for (const lr of ledgerRows) { |
| 439 | if (idMatchedLedger.has(lr)) continue; // already pushed, above |
| 440 | if (lr.tid) { out.push(lr); continue; } // has an id; not a guessing candidate |
| 441 | let match = null; |
| 442 | for (const te of turnEndRows) { |
| 443 | if (usedTurnEnd.has(te)) continue; |
| 444 | if (te.device !== lr.device) continue; |
| 445 | if (Math.abs((te.ts || 0) - (lr.ts || 0)) >= 5000) continue; |
| 446 | if ((te.prompt || 0) !== (lr.prompt || 0)) continue; |
| 447 | match = te; |
| 448 | break; |
| 449 | } |
| 450 | if (match) { |
| 451 | usedTurnEnd.add(match); |
| 452 | out.push(Object.assign({}, lr, { turn: match.turn, rounds: match.rounds, reported: false })); |
| 453 | } else { |
| 454 | out.push(lr); |
| 455 | } |
| 456 | } |
| 457 | for (const te of turnEndRows) if (!usedTurnEnd.has(te)) out.push(te); |
| 458 | for (const r of out) { |
| 459 | if (!r.turn) continue; |
| 460 | const key = String(r.turn); |
| 461 | r.maxPrompt = maxPromptByTurn.has(key) ? maxPromptByTurn.get(key) : null; |
| 462 | r.folds = foldsByTurn.get(key) || 0; |
| 463 | // The provider's own trace facts for this turn's last traced round, or `null` |
| 464 | // where none arrived (an older build, or a dialect off OpenRouter). See |
| 465 | // `lastRoundByTurn` above. |
| 466 | r.lastRound = lastRoundByTurn.has(key) ? lastRoundByTurn.get(key) : null; |
| 467 | } |
| 468 | for (const r of cumulativeRecords(filter, out)) out.push(r); |
| 469 | out.sort((a, b) => a.ts - b.ts); |
| 470 | return since ? out.filter(r => r.ts >= since) : out; |
| 471 | } |
| 472 | |
| 473 | /// Turns recovered by differencing the per-model running totals every telemetry |
| 474 | /// tick carries. `known` is every turn the other two sources already found; an |
| 475 | /// interval one of them covers for the same device and model yields nothing |
| 476 | /// here, so the two never add up to the same turn twice. |
| 477 | function cumulativeRecords(filter, known) { |
| 478 | const byDevice = new Map(); |
| 479 | for (const t of loadTelemetry(filter, 0)) { |
| 480 | if (!t.stats || !Array.isArray(t.stats.models)) continue; |
| 481 | const list = byDevice.get(t.device) || []; |
| 482 | list.push(t); |
| 483 | byDevice.set(t.device, list); |
| 484 | } |
| 485 | const out = []; |
| 486 | for (const [device, ticks] of byDevice) { |
| 487 | ticks.sort((a, b) => (a.ts || a.bt || 0) - (b.ts || b.bt || 0)); |
| 488 | let prev = null; |
| 489 | for (const tick of ticks) { |
| 490 | const now = new Map((tick.stats.models || []).map(m => [m.model, m])); |
| 491 | if (prev) { |
| 492 | const from = prev.ts || prev.bt || 0, to = tick.ts || tick.bt || 0; |
| 493 | for (const [model, m] of now) { |
| 494 | const was = prev.models.get(model); |
| 495 | // A model absent from the previous tick is new, so its running |
| 496 | // total starts at zero; a total that went DOWN is a ledger that |
| 497 | // was cleared, and a reset is not a turn. |
| 498 | const dTurns = (m.turns || 0) - (was ? (was.turns || 0) : 0); |
| 499 | if (dTurns <= 0) continue; |
| 500 | if (known.some(r => r.device === device && r.model === model |
| 501 | && r.ts > from && r.ts <= to)) continue; |
| 502 | out.push({ |
| 503 | ts: to, device, model, turns: dTurns, rounds: null, |
| 504 | prompt: Math.max(0, (m.prompt || 0) - (was ? (was.prompt || 0) : 0)), |
| 505 | completion: Math.max(0, (m.completion || 0) - (was ? (was.completion || 0) : 0)), |
| 506 | cached: Math.max(0, (m.cached || 0) - (was ? (was.cached || 0) : 0)), |
| 507 | usd: Math.max(0, (m.usd || 0) - (was ? (was.usd || 0) : 0)), |
| 508 | // A running total says how many and how much, never how they |
| 509 | // ended. Claiming `done` here would invent a fact. |
| 510 | outcome: '?', reported: false, from, src: 'cumulative', |
| 511 | }); |
| 512 | } |
| 513 | } |
| 514 | prev = { ts: tick.ts || tick.bt || 0, bt: tick.bt, models: now }; |
| 515 | } |
| 516 | } |
| 517 | return out; |
| 518 | } |
| 519 | |
| 520 | /// The newest live picture per device: build, last beat, model, context, worker. |
| 521 | function deviceStates(filter) { |
| 522 | const by = {}; |
| 523 | const dev = (d) => by[d] || (by[d] = { |
| 524 | device: d, build: '', name: '', lastBeat: 0, live: null, outbox: null, |
| 525 | models: [], activity: '', account: '', evSeq: null, health: null, |
| 526 | screen: null, screenAt: 0, |
| 527 | }); |
| 528 | for (const d of archiveDevices()) if (wantDevice(d, filter)) dev(d); |
| 529 | |
| 530 | for (const t of loadTelemetry(filter, 0)) { |
| 531 | const s = dev(t.device); |
| 532 | const ts = t.ts || t.bt || 0; |
| 533 | if (ts >= s.lastBeat) { |
| 534 | s.lastBeat = ts; |
| 535 | if (t.stats && t.stats.live) s.live = t.stats.live; |
| 536 | if (t.stats && t.stats.models) s.models = t.stats.models; |
| 537 | } |
| 538 | // The build is announced once per boot on the breadcrumb trail, not on |
| 539 | // every tick, so it is taken from the newest trail row that carries it. |
| 540 | for (const r of (t.trail || [])) { |
| 541 | if (!r || r.w !== 'build') continue; |
| 542 | const id = String(r.d || '').split(' ')[0]; |
| 543 | if (id && (r.t || 0) >= (s.buildAt || 0)) { s.build = id; s.buildAt = r.t || 0; } |
| 544 | } |
| 545 | } |
| 546 | // A snapshot's device roster states the build of every device the account |
| 547 | // knows, which is the only build the feed carries until `ev` rows ship. |
| 548 | for (const r of loadSnapshotIndex(filter)) { |
| 549 | if (!r.builds) continue; |
| 550 | for (const id of Object.keys(r.builds)) { |
| 551 | for (const s of Object.values(by)) { |
| 552 | if (!idsMatch(id, s.device)) continue; |
| 553 | if ((r.ts || 0) >= (s.buildAt || 0)) { s.build = r.builds[id]; s.buildAt = r.ts || 0; } |
| 554 | } |
| 555 | } |
| 556 | } |
| 557 | // The short NAME ("argonaut", "gilgamesh", "iOS") a reader recognises, from |
| 558 | // the same roster -- via `nameFor`, which is where the fallback lives for an |
| 559 | // archive whose index predates this lane. |
| 560 | for (const s of Object.values(by)) s.name = nameFor(s.device); |
| 561 | for (const e of loadEvents(filter, 0)) { |
| 562 | if (e.src !== 'ev') continue; |
| 563 | const s = dev(e.device); |
| 564 | const p = e.ev || {}; |
| 565 | if (p.b && (e.ts || 0) >= (s.buildAt || 0)) { s.build = String(p.b); s.buildAt = e.ts || 0; } |
| 566 | if (Number.isFinite(p.n)) s.evSeq = Math.max(s.evSeq ?? 0, p.n); |
| 567 | if (e.kind === 'beat' || e.kind === 'boot') { |
| 568 | // FEED HEALTH, which only the beat carries: posts the gateway refused |
| 569 | // for rate, posts that failed for anything else, and console lines the |
| 570 | // client's own cap dropped. A steady state shows zeroes; anything else |
| 571 | // is the feed in trouble, and the feed in trouble is why a question |
| 572 | // below it would have no answer. Taken from the newest BEAT on its own |
| 573 | // clock -- a telemetry tick arriving after it says nothing about the |
| 574 | // feed, and must not hide what the beat said. |
| 575 | if ((p.throttled !== undefined || p.postFail !== undefined || p.cdrop !== undefined) |
| 576 | && (e.ts || 0) >= (s.healthAt || 0)) { |
| 577 | s.health = { |
| 578 | throttled: Number(p.throttled) || 0, |
| 579 | postFail: Number(p.postFail) || 0, |
| 580 | cdrop: Number(p.cdrop) || 0, |
| 581 | at: e.ts || 0, |
| 582 | }; |
| 583 | s.healthAt = e.ts || 0; |
| 584 | } |
| 585 | if ((e.ts || 0) >= s.lastBeat) { |
| 586 | s.lastBeat = e.ts || 0; |
| 587 | if (p.outbox !== undefined) s.outbox = p.outbox; |
| 588 | if (p.ob !== undefined) s.outbox = p.ob; |
| 589 | if (p.context !== undefined || p.window !== undefined) { |
| 590 | s.live = Object.assign({}, s.live, { |
| 591 | contextActual: p.context ?? (s.live && s.live.contextActual), |
| 592 | contextWindow: p.window ?? (s.live && s.live.contextWindow), |
| 593 | foldAt: p.fold_at ?? (s.live && s.live.foldAt), |
| 594 | activeModel: p.model ?? (s.live && s.live.activeModel), |
| 595 | }); |
| 596 | } |
| 597 | } |
| 598 | } |
| 599 | // THE SCREEN (added 2026-09-13): what the person is actually looking at, |
| 600 | // shipped only on change or a five-minute floor (see debugshare.js |
| 601 | // `screenTick`), so the newest one by timestamp is simply the current |
| 602 | // picture -- there is no aggregation to do, unlike the beat's health. |
| 603 | if (e.kind === 'screen' && (e.ts || 0) >= s.screenAt) { |
| 604 | s.screen = p; |
| 605 | s.screenAt = e.ts || 0; |
| 606 | } |
| 607 | } |
| 608 | return Object.values(by).sort((a, b) => b.lastBeat - a.lastBeat); |
| 609 | } |
| 610 | |
| 611 | /// Errors, deduplicated by a normalised message so twenty copies of one fault |
| 612 | /// are one line with a count. Numbers are flattened to `N` for that match, since |
| 613 | /// a line differing only in an id or a byte count is the same fault. |
| 614 | function errorRecords(filter, since, opts) { |
| 615 | const withAborted = !!(opts && opts.aborted); |
| 616 | const byMsg = new Map(); |
| 617 | const add = (ts, device, kind, msg, weight, groupKey) => { |
| 618 | const text = String(msg || '').replace(/\s+/g, ' ').trim().slice(0, 200); |
| 619 | if (!text) return; |
| 620 | const key = groupKey || (kind + '|' + text.replace(/\d+/g, 'N')); |
| 621 | let r = byMsg.get(key); |
| 622 | if (!r) byMsg.set(key, r = { kind, msg: text, count: 0, first: ts, last: ts, devices: new Set() }); |
| 623 | r.count += (weight > 0 ? weight : 1); |
| 624 | r.first = Math.min(r.first, ts); |
| 625 | r.last = Math.max(r.last, ts); |
| 626 | r.devices.add(String(device).slice(0, 7)); |
| 627 | }; |
| 628 | |
| 629 | const evs = loadEvents(filter, since); |
| 630 | // When each device last came up, for the reload inference below. A boot is an |
| 631 | // `ev` row like any other, so it is read off the same pass the errors are. |
| 632 | const boots = new Map(); |
| 633 | for (const e of evs) { |
| 634 | if (e.src !== 'ev' || e.kind !== 'boot') continue; |
| 635 | const k = String(e.device); |
| 636 | if (!boots.has(k)) boots.set(k, []); |
| 637 | boots.get(k).push(e.ts || 0); |
| 638 | } |
| 639 | |
| 640 | /// Did this device RELOAD within the grace window after `ts`? |
| 641 | /// |
| 642 | /// A reload kills every call in flight and each arrives as a status 0 that |
| 643 | /// means nothing was wrong. The client tags one `aborted` when the page has |
| 644 | /// already said `pagehide` or gone hidden -- but a reload the page never saw |
| 645 | /// coming (the updater's own `location.reload`, a crash-and-restart, iOS |
| 646 | /// re-instantiating a backgrounded tab) kills the same calls with nothing set, |
| 647 | /// and those arrived in `errors` as faults. The boot that follows is the proof |
| 648 | /// the client could not give: a status 0 seconds before a device came up is a |
| 649 | /// request the reload took, not a gateway that failed. |
| 650 | const reloadFollowed = (device, ts) => { |
| 651 | const a = boots.get(String(device)); |
| 652 | if (!a) return false; |
| 653 | return a.some((b) => b >= ts && (b - ts) <= RELOAD_GRACE_MS); |
| 654 | }; |
| 655 | |
| 656 | for (const e of evs) { |
| 657 | if (e.src !== 'ev') continue; |
| 658 | const p = e.ev || {}; |
| 659 | // A console warning or error is an error the app printed and nobody read; |
| 660 | // the other three levels are the transcript and live under `console`. The |
| 661 | // repeat count rides through, so twenty of one line count as twenty. |
| 662 | if (e.kind === 'console') { |
| 663 | if (p.lvl !== 'warn' && p.lvl !== 'error') continue; |
| 664 | add(e.ts, e.device, 'console.' + p.lvl, (p.src ? p.src + ' ' : '') + (p.msg || ''), |
| 665 | Number(p.x) || 1); |
| 666 | continue; |
| 667 | } |
| 668 | // `truncated` and `stalled` are here since proposal 15 (2026-09-15): a provider |
| 669 | // length cut or a dead stream is worth exactly the same "go and look" treatment |
| 670 | // as a fetch failure, and until this fix neither reached the feed at all. |
| 671 | if (!/^(error|fetch\.fail|feed\.fault|feed\.drop|feed\.throttled|truncated|stalled)$/ |
| 672 | .test(e.kind)) continue; |
| 673 | if (e.kind === 'truncated' || e.kind === 'stalled') { |
| 674 | add(e.ts, e.device, e.kind, |
| 675 | 'turn ' + (p.turn || '?') + ' round ' + (p.r ?? '?') |
| 676 | + (p.provider ? ' via ' + p.provider : '') |
| 677 | + (p.finish ? ' finish=' + p.finish : ''), |
| 678 | 1, e.kind + '|' + (p.turn || '?') + '|' + (p.r ?? '?')); |
| 679 | continue; |
| 680 | } |
| 681 | if (e.kind === 'fetch.fail') { |
| 682 | // A request the page abandoned on its way out is not a fault. It is kept |
| 683 | // in the archive and shown on request, never counted by default -- whether |
| 684 | // the client tagged it or the boot that followed gave it away. |
| 685 | if (!withAborted && (p.aborted || (!(p.status | 0) && reloadFollowed(e.device, e.ts || 0)))) continue; |
| 686 | // THE PATH AND THE STATUS ARE THE FAULT, so neither is flattened. The |
| 687 | // generic key above turns every digit into `N`, which made `/api/sync 409`, |
| 688 | // `413` and `502` one line with a count -- three different faults with three |
| 689 | // different remedies, reported as one. A thrown fetch has no status and its |
| 690 | // message is what distinguishes it, so that part is still normalised. |
| 691 | add(e.ts, e.device, e.kind, |
| 692 | p.msg || p.message || p.err || ((p.path || '?') + ' ' + (p.status || '')), |
| 693 | 1, e.kind + '|' + (p.path || '?') + '|' + (p.status | 0) + '|' |
| 694 | + String(p.msg || p.message || p.err || '').replace(/\d+/g, 'N')); |
| 695 | continue; |
| 696 | } |
| 697 | add(e.ts, e.device, e.kind, |
| 698 | p.msg || p.message || p.err || (p.path ? p.path + ' ' + (p.status || '') : JSON.stringify(p))); |
| 699 | } |
| 700 | // Without `ev` rows the trail and the diagnostics ring are what there is: the |
| 701 | // trail records `page error` and `unhandled rejection` verbatim, and a ledger |
| 702 | // entry flagged `e` is a turn that ended badly. |
| 703 | const seen = new Set(); |
| 704 | for (const t of loadTelemetry(filter, 0)) { |
| 705 | for (const r of (t.trail || [])) { |
| 706 | if (!r || !r.w) continue; |
| 707 | if (!/error|rejection|fail|fault|refus|denied/i.test(r.w)) continue; |
| 708 | const key = t.device + '|' + r.t + '|' + r.w; |
| 709 | if (seen.has(key)) continue; |
| 710 | seen.add(key); |
| 711 | if (since && (r.t || 0) < since) continue; |
| 712 | add(r.t || t.ts, t.device, 'trail', r.w + (r.d ? ': ' + r.d : '')); |
| 713 | } |
| 714 | for (const r of (t.diag || [])) { |
| 715 | if (!r || !r.tag) continue; |
| 716 | const text = r.tag + ' ' + (r.data || ''); |
| 717 | if (!/error|fail|fault|refus|reject|\b(4\d\d|5\d\d)\b/i.test(text)) continue; |
| 718 | const key = t.device + '|' + r.ts + '|' + r.tag; |
| 719 | if (seen.has(key)) continue; |
| 720 | seen.add(key); |
| 721 | if (since && (r.ts || 0) < since) continue; |
| 722 | add(r.ts || t.ts, t.device, 'diag', text); |
| 723 | } |
| 724 | } |
| 725 | for (const t of turnRecords(filter, since)) { |
| 726 | if (t.outcome === 'error') add(t.ts, t.device, 'turn', 'turn ended in error on ' + t.model); |
| 727 | // A turn that reasoned itself to nothing, or answered nothing at all, is exactly |
| 728 | // what proposal 15 needed surfaced here: `endedHow` (www/js/daimond.js) names |
| 729 | // these words and none of them is `done`, so a reader chasing "what went wrong |
| 730 | // today" sees them without having to know the turn's `how` field exists. |
| 731 | if (t.outcome === 'reasoned_only' || t.outcome === 'silent') |
| 732 | add(t.ts, t.device, 'turn', 'turn ended ' + t.outcome + ' on ' + t.model); |
| 733 | } |
| 734 | const out = [...byMsg.values()].map(r => ({ ...r, devices: [...r.devices] })); |
| 735 | out.sort((a, b) => b.last - a.last); |
| 736 | return out; |
| 737 | } |
| 738 | |
| 739 | /// The console as the device printed it, newest last, one record per event -- |
| 740 | /// which is already one record per distinct line per minute, with `x` carrying |
| 741 | /// the repeats. `lvls` is the set of levels wanted; the default is the two a |
| 742 | /// reader is looking for. |
| 743 | function consoleRecords(filter, since, lvls) { |
| 744 | const out = []; |
| 745 | for (const e of loadEvents(filter, since)) { |
| 746 | if (e.src !== 'ev' || e.kind !== 'console') continue; |
| 747 | const p = e.ev || {}; |
| 748 | const lvl = String(p.lvl || '?'); |
| 749 | if (lvls && !lvls.has(lvl)) continue; |
| 750 | out.push({ ts: e.ts, device: e.device, lvl, msg: String(p.msg || ''), |
| 751 | src: String(p.src || ''), x: Number(p.x) || 1, build: p.b || '' }); |
| 752 | } |
| 753 | return out; |
| 754 | } |
| 755 | |
| 756 | /// How much the console said in a window, by level. `lines` counts repeats, |
| 757 | /// which is the honest volume; `distinct` counts the events, which is what a |
| 758 | /// reader will actually have to look at. |
| 759 | function consoleCounts(filter, since) { |
| 760 | const c = { warn: 0, error: 0, other: 0, distinct: 0, lines: 0 }; |
| 761 | for (const r of consoleRecords(filter, since, null)) { |
| 762 | c.distinct += 1; |
| 763 | c.lines += r.x; |
| 764 | if (r.lvl === 'warn' || r.lvl === 'error') c[r.lvl] += r.x; |
| 765 | else c.other += r.x; |
| 766 | } |
| 767 | return c; |
| 768 | } |
| 769 | |
| 770 | /// The levels `--lvl` asks for. `all` is every level; anything else is the two |
| 771 | /// defaults widened by the ones named, comma-separated. |
| 772 | function levelSet(v) { |
| 773 | if (v === undefined || v === null || v === '') return new Set(['warn', 'error']); |
| 774 | const s = String(v).toLowerCase(); |
| 775 | if (s === 'all' || s === '*') return null; |
| 776 | return new Set(s.split(',').map(x => x.trim()).filter(Boolean)); |
| 777 | } |
| 778 | |
| 779 | // ── Formatting ─────────────────────────────────────────────────────── |
| 780 | |
| 781 | |
| 782 | function usd(v) { |
| 783 | v = Number(v) || 0; |
| 784 | if (v === 0) return '$0'; |
| 785 | return '$' + (v < 0.01 ? v.toFixed(4) : v.toFixed(2)); |
| 786 | } |
| 787 | |
| 788 | |
| 789 | |
| 790 | |
| 791 | |
| 792 | /// A model id short enough to read in a column, keeping the part that varies. |
| 793 | function shortModel(m) { |
| 794 | const s = String(m || '?'); |
| 795 | const cut = s.lastIndexOf('/'); |
| 796 | return (cut < 0 ? s : s.slice(cut + 1)).slice(0, 22); |
| 797 | } |
| 798 | |
| 799 | |
| 800 | |
| 801 | let NAME_INDEX = null; |
| 802 | |
| 803 | /// Every device id this archive has ever had a roster name for, newest wins. |
| 804 | /// A device's own roster lists the WHOLE account it belongs to, so any ONE |
| 805 | /// snapshot -- not necessarily the named device's own -- names every device. |
| 806 | /// Memoised for the process's life; a fresh `node lens.mjs` invocation per |
| 807 | /// command is what makes that safe. |
| 808 | function nameIndex() { |
| 809 | if (NAME_INDEX) return NAME_INDEX; |
| 810 | const byId = {}; |
| 811 | const rows = loadSnapshotIndex(null); |
| 812 | for (const r of rows) { |
| 813 | if (!r.names) continue; |
| 814 | for (const id of Object.keys(r.names)) { |
| 815 | const cur = byId[id]; |
| 816 | if (!cur || (r.ts || 0) >= cur.ts) byId[id] = { name: r.names[id], ts: r.ts || 0 }; |
| 817 | } |
| 818 | } |
| 819 | // Fall back to a snapshot BODY when every index row predates this lane |
| 820 | // (2026-09-13, `names` added to `summariseSnapshot`): the lightweight index |
| 821 | // carries nothing to read, but the body it already points at is still on |
| 822 | // disk, and any ONE account-wide roster names every device -- so the newest |
| 823 | // body with a non-empty roster is enough, not a re-pull and not a scan of |
| 824 | // them all. |
| 825 | if (!Object.keys(byId).length) { |
| 826 | const withBody = rows.filter(r => r.body).sort((a, b) => (b.ts || 0) - (a.ts || 0)); |
| 827 | for (const r of withBody) { |
| 828 | const bundle = readJson(path.join(SNAPS, r.body), null); |
| 829 | const reg = bundle && bundle.roster; |
| 830 | if (!reg || typeof reg !== 'object' || !Object.keys(reg).length) continue; |
| 831 | for (const id of Object.keys(reg)) { |
| 832 | const nm = reg[id] && reg[id].name; |
| 833 | if (nm) byId[id] = { name: shortRosterName(nm), ts: r.ts || 0 }; |
| 834 | } |
| 835 | break; |
| 836 | } |
| 837 | } |
| 838 | NAME_INDEX = byId; |
| 839 | return byId; |
| 840 | } |
| 841 | |
| 842 | /// The short roster name for a device id (`argonaut`, `gilgamesh`, `iOS`), or |
| 843 | /// `''` when this archive has never seen one -- an id typed by hand, or a |
| 844 | /// device too new to have shipped a snapshot yet. |
| 845 | function nameFor(device) { |
| 846 | const idx = nameIndex(); |
| 847 | if (idx[device]) return idx[device].name; |
| 848 | for (const id of Object.keys(idx)) { |
| 849 | if (idsMatch(id, device)) return idx[id].name; |
| 850 | } |
| 851 | return ''; |
| 852 | } |
| 853 | |
| 854 | /// The 7-hex id beside its roster name, when one is known. This is what two |
| 855 | /// sessions today needed and did not have: `dev7` alone is what mis-read |
| 856 | /// a7b34e2 (gilgamesh) as argonaut (96a1474, the actual argonaut). |
| 857 | function devLabel(id) { |
| 858 | const nm = nameFor(id); |
| 859 | return nm ? `${dev7(id)} ${nm}` : dev7(id); |
| 860 | } |
| 861 | |
| 862 | |
| 863 | // ── Questions ──────────────────────────────────────────────────────── |
| 864 | |
| 865 | /// One line of what a device's `screen` event says the person is looking at, or |
| 866 | /// `null` when none has arrived yet (an older build, or one that has not shipped |
| 867 | /// its first beat). `tile` rides the wire as one string, `"role: text"` -- split |
| 868 | /// back into the two words a reader wants, `tile <role> '<text>'`, rather than |
| 869 | /// printed as the raw joined form. |
| 870 | function screenLine(s) { |
| 871 | if (!s) return null; |
| 872 | const parts = [String(s.view || '?')]; |
| 873 | if (s.seat) parts.push(`seat '${s.seat}'`); |
| 874 | if (s.tile) { |
| 875 | const m = /^([^:]{1,20}): ([\s\S]*)$/.exec(s.tile); |
| 876 | parts.push(m ? `tile ${m[1]} '${m[2]}'` : `tile '${s.tile}'`); |
| 877 | } |
| 878 | parts.push(`dlg ${s.dlg || 'none'}`); |
| 879 | if (s.locked) parts.push('locked'); |
| 880 | if (s.upd && s.upd !== 'none') parts.push(`upd ${s.upd}`); |
| 881 | return parts.join(' · '); |
| 882 | } |
| 883 | |
| 884 | function cmdStatus(args) { |
| 885 | const filter = args.flags.device; |
| 886 | const hour = Date.now() - 3600000; |
| 887 | const midnight = (() => { const d = new Date(); d.setHours(0, 0, 0, 0); return d.getTime(); })(); |
| 888 | const since = sinceMs(args.flags.since, midnight); |
| 889 | const cov = coverage(filter); |
| 890 | const states = deviceStates(filter); |
| 891 | const turns = turnRecords(filter, since); |
| 892 | const errs = errorRecords(filter, hour); |
| 893 | const con = consoleCounts(filter, hour); |
| 894 | const count = (rs) => rs.reduce((a, t) => a + (t.turns || 1), 0); |
| 895 | |
| 896 | if (JSONOUT) { |
| 897 | console.log(JSON.stringify({ |
| 898 | now: Date.now(), since, coverage: cov, devices: states, |
| 899 | windowTurns: count(turns), turnRecords: turns.length, errors1h: errs, |
| 900 | console1h: con, |
| 901 | })); |
| 902 | return 0; |
| 903 | } |
| 904 | if (!states.length) { console.log('status: no devices in the archive; run `lens.mjs pull`.'); return 0; } |
| 905 | |
| 906 | // Coverage first, and in UTC. Everything below it is a count over a window |
| 907 | // this archive can actually see; a zero here means "none recorded since", |
| 908 | // never "none happened". |
| 909 | // The header quotes the TICK coverage, never the oldest row: an election |
| 910 | // trace from three days ago says nothing about whether a turn was seen. |
| 911 | const lines = [`archive since ${utcLabel(cov.ticks)} (telemetry/ev)` |
| 912 | + (cov.rows && cov.rows < cov.ticks ? `, rows from ${utcLabel(cov.rows)}` : '') |
| 913 | + ` — ${states.length} device(s)`]; |
| 914 | for (const s of states) { |
| 915 | const l = s.live || {}; |
| 916 | const win = Number(l.contextWindow) || 0; |
| 917 | const act = Number(l.contextActual) || 0; |
| 918 | const pct = win ? Math.round(100 * act / win) : 0; |
| 919 | const mine = turns.filter(t => t.device === s.device); |
| 920 | const spend = mine.reduce((a, t) => a + (t.usd || 0), 0); |
| 921 | const myErr = errs.filter(e => e.devices.includes(dev7(s.device))) |
| 922 | .reduce((a, e) => a + e.count, 0); |
| 923 | const c = (cov.byDevice[s.device] || {}).ticks || 0; |
| 924 | lines.push(`${s.name ? dev7(s.device) + ' ' + s.name : dev7(s.device)} build ${s.build || '?'}` |
| 925 | + ` beat ${hm(s.lastBeat)} (${ago(s.lastBeat)} ago) since ${covLabel(c)}`); |
| 926 | const sees = screenLine(s.screen); |
| 927 | if (sees) lines.push(` sees ${sees}`); |
| 928 | lines.push(` live ${shortModel(l.activeModel)} ctx ${num(act)}/${num(win)} (${pct}%)` |
| 929 | + ` fold@${l.foldAt ? Math.round(l.foldAt * 100) + '%' : '?'}` |
| 930 | + ` ${l.activity || '?'}` |
| 931 | + (l.workerState ? ` workers ${l.workerState.active}/${l.workerState.queued}` : '') |
| 932 | + (s.outbox === null ? '' : ` outbox ${s.outbox}`)); |
| 933 | // A zero that the archive could not have seen is not a zero. Where the |
| 934 | // window starts before this device's first tick, the count is replaced by |
| 935 | // the point its record actually begins. |
| 936 | const blind = !count(mine) && c && since < c; |
| 937 | lines.push(` window ${blind ? `(coverage from ${covLabel(c)})` : count(mine) + ' turn(s)'}` |
| 938 | + ` ${usd(spend)} errors(1h) ${myErr}`); |
| 939 | } |
| 940 | const tot = turns.reduce((a, t) => a + (t.usd || 0), 0); |
| 941 | lines.push(`all devices: ${count(turns)} turn(s) since ${windowLabel(since, cov.ticks)}, ${usd(tot)}; ` |
| 942 | + `${errs.reduce((a, e) => a + e.count, 0)} error(s) in the last hour`); |
| 943 | // What the console said, and whether the feed itself is healthy. A feed |
| 944 | // dropping posts or console lines is the reason the numbers above would be |
| 945 | // short, so it is stated beside them rather than left to be asked for. |
| 946 | lines.push(`console (1h): ${con.warn} warn, ${con.error} err` |
| 947 | + (con.other ? `, ${con.other} other` : '') + ` (${con.distinct} distinct)`); |
| 948 | const health = states.map(s => s.health).filter(Boolean); |
| 949 | if (health.length) { |
| 950 | const sum = (k) => health.reduce((a, h) => a + (h[k] || 0), 0); |
| 951 | lines.push(`feed: throttled ${sum('throttled')}, postFail ${sum('postFail')},` |
| 952 | + ` console dropped ${sum('cdrop')} (last beat)`); |
| 953 | } |
| 954 | emit(lines, STATUS_MAX_LINES); |
| 955 | return 0; |
| 956 | } |
| 957 | |
| 958 | function cmdTurns(args) { |
| 959 | const since = sinceMs(args.flags.since, Date.now() - 2 * 3600000); |
| 960 | const cov = coverage(args.flags.device).ticks; |
| 961 | const rows = turnRecords(args.flags.device, since); |
| 962 | const limit = Number(args.flags.limit) || 30; |
| 963 | if (JSONOUT) { console.log(JSON.stringify(rows)); return 0; } |
| 964 | if (!rows.length) { console.log(`turns: none since ${windowLabel(since, cov)}.`); return 0; } |
| 965 | const lines = []; |
| 966 | for (const t of rows.slice(-limit)) { |
| 967 | const cp = t.prompt ? Math.round(100 * t.cached / t.prompt) : 0; |
| 968 | const src = t.src === 'cumulative' ? 'cum' : t.src === 'ended' ? 'end' : (t.reported ? 'rep' : 'est'); |
| 969 | // `maxPrompt`/`folds` come from `round`/`fold` events joined by turn id, so they |
| 970 | // are blank wherever those events do not exist for a turn (an older build, or a |
| 971 | // source -- ledger/cumulative -- that carries no turn id to join on at all). |
| 972 | const maxP = t.maxPrompt == null ? '-' : num(t.maxPrompt); |
| 973 | const folds = t.folds || 0; |
| 974 | const inTok = t.prompt == null ? '-' : num(t.prompt).padStart(6); |
| 975 | const outTok = t.completion == null ? '-' : num(t.completion).padStart(5); |
| 976 | const spend = t.usd == null ? '-'.padStart(7) : usd(t.usd).padStart(7); |
| 977 | lines.push(`${hm(t.ts)} ${devLabel(t.device)} ${shortModel(t.model).padEnd(22)}` |
| 978 | + ` r${t.rounds ?? '-'} max ${maxP} f${folds} in ${inTok} out ${outTok}` |
| 979 | + ` cache ${String(cp).padStart(3)}% ${spend}` |
| 980 | + ` ${src} ${t.outcome}` + ((t.turns || 1) > 1 ? ` x${t.turns}` : '') |
| 981 | // The provider's own trace facts for the last traced round, where any |
| 982 | // arrived. Blank for an older build or a dialect off OpenRouter. |
| 983 | + (t.lastRound ? ` ${t.lastRound.stalled ? 'STALLED' : (t.lastRound.finish || '-')}` |
| 984 | + (t.lastRound.provider ? ` via ${t.lastRound.provider}` : '') : '')); |
| 985 | } |
| 986 | const spend = rows.reduce((a, t) => a + (t.usd || 0), 0); |
| 987 | const count = rows.reduce((a, t) => a + (t.turns || 1), 0); |
| 988 | lines.push(`${count} turn(s) in ${rows.length} record(s), ${usd(spend)} since ${windowLabel(since, cov)}` |
| 989 | + (rows.length > limit ? ` (showing last ${limit})` : '')); |
| 990 | emit(lines, limit + 2); |
| 991 | return 0; |
| 992 | } |
| 993 | |
| 994 | function cmdErrors(args) { |
| 995 | const since = sinceMs(args.flags.since, Date.now() - 3600000); |
| 996 | // The tick coverage, not the row coverage: an error is read off the trail and |
| 997 | // the ledger, both of which arrive on a telemetry tick. |
| 998 | const cov = coverage(args.flags.device).ticks; |
| 999 | let rows = errorRecords(args.flags.device, since, { aborted: !!args.bool.aborted }); |
| 1000 | if (args.flags.grep) { |
| 1001 | const re = new RegExp(args.flags.grep, 'i'); |
| 1002 | rows = rows.filter(r => re.test(r.msg) || re.test(r.kind)); |
| 1003 | } |
| 1004 | if (JSONOUT) { console.log(JSON.stringify(rows)); return 0; } |
| 1005 | if (!rows.length) { console.log(`errors: none since ${windowLabel(since, cov)}.`); return 0; } |
| 1006 | const limit = Number(args.flags.limit) || 15; |
| 1007 | const lines = rows.slice(0, limit).map(r => |
| 1008 | `${hm(r.last)} x${String(r.count).padStart(3)} ${r.kind.padEnd(13)}` |
| 1009 | + ` ${r.devices.map(devLabel).join(',')} ${r.msg.slice(0, 110)}`); |
| 1010 | lines.push(`${rows.length} distinct message(s) since ${windowLabel(since, cov)}`); |
| 1011 | emit(lines, limit + 1); |
| 1012 | return 0; |
| 1013 | } |
| 1014 | |
| 1015 | /// A readable body for one `ev <kind>` row -- generic `JSON.stringify` for most |
| 1016 | /// kinds, but `compile` and `diamond` get a one-line rendering so a scroll of |
| 1017 | /// `events` reads without reaching for `--json`. |
| 1018 | function fmtEventBody(r) { |
| 1019 | if (r.src !== 'ev') return r.data || ''; |
| 1020 | const p = r.ev || {}; |
| 1021 | if (r.kind === 'compile') { |
| 1022 | // `producer` replaced `engine` outright (the old name carried the live-view |
| 1023 | // flag, not the door that actually compiled -- see `www/js/daimond.js` |
| 1024 | // `pdfHasPages` and `www/js/typstwatch.js` `liveCompileTelemetry`). The |
| 1025 | // fallback to `p.engine` reads ONLY rows archived before that change; a row |
| 1026 | // written since always has `producer` and never falls back. |
| 1027 | const producer = p.producer || p.engine || '-'; |
| 1028 | const via = p.via ? ` via=${p.via}` : ''; |
| 1029 | return `main=${p.main || '-'} producer=${producer}${via} ok=${p.ok} ms=${p.ms} bytes=${p.bytes} ${p.err || ''}`.trimEnd(); |
| 1030 | } |
| 1031 | if (r.kind === 'diamond') { |
| 1032 | const items = Array.isArray(p.items) ? p.items : []; |
| 1033 | return `${p.id || '-'} items=${items.length}: ${items.slice(0, 4).join(', ')}`; |
| 1034 | } |
| 1035 | return JSON.stringify(p); |
| 1036 | } |
| 1037 | |
| 1038 | function cmdEvents(args) { |
| 1039 | const since = sinceMs(args.flags.since, Date.now() - 3600000); |
| 1040 | // The ROW coverage here, because that is what this command reads. |
| 1041 | const cov = coverage(args.flags.device).rows; |
| 1042 | let rows = loadEvents(args.flags.device, since); |
| 1043 | if (args.flags.kind) { |
| 1044 | // A comma-separated list, not just one kind -- `--kind update,boot` was an |
| 1045 | // exact single match against the whole string "update,boot", so it matched |
| 1046 | // nothing and printed "none" with no hint that the flag was misread. |
| 1047 | const ks = String(args.flags.kind).split(',').map(s => s.trim()).filter(Boolean); |
| 1048 | rows = rows.filter(r => ks.some(k => |
| 1049 | (r.src === 'ev' ? r.kind === k : String(r.tag || '').startsWith(k)))); |
| 1050 | } |
| 1051 | if (args.flags.grep) { |
| 1052 | const re = new RegExp(args.flags.grep, 'i'); |
| 1053 | rows = rows.filter(r => re.test(JSON.stringify(r.src === 'ev' ? r.ev : r.data || '')) || re.test(r.kind || r.tag || '')); |
| 1054 | } |
| 1055 | if (JSONOUT) { console.log(JSON.stringify(rows)); return 0; } |
| 1056 | if (!rows.length) { console.log(`events: none since ${windowLabel(since, cov)}.`); return 0; } |
| 1057 | const limit = Number(args.flags.limit) || 30; |
| 1058 | const lines = rows.slice(-limit).map((r) => { |
| 1059 | const body = fmtEventBody(r); |
| 1060 | return `${hm(r.ts)} ${devLabel(r.device)} ${(r.src === 'ev' ? r.kind : r.tag).slice(0, 24).padEnd(24)} ${body.slice(0, 100)}`; |
| 1061 | }); |
| 1062 | // `--kind round` is throttled at the source (every 5th round plus the one a |
| 1063 | // turn ended on, see daimond.js), so the raw rows above are samples and not |
| 1064 | // a round-by-round trace. One line per turn id -- round range and the |
| 1065 | // highest `ctx` seen -- says what the samples actually cover. |
| 1066 | let summaryLines = 0; |
| 1067 | if (String(args.flags.kind || '') === 'round') { |
| 1068 | const byTurn = new Map(); |
| 1069 | for (const r of rows) { |
| 1070 | if (r.src !== 'ev') continue; |
| 1071 | const p = r.ev || {}; |
| 1072 | if (!p.turn) continue; |
| 1073 | const s = byTurn.get(p.turn) || { lo: Infinity, hi: -Infinity, max: 0, n: 0 }; |
| 1074 | const rr = Number(p.r) || 0, ctx = Number(p.ctx) || 0; |
| 1075 | s.lo = Math.min(s.lo, rr); s.hi = Math.max(s.hi, rr); |
| 1076 | s.max = Math.max(s.max, ctx); s.n += 1; |
| 1077 | byTurn.set(p.turn, s); |
| 1078 | } |
| 1079 | for (const [turn, s] of byTurn) { |
| 1080 | lines.push(` turn ${turn}: rounds ${s.lo}-${s.hi} sampled (${s.n}), max prompt ${num(s.max)}`); |
| 1081 | } |
| 1082 | // A WORKER'S OWN ROUNDS, keyed `w` rather than `turn` -- see daimond.js's |
| 1083 | // worker sink. No `ctx`: a worker's round row carries only its id and the |
| 1084 | // round number, so the summary is the range and the sample count alone. |
| 1085 | const byWorker = new Map(); |
| 1086 | for (const r of rows) { |
| 1087 | if (r.src !== 'ev') continue; |
| 1088 | const p = r.ev || {}; |
| 1089 | if (!p.w) continue; |
| 1090 | const s = byWorker.get(p.w) || { lo: Infinity, hi: -Infinity, n: 0 }; |
| 1091 | const rr = Number(p.r) || 0; |
| 1092 | s.lo = Math.min(s.lo, rr); s.hi = Math.max(s.hi, rr); s.n += 1; |
| 1093 | byWorker.set(p.w, s); |
| 1094 | } |
| 1095 | for (const [w, s] of byWorker) { |
| 1096 | lines.push(` worker ${w}: rounds ${s.lo}-${s.hi} sampled (${s.n})`); |
| 1097 | } |
| 1098 | summaryLines = byTurn.size + byWorker.size; |
| 1099 | } |
| 1100 | lines.push(`${rows.length} row(s) since ${windowLabel(since, cov)}` |
| 1101 | + (rows.length > limit ? ` (showing last ${limit})` : '')); |
| 1102 | emit(lines, limit + 1 + summaryLines); |
| 1103 | return 0; |
| 1104 | } |
| 1105 | |
| 1106 | /// What the tab printed. Default warn and error, because that is the question; |
| 1107 | /// `--lvl all` is the whole transcript, which on a busy device is long. |
| 1108 | function cmdConsole(args) { |
| 1109 | const since = sinceMs(args.flags.since, Date.now() - 3600000); |
| 1110 | const cov = coverage(args.flags.device).rows; |
| 1111 | const lvls = levelSet(args.flags.lvl); |
| 1112 | let rows = consoleRecords(args.flags.device, since, lvls); |
| 1113 | if (args.flags.grep) { |
| 1114 | const re = new RegExp(args.flags.grep, 'i'); |
| 1115 | rows = rows.filter(r => re.test(r.msg) || re.test(r.src)); |
| 1116 | } |
| 1117 | if (JSONOUT) { console.log(JSON.stringify(rows)); return 0; } |
| 1118 | if (!rows.length) { |
| 1119 | console.log(`console: nothing at ${lvls ? [...lvls].join('/') : 'any level'}` |
| 1120 | + ` since ${windowLabel(since, cov)}.`); |
| 1121 | return 0; |
| 1122 | } |
| 1123 | const limit = Number(args.flags.limit) || 40; |
| 1124 | const lines = rows.slice(-limit).map(r => |
| 1125 | `${hm(r.ts)} ${devLabel(r.device)} ${r.lvl.padEnd(5)}` |
| 1126 | + ` ${(r.x > 1 ? 'x' + r.x : '').padStart(4)} ${r.src.slice(0, 28).padEnd(28)}` |
| 1127 | + ` ${r.msg.slice(0, 100)}`); |
| 1128 | const lines2 = rows.reduce((a, r) => a + r.x, 0); |
| 1129 | lines.push(`${rows.length} distinct line(s), ${lines2} printed, since ${windowLabel(since, cov)}` |
| 1130 | + (rows.length > limit ? ` (showing last ${limit})` : '')); |
| 1131 | emit(lines, limit + 1); |
| 1132 | return 0; |
| 1133 | } |
| 1134 | |
| 1135 | function cmdSnapshot(args) { |
| 1136 | const idx = loadSnapshotIndex(args.flags.device); |
| 1137 | const pend = pendingSets(args.flags.device).filter(p => p.kind === 'snapshot'); |
| 1138 | if (!idx.length) { |
| 1139 | if (JSONOUT) { console.log(JSON.stringify({ sets: [], pending: pend })); return 0; } |
| 1140 | console.log('snapshot: no completed set in the archive' |
| 1141 | + (pend.length ? `; ${pend.length} still assembling` : '') + '.'); |
| 1142 | for (const p of pend.slice(0, 6)) { |
| 1143 | console.log(` ${dev7(p.device)} ${p.id} ${p.have}/${p.n} chunk(s), last ${ago(p.lastBt)} ago`); |
| 1144 | } |
| 1145 | return 0; |
| 1146 | } |
| 1147 | const complete = idx.filter(r => r.complete && r.body); |
| 1148 | const pick = args.flags.id |
| 1149 | ? idx.find(r => r.id === args.flags.id) |
| 1150 | : (complete.length ? complete[complete.length - 1] : idx[idx.length - 1]); |
| 1151 | if (!pick) { console.log('snapshot: no such set.'); return 1; } |
| 1152 | |
| 1153 | if (!pick.complete || !pick.body) { |
| 1154 | // A gap is reported, never guessed at: the rows are gone because jarrah |
| 1155 | // rotated the file before a pull reached it, which is a fact about the |
| 1156 | // pull cadence and worth saying plainly. |
| 1157 | const msg = { |
| 1158 | id: pick.id, device: pick.device, ts: pick.ts, |
| 1159 | complete: false, have: pick.have, of: pick.n, missing: pick.missing, |
| 1160 | note: 'chunks lost to rotation on jarrah before a pull reached them', |
| 1161 | }; |
| 1162 | if (JSONOUT) { console.log(JSON.stringify(msg)); return 0; } |
| 1163 | console.log(`snapshot ${pick.id} (${dev7(pick.device)}, ${hm(pick.ts)}): INCOMPLETE` |
| 1164 | + ` — ${pick.have} of ${pick.n} chunks, missing ${pick.missing}`); |
| 1165 | console.log(` ${complete.length} of ${idx.length} set(s) in the archive are complete.`); |
| 1166 | return 0; |
| 1167 | } |
| 1168 | const body = readJson(path.join(SNAPS, pick.body), null); |
| 1169 | if (!body) { console.log('snapshot: the body file is missing or unreadable.'); return 1; } |
| 1170 | if (args.flags.out) { |
| 1171 | fs.writeFileSync(args.flags.out, JSON.stringify(body, null, 1)); |
| 1172 | console.log(`snapshot ${pick.id} → ${args.flags.out} (${fs.statSync(args.flags.out).size} bytes)`); |
| 1173 | return 0; |
| 1174 | } |
| 1175 | if (JSONOUT) { console.log(JSON.stringify(body)); return 0; } |
| 1176 | // A snapshot is megabytes; printing it whole would blow the reader's budget |
| 1177 | // for no gain, so the default is its shape and `--out` writes the real thing. |
| 1178 | const lines = [`snapshot ${pick.id} ${dev7(pick.device)} ${pick.iso || hm(pick.ts)}` |
| 1179 | + ` ${pick.n} chunk(s)`]; |
| 1180 | for (const k of Object.keys(body)) { |
| 1181 | const v = body[k]; |
| 1182 | let shape; |
| 1183 | if (Array.isArray(v)) shape = `[${v.length}]`; |
| 1184 | else if (v && typeof v === 'object') shape = `{${Object.keys(v).length} keys}`; |
| 1185 | else shape = JSON.stringify(v); |
| 1186 | lines.push(` ${k.padEnd(14)} ${String(shape).slice(0, 60)}`); |
| 1187 | } |
| 1188 | lines.push(` (--out FILE for the whole bundle, --json for stdout)`); |
| 1189 | emit(lines, 40); |
| 1190 | return 0; |
| 1191 | } |
| 1192 | |
| 1193 | function cmdDigest(args) { |
| 1194 | const day = Date.now() - 86400000; |
| 1195 | const hour = Date.now() - 3600000; |
| 1196 | const states = deviceStates(args.flags.device); |
| 1197 | const turns = turnRecords(args.flags.device, day); |
| 1198 | const errs = errorRecords(args.flags.device, day); |
| 1199 | const con = consoleCounts(args.flags.device, hour); |
| 1200 | const tel = loadTelemetry(args.flags.device, 0); |
| 1201 | const lastTel = tel.length ? tel[tel.length - 1] : null; |
| 1202 | const snaps = loadSnapshotIndex(args.flags.device); |
| 1203 | |
| 1204 | const cov = coverage(args.flags.device); |
| 1205 | const byModel = new Map(); |
| 1206 | for (const t of turns) { |
| 1207 | const r = byModel.get(t.model) || { turns: 0, usd: 0, prompt: 0, cached: 0 }; |
| 1208 | r.turns += t.turns || 1; r.usd += t.usd || 0; |
| 1209 | r.prompt += t.prompt || 0; r.cached += t.cached || 0; |
| 1210 | byModel.set(t.model, r); |
| 1211 | } |
| 1212 | const spend = turns.reduce((a, t) => a + (t.usd || 0), 0); |
| 1213 | const count = turns.reduce((a, t) => a + (t.turns || 1), 0); |
| 1214 | const errs1h = errs.filter(e => e.last >= hour); |
| 1215 | |
| 1216 | if (JSONOUT) { |
| 1217 | console.log(JSON.stringify({ |
| 1218 | now: Date.now(), coverage: cov, |
| 1219 | devices: states.map(s => ({ device: s.device, build: s.build, lastBeat: s.lastBeat, live: s.live })), |
| 1220 | turns24h: count, spend24h: spend, |
| 1221 | models: [...byModel].map(([m, r]) => ({ model: m, ...r })), |
| 1222 | errors1h: errs1h.length, errors24h: errs.length, |
| 1223 | console1h: con, |
| 1224 | feedHealth: states.map(s => s.health).filter(Boolean), |
| 1225 | telemetryTicks: tel.length, snapshotSets: snaps.length, |
| 1226 | snapshotComplete: snaps.filter(s => s.complete).length, |
| 1227 | pendingSets: pendingSets(args.flags.device).length, |
| 1228 | })); |
| 1229 | return 0; |
| 1230 | } |
| 1231 | |
| 1232 | const lines = []; |
| 1233 | lines.push(`daimond lens — ${utcLabel(Date.now())} ${states.length} device(s)` |
| 1234 | + ` archive since ${utcLabel(cov.ticks)}` |
| 1235 | + (cov.rows && cov.rows < cov.ticks ? ` (rows ${utcLabel(cov.rows)})` : '')); |
| 1236 | for (const s of states.slice(0, 4)) { |
| 1237 | const l = s.live || {}; |
| 1238 | const win = Number(l.contextWindow) || 0, act = Number(l.contextActual) || 0; |
| 1239 | lines.push(`${dev7(s.device)} b${s.build || '?'} ${ago(s.lastBeat)} ago` |
| 1240 | + ` since ${covLabel((cov.byDevice[s.device] || {}).ticks || 0)}` |
| 1241 | + ` ${shortModel(l.activeModel)} ctx ${num(act)}/${num(win)}` |
| 1242 | + (win ? ` ${Math.round(100 * act / win)}%` : '') |
| 1243 | + ` fold@${l.foldAt ? Math.round(l.foldAt * 100) + '%' : '?'} ${l.activity || '?'}`); |
| 1244 | } |
| 1245 | // The window says what it actually covered. "24h: 0" over an archive four |
| 1246 | // hours old is a lie a pickup would act on. |
| 1247 | lines.push(`${cov.ticks && cov.ticks > day ? 'since ' + utcLabel(cov.ticks) : '24h'}:` |
| 1248 | + ` ${count} turn(s) ${usd(spend)}`); |
| 1249 | const models = [...byModel].sort((a, b) => b[1].usd - a[1].usd).slice(0, 4); |
| 1250 | for (const [m, r] of models) { |
| 1251 | lines.push(` ${shortModel(m).padEnd(22)} ${String(r.turns).padStart(4)} turn(s)` |
| 1252 | + ` ${usd(r.usd).padStart(7)} cache ${r.prompt ? Math.round(100 * r.cached / r.prompt) : 0}%`); |
| 1253 | } |
| 1254 | // An empty window over an archive that begins after the turns it is missing |
| 1255 | // would otherwise read as a quiet account. The running counters on the newest |
| 1256 | // tick are all-time totals, so they still say what this account has done. |
| 1257 | if (!count && lastTel && lastTel.stats && Array.isArray(lastTel.stats.models)) { |
| 1258 | const all = lastTel.stats.models.slice().sort((a, b) => (b.usd || 0) - (a.usd || 0)); |
| 1259 | const tt = all.reduce((a, m) => a + (m.turns || 0), 0); |
| 1260 | const tu = all.reduce((a, m) => a + (m.usd || 0), 0); |
| 1261 | if (tt) { |
| 1262 | lines.push(` counters (all time, ${dev7(lastTel.device)}): ${tt} turn(s) ${usd(tu)}` |
| 1263 | + ' — ' + all.slice(0, 2).map(m => `${shortModel(m.model)} ${m.turns} ${usd(m.usd)}`).join(', ')); |
| 1264 | } |
| 1265 | } |
| 1266 | lines.push(`errors: ${errs1h.reduce((a, e) => a + e.count, 0)} in 1h,` |
| 1267 | + ` ${errs.reduce((a, e) => a + e.count, 0)} in 24h, ${errs.length} distinct`); |
| 1268 | for (const e of errs.slice(0, 3)) lines.push(` x${e.count} ${e.kind}: ${e.msg.slice(0, 88)}`); |
| 1269 | lines.push(`console (1h): ${con.warn} warn, ${con.error} err` |
| 1270 | + (con.other ? `, ${con.other} other` : '') + ` (${con.distinct} distinct)`); |
| 1271 | const pend = pendingSets(args.flags.device); |
| 1272 | const health = states.map(s => s.health).filter(Boolean); |
| 1273 | const sum = (k) => health.reduce((a, h) => a + (h[k] || 0), 0); |
| 1274 | lines.push(`feed: ${tel.length} telemetry tick(s), last ${lastTel ? ago(lastTel.ts || lastTel.bt) + ' ago' : 'never'};` |
| 1275 | + ` ${snaps.filter(s => s.complete).length}/${snaps.length} snapshot set(s) complete` |
| 1276 | + (pend.length ? `, ${pend.length} assembling` : '') |
| 1277 | + (health.length ? `; throttled ${sum('throttled')} postFail ${sum('postFail')} cdrop ${sum('cdrop')}` : '')); |
| 1278 | |
| 1279 | // The cap is the contract. Drop from the middle -- the device block and the |
| 1280 | // feed line are what a pickup needs -- rather than truncating the tail. |
| 1281 | let out = lines.join('\n'); |
| 1282 | while (Buffer.byteLength(out) > DIGEST_MAX_BYTES && lines.length > 4) { |
| 1283 | lines.splice(Math.max(1, lines.length - 3), 1); |
| 1284 | out = lines.join('\n'); |
| 1285 | } |
| 1286 | console.log(out); |
| 1287 | return 0; |
| 1288 | } |
| 1289 | |
| 1290 | function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } |
| 1291 | |
| 1292 | async function cmdWatch(args) { |
| 1293 | let mark = sinceMs(args.flags.since, Date.now() - 5 * 60000); |
| 1294 | const once = !!args.bool.once; |
| 1295 | for (;;) { |
| 1296 | if (!args.bool['no-pull']) cmdPullQuiet(args); |
| 1297 | const rows = loadEvents(args.flags.device, mark); |
| 1298 | for (const r of rows) { |
| 1299 | const body = r.src === 'ev' ? JSON.stringify(r.ev) : (r.data || ''); |
| 1300 | console.log(`${hm(r.ts)} ${dev7(r.device)} ${(r.src === 'ev' ? r.kind : r.tag).slice(0, 24).padEnd(24)} ${body.slice(0, 100)}`); |
| 1301 | mark = Math.max(mark, (r.ts || 0) + 1); |
| 1302 | } |
| 1303 | const tel = loadTelemetry(args.flags.device, mark); |
| 1304 | for (const t of tel) { |
| 1305 | const l = (t.stats && t.stats.live) || {}; |
| 1306 | console.log(`${hm(t.ts)} ${dev7(t.device)} tick`.padEnd(45) |
| 1307 | + ` ${shortModel(l.activeModel)} ctx ${num(l.contextActual)}/${num(l.contextWindow)}` |
| 1308 | + ` ${l.activity || '?'} +${(t.ledger || []).length} ledger`); |
| 1309 | mark = Math.max(mark, (t.ts || 0) + 1); |
| 1310 | } |
| 1311 | if (once) return 0; |
| 1312 | await sleep(15000); |
| 1313 | } |
| 1314 | } |
| 1315 | |
| 1316 | /// `watch` pulls on every cycle but must not print a pull report each time. |
| 1317 | function cmdPullQuiet(args) { |
| 1318 | const saved = console.log; |
| 1319 | console.log = () => {}; |
| 1320 | try { cmdPull(args); } finally { console.log = saved; } |
| 1321 | } |
| 1322 | |
| 1323 | // ── Dispatch ───────────────────────────────────────────────────────── |
| 1324 | |
| 1325 | const COMMANDS = { |
| 1326 | pull: cmdPull, |
| 1327 | status: cmdStatus, |
| 1328 | turns: cmdTurns, |
| 1329 | errors: cmdErrors, |
| 1330 | console: cmdConsole, |
| 1331 | events: cmdEvents, |
| 1332 | snapshot: cmdSnapshot, |
| 1333 | watch: cmdWatch, |
| 1334 | digest: cmdDigest, |
| 1335 | }; |
| 1336 | |
| 1337 | const fn = COMMANDS[ARGS.cmd]; |
| 1338 | if (!fn) { |
| 1339 | console.log('lens.mjs — commands: ' + Object.keys(COMMANDS).join(', ') |
| 1340 | + '\n see the usage block at the top of this file'); |
| 1341 | process.exit(ARGS.cmd === 'help' || ARGS.bool.help ? 0 : 2); |
| 1342 | } |
| 1343 | const rc = await fn(ARGS); |
| 1344 | process.exit(rc || 0); |