oxedyne/fe2o3/fe2o3_net/lens/js/test/writer.test.mjs
32.8 KiB, 1 run
created by r1870400018:65632, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // writer.test.mjs -- the browser writer core, `../writer.js`, ported from the event-feed, |
| 2 | // console, rate and scrubber sections of Daimond's `www/js/debugshare.test.mjs`. |
| 3 | // |
| 4 | // What carried over is every property of the core: the envelope, the 360-byte fit, the durable |
| 5 | // outbox (exactly once, in order, across an outage and a reload), the feed that cannot break the |
| 6 | // app, the caps, the overflow, error and console capture, the one post clock and its 429 |
| 7 | // handling. What stayed behind is what is Daimond's: the Settings flag and its sync, the |
| 8 | // snapshot and telemetry lanes, the beat, the screen, the indicator. Those run in Daimond's own |
| 9 | // test, through `test/adapter` (see the record). |
| 10 | // |
| 11 | // node test/writer.test.mjs |
| 12 | import { readFileSync } from 'node:fs'; |
| 13 | import { dirname, join } from 'node:path'; |
| 14 | import { fileURLToPath } from 'node:url'; |
| 15 | import { createRedactor, createScrubber } from '../redact.js'; |
| 16 | import { check, finish, part, section } from './harness.mjs'; |
| 17 | import { makeHost, makeWriter, parse, sleep, until } from './host.mjs'; |
| 18 | |
| 19 | const HERE = dirname(fileURLToPath(import.meta.url)); |
| 20 | const RAW_API_KEY = 'sk-or-v1-ZZZTOPSECRETproviderKEY0123456789abcdef'; // allowlist secret |
| 21 | |
| 22 | async function main() { |
| 23 | |
| 24 | section('writer: events -- the envelope, the tag and the 360-byte cap'); |
| 25 | await part(async () => { |
| 26 | const h = makeHost({ respond: () => 500, fastTimers: true }); |
| 27 | const w = makeWriter(h); |
| 28 | w.state.set(false); |
| 29 | check('event() is a no-op while the gate is shut', w.event('tool', { name: 'read' }) === false); |
| 30 | check('nothing is queued while shut', w.depth() === 0); |
| 31 | w.state.set(true); |
| 32 | check('event() queues while open', w.event('tool', { name: 'read', out: 'done' }) === true); |
| 33 | const rows = w.outbox(); |
| 34 | check('exactly one event was queued', rows.length === 1); |
| 35 | check('the row is tagged "ev <kind>"', rows[0].tag === 'ev tool'); |
| 36 | check('the data is plain JSON, not base64', rows[0].data.charAt(0) === '{'); |
| 37 | const e1 = JSON.parse(rows[0].data); |
| 38 | check('the envelope is {v,d,n,b,t} in that order, then the payload', |
| 39 | Object.keys(e1).join(',') === 'v,d,n,b,t,name,out'); |
| 40 | check('v is 1, n a positive count, t a millisecond clock', |
| 41 | e1.v === 1 && e1.n > 0 && typeof e1.t === 'number' && e1.t > 1e12); |
| 42 | check('d and b come from the caller and are bounded (12 and 24 characters)', |
| 43 | e1.d === 'devAAAA11112' && e1.b === 'build0123456789abcdef012'); |
| 44 | check('the sequence advances by one per event', (w.event('tool', {}), JSON.parse(w.outbox()[1].data).n === e1.n + 1)); |
| 45 | check('the sequence is persisted before the event is queued', |
| 46 | Number(h.store.get('lens-seq')) === JSON.parse(w.outbox()[1].data).n); |
| 47 | w.event('tool', { n: 999, t: 1, v: 9, d: 'x', b: 'y', own: 1 }); |
| 48 | const e3 = JSON.parse(w.outbox()[2].data); |
| 49 | check('a payload cannot overwrite the envelope', e3.v === 1 && e3.n !== 999 && e3.t > 1e12 && e3.d.startsWith('devAAAA') && e3.own === 1); |
| 50 | w.event('tool', { skip: undefined, keep: 0 }); |
| 51 | check('an undefined payload field is left out, a zero is kept', |
| 52 | !('skip' in JSON.parse(w.outbox()[3].data)) && JSON.parse(w.outbox()[3].data).keep === 0); |
| 53 | |
| 54 | w.event('tool', { name: 'small', msg: 'x'.repeat(900) }); |
| 55 | const big = w.outbox()[4]; |
| 56 | const e4 = JSON.parse(big.data); |
| 57 | check('a giant event is cut to the 360-byte cap', new TextEncoder().encode(big.data).length <= 360); |
| 58 | check('a cut event is marked tr:1 and keeps its whole envelope', e4.tr === 1 && e4.v === 1 && e4.n > 0 && typeof e4.t === 'number'); |
| 59 | check('the cut trims the large field and keeps the small one', e4.name === 'small' && e4.msg.length < 900); |
| 60 | w.event('tool', { m: 'é'.repeat(500) }); |
| 61 | check('a multi-byte string still lands inside the cap', new TextEncoder().encode(w.outbox()[5].data).length <= 360); |
| 62 | check('every queued event is within the cap', w.outbox().every((r) => new TextEncoder().encode(r.data).length <= 360)); |
| 63 | check('the cut is a pure function of the object: fit(obj) returns what event() queued', |
| 64 | w.fit({ v: 1, d: 'd', n: 1, b: 'b', t: 1, a: 'y'.repeat(800) }).length <= 360); |
| 65 | w.event('x', { k: 'a'.repeat(100), j: 'b'.repeat(400) }); |
| 66 | check('the largest string goes first, so a short sibling survives', JSON.parse(w.outbox()[6].data).k === 'a'.repeat(100)); |
| 67 | h.halt(); |
| 68 | }); |
| 69 | |
| 70 | section('writer: events -- the redactor, with the caller\'s two hooks'); |
| 71 | await part(async () => { |
| 72 | const h = makeHost({ respond: () => 500, fastTimers: true }); |
| 73 | const w = makeWriter(h); |
| 74 | w.event('cfg', { apiKey: RAW_API_KEY, model: 'anthropic/claude-3.5' }); |
| 75 | const data = w.outbox()[0].data; |
| 76 | check('a secret-named field is not carried raw', data.indexOf(RAW_API_KEY) === -1); |
| 77 | check('it appears as a fingerprint keeping six characters', /"apiKey":"\[redacted sk-or-…#[0-9a-f]+\/\d+\]"/.test(data)); |
| 78 | check('a field beside it is untouched', data.indexOf('anthropic/claude-3.5') !== -1); |
| 79 | |
| 80 | const eight = /^(?:[a-z]+ ){7}[a-z]+$/; |
| 81 | const h2 = makeHost({ respond: () => 500, fastTimers: true }); |
| 82 | const w2 = makeWriter(h2, { redact: createRedactor({ deny: ['draft'], test: (s) => eight.test(s), head: 0 }) }); |
| 83 | w2.event('screen', { draft: 'typed so far', phrase: 'one two three four five six seven eight', ok: 'fine' }); |
| 84 | const d2 = w2.outbox()[0].data; |
| 85 | check('the caller\'s deny-list hook covers its field', d2.indexOf('typed so far') === -1 && /"draft":"\[redacted …#/.test(d2)); |
| 86 | check('the caller\'s string test covers a phrase under any field name', d2.indexOf('one two three') === -1 && /"phrase":"\[redacted …#/.test(d2)); |
| 87 | check('an innocent field is left', d2.indexOf('"ok":"fine"') !== -1); |
| 88 | h.halt(); h2.halt(); |
| 89 | }); |
| 90 | |
| 91 | section('writer: events -- the content scrubber meets every row at the egress seam, BEFORE the cut'); |
| 92 | await part(async () => { |
| 93 | const KEY = 'sk-' + 'Zq9Xw8Vu7Ts6Rq5Po4Nm3Lk2Jh1'; |
| 94 | const h = makeHost({ respond: () => 500, fastTimers: true }); |
| 95 | const w = makeWriter(h); |
| 96 | w.event('tool', { out: 'call used ' + KEY + ' today' }); |
| 97 | check('a key shape inside a free-text field is replaced by a marker naming its shape', |
| 98 | w.outbox()[0].data.indexOf(KEY) === -1 && /\[redacted sk #[0-9a-f]+\/\d+\]/.test(w.outbox()[0].data)); |
| 99 | w.event('tool', { msg: 'a'.repeat(300) + ' ' + KEY + ' ' + 'b'.repeat(200) }); |
| 100 | check('a key that the 360-byte cut would have split leaves no prefix of itself behind', |
| 101 | w.outbox()[1].data.indexOf('sk-Zq9') === -1 && new TextEncoder().encode(w.outbox()[1].data).length <= 360); |
| 102 | w.captureConsole(h.console); |
| 103 | h.console.warn('x'.repeat(190) + ' ' + KEY + ' ' + 'tail '.repeat(40)); |
| 104 | w.flushConsole(true); |
| 105 | check('nor does a console line clipped at 200 characters, because the clip scrubs first', |
| 106 | w.outbox().filter((r) => r.tag === 'ev console').every((r) => r.data.indexOf('sk-Zq9') === -1)); |
| 107 | check('and a key in an object key is taken too', (w.event('t', { ['k ' + KEY]: 1 }), w.outbox().pop().data.indexOf(KEY) === -1)); |
| 108 | h.halt(); |
| 109 | }); |
| 110 | |
| 111 | section('writer: events -- RULE 1, exactly once and in order across an outage'); |
| 112 | await part(async () => { |
| 113 | const h = makeHost({ fastTimers: true, respond: (i) => (i < 6 ? 503 : 200) }); |
| 114 | const w = makeWriter(h); |
| 115 | for (let i = 0; i < 300; i++) w.event('round', { turn: 'T', r: i }); |
| 116 | check('300 events are queued (some may already be in flight)', w.depth() + h.posts.length >= 1 && w.depth() <= 300); |
| 117 | check('the outbox empties once the endpoint recovers', await until(() => w.depth() === 0)); |
| 118 | check('the endpoint really did refuse a window of posts', h.posts.filter((p) => p.status === 503).length >= 6); |
| 119 | const arrived = parse(h.landed()); |
| 120 | const ns = arrived.map((e) => e.n); |
| 121 | check('every one of the 300 events arrived', arrived.length === 300); |
| 122 | check('no n arrived twice', new Set(ns).size === ns.length); |
| 123 | check('they arrived in sequence', ns.every((n, i) => i === 0 || n > ns[i - 1])); |
| 124 | check('their payloads arrived in the order they were made', arrived.every((e, i) => e.r === i)); |
| 125 | check('the run carries one device id', new Set(arrived.map((e) => e.d)).size === 1); |
| 126 | check('a repeated failure backs the next attempt off past the ordinary gap', h.timerDelays.some((d) => d > 3000)); |
| 127 | check('the first failure waits the ordinary gap', w.backoffMs(1) === 3000); |
| 128 | check('the second waits twice it', w.backoffMs(2) === 6000); |
| 129 | check('the backoff is capped at five minutes', w.backoffMs(30) === 300000); |
| 130 | }); |
| 131 | |
| 132 | section('writer: events -- RULE 2, the outbox survives a reload'); |
| 133 | await part(async () => { |
| 134 | const h1 = makeHost({ fastTimers: true, respond: () => 500 }); |
| 135 | const w1 = makeWriter(h1); |
| 136 | for (let i = 0; i < 40; i++) w1.event('round', { turn: 'T', r: i }); |
| 137 | await sleep(30); |
| 138 | w1.persistNow(); |
| 139 | const depthBefore = w1.depth(); |
| 140 | const lastPostBefore = w1.health().lastPostAt; |
| 141 | check('the queue is held while the endpoint refuses', depthBefore === 40); |
| 142 | check('the post clock is itself persisted before the reload', lastPostBefore > 0 && Number(h1.store.get('lens-lastpost')) === lastPostBefore); |
| 143 | h1.halt(); |
| 144 | |
| 145 | const h2 = makeHost({ store: h1.store, fastTimers: true }); |
| 146 | const w2 = makeWriter(h2); |
| 147 | check('the reloaded writer recovered the whole outbox', w2.depth() === 40); |
| 148 | check('and the post clock, so a post right after boot still knows a gap is owed', |
| 149 | w2.health().lastPostAt === lastPostBefore && w2.gapLeft() > 0); |
| 150 | w2.event('round', { turn: 'T', r: 40 }); |
| 151 | const n41 = JSON.parse(w2.outbox()[40].data).n; |
| 152 | check('the sequence continues where it left off, never restarting', n41 === 41); |
| 153 | w2.drain(); |
| 154 | check('the recovered outbox drains', await until(() => w2.depth() === 0)); |
| 155 | const landed = parse(h2.landed()); |
| 156 | check('all 41 events reach the wire after the reload', landed.length === 41); |
| 157 | check('nothing queued before the reload was lost, and none arrived twice', |
| 158 | new Set(landed.map((e) => e.n)).size === 41 && landed.every((e, i) => e.r === i)); |
| 159 | }); |
| 160 | |
| 161 | section('writer: events -- a corrupt or odd stored outbox cannot stop the boot'); |
| 162 | await part(async () => { |
| 163 | const store = new Map([['lens-outbox', '{not json'], ['lens-seq', 'banana']]); |
| 164 | const w = makeWriter(makeHost({ store, respond: () => 500 })); |
| 165 | check('unparseable storage yields an empty queue, not a throw', w.depth() === 0); |
| 166 | check('an unreadable sequence restarts from one', (w.event('x', {}), JSON.parse(w.outbox()[0].data).n === 1)); |
| 167 | const store2 = new Map([['lens-outbox', JSON.stringify([{ ts: 1, tag: 'ev a', data: '{}' }, { tag: 5 }, null, 'x', { ts: 2, tag: 'ev b' }])]]); |
| 168 | const w2 = makeWriter(makeHost({ store: store2, respond: () => 500 })); |
| 169 | check('only well-formed rows are recovered', w2.depth() === 1 && w2.outbox()[0].tag === 'ev a'); |
| 170 | const big = Array.from({ length: 5200 }, (_, i) => ({ ts: i + 1, tag: 'ev r', data: '{"i":' + i + '}' })); |
| 171 | const w3 = makeWriter(makeHost({ store: new Map([['lens-outbox', JSON.stringify(big)]]), respond: () => 500 })); |
| 172 | check('an oversized stored outbox keeps its newest 5,000', w3.depth() === 5000 && w3.outbox()[0].data === '{"i":200}'); |
| 173 | const w4 = makeWriter(makeHost({ store: undefined, respond: () => 500 }), { host: { storage: { getItem() { throw new Error('blocked'); }, setItem() { throw new Error('blocked'); } }, setTimeout, clearTimeout, console: {}, now: Date.now } }); |
| 174 | check('a storage that throws on every call still lets the writer run in memory', (w4.event('x', {}), w4.depth() === 1)); |
| 175 | }); |
| 176 | |
| 177 | section('writer: events -- RULE 3, the feed cannot break the app'); |
| 178 | await part(async () => { |
| 179 | const h = makeHost({ fastTimers: true }); |
| 180 | const w = makeWriter(h); |
| 181 | check('the feed is healthy before the fault', w.ok() === true); |
| 182 | let appDid = 0; |
| 183 | const poison = { get name() { throw new Error('provider blew up'); } }; |
| 184 | const appCall = () => { w.event('tool', poison); appDid += 1; return 'the answer'; }; |
| 185 | check('the app\'s own call returns its own answer', appCall() === 'the answer'); |
| 186 | check('the app\'s own call completes', appDid === 1); |
| 187 | check('the feed turned itself off', w.ok() === false); |
| 188 | await until(() => w.depth() === 0); |
| 189 | const faults = h.landed().filter((r) => r.tag === 'ev feed.fault'); |
| 190 | check('exactly one feed.fault reached the wire (the drainer was left running)', faults.length === 1); |
| 191 | check('the fault says what threw', JSON.parse(faults[0].data).msg.indexOf('provider blew up') !== -1); |
| 192 | check('the fault carries the ordinary envelope', JSON.parse(faults[0].data).n > 0 && JSON.parse(faults[0].data).v === 1); |
| 193 | check('a later event is refused by the off feed', w.event('tool', { name: 'after' }) === false); |
| 194 | const depth = w.depth(); |
| 195 | appCall(); |
| 196 | check('a second fault queues nothing more and the app still completes', w.depth() === depth && appDid === 2); |
| 197 | |
| 198 | let faulted = null; |
| 199 | const w2 = makeWriter(makeHost({ fastTimers: true }), { onFault: (e) => { faulted = e; } }); |
| 200 | w2.event('tool', poison); |
| 201 | check('the caller is told of the fault, so it can stop its own timers', faulted && faulted.message === 'provider blew up'); |
| 202 | let after = 0; |
| 203 | const w3 = makeWriter(makeHost({ fastTimers: true }), { after: () => { after += 1; } }); |
| 204 | w3.event('round', { r: 1 }); |
| 205 | w3.state.set(false); |
| 206 | w3.event('round', { r: 2 }); |
| 207 | check('the after-hook runs for an accepted event only', after === 1); |
| 208 | }); |
| 209 | |
| 210 | section('writer: events -- a post never exceeds 400 rows or 200 KiB'); |
| 211 | await part(async () => { |
| 212 | const h = makeHost({ fastTimers: true }); |
| 213 | const w = makeWriter(h); |
| 214 | for (let i = 0; i < 900; i++) w.event('round', { turn: 'T', r: i, note: 'n'.repeat(40) }); |
| 215 | check('900 events all drain', await until(() => w.depth() === 0)); |
| 216 | check('they went in more than one post', h.posts.length >= 3); |
| 217 | check('no post exceeds 400 rows', h.posts.every((p) => p.rows.length <= 400)); |
| 218 | check('no post exceeds 200 KiB', h.posts.every((p) => JSON.stringify(p.rows).length <= 200 * 1024)); |
| 219 | check('a post carries only whole events', h.landed().every((r) => { try { return JSON.parse(r.data).v === 1; } catch (e) { return false; } })); |
| 220 | const h2 = makeHost({ respond: () => 500, fastTimers: true }); |
| 221 | const w2 = makeWriter(h2); |
| 222 | w2.state.set(false); |
| 223 | for (let i = 0; i < 450; i++) w2.pushRow('ev x', '{"i":' + i + '}'); |
| 224 | check('one batch is capped at 400 rows', w2.eventBatch().length === 400); |
| 225 | const w3 = makeWriter(makeHost({ respond: () => 500 }), { caps: { maxBodyBytes: 2000 } }); |
| 226 | w3.state.set(false); |
| 227 | for (let i = 0; i < 50; i++) w3.pushRow('ev x', '{"pad":"' + 'p'.repeat(200) + '"}'); |
| 228 | check('a smaller body cap shortens the batch, and a lone row always goes', |
| 229 | w3.eventBatch().length < 50 && w3.eventBatch().length >= 1); |
| 230 | h2.halt(); |
| 231 | }); |
| 232 | |
| 233 | section('writer: events -- overflow drops the OLDEST and says how many'); |
| 234 | await part(async () => { |
| 235 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 236 | const w = makeWriter(h); |
| 237 | for (let i = 0; i < 5040; i++) w.event('round', { turn: 'T', r: i }); |
| 238 | const rows = w.outbox(); |
| 239 | check('the outbox is capped at 5,000 events', rows.length === 5000); |
| 240 | const drops = rows.filter((r) => r.tag === 'ev feed.drop'); |
| 241 | check('a feed.drop records the overflow', drops.length >= 1); |
| 242 | check('the drop carries a count', JSON.parse(drops[0].data).count >= 1); |
| 243 | check('the oldest events are the ones dropped', JSON.parse(rows.find((r) => r.tag === 'ev round').data).r > 0); |
| 244 | check('the newest event survived', JSON.parse(rows[rows.length - 1].data).r === 5039 || rows[rows.length - 1].tag === 'ev feed.drop'); |
| 245 | w.reset(); |
| 246 | check('reset() empties the outbox, so nothing more leaves the device', w.depth() === 0); |
| 247 | h.halt(); |
| 248 | }); |
| 249 | |
| 250 | section('writer: a quota that halves the outbox says so'); |
| 251 | await part(async () => { |
| 252 | let quotaLeft = 1; |
| 253 | const h = makeHost({ fastTimers: true, respond: () => 500, quota: (k) => (k === 'lens-outbox' && quotaLeft > 0) ? (quotaLeft -= 1, true) : false }); |
| 254 | const w = makeWriter(h); |
| 255 | for (let i = 0; i < 8; i++) w.event('tool', { turn: 'Q1', name: 'run' }); |
| 256 | await sleep(40); |
| 257 | h.halt(); |
| 258 | const drops = parse(w.outbox().filter((r) => r.tag === 'ev feed.drop')); |
| 259 | check('a quota drop is announced once', drops.length === 1); |
| 260 | check('it names the quota and says how many rows went', drops.length === 1 && drops[0].why === 'quota' && drops[0].count >= 1); |
| 261 | check('the rows that survived are still there', w.outbox().some((r) => r.tag === 'ev tool')); |
| 262 | }); |
| 263 | |
| 264 | section('writer: errors -- rate-capped, with a dropped count'); |
| 265 | await part(async () => { |
| 266 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 267 | const w = makeWriter(h); |
| 268 | w.noteError('x is not a function', 'js/app.js:120'); |
| 269 | const first = JSON.parse(w.outbox()[0].data); |
| 270 | check('an error becomes an `error` event', w.outbox()[0].tag === 'ev error'); |
| 271 | check('it carries its message and where', first.msg.indexOf('x is not a function') !== -1 && first.at === 'js/app.js:120'); |
| 272 | w.noteError('y'.repeat(900), ''); |
| 273 | check('a long message is clipped to 200 characters', JSON.parse(w.outbox()[1].data).msg.length <= 200); |
| 274 | const before = w.depth(); |
| 275 | for (let i = 0; i < 100; i++) w.noteError('flood ' + i, ''); |
| 276 | check('the rate cap admits at most 20 a minute', w.depth() - before <= 18); |
| 277 | check('every admitted error is well formed and inside the cap', |
| 278 | w.outbox().filter((r) => r.tag === 'ev error').every((r) => JSON.parse(r.data).n > 0 && new TextEncoder().encode(r.data).length <= 360)); |
| 279 | w.state.set(false); |
| 280 | const q = w.depth(); |
| 281 | w.noteError('quiet', ''); |
| 282 | check('nothing is captured while the gate is shut', w.depth() === q); |
| 283 | h.halt(); |
| 284 | }); |
| 285 | |
| 286 | section('writer: fetch.fail -- no body, an abort is marked, the feed never reports itself'); |
| 287 | await part(async () => { |
| 288 | let hidden = false; |
| 289 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 290 | const w = makeWriter(h, { host: Object.assign({}, h.host, { document: { get visibilityState() { return hidden ? 'hidden' : 'visible'; } } }) }); |
| 291 | w.noteFetchFail('/api/sync?token=abc', 502, 40, ''); |
| 292 | w.noteFetchFail('/api/x', 0, 7, 'Failed to fetch'); |
| 293 | const fails = () => parse(w.outbox().filter((r) => r.tag === 'ev fetch.fail')); |
| 294 | check('a path and status and elapsed ms, with the query dropped', fails()[0].path === '/api/sync' && fails()[0].status === 502 && fails()[0].ms === 40); |
| 295 | check('a thrown fetch carries status 0 and its message', fails()[1].status === 0 && fails()[1].err === 'Failed to fetch'); |
| 296 | check('no request body travels', w.outbox().every((r) => r.data.indexOf('"body"') === -1)); |
| 297 | check('a status 0 on a live page is a plain failure', fails()[1].aborted === undefined); |
| 298 | hidden = true; |
| 299 | w.noteFetchFail('/api/y', 0, 1, ''); |
| 300 | check('a status 0 while the page is hidden is marked aborted', fails().pop().aborted === 1); |
| 301 | w.noteFetchFail('/api/y', 500, 1, ''); |
| 302 | check('a real status while hidden is not', fails().pop().aborted === undefined); |
| 303 | hidden = false; |
| 304 | const pageHideListeners = []; |
| 305 | w.captureLifecycle({ addEventListener: (t, f) => pageHideListeners.push([t, f]) }, undefined); |
| 306 | pageHideListeners.filter((x) => x[0] === 'pagehide').forEach((x) => x[1]({})); |
| 307 | w.noteFetchFail('/api/z', 0, 1, ''); |
| 308 | check('a status 0 just after pagehide is marked aborted', fails().pop().aborted === 1); |
| 309 | const n = fails().length; |
| 310 | w.noteFetchFail('/lens/post?x=1', 429, 1, ''); |
| 311 | check('the feed\'s own endpoint never becomes a fetch.fail', fails().length === n); |
| 312 | h.halt(); |
| 313 | }); |
| 314 | |
| 315 | section('writer: console -- every level, the ORIGINAL first, bounded three ways'); |
| 316 | await part(async () => { |
| 317 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 318 | const w = makeWriter(h); |
| 319 | w.captureConsole(h.console); |
| 320 | h.console.log('plain line'); |
| 321 | h.console.info('an info line'); |
| 322 | h.console.debug('[improve] a debug line'); |
| 323 | h.console.warn('i18n: no string for "home.sec_diag"'); |
| 324 | h.console.error('TypeError: boom'); |
| 325 | w.flushConsole(true); |
| 326 | const rows = parse(w.outbox().filter((r) => r.tag === 'ev console')); |
| 327 | check('all five levels become events', rows.length === 5); |
| 328 | check('each event names its level', ['log', 'info', 'debug', 'warn', 'error'].every((l) => rows.some((r) => r.lvl === l))); |
| 329 | check('the warning is carried verbatim', rows.some((r) => r.lvl === 'warn' && r.msg === 'i18n: no string for "home.sec_diag"')); |
| 330 | check('the original ran first, for every level', h.logged.length === 5 && h.logged[0] === 'log:plain line'); |
| 331 | check('a console event carries a bounded source', rows.every((r) => typeof r.src === 'string' && r.src.length <= 60)); |
| 332 | check('console.error also becomes an `error` event', w.outbox().some((r) => r.tag === 'ev error' && JSON.parse(r.data).at === 'console.error')); |
| 333 | check('a console event stays inside the cap', w.outbox().every((r) => new TextEncoder().encode(r.data).length <= 360)); |
| 334 | h.console.warn('a', 'b', 3, true, 'fifth is dropped'); |
| 335 | w.flushConsole(true); |
| 336 | check('several arguments join into one message, to four', parse(w.outbox().filter((r) => r.tag === 'ev console')).pop().msg === 'a b 3 true'); |
| 337 | h.console.log('z'.repeat(900)); |
| 338 | w.flushConsole(true); |
| 339 | check('a long console line is clipped to 200 characters', parse(w.outbox().filter((r) => r.tag === 'ev console')).pop().msg.length <= 200); |
| 340 | check('wrapping twice does not wrap twice', (() => { const before = h.console.log; w.captureConsole(h.console); return h.console.log === before; })()); |
| 341 | h.halt(); |
| 342 | }); |
| 343 | await part(async () => { |
| 344 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 345 | const w = makeWriter(h); |
| 346 | w.captureConsole(h.console); |
| 347 | h.console.warn('held'); |
| 348 | w.state.set(false); |
| 349 | w.reset(); |
| 350 | w.state.set(true); |
| 351 | w.flushConsole(true); |
| 352 | check('a line still held when the writer is reset is dropped, not delivered later', w.outbox().filter((r) => r.tag === 'ev console').length === 0); |
| 353 | w.state.set(false); |
| 354 | const q = w.depth(); |
| 355 | h.console.warn('nobody is listening'); |
| 356 | check('nothing is captured while the gate is shut, but the original still ran', |
| 357 | w.depth() === q && h.logged.pop() === 'warn:nobody is listening'); |
| 358 | h.halt(); |
| 359 | }); |
| 360 | await part(async () => { |
| 361 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 362 | const w = makeWriter(h); |
| 363 | w.noteConsole('warn', ['[sync] chunk index not merged on this device']); |
| 364 | w.flushConsole(true); |
| 365 | check('with no stack to read, the bracketed prefix is the source', parse(w.outbox().filter((r) => r.tag === 'ev console'))[0].src === '[sync]' || /\.(?:js|mjs):\d+$/.test(parse(w.outbox().filter((r) => r.tag === 'ev console'))[0].src)); |
| 366 | h.halt(); |
| 367 | }); |
| 368 | await part(async () => { |
| 369 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 370 | const w = makeWriter(h); |
| 371 | for (let i = 0; i < 40; i++) w.noteConsole('warn', ['repeat me']); |
| 372 | w.noteConsole('warn', ['once']); |
| 373 | check('a repeat queues nothing until the window closes', w.depth() === 0); |
| 374 | w.flushConsole(true); |
| 375 | const rows = parse(w.outbox().filter((r) => r.tag === 'ev console')); |
| 376 | check('forty identical lines are one event carrying their count, a single line none', rows.length === 2 && rows[0].x === 40 && rows[1].x === undefined); |
| 377 | w.noteConsole('error', ['repeat me']); |
| 378 | w.flushConsole(true); |
| 379 | check('the same text at another level is its own event', parse(w.outbox().filter((r) => r.tag === 'ev console')).length === 3); |
| 380 | h.halt(); |
| 381 | }); |
| 382 | await part(async () => { |
| 383 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 384 | const w = makeWriter(h); |
| 385 | for (let i = 0; i < 75; i++) w.noteConsole('log', ['distinct ' + i]); |
| 386 | w.flushConsole(true); |
| 387 | check('the cap admits sixty distinct lines a minute', parse(w.outbox().filter((r) => r.tag === 'ev console')).length === 60); |
| 388 | check('the fifteen it dropped are counted', w.health().cdrop === 15); |
| 389 | check('and the count is taken and cleared by whoever reports it', w.takeConsoleDropped() === 15 && w.health().cdrop === 0); |
| 390 | h.halt(); |
| 391 | }); |
| 392 | await part(async () => { |
| 393 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 394 | const w = makeWriter(h); |
| 395 | w.captureConsole(h.console); |
| 396 | // An argument that logs WHILE it is being read: the capture reaches for `.message`, and this |
| 397 | // getter answers by logging. Without the guard that is the capture calling itself from inside. |
| 398 | let reentries = 0; |
| 399 | const noisy = { get message() { reentries++; h.console.error('re-entrant line'); return 'noisy'; } }; |
| 400 | h.console.log(noisy); |
| 401 | w.flushConsole(true); |
| 402 | const rows = parse(w.outbox().filter((r) => r.tag === 'ev console')); |
| 403 | check('the re-entrant log really did happen', reentries >= 1); |
| 404 | check('only the outer line became an event', rows.length === 1 && rows[0].msg === 'noisy'); |
| 405 | check('the re-entrant line is not in the feed', w.outbox().every((r) => r.data.indexOf('re-entrant') === -1)); |
| 406 | check('the feed is still collecting', w.ok() === true); |
| 407 | h.halt(); |
| 408 | }); |
| 409 | await part(async () => { |
| 410 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 411 | const w = makeWriter(h); |
| 412 | w.captureConsole(h.console); |
| 413 | h.console.log('api_key: hunter2'); |
| 414 | h.console.log('call used sk-' + 'Zq9Xw8Vu7Ts6Rq5Po4Nm3Lk2' + ' today'); |
| 415 | h.console.log('the token expired at noon'); |
| 416 | w.flushConsole(true); |
| 417 | const wire = w.outbox().map((r) => r.data).join('\n'); |
| 418 | check('a key logged beside its name never reaches the outbox', wire.indexOf('hunter2') === -1); |
| 419 | check('what is kept is a fingerprint or a marker', /\[redacted [^"]+\]/.test(wire)); |
| 420 | check('a key-shaped token is caught by its shape alone', wire.indexOf('Zq9Xw8Vu7Ts6Rq5Po4Nm3Lk2') === -1); |
| 421 | check('a message that merely mentions a token is kept whole', wire.indexOf('the token expired at noon') !== -1); |
| 422 | const h2 = makeHost({ fastTimers: true, respond: () => 500 }); |
| 423 | const w2 = makeWriter(h2, { redact: createRedactor({ test: (s) => /correct horse/.test(s), head: 0 }) }); |
| 424 | w2.noteConsole('log', ['typed correct horse battery staple']); |
| 425 | w2.flushConsole(true); |
| 426 | check('the caller\'s string test reaches console lines too', w2.outbox().map((r) => r.data).join('').indexOf('correct horse') === -1); |
| 427 | h.halt(); h2.halt(); |
| 428 | }); |
| 429 | |
| 430 | section('writer: error hooks and lifecycle'); |
| 431 | await part(async () => { |
| 432 | const h = makeHost({ fastTimers: true, respond: () => 500 }); |
| 433 | const w = makeWriter(h); |
| 434 | const L = {}; |
| 435 | const win = { addEventListener: (t, f) => { (L[t] = L[t] || []).push(f); } }; |
| 436 | w.captureErrors(win); |
| 437 | L.error[0]({ message: 'Uncaught boom', filename: 'https://x.test/js/a.js', lineno: 12 }); |
| 438 | L.unhandledrejection[0]({ reason: { message: 'rejected thing' } }); |
| 439 | const evs = parse(w.outbox().filter((r) => r.tag === 'ev error')); |
| 440 | check('an uncaught error becomes an event with its path and line, origin stripped', evs[0].msg === 'Uncaught boom' && evs[0].at === '/js/a.js:12'); |
| 441 | check('an unhandled rejection becomes one too', evs[1].msg === 'rejected thing' && evs[1].at === 'unhandledrejection'); |
| 442 | let persists = 0; |
| 443 | const h2 = makeHost({ fastTimers: true, respond: () => 500, quota: () => { persists += 1; return false; } }); |
| 444 | const w2 = makeWriter(h2); |
| 445 | const L2 = {}; |
| 446 | const doc = { visibilityState: 'visible' }; |
| 447 | w2.captureLifecycle({ addEventListener: (t, f) => { (L2[t] = L2[t] || []).push(f); } }, doc); |
| 448 | w2.event('a', {}); |
| 449 | const beforeWrites = h2.store.get('lens-outbox'); |
| 450 | L2.pagehide.forEach((f) => f({})); |
| 451 | check('pagehide writes the outbox out at once, not on the debounce', beforeWrites === undefined && JSON.parse(h2.store.get('lens-outbox')).length === 1); |
| 452 | doc.visibilityState = 'hidden'; |
| 453 | w2.event('b', {}); |
| 454 | L2.visibilitychange.forEach((f) => f({})); |
| 455 | check('and so does the page going hidden', JSON.parse(h2.store.get('lens-outbox')).length === 2); |
| 456 | h.halt(); h2.halt(); |
| 457 | }); |
| 458 | |
| 459 | section('writer: rate -- ONE post clock for every lane, and what a 429 does to it'); |
| 460 | await part(async () => { |
| 461 | // A second lane: a caller's own big payload, drained after the outbox, on the same clock. |
| 462 | let laneRows = [[{ ts: 1, tag: 'ds a 1/3', data: 'x' }], [{ ts: 2, tag: 'ds a 2/3', data: 'y' }], [{ ts: 3, tag: 'ds a 3/3', data: 'z' }]]; |
| 463 | const lane = { pending: () => laneRows.length > 0, next: () => laneRows.shift() }; |
| 464 | const h = makeHost({ fastTimers: true }); |
| 465 | const w = makeWriter(h, { lane }); |
| 466 | w.event('turn.start', { turn: 'T1' }); |
| 467 | w.event('turn.end', { turn: 'T1' }); |
| 468 | w.drain(); |
| 469 | check('both lanes empty', await until(() => w.depth() === 0 && laneRows.length === 0)); |
| 470 | check('the outbox went first, then the lane', h.posts[0].rows[0].tag.startsWith('ev ') && h.posts[h.posts.length - 1].rows[0].tag === 'ds a 3/3'); |
| 471 | check('more than one post was needed', h.posts.length >= 4); |
| 472 | const pacing = h.timerDelays.filter((d) => d >= 3000); |
| 473 | check('every post after the first waited a full window', pacing.length >= h.posts.length - 1); |
| 474 | check('nothing asked to post inside another post\'s window', |
| 475 | h.timerDelays.every((d) => d === 0 || d === 250 || d >= 3000), JSON.stringify(h.timerDelays.filter((d) => d !== 0 && d !== 250 && d < 3000))); |
| 476 | |
| 477 | const h2 = makeHost({ fastTimers: true }); |
| 478 | const w2 = makeWriter(h2); |
| 479 | w2.event('x', {}); |
| 480 | await until(() => w2.depth() === 0); |
| 481 | const sent = h2.posts.length; |
| 482 | h2.timerDelays.length = 0; |
| 483 | w2.event('y', {}); |
| 484 | check('a drain that STARTS inside the window waits out the remainder first', await until(() => h2.timerDelays.some((d) => d > 0 && d <= 3000))); |
| 485 | check('and then it posts, with no 429 provoked', await until(() => h2.posts.length === sent + 1) && w2.health().throttled === 0); |
| 486 | }); |
| 487 | await part(async () => { |
| 488 | const h = makeHost({ fastTimers: true, respond: (i) => (i < 3 ? 429 : 200) }); |
| 489 | const w = makeWriter(h); |
| 490 | w.event('a', {}); |
| 491 | w.event('b', {}); |
| 492 | await until(() => w.depth() === 0); |
| 493 | const throttles = parse(h.landed().filter((r) => r.tag === 'ev feed.throttled')); |
| 494 | check('a 429 is announced, once per burst', throttles.length === 1); |
| 495 | check('the 429s are counted, and a 429 is not a post failure', w.health().throttled >= 1 && w.health().postFail === 0); |
| 496 | check('the events themselves were unlost', parse(h.landed()).filter((e) => e.n > 0 && !e.gap).length >= 2); |
| 497 | check('the announcement says the gap it widened to', throttles[0].gap >= 3000); |
| 498 | check('a post that lands clears the widening', w.health().gap === 3000); |
| 499 | }); |
| 500 | await part(async () => { |
| 501 | const h = makeHost({ fastTimers: true, respond: (i) => (i < 2 ? 429 : 200) }); |
| 502 | const w = makeWriter(h); |
| 503 | w.event('a', {}); |
| 504 | await sleep(20); |
| 505 | check('while refused the gap has widened past the ordinary interval', w.health().gap > 3000 || w.health().throttled >= 1); |
| 506 | h.halt(); |
| 507 | const h2 = makeHost({ fastTimers: true, respond: () => 500 }); |
| 508 | const w2 = makeWriter(h2); |
| 509 | w2.event('a', {}); |
| 510 | await sleep(20); |
| 511 | h2.halt(); |
| 512 | check('a 500 is a post failure and not a throttle', w2.health().postFail >= 1 && w2.health().throttled === 0); |
| 513 | check('and the gap stays at the ordinary interval', w2.health().gap === 3000); |
| 514 | }); |
| 515 | |
| 516 | section('writer: what send() may answer'); |
| 517 | await part(async () => { |
| 518 | const bare = makeHost({ fastTimers: true, reply: 'bare' }); |
| 519 | const w = makeWriter(bare); |
| 520 | w.event('a', {}); |
| 521 | check('a send that resolves nothing counts as delivered, or the outbox would never empty', await until(() => w.depth() === 0)); |
| 522 | const th = makeHost({ fastTimers: true, reply: 'throw' }); |
| 523 | const w2 = makeWriter(th); |
| 524 | w2.event('a', {}); |
| 525 | await sleep(20); |
| 526 | th.halt(); |
| 527 | check('a send that rejects is a failure and the event stays', w2.depth() === 1 && w2.health().postFail >= 1); |
| 528 | const r1 = makeHost({ fastTimers: true }); |
| 529 | const w3 = makeWriter(r1, { send: (rows) => { r1.send(rows); return Promise.resolve({ status: 204 }); } }); |
| 530 | w3.event('a', {}); |
| 531 | check('a reply with only a status below 400 is delivered', await until(() => w3.depth() === 0)); |
| 532 | const r2 = makeHost({ fastTimers: true }); |
| 533 | const w4 = makeWriter(r2, { send: (rows) => { r2.send(rows); return Promise.resolve({ status: 503 }); } }); |
| 534 | w4.event('a', {}); |
| 535 | await sleep(20); |
| 536 | r2.halt(); |
| 537 | check('a reply with a status of 400 or more is not', w4.depth() === 1); |
| 538 | const calls = []; |
| 539 | const r3 = makeHost({ fastTimers: true }); |
| 540 | const w5 = makeWriter(r3, { send: (rows, info) => { calls.push(info); return r3.send(rows); }, caps: { gapMs: 2000 } }); |
| 541 | w5.event('a', {}); |
| 542 | await until(() => w5.depth() === 0); |
| 543 | check('send receives the device in its second argument', calls[0] && calls[0].device === w5.device()); |
| 544 | check('the pacing gap is the caller\'s: a socket drain may run at two seconds', w5.gapMs() === 2000); |
| 545 | check('postBody builds the body the Rust sink takes: {v:1,device,rows}', |
| 546 | JSON.stringify(JSON.parse(w5.postBody([{ ts: 1, tag: 't', data: 'd' }]))) === '{"v":1,"device":"devAAAA1111222233334444","rows":[{"ts":1,"tag":"t","data":"d"}]}'); |
| 547 | }); |
| 548 | |
| 549 | section('writer: nothing of an app in the core'); |
| 550 | await part(async () => { |
| 551 | // Code lines only: a comment may say where the code came from, the code may not name an app. |
| 552 | const code = (f) => readFileSync(join(HERE, '..', f), 'utf8').split('\n') |
| 553 | .filter((l) => !/^\s*(\/\/|\*|\/\*)/.test(l)).map((l) => l.replace(/\s\/\/ .*$/, '')).join('\n'); |
| 554 | check('no Daimond or Oxegen name in the code of the writer or the redactor', !/daimond|oxegen|DEBUG_SHARE/i.test(code('writer.js') + code('redact.js'))); |
| 555 | |
| 556 | // And no global it was not handed: every one of these throws on touch, and a full scenario runs. |
| 557 | const touched = []; |
| 558 | for (const name of ['localStorage', 'sessionStorage', 'fetch', 'document', 'window', 'XMLHttpRequest', 'navigator']) { |
| 559 | Object.defineProperty(globalThis, name, { configurable: true, get() { touched.push(name); throw new Error('touched ' + name); } }); |
| 560 | } |
| 561 | try { |
| 562 | const h = makeHost({ fastTimers: true }); |
| 563 | const w = makeWriter(h); |
| 564 | w.captureConsole(h.console); |
| 565 | w.captureErrors({ addEventListener() {} }); |
| 566 | w.captureLifecycle({ addEventListener() {} }, { visibilityState: 'visible' }); |
| 567 | w.event('a', { k: 1 }); |
| 568 | h.console.warn('line'); |
| 569 | w.noteError('e', 'w'); |
| 570 | w.noteFetchFail('/x', 500, 1, ''); |
| 571 | w.flushConsole(true); |
| 572 | await until(() => w.depth() === 0); |
| 573 | w.reset(); |
| 574 | } finally { |
| 575 | for (const name of ['localStorage', 'sessionStorage', 'fetch', 'document', 'window', 'XMLHttpRequest', 'navigator']) delete globalThis[name]; |
| 576 | } |
| 577 | check('a full scenario runs without touching localStorage, fetch, document, window or navigator', touched.length === 0, touched.join(',')); |
| 578 | }); |
| 579 | |
| 580 | } |
| 581 | main().then(() => finish('writer')).catch((e) => { console.error(e); process.exit(1); }); |