oxedyne/fe2o3/fe2o3_net/lens/js/test/redact.test.mjs
17.2 KiB, 3 runs
created by r1870400018:65628, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // redact.test.mjs -- the redactor and the content scrubber of `../redact.js`. |
| 2 | // |
| 3 | // Two halves. The first is the scrubber section of Daimond's `www/js/debugshare.test.mjs` |
| 4 | // (one fixture per credential shape, the entropy catch, what the feed is supposed to carry), |
| 5 | // lifted with only these changes: `DS._scrubText` and `DS._scrubDeep` became the scrubber |
| 6 | // below, and that scrubber is built the way Daimond would build it, naming its own session |
| 7 | // cookie and its own correlation-id fields as parameters (the core knows neither). |
| 8 | // The second half tests what the extraction added: the caller deny-list hook, the caller string |
| 9 | // test, the fingerprint head, and the name rule that `lens_secret_re.tsv` pins for Rust. |
| 10 | // |
| 11 | // node test/redact.test.mjs |
| 12 | import { |
| 13 | SECRET_RE, |
| 14 | SECRET_RE_LOOSE, |
| 15 | byteLen, |
| 16 | createRedactor, |
| 17 | createScrubber, |
| 18 | fingerprint, |
| 19 | scrubDeep as baseDeep, |
| 20 | scrubText as baseText, |
| 21 | secretName, |
| 22 | } from '../redact.js'; |
| 23 | import { check, finish, section } from './harness.mjs'; |
| 24 | |
| 25 | const NL = String.fromCharCode(10); |
| 26 | |
| 27 | // Daimond's own additions to the scrubber: the gateway's session cookie, and the fields whose |
| 28 | // values are ids it navigates by. |
| 29 | const D = createScrubber({ |
| 30 | pairs: [{ k: 'cookie', g: 2, re: /(daimond_gw_sess=)([^;,\s"'\[\]]{4,})/g }], |
| 31 | idKeys: { holder: 1, self: 1, nominated: 1, body: 1 }, |
| 32 | }); |
| 33 | const scrubText = D.text; |
| 34 | const scrubDeep = D.deep; |
| 35 | |
| 36 | function lifted() { |
| 37 | // ══ THE CONTENT SCRUBBER ══════════════════════════════════════════ |
| 38 | // |
| 39 | // The name-based redactors answer for `config`, whose fields are NAMED. These |
| 40 | // blocks answer the owner's general question -- "the debug feed does not |
| 41 | // contain keys" -- for everything else the feed carries, which is free text: a |
| 42 | // console line, the tile of the daimon's own answer, a failed fetch's URL, a |
| 43 | // tool argument, a stack frame. Every fixture below is INVENTED here and wears |
| 44 | // only the SHAPE of a real credential; none has ever been a live key, and each |
| 45 | // is BUILT from a seed rather than typed out, so no line of this file is a |
| 46 | // paste-able credential and a scanner reading the repository finds none. |
| 47 | |
| 48 | // A deterministic run of characters from a seed. Not random: a fixture that |
| 49 | // changes between runs cannot be asserted about. |
| 50 | const fakeRun = (n, seed, alpha) => { |
| 51 | alpha = alpha || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; |
| 52 | let s = '', h = (seed * 2654435761) >>> 0; |
| 53 | for (let i = 0; i < n; i++) { h = (h * 1103515245 + 12345) >>> 0; s += alpha[(h >>> 8) % alpha.length]; } |
| 54 | return s; |
| 55 | }; |
| 56 | const fakeHex = (n, seed) => fakeRun(n, seed, '0123456789abcdef'); |
| 57 | const fakeB64 = (n, seed) => fakeRun(n, seed, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'); |
| 58 | |
| 59 | console.log('debugshare: scrubber — one fixture per credential shape, none survives'); |
| 60 | { |
| 61 | |
| 62 | // One entry per shape the scrubber is asked to know. |
| 63 | const CORPUS = [ |
| 64 | ['github classic', 'ghp_' + fakeRun(36, 11)], |
| 65 | ['github oauth', 'gho_' + fakeRun(36, 12)], |
| 66 | ['github user', 'ghu_' + fakeRun(36, 13)], |
| 67 | ['github server', 'ghs_' + fakeRun(36, 14)], |
| 68 | ['github refresh', 'ghr_' + fakeRun(36, 15)], |
| 69 | ['github fine PAT', 'github_pat_' + fakeRun(22, 16) + '_' + fakeRun(59, 17)], |
| 70 | ['openai', 'sk-' + fakeRun(48, 21)], |
| 71 | ['openrouter', 'sk-or-v1-' + fakeHex(64, 22)], |
| 72 | ['anthropic', 'sk-ant-api03-' + fakeB64(95, 23)], |
| 73 | ['stripe live', 'sk_live_' + fakeRun(24, 31)], |
| 74 | ['stripe test', 'sk_test_' + fakeRun(24, 32)], |
| 75 | ['stripe restricted', 'rk_live_' + fakeRun(24, 33)], |
| 76 | ['stripe webhook', 'whsec_' + fakeRun(32, 34)], |
| 77 | ['aws access key', 'AKIA' + fakeRun(16, 41).toUpperCase()], |
| 78 | ['google api key', 'AIza' + fakeB64(35, 51)], |
| 79 | ['slack bot', 'xoxb-' + fakeRun(12, 61) + '-' + fakeRun(12, 62) + '-' + fakeRun(24, 63)], |
| 80 | ['jwt', 'eyJ' + fakeB64(28, 71) + '.eyJ' + fakeB64(40, 72) + '.' + fakeB64(43, 73)], |
| 81 | ['tune relay run token', 'tune-' + fakeHex(32, 81)], |
| 82 | ['high-entropy blob', fakeB64(48, 91)], |
| 83 | ]; |
| 84 | const survived = []; |
| 85 | for (const [name, val] of CORPUS) { |
| 86 | if (scrubText('the call failed: ' + val + ' was refused').indexOf(val) !== -1) survived.push(name); |
| 87 | } |
| 88 | if (survived.length) console.log(' survived: ' + survived.join(', ')); |
| 89 | check('every enumerated shape is replaced, not passed through', survived.length === 0); |
| 90 | |
| 91 | // The three whose shape only exists NEXT TO something else. |
| 92 | const AWS_ID = 'AKIA' + fakeRun(16, 42).toUpperCase(); |
| 93 | const AWS_SEC = fakeRun(40, 43); |
| 94 | const PEM_BODY = fakeB64(64, 101); |
| 95 | // The PEM ARMOUR only -- the body between the lines is generated above and |
| 96 | // is not key material. allowlist secret |
| 97 | const PEM = '-----BEGIN RSA PRIVATE KEY-----' + NL + PEM_BODY + NL + '-----END RSA PRIVATE KEY-----'; |
| 98 | check('a PEM private key block goes whole', |
| 99 | scrubText('key material:' + NL + PEM + NL + 'done').indexOf(PEM_BODY) === -1); |
| 100 | check('a PEM block clipped by elision still goes', // allowlist secret |
| 101 | scrubText('-----BEGIN EC PRIVATE KEY-----' + NL + fakeB64(64, 103)).indexOf(fakeB64(64, 103)) === -1); |
| 102 | check('an AWS secret beside its access-key id goes', |
| 103 | scrubText('creds: ' + AWS_ID + ', "' + AWS_SEC + '"').indexOf(AWS_SEC) === -1); |
| 104 | check('an AWS secret beside its own field name goes', |
| 105 | scrubText('aws_secret_access_key = ' + AWS_SEC).indexOf(AWS_SEC) === -1); |
| 106 | |
| 107 | // The marker: the shape, a stable hash and the length -- so two sightings of |
| 108 | // one key are known to be one key, and nothing more. |
| 109 | const GH = 'ghp_' + fakeRun(36, 11); |
| 110 | check('the marker keeps the redactConfig style', |
| 111 | /^\[redacted gh #[0-9a-f]+\/40\]$/.test(scrubText(GH))); |
| 112 | check('the same value marks identically in two places', |
| 113 | scrubText('a ' + GH) === 'a ' + scrubText('b ' + GH).slice(2)); |
| 114 | check('two different values mark differently', |
| 115 | scrubText(GH) !== scrubText('ghp_' + fakeRun(36, 12))); |
| 116 | |
| 117 | // The name-against-a-value rules, which keep the LINE and lose the VALUE -- |
| 118 | // the thing fingerprinting a whole console message could not do. |
| 119 | const BEAR = fakeRun(40, 111); |
| 120 | const line = scrubText('[gw] Authorization: Bearer ' + BEAR + ' rejected'); |
| 121 | check('a bearer token goes and the line survives', |
| 122 | line.indexOf(BEAR) === -1 && line.indexOf('[gw]') === 0 && line.indexOf('rejected') > 0); |
| 123 | const TOK = fakeRun(30, 112); |
| 124 | const url = scrubText('POST /api/sync?access_token=' + TOK + '&v=2 failed 401'); |
| 125 | check('a token in a URL query goes and the URL survives', |
| 126 | url.indexOf(TOK) === -1 && url.indexOf('POST /api/sync?access_token=') === 0 |
| 127 | && url.indexOf('&v=2 failed 401') > 0); |
| 128 | check('an `&key=` argument goes too', |
| 129 | scrubText('GET /x?a=1&key=' + fakeRun(30, 113)).indexOf(fakeRun(30, 113)) === -1); |
| 130 | check('the gateway session cookie goes', |
| 131 | scrubText('Cookie: daimond_gw_sess=' + fakeRun(43, 114) + '; Path=/').indexOf(fakeRun(43, 114)) === -1); |
| 132 | check('a secret name sitting against a value goes', |
| 133 | scrubText('config apiKey="' + fakeRun(30, 115) + '" loaded').indexOf(fakeRun(30, 115)) === -1); |
| 134 | |
| 135 | // IDEMPOTENCE, which is not a nicety here: `dev/lens.mjs` runs this same |
| 136 | // block AGAIN on ingest, over output the client has already scrubbed. A rule |
| 137 | // whose replacement its own pattern can match nests its markers one deep per |
| 138 | // pass -- which the `?token=` rule did, until `[` and `]` were taken out of |
| 139 | // its value class. |
| 140 | const twice = [ |
| 141 | 'POST /api/sync?access_token=' + fakeRun(30, 121) + ' failed', |
| 142 | 'Cookie: daimond_gw_sess=' + fakeRun(43, 122) + '; Path=/', |
| 143 | 'Authorization: Bearer ' + fakeRun(40, 123), |
| 144 | 'ghp_' + fakeRun(36, 124), |
| 145 | 'apiKey=' + fakeRun(30, 125), |
| 146 | 'blob ' + fakeB64(48, 126) + ' end', |
| 147 | ]; |
| 148 | const nested = twice.filter((t) => scrubText(scrubText(t)) !== scrubText(t)); |
| 149 | check('scrubbing twice is scrubbing once, for every rule', nested.length === 0); |
| 150 | } |
| 151 | |
| 152 | console.log('debugshare: scrubber — what the feed is SUPPOSED to carry still reads'); |
| 153 | { |
| 154 | // The legibility half. A scrubber that eats the archive's own index is no |
| 155 | // use, so every shape the reader navigates by is whitelisted BY SHAPE. |
| 156 | const KEEP = [ |
| 157 | ['a sha256 digest', '3b1f8c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c'], |
| 158 | ['a build id', 'dae1ed646c3f'], |
| 159 | ['a device id', 'a7b34e2d5181e710301a30ebbc3ee062'], |
| 160 | ['a UUID', '0a1b2c3d-4e5f-6071-8293-a4b5c6d7e8f9'], |
| 161 | ['a message id', 'mgk3j2a-1f-x7q2p'], |
| 162 | ['a model id', 'anthropic/claude-sonnet-4-5-20250929'], |
| 163 | ['a chunk tag', 'ds snapshot smtxxyvgy988 41/2377'], |
| 164 | ['a sentence', '[sync] chunk index not merged on this device; refusing to commit'], |
| 165 | ['a constant', 'TRANSCRIPT_BUDGET_BYTES exceeded by 41 bytes'], |
| 166 | ['a worded id', 'worker_pool_seat_000412'], |
| 167 | ['a stack frame', 'sync.js:1511'], |
| 168 | ['a long decimal', '178930080399617893008039961789300803996'], |
| 169 | ]; |
| 170 | const lost = []; |
| 171 | for (const [name, val] of KEEP) if (scrubText(val) !== val) lost.push(name); |
| 172 | if (lost.length) console.log(' lost: ' + lost.join(', ')); |
| 173 | check('every legible shape survives untouched', lost.length === 0); |
| 174 | |
| 175 | // And the correlation-id FIELDS, whose values a provider mints out of an |
| 176 | // alphabet a key would be at home in. |
| 177 | const callId = 'call_00_AbC9dEf1GhI2jKl3MnO4pQr5'; |
| 178 | const walked = scrubDeep({ id: callId, tool_call_id: callId, callId: callId, |
| 179 | turn: 'smtxxyvgy988', notAnId: callId }); |
| 180 | check('a tool-call id survives in an id field', |
| 181 | walked.id === callId && walked.tool_call_id === callId && walked.callId === callId); |
| 182 | check('but the same run in an ordinary field does not', walked.notAnId !== callId); |
| 183 | check('and an id field holding a real key shape is STILL scrubbed', |
| 184 | scrubDeep({ id: 'ghp_' + fakeRun(36, 11) }).id.indexOf(fakeRun(36, 11)) === -1); |
| 185 | } |
| 186 | } |
| 187 | lifted(); |
| 188 | |
| 189 | // ── What the extraction added ──────────────────────────────────────── |
| 190 | |
| 191 | section('redact: the core carries nothing of the app that lent it'); |
| 192 | { |
| 193 | check('the base scrubber does not know Daimond\'s session cookie', |
| 194 | baseText('cookie: daimond_gw_sess=abcd1234efgh') === 'cookie: daimond_gw_sess=abcd1234efgh'); |
| 195 | check('the configured scrubber does', D.text('cookie: daimond_gw_sess=abcd1234efgh').indexOf('abcd1234efgh') === -1); |
| 196 | check('the base scrubber still takes a key shape', baseText('x ghp_' + 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8') .indexOf('A1b2C3d4') === -1); |
| 197 | check('the base and a default-built scrubber agree', |
| 198 | createScrubber().text('Bearer abcdefghijkl') === baseText('Bearer abcdefghijkl')); |
| 199 | check('the base deep walk copies and does not mutate', |
| 200 | (() => { const o = { a: ['sk-' + 'x'.repeat(20) + 'Q9'] }; const c = baseDeep(o); return c !== o && o.a[0].startsWith('sk-x'); })()); |
| 201 | } |
| 202 | |
| 203 | section('redact: the name rule, strict and loose'); |
| 204 | { |
| 205 | check('apikey, key, token are strict hits', ['apikey', 'key', 'token', 'x_secret', 'a.password', 'b-salt'].every((n) => SECRET_RE.test(n))); |
| 206 | check('a separator-free camelCase name slips the strict rule', !SECRET_RE.test('pushToken') && !SECRET_RE.test('refreshSecret')); |
| 207 | check('and the loose rule catches it', SECRET_RE_LOOSE.test('pushToken') && SECRET_RE_LOOSE.test('refreshSecret')); |
| 208 | check('keys and monkey are not secrets', !SECRET_RE.test('keys') && !SECRET_RE.test('monkey')); |
| 209 | check('secretName(name) is the strict rule, secretName(name, true) the strict-or-loose', |
| 210 | secretName('apikeyenc') === true && secretName('pushToken') === false && secretName('pushToken', true) === true); |
| 211 | check('an "enc" tail counts only at a word edge', secretName('keyenc') === true && secretName('keyencx') === false); |
| 212 | } |
| 213 | |
| 214 | section('redact: fingerprints'); |
| 215 | { |
| 216 | check('empty, absent and typed markers', fingerprint('') === '[redacted:empty]' && fingerprint(null) === '[redacted:absent]' |
| 217 | && fingerprint(undefined) === '[redacted:absent]'); |
| 218 | check('a number is fingerprinted as its text', fingerprint(12345) === '[redacted 12345…#' + (function () { |
| 219 | var h = 5381; '12345'.split('').forEach(function (c) { h = ((h << 5) + h + c.charCodeAt(0)) >>> 0; }); return h.toString(16); })() + '/5]'); |
| 220 | check('the default head is six characters', /^\[redacted abcdef…#[0-9a-f]+\/9\]$/.test(fingerprint('abcdefghi'))); |
| 221 | check('a head of zero shows no character of the value', /^\[redacted …#[0-9a-f]+\/9\]$/.test(fingerprint('abcdefghi', 0))); |
| 222 | check('the hash is the same whatever the head, so two sightings still match', |
| 223 | fingerprint('abcdefghi', 0).split('#')[1] === fingerprint('abcdefghi', 6).split('#')[1]); |
| 224 | check('two different values differ', fingerprint('aaaaaaa') !== fingerprint('aaaaaab')); |
| 225 | } |
| 226 | |
| 227 | section('redact: the walk, the deny-list hook and the string test'); |
| 228 | { |
| 229 | const R = createRedactor(); |
| 230 | const src = { name: 'ok', apiKey: 'sk-or-v1-abcdefghij', cfg: { token: 'T0K3Nvalue', n: 3 }, |
| 231 | list: [{ salt: 'saltsalt' }, 'plain'], wrapped: { a: 1 } }; |
| 232 | const out = R.value(src); |
| 233 | check('a secret-named scalar becomes a fingerprint', /^\[redacted sk-or-…#[0-9a-f]+\/19\]$/.test(out.apiKey)); |
| 234 | check('a nested one too, and the sibling is left alone', /^\[redacted T0K3Nv/.test(out.cfg.token) && out.cfg.n === 3); |
| 235 | check('an array is walked', /^\[redacted /.test(out.list[0].salt) && out.list[1] === 'plain'); |
| 236 | check('a secret-named object is noted as present, not walked', out.wrapped === '[redacted:object]'); |
| 237 | check('the source is not mutated', src.apiKey === 'sk-or-v1-abcdefghij' && src.cfg.token === 'T0K3Nvalue'); |
| 238 | check('an already-redacted marker in a secret field is not fingerprinted again', |
| 239 | R.value({ apiKey: out.apiKey }).apiKey === out.apiKey); |
| 240 | |
| 241 | const cyc = { a: 1 }; cyc.self = cyc; |
| 242 | check('a cycle ends in a marker, not a stack overflow', R.value(cyc).self === '[redacted:cycle-or-deep]'); |
| 243 | let deep = { x: 1 }; for (let i = 0; i < 60; i++) deep = { d: deep }; |
| 244 | check('past forty levels the walk ends in a marker', JSON.stringify(R.value(deep)).indexOf('[redacted:cycle-or-deep]') !== -1); |
| 245 | |
| 246 | const L = createRedactor({ loose: true }); |
| 247 | check('loose catches camelCase names', /^\[redacted /.test(L.value({ pushToken: 'abcdefgh' }).pushToken) |
| 248 | && R.value({ pushToken: 'abcdefgh' }).pushToken === 'abcdefgh'); |
| 249 | |
| 250 | const N = createRedactor({ deny: ['pass'] }); |
| 251 | check('a deny-listed NAME is covered at any depth', |
| 252 | /^\[redacted /.test(N.value({ a: { b: { pass: 'hunter22' } } }).a.b.pass) && N.value({ passage: 'x' }).passage === 'x'); |
| 253 | const P = createRedactor({ deny: [['user', '*', 'phrase']] }); |
| 254 | const pp = P.value({ user: { u1: { phrase: 'correct horse', other: 'k' }, u2: 'z' }, phrase: 'top-level stays' }); |
| 255 | check('a deny-listed PATH is covered, with * for any one name', /^\[redacted /.test(pp.user.u1.phrase) && pp.user.u1.other === 'k'); |
| 256 | check('and the same name off the path is not', pp.phrase === 'top-level stays'); |
| 257 | check('a path of the wrong length does not match', P.value({ user: { phrase: 'x' } }).user.phrase === 'x'); |
| 258 | |
| 259 | const eight = /^(?:[a-z]+ ){7}[a-z]+$/; |
| 260 | const T = createRedactor({ test: (s) => eight.test(s) }); |
| 261 | const tt = T.value({ note: 'one two three four five six seven eight', short: 'one two', list: ['a b c d e f g h'] }); |
| 262 | check('a string the caller\'s test calls secret is fingerprinted whole', /^\[redacted one tw/.test(tt.note)); |
| 263 | check('a string it does not is left', tt.short === 'one two'); |
| 264 | check('the test sees strings inside arrays', /^\[redacted a b c/.test(tt.list[0])); |
| 265 | check('the test sees keys too', /^\[redacted /.test(Object.keys(T.value({ ['one two three four five six seven eight']: 1 }))[0])); |
| 266 | check('a marker is never tested again', T.text('[redacted x…#1/1]') === null); |
| 267 | check('text() answers a fingerprint for a hit and null for a miss', |
| 268 | /^\[redacted /.test(T.text('one two three four five six seven eight')) && T.text('fine') === null); |
| 269 | check('head applies to the string test as well', |
| 270 | createRedactor({ test: (s) => s === 'abcdef', head: 0 }).value({ a: 'abcdef' }).a.startsWith('[redacted …#')); |
| 271 | check('a byte length counts UTF-8, not characters', byteLen('é') === 2 && byteLen('abc') === 3); |
| 272 | } |
| 273 | |
| 274 | section('redact: SCRUB-ONE, a typed password field is marked whole, with no head'); |
| 275 | { |
| 276 | // Mail posts a mailbox's app password as `password` and keeps it sealed as `pass`; a passphrase's |
| 277 | // first six characters are its first word. Such a field keeps nothing of its value (D-20261006-24). |
| 278 | const APP = 'abcd efgh ijkl mnop', PHRASE = 'correct horse battery staple'; |
| 279 | const posted = scrubDeep({ address: 'me@x.org', user: 'me', password: APP, pass: APP, smtpPassword: APP, |
| 280 | newPassphrase: PHRASE, passAt: 1789300000000, passwordHint: 'the usual', bypass: 'on' }); |
| 281 | const wire = JSON.stringify(posted); |
| 282 | check('ARM mailbox: a password-named field goes whole in the scrubber\'s walk', wire.indexOf('efgh') === -1 && wire.indexOf('horse') === -1, wire); |
| 283 | check('ARM mailbox: as a mark with no head', /^\[redacted named #[0-9a-f]+\/19\]$/.test(String(posted.pass)), String(posted.pass)); |
| 284 | check('ARM mailbox: and the fields beside it still read', posted.address === 'me@x.org' && posted.user === 'me' |
| 285 | && posted.passAt === 1789300000000 && posted.passwordHint === 'the usual' && posted.bypass === 'on'); |
| 286 | const red = createRedactor().value({ password: PHRASE, pass: APP, smtpPassword: APP, seed_phrase: PHRASE, apiKey: 'sk-or-v1-abcdefghij' }); |
| 287 | const rw = JSON.stringify(red); |
| 288 | check('ARM name walk: the redactor keeps no head of a typed field', rw.indexOf('correc') === -1 && rw.indexOf('abcd') === -1, rw); |
| 289 | check('ARM name walk: a key keeps its six', red.apiKey.indexOf('[redacted sk-or-…#') === 0, red.apiKey); |
| 290 | check('ARM name walk: and a second walk changes nothing', JSON.stringify(createRedactor().value(red)) === rw); |
| 291 | } |
| 292 | |
| 293 | finish('redact'); |