Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/lens/js/test/redact.test.mjs

17.2 KiB, 3 runs

created by r1870400018:65628, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// redact.test.mjs -- the redactor and the content scrubber of `../redact.js`.
2//
3// Two halves. The first is the scrubber section of Daimond's `www/js/debugshare.test.mjs`
4// (one fixture per credential shape, the entropy catch, what the feed is supposed to carry),
5// lifted with only these changes: `DS._scrubText` and `DS._scrubDeep` became the scrubber
6// below, and that scrubber is built the way Daimond would build it, naming its own session
7// cookie and its own correlation-id fields as parameters (the core knows neither).
8// The second half tests what the extraction added: the caller deny-list hook, the caller string
9// test, the fingerprint head, and the name rule that `lens_secret_re.tsv` pins for Rust.
10//
11// node test/redact.test.mjs
12import {
13 SECRET_RE,
14 SECRET_RE_LOOSE,
15 byteLen,
16 createRedactor,
17 createScrubber,
18 fingerprint,
19 scrubDeep as baseDeep,
20 scrubText as baseText,
21 secretName,
22} from '../redact.js';
23import { check, finish, section } from './harness.mjs';
24
25const NL = String.fromCharCode(10);
26
27// Daimond's own additions to the scrubber: the gateway's session cookie, and the fields whose
28// values are ids it navigates by.
29const D = createScrubber({
30 pairs: [{ k: 'cookie', g: 2, re: /(daimond_gw_sess=)([^;,\s"'\[\]]{4,})/g }],
31 idKeys: { holder: 1, self: 1, nominated: 1, body: 1 },
32});
33const scrubText = D.text;
34const scrubDeep = D.deep;
35
36function lifted() {
37 // ══ THE CONTENT SCRUBBER ══════════════════════════════════════════
38 //
39 // The name-based redactors answer for `config`, whose fields are NAMED. These
40 // blocks answer the owner's general question -- "the debug feed does not
41 // contain keys" -- for everything else the feed carries, which is free text: a
42 // console line, the tile of the daimon's own answer, a failed fetch's URL, a
43 // tool argument, a stack frame. Every fixture below is INVENTED here and wears
44 // only the SHAPE of a real credential; none has ever been a live key, and each
45 // is BUILT from a seed rather than typed out, so no line of this file is a
46 // paste-able credential and a scanner reading the repository finds none.
47
48 // A deterministic run of characters from a seed. Not random: a fixture that
49 // changes between runs cannot be asserted about.
50 const fakeRun = (n, seed, alpha) => {
51 alpha = alpha || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
52 let s = '', h = (seed * 2654435761) >>> 0;
53 for (let i = 0; i < n; i++) { h = (h * 1103515245 + 12345) >>> 0; s += alpha[(h >>> 8) % alpha.length]; }
54 return s;
55 };
56 const fakeHex = (n, seed) => fakeRun(n, seed, '0123456789abcdef');
57 const fakeB64 = (n, seed) => fakeRun(n, seed, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_');
58
59 console.log('debugshare: scrubber — one fixture per credential shape, none survives');
60 {
61
62 // One entry per shape the scrubber is asked to know.
63 const CORPUS = [
64 ['github classic', 'ghp_' + fakeRun(36, 11)],
65 ['github oauth', 'gho_' + fakeRun(36, 12)],
66 ['github user', 'ghu_' + fakeRun(36, 13)],
67 ['github server', 'ghs_' + fakeRun(36, 14)],
68 ['github refresh', 'ghr_' + fakeRun(36, 15)],
69 ['github fine PAT', 'github_pat_' + fakeRun(22, 16) + '_' + fakeRun(59, 17)],
70 ['openai', 'sk-' + fakeRun(48, 21)],
71 ['openrouter', 'sk-or-v1-' + fakeHex(64, 22)],
72 ['anthropic', 'sk-ant-api03-' + fakeB64(95, 23)],
73 ['stripe live', 'sk_live_' + fakeRun(24, 31)],
74 ['stripe test', 'sk_test_' + fakeRun(24, 32)],
75 ['stripe restricted', 'rk_live_' + fakeRun(24, 33)],
76 ['stripe webhook', 'whsec_' + fakeRun(32, 34)],
77 ['aws access key', 'AKIA' + fakeRun(16, 41).toUpperCase()],
78 ['google api key', 'AIza' + fakeB64(35, 51)],
79 ['slack bot', 'xoxb-' + fakeRun(12, 61) + '-' + fakeRun(12, 62) + '-' + fakeRun(24, 63)],
80 ['jwt', 'eyJ' + fakeB64(28, 71) + '.eyJ' + fakeB64(40, 72) + '.' + fakeB64(43, 73)],
81 ['tune relay run token', 'tune-' + fakeHex(32, 81)],
82 ['high-entropy blob', fakeB64(48, 91)],
83 ];
84 const survived = [];
85 for (const [name, val] of CORPUS) {
86 if (scrubText('the call failed: ' + val + ' was refused').indexOf(val) !== -1) survived.push(name);
87 }
88 if (survived.length) console.log(' survived: ' + survived.join(', '));
89 check('every enumerated shape is replaced, not passed through', survived.length === 0);
90
91 // The three whose shape only exists NEXT TO something else.
92 const AWS_ID = 'AKIA' + fakeRun(16, 42).toUpperCase();
93 const AWS_SEC = fakeRun(40, 43);
94 const PEM_BODY = fakeB64(64, 101);
95 // The PEM ARMOUR only -- the body between the lines is generated above and
96 // is not key material. allowlist secret
97 const PEM = '-----BEGIN RSA PRIVATE KEY-----' + NL + PEM_BODY + NL + '-----END RSA PRIVATE KEY-----';
98 check('a PEM private key block goes whole',
99 scrubText('key material:' + NL + PEM + NL + 'done').indexOf(PEM_BODY) === -1);
100 check('a PEM block clipped by elision still goes', // allowlist secret
101 scrubText('-----BEGIN EC PRIVATE KEY-----' + NL + fakeB64(64, 103)).indexOf(fakeB64(64, 103)) === -1);
102 check('an AWS secret beside its access-key id goes',
103 scrubText('creds: ' + AWS_ID + ', "' + AWS_SEC + '"').indexOf(AWS_SEC) === -1);
104 check('an AWS secret beside its own field name goes',
105 scrubText('aws_secret_access_key = ' + AWS_SEC).indexOf(AWS_SEC) === -1);
106
107 // The marker: the shape, a stable hash and the length -- so two sightings of
108 // one key are known to be one key, and nothing more.
109 const GH = 'ghp_' + fakeRun(36, 11);
110 check('the marker keeps the redactConfig style',
111 /^\[redacted gh #[0-9a-f]+\/40\]$/.test(scrubText(GH)));
112 check('the same value marks identically in two places',
113 scrubText('a ' + GH) === 'a ' + scrubText('b ' + GH).slice(2));
114 check('two different values mark differently',
115 scrubText(GH) !== scrubText('ghp_' + fakeRun(36, 12)));
116
117 // The name-against-a-value rules, which keep the LINE and lose the VALUE --
118 // the thing fingerprinting a whole console message could not do.
119 const BEAR = fakeRun(40, 111);
120 const line = scrubText('[gw] Authorization: Bearer ' + BEAR + ' rejected');
121 check('a bearer token goes and the line survives',
122 line.indexOf(BEAR) === -1 && line.indexOf('[gw]') === 0 && line.indexOf('rejected') > 0);
123 const TOK = fakeRun(30, 112);
124 const url = scrubText('POST /api/sync?access_token=' + TOK + '&v=2 failed 401');
125 check('a token in a URL query goes and the URL survives',
126 url.indexOf(TOK) === -1 && url.indexOf('POST /api/sync?access_token=') === 0
127 && url.indexOf('&v=2 failed 401') > 0);
128 check('an `&key=` argument goes too',
129 scrubText('GET /x?a=1&key=' + fakeRun(30, 113)).indexOf(fakeRun(30, 113)) === -1);
130 check('the gateway session cookie goes',
131 scrubText('Cookie: daimond_gw_sess=' + fakeRun(43, 114) + '; Path=/').indexOf(fakeRun(43, 114)) === -1);
132 check('a secret name sitting against a value goes',
133 scrubText('config apiKey="' + fakeRun(30, 115) + '" loaded').indexOf(fakeRun(30, 115)) === -1);
134
135 // IDEMPOTENCE, which is not a nicety here: `dev/lens.mjs` runs this same
136 // block AGAIN on ingest, over output the client has already scrubbed. A rule
137 // whose replacement its own pattern can match nests its markers one deep per
138 // pass -- which the `?token=` rule did, until `[` and `]` were taken out of
139 // its value class.
140 const twice = [
141 'POST /api/sync?access_token=' + fakeRun(30, 121) + ' failed',
142 'Cookie: daimond_gw_sess=' + fakeRun(43, 122) + '; Path=/',
143 'Authorization: Bearer ' + fakeRun(40, 123),
144 'ghp_' + fakeRun(36, 124),
145 'apiKey=' + fakeRun(30, 125),
146 'blob ' + fakeB64(48, 126) + ' end',
147 ];
148 const nested = twice.filter((t) => scrubText(scrubText(t)) !== scrubText(t));
149 check('scrubbing twice is scrubbing once, for every rule', nested.length === 0);
150 }
151
152 console.log('debugshare: scrubber — what the feed is SUPPOSED to carry still reads');
153 {
154 // The legibility half. A scrubber that eats the archive's own index is no
155 // use, so every shape the reader navigates by is whitelisted BY SHAPE.
156 const KEEP = [
157 ['a sha256 digest', '3b1f8c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c'],
158 ['a build id', 'dae1ed646c3f'],
159 ['a device id', 'a7b34e2d5181e710301a30ebbc3ee062'],
160 ['a UUID', '0a1b2c3d-4e5f-6071-8293-a4b5c6d7e8f9'],
161 ['a message id', 'mgk3j2a-1f-x7q2p'],
162 ['a model id', 'anthropic/claude-sonnet-4-5-20250929'],
163 ['a chunk tag', 'ds snapshot smtxxyvgy988 41/2377'],
164 ['a sentence', '[sync] chunk index not merged on this device; refusing to commit'],
165 ['a constant', 'TRANSCRIPT_BUDGET_BYTES exceeded by 41 bytes'],
166 ['a worded id', 'worker_pool_seat_000412'],
167 ['a stack frame', 'sync.js:1511'],
168 ['a long decimal', '178930080399617893008039961789300803996'],
169 ];
170 const lost = [];
171 for (const [name, val] of KEEP) if (scrubText(val) !== val) lost.push(name);
172 if (lost.length) console.log(' lost: ' + lost.join(', '));
173 check('every legible shape survives untouched', lost.length === 0);
174
175 // And the correlation-id FIELDS, whose values a provider mints out of an
176 // alphabet a key would be at home in.
177 const callId = 'call_00_AbC9dEf1GhI2jKl3MnO4pQr5';
178 const walked = scrubDeep({ id: callId, tool_call_id: callId, callId: callId,
179 turn: 'smtxxyvgy988', notAnId: callId });
180 check('a tool-call id survives in an id field',
181 walked.id === callId && walked.tool_call_id === callId && walked.callId === callId);
182 check('but the same run in an ordinary field does not', walked.notAnId !== callId);
183 check('and an id field holding a real key shape is STILL scrubbed',
184 scrubDeep({ id: 'ghp_' + fakeRun(36, 11) }).id.indexOf(fakeRun(36, 11)) === -1);
185 }
186}
187lifted();
188
189// ── What the extraction added ────────────────────────────────────────
190
191section('redact: the core carries nothing of the app that lent it');
192{
193 check('the base scrubber does not know Daimond\'s session cookie',
194 baseText('cookie: daimond_gw_sess=abcd1234efgh') === 'cookie: daimond_gw_sess=abcd1234efgh');
195 check('the configured scrubber does', D.text('cookie: daimond_gw_sess=abcd1234efgh').indexOf('abcd1234efgh') === -1);
196 check('the base scrubber still takes a key shape', baseText('x ghp_' + 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8') .indexOf('A1b2C3d4') === -1);
197 check('the base and a default-built scrubber agree',
198 createScrubber().text('Bearer abcdefghijkl') === baseText('Bearer abcdefghijkl'));
199 check('the base deep walk copies and does not mutate',
200 (() => { const o = { a: ['sk-' + 'x'.repeat(20) + 'Q9'] }; const c = baseDeep(o); return c !== o && o.a[0].startsWith('sk-x'); })());
201}
202
203section('redact: the name rule, strict and loose');
204{
205 check('apikey, key, token are strict hits', ['apikey', 'key', 'token', 'x_secret', 'a.password', 'b-salt'].every((n) => SECRET_RE.test(n)));
206 check('a separator-free camelCase name slips the strict rule', !SECRET_RE.test('pushToken') && !SECRET_RE.test('refreshSecret'));
207 check('and the loose rule catches it', SECRET_RE_LOOSE.test('pushToken') && SECRET_RE_LOOSE.test('refreshSecret'));
208 check('keys and monkey are not secrets', !SECRET_RE.test('keys') && !SECRET_RE.test('monkey'));
209 check('secretName(name) is the strict rule, secretName(name, true) the strict-or-loose',
210 secretName('apikeyenc') === true && secretName('pushToken') === false && secretName('pushToken', true) === true);
211 check('an "enc" tail counts only at a word edge', secretName('keyenc') === true && secretName('keyencx') === false);
212}
213
214section('redact: fingerprints');
215{
216 check('empty, absent and typed markers', fingerprint('') === '[redacted:empty]' && fingerprint(null) === '[redacted:absent]'
217 && fingerprint(undefined) === '[redacted:absent]');
218 check('a number is fingerprinted as its text', fingerprint(12345) === '[redacted 12345…#' + (function () {
219 var h = 5381; '12345'.split('').forEach(function (c) { h = ((h << 5) + h + c.charCodeAt(0)) >>> 0; }); return h.toString(16); })() + '/5]');
220 check('the default head is six characters', /^\[redacted abcdef…#[0-9a-f]+\/9\]$/.test(fingerprint('abcdefghi')));
221 check('a head of zero shows no character of the value', /^\[redacted …#[0-9a-f]+\/9\]$/.test(fingerprint('abcdefghi', 0)));
222 check('the hash is the same whatever the head, so two sightings still match',
223 fingerprint('abcdefghi', 0).split('#')[1] === fingerprint('abcdefghi', 6).split('#')[1]);
224 check('two different values differ', fingerprint('aaaaaaa') !== fingerprint('aaaaaab'));
225}
226
227section('redact: the walk, the deny-list hook and the string test');
228{
229 const R = createRedactor();
230 const src = { name: 'ok', apiKey: 'sk-or-v1-abcdefghij', cfg: { token: 'T0K3Nvalue', n: 3 },
231 list: [{ salt: 'saltsalt' }, 'plain'], wrapped: { a: 1 } };
232 const out = R.value(src);
233 check('a secret-named scalar becomes a fingerprint', /^\[redacted sk-or-…#[0-9a-f]+\/19\]$/.test(out.apiKey));
234 check('a nested one too, and the sibling is left alone', /^\[redacted T0K3Nv/.test(out.cfg.token) && out.cfg.n === 3);
235 check('an array is walked', /^\[redacted /.test(out.list[0].salt) && out.list[1] === 'plain');
236 check('a secret-named object is noted as present, not walked', out.wrapped === '[redacted:object]');
237 check('the source is not mutated', src.apiKey === 'sk-or-v1-abcdefghij' && src.cfg.token === 'T0K3Nvalue');
238 check('an already-redacted marker in a secret field is not fingerprinted again',
239 R.value({ apiKey: out.apiKey }).apiKey === out.apiKey);
240
241 const cyc = { a: 1 }; cyc.self = cyc;
242 check('a cycle ends in a marker, not a stack overflow', R.value(cyc).self === '[redacted:cycle-or-deep]');
243 let deep = { x: 1 }; for (let i = 0; i < 60; i++) deep = { d: deep };
244 check('past forty levels the walk ends in a marker', JSON.stringify(R.value(deep)).indexOf('[redacted:cycle-or-deep]') !== -1);
245
246 const L = createRedactor({ loose: true });
247 check('loose catches camelCase names', /^\[redacted /.test(L.value({ pushToken: 'abcdefgh' }).pushToken)
248 && R.value({ pushToken: 'abcdefgh' }).pushToken === 'abcdefgh');
249
250 const N = createRedactor({ deny: ['pass'] });
251 check('a deny-listed NAME is covered at any depth',
252 /^\[redacted /.test(N.value({ a: { b: { pass: 'hunter22' } } }).a.b.pass) && N.value({ passage: 'x' }).passage === 'x');
253 const P = createRedactor({ deny: [['user', '*', 'phrase']] });
254 const pp = P.value({ user: { u1: { phrase: 'correct horse', other: 'k' }, u2: 'z' }, phrase: 'top-level stays' });
255 check('a deny-listed PATH is covered, with * for any one name', /^\[redacted /.test(pp.user.u1.phrase) && pp.user.u1.other === 'k');
256 check('and the same name off the path is not', pp.phrase === 'top-level stays');
257 check('a path of the wrong length does not match', P.value({ user: { phrase: 'x' } }).user.phrase === 'x');
258
259 const eight = /^(?:[a-z]+ ){7}[a-z]+$/;
260 const T = createRedactor({ test: (s) => eight.test(s) });
261 const tt = T.value({ note: 'one two three four five six seven eight', short: 'one two', list: ['a b c d e f g h'] });
262 check('a string the caller\'s test calls secret is fingerprinted whole', /^\[redacted one tw/.test(tt.note));
263 check('a string it does not is left', tt.short === 'one two');
264 check('the test sees strings inside arrays', /^\[redacted a b c/.test(tt.list[0]));
265 check('the test sees keys too', /^\[redacted /.test(Object.keys(T.value({ ['one two three four five six seven eight']: 1 }))[0]));
266 check('a marker is never tested again', T.text('[redacted x…#1/1]') === null);
267 check('text() answers a fingerprint for a hit and null for a miss',
268 /^\[redacted /.test(T.text('one two three four five six seven eight')) && T.text('fine') === null);
269 check('head applies to the string test as well',
270 createRedactor({ test: (s) => s === 'abcdef', head: 0 }).value({ a: 'abcdef' }).a.startsWith('[redacted …#'));
271 check('a byte length counts UTF-8, not characters', byteLen('é') === 2 && byteLen('abc') === 3);
272}
273
274section('redact: SCRUB-ONE, a typed password field is marked whole, with no head');
275{
276 // Mail posts a mailbox's app password as `password` and keeps it sealed as `pass`; a passphrase's
277 // first six characters are its first word. Such a field keeps nothing of its value (D-20261006-24).
278 const APP = 'abcd efgh ijkl mnop', PHRASE = 'correct horse battery staple';
279 const posted = scrubDeep({ address: 'me@x.org', user: 'me', password: APP, pass: APP, smtpPassword: APP,
280 newPassphrase: PHRASE, passAt: 1789300000000, passwordHint: 'the usual', bypass: 'on' });
281 const wire = JSON.stringify(posted);
282 check('ARM mailbox: a password-named field goes whole in the scrubber\'s walk', wire.indexOf('efgh') === -1 && wire.indexOf('horse') === -1, wire);
283 check('ARM mailbox: as a mark with no head', /^\[redacted named #[0-9a-f]+\/19\]$/.test(String(posted.pass)), String(posted.pass));
284 check('ARM mailbox: and the fields beside it still read', posted.address === 'me@x.org' && posted.user === 'me'
285 && posted.passAt === 1789300000000 && posted.passwordHint === 'the usual' && posted.bypass === 'on');
286 const red = createRedactor().value({ password: PHRASE, pass: APP, smtpPassword: APP, seed_phrase: PHRASE, apiKey: 'sk-or-v1-abcdefghij' });
287 const rw = JSON.stringify(red);
288 check('ARM name walk: the redactor keeps no head of a typed field', rw.indexOf('correc') === -1 && rw.indexOf('abcd') === -1, rw);
289 check('ARM name walk: a key keeps its six', red.apiKey.indexOf('[redacted sk-or-…#') === 0, red.apiKey);
290 check('ARM name walk: and a second walk changes nothing', JSON.stringify(createRedactor().value(red)) === rw);
291}
292
293finish('redact');