oxedyne/fe2o3/fe2o3_net/lens/js/test/crosscheck.test.mjs
6.3 KiB, 5 runs
created by r1870400018:65616, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // crosscheck.test.mjs -- the JavaScript redactor against the oracle tables T5a's Rust redactor |
| 2 | // is tested on, so that the two halves of the lens cannot drift apart. |
| 3 | // |
| 4 | // `fe2o3_net/tests/data/lens_secret_re.tsv` and `lens_fingerprint.tsv` were generated by |
| 5 | // `lens_oracle.mjs` from Daimond's own `SECRET_RE`, `SECRET_RE_LOOSE` and `fingerprint`, and |
| 6 | // `tests/lens.rs` holds `secret_name`, `secret_name_loose` and `fingerprint` to them in Rust. |
| 7 | // Here the same tables hold `redact.js`: if it, Daimond's source and the Rust agree on every |
| 8 | // row, a name or a value is covered identically on the page and at the peer. |
| 9 | // |
| 10 | // node test/crosscheck.test.mjs |
| 11 | import { readFileSync } from 'node:fs'; |
| 12 | import { dirname, join } from 'node:path'; |
| 13 | import { fileURLToPath } from 'node:url'; |
| 14 | import { createRedactor, fingerprint, scrubText, secretName } from '../redact.js'; |
| 15 | import { check, finish, section } from './harness.mjs'; |
| 16 | import { LEAKS, TEMPLATES, expand } from './scrub_corpus.mjs'; |
| 17 | |
| 18 | const DATA = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'tests', 'data'); |
| 19 | const lines = (f) => readFileSync(join(DATA, f), 'utf8').split('\n').filter((l) => l.length); |
| 20 | |
| 21 | section('crosscheck: secret names, strict and loose, against lens_secret_re.tsv'); |
| 22 | { |
| 23 | const rows = lines('lens_secret_re.tsv').map((l) => l.split('\t')); |
| 24 | let strictBad = 0, looseBad = 0, strictHits = 0, firstBad = ''; |
| 25 | const R = createRedactor(), L = createRedactor({ loose: true }); |
| 26 | for (const [name, strict, loose] of rows) { |
| 27 | const s = secretName(name) ? 1 : 0, lo = secretName(name, true) ? 1 : 0; |
| 28 | if (s !== Number(strict)) { strictBad += 1; firstBad = firstBad || name; } |
| 29 | if (lo !== Number(loose)) { looseBad += 1; firstBad = firstBad || name; } |
| 30 | if (R.secret(name) !== (Number(strict) === 1) || L.secret(name) !== (Number(loose) === 1)) { strictBad += 1; firstBad = firstBad || name; } |
| 31 | strictHits += Number(strict); |
| 32 | } |
| 33 | check('the table has the 3,136 names Rust is tested on', rows.length === 3136, rows.length + ' rows'); |
| 34 | check('the strict rule agrees on every name', strictBad === 0, strictBad + ' differ, first ' + JSON.stringify(firstBad)); |
| 35 | check('the loose rule agrees on every name', looseBad === 0, looseBad + ' differ'); |
| 36 | check('the table is not one answer: it holds both hits and misses', strictHits > 500 && strictHits < rows.length - 500, strictHits + ' strict hits'); |
| 37 | } |
| 38 | |
| 39 | section('crosscheck: fingerprints against lens_fingerprint.tsv'); |
| 40 | { |
| 41 | const rows = lines('lens_fingerprint.tsv').map((l) => l.split('\t').map((c) => JSON.parse(c))); |
| 42 | const bad = rows.filter(([input, want]) => fingerprint(input) !== want); |
| 43 | check('the table has its 21 inputs, including a number, a null, an astral and a long string', rows.length === 21, rows.length + ' rows'); |
| 44 | check('every fingerprint agrees, character for character', bad.length === 0, JSON.stringify(bad.slice(0, 2))); |
| 45 | check('the table has a surrogate pair, whose length counts in UTF-16 units as the Rust does', |
| 46 | rows.some(([i]) => typeof i === 'string' && /[\u{10000}-\u{10ffff}]/u.test(i))); |
| 47 | check('a redactor walk fingerprints through the same function', |
| 48 | createRedactor().value({ apiKey: 'abcdefghi' }).apiKey === fingerprint('abcdefghi')); |
| 49 | } |
| 50 | |
| 51 | section('crosscheck: the content scrubber against lens_scrub.tsv, the table Rust\'s Shapes is held to'); |
| 52 | { |
| 53 | // Template, scrubbed output and the id flag, from `gen_scrub_oracle.mjs`. The table holds the recipe |
| 54 | // of each input and never a credential; `scrub_corpus.mjs` says how to expand one. |
| 55 | const rows = lines('lens_scrub.tsv').map((l) => l.split('\t')).map(([t, o, id]) => [JSON.parse(t), JSON.parse(o), id === '1']); |
| 56 | const bad = rows.filter(([t, o, id]) => scrubText(expand(t), id) !== o); |
| 57 | check('every case scrubs to the table\'s output, character for character', bad.length === 0, bad.length + ' differ, first ' + JSON.stringify(bad[0] && bad[0][0])); |
| 58 | check('the table is the corpus, expanded and scrubbed now (regenerate it if redact.js moved)', |
| 59 | rows.length === TEMPLATES.length && rows.every(([t, , id], i) => t === TEMPLATES[i][0] && id === (TEMPLATES[i][1] === 1)), rows.length + ' rows'); |
| 60 | const kinds = new Set(); |
| 61 | for (const [, o] of rows) for (const m of o.matchAll(/\[redacted (\w+) #/g)) kinds.add(m[1]); |
| 62 | check('it exercises every shape, pair and the entropy catch', |
| 63 | ['pem', 'gh', 'ghpat', 'stripe', 'whsec', 'sk', 'aws', 'gcp', 'slack', 'jwt', 'tune', 'bearer', 'urlarg', 'named', 'awssec', 'hi'].every((k) => kinds.has(k)), [...kinds].join()); |
| 64 | const same = rows.filter(([t, o]) => o === expand(t)).length; |
| 65 | check('and it holds decoys that must come through untouched', same >= 15 && same < rows.length - 50, same + ' of ' + rows.length + ' unchanged'); |
| 66 | check('a case that was already scrubbed stays as it is (the second pass is idempotent)', |
| 67 | rows.every(([, o, id]) => scrubText(o, id) === o)); |
| 68 | check('the table holds no literal credential: no template has a long run of characters outside a token', |
| 69 | rows.every(([t]) => { const r = t.replace(/\{\{[^}]*\}\}/g, ''); return !/[A-Za-z0-9]{20,}/.test(r) && !/(?:sk_|rk_|pk_|sk-|ghp_|gho_|AKIA|ASIA|eyJ|xox.-|AIza|whsec_|tune-)[A-Za-z0-9]{8,}/.test(r); })); |
| 70 | // Pinned expansions, so a port of the expander can be checked before it is trusted with the table. |
| 71 | check('the expander gives the pinned vectors', |
| 72 | expand('{{8:1:hex}}') === '8388dbc2' && expand('{{12:7:alnum}}') === 'Dn8QmkIRy6vU' && expand('{{10:3:digit}}') === '0086285001' |
| 73 | && expand('{{6:5:upper36}}') === 'PJMEIR' && expand('a{{dash5}}b') === 'a-----b'); |
| 74 | } |
| 75 | |
| 76 | section('crosscheck: SCRUB-ONE, what must not survive, against lens_scrub_leaks.tsv'); |
| 77 | { |
| 78 | // Each leak is a template and what of it must be gone once scrubbed: a multi-word passphrase, a |
| 79 | // short password, a quoted value, a lower-case `bearer`. Held to the absence, not to the table's |
| 80 | // own output, so a table generated by a leaking scrubber cannot pass it. |
| 81 | const rows = lines('lens_scrub_leaks.tsv').map((l) => l.split('\t').map((c) => JSON.parse(c))); |
| 82 | check('the leak table is the corpus\'s LEAKS', rows.length === LEAKS.length && rows.length >= 20, rows.length + ' rows'); |
| 83 | const kept = rows.filter(([t, g]) => scrubText(expand(t)).indexOf(expand(g)) !== -1); |
| 84 | check('no leak survives the scrubber', kept.length === 0, kept.length + ' kept: ' + kept.map(([t]) => JSON.stringify(t)).join(' | ')); |
| 85 | check('and a second pass over each changes nothing', rows.every(([t]) => { const o = scrubText(expand(t)); return scrubText(o) === o; })); |
| 86 | } |
| 87 | |
| 88 | finish('crosscheck'); |