oxedyne/fe2o3/fe2o3_net/lens/js/test/scrub_corpus.mjs
8.9 KiB, 2 runs
created by r1870400018:65630, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // scrub_corpus.mjs -- the inputs of `tests/data/lens_scrub.tsv`, as templates, and how to expand them. |
| 2 | // |
| 3 | // A template is built, never typed: a credential-shaped literal in a repository is what a scanner |
| 4 | // is for, so the table holds `{{n:seed:alphabet}}` where a run of characters belongs, and the |
| 5 | // scrubbed output, which holds markers and no credential. Whoever reads the table (the JavaScript |
| 6 | // test, and the Rust `lens::Shapes`) expands a template with the same rule and compares: |
| 7 | // |
| 8 | // {{n:seed:alphabet}} n characters; state h = (seed * 2654435761) mod 2^32; for each character, |
| 9 | // h = (h * 1103515245 + 12345) mod 2^32 (u32 wrapping arithmetic, exact) and |
| 10 | // take alphabet[(h >> 16) mod len] |
| 11 | // {{dash5}} five hyphens, which a PEM header is made of |
| 12 | // |
| 13 | // The alphabets are `alnum` (A-Z a-z 0-9), `hex` (0-9 a-f), `b64` (A-Z a-z 0-9 - _), `b64std` |
| 14 | // (A-Z a-z 0-9 / +), `upper36` (A-Z 0-9) and `digit` (0-9), in those orders. |
| 15 | export const ALPHABETS = { |
| 16 | alnum: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', |
| 17 | hex: '0123456789abcdef', |
| 18 | b64: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', |
| 19 | b64std: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/+', |
| 20 | upper36: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', |
| 21 | digit: '0123456789', |
| 22 | }; |
| 23 | |
| 24 | function run(n, seed, alpha) { |
| 25 | const a = ALPHABETS[alpha]; |
| 26 | if (!a) throw new Error('no alphabet ' + alpha); |
| 27 | let s = '', h = (seed * 2654435761) >>> 0; |
| 28 | for (let i = 0; i < n; i++) { h = (Math.imul(h, 1103515245) + 12345) >>> 0; s += a[(h >>> 16) % a.length]; } |
| 29 | return s; |
| 30 | } |
| 31 | |
| 32 | export function expand(t) { |
| 33 | return t.replace(/\{\{(\d+):(\d+):(\w+)\}\}/g, (_, n, seed, alpha) => run(Number(n), Number(seed), alpha)) |
| 34 | .replace(/\{\{dash5\}\}/g, '-----'); |
| 35 | } |
| 36 | |
| 37 | // [template, isId]. Each credential shape at its minimum length, well above it, one short of it, |
| 38 | // in a sentence, twice in a string; the pairs; the entropy catch and what it must leave alone. |
| 39 | let seed = 100; |
| 40 | const q = (n, alpha) => '{{' + n + ':' + (seed++) + ':' + alpha + '}}'; |
| 41 | export const TEMPLATES = []; |
| 42 | const add = (t, id) => TEMPLATES.push([t, id ? 1 : 0]); |
| 43 | |
| 44 | // By shape. |
| 45 | add('{{dash5}}BEGIN RSA PRIVATE KEY{{dash5}}\n' + q(64, 'b64std') + '\n' + q(64, 'b64std') + '\n{{dash5}}END RSA PRIVATE KEY{{dash5}}'); |
| 46 | add('log: {{dash5}}BEGIN PRIVATE KEY{{dash5}}\n' + q(40, 'b64std') + '… (clipped)'); |
| 47 | add('{{dash5}}BEGIN CERTIFICATE{{dash5}}\n' + q(40, 'b64std') + '\n{{dash5}}END CERTIFICATE{{dash5}}'); |
| 48 | for (const [pre, n] of [['ghp_', 16], ['gho_', 36], ['ghu_', 20], ['ghs_', 40], ['ghr_', 16], ['ghp_', 15]]) add('token ' + pre + q(n, 'alnum') + ' end'); |
| 49 | for (const n of [20, 22, 40, 19]) add('github_pat_' + q(n, 'alnum')); |
| 50 | for (const [pre, n] of [['sk_test_', 10], ['sk_live_', 24], ['rk_live_', 24], ['rk_test_', 10], ['pk_test_', 24], ['sk_test_', 9]]) add('key=' + pre + q(n, 'alnum') + '&x=1'); |
| 51 | for (const n of [16, 32, 15]) add('hook whsec_' + q(n, 'alnum')); |
| 52 | for (const [pre, n, a] of [['sk-', 16, 'b64'], ['sk-', 48, 'b64'], ['sk-or-v1-', 48, 'hex'], ['sk-ant-api03-', 80, 'b64'], ['sk-', 15, 'b64'], ['task-', 40, 'alnum']]) add(pre + q(n, a)); |
| 53 | for (const [pre, n] of [['AKIA', 16], ['ASIA', 12], ['ABIA', 20], ['ACCA', 16], ['AGPA', 16], ['AIDA', 16], ['AIPA', 16], ['ANPA', 16], ['ANVA', 16], ['AROA', 16], ['APKA', 16], ['AKIA', 11], ['AKIB', 16]]) add('id ' + pre + q(n, 'upper36') + ' used'); |
| 54 | for (const n of [35, 39, 29]) add('AIza' + q(n, 'b64')); |
| 55 | for (const [a, b] of [[12, 12], [10, 10], [9, 12]]) add('xoxb-' + q(a, 'alnum') + '-' + q(b, 'alnum')); |
| 56 | add('xoxp-' + q(12, 'alnum') + '-' + q(12, 'alnum') + '-' + q(24, 'alnum')); |
| 57 | for (const [a, b, c] of [[6, 6, 4], [20, 40, 43], [36, 120, 43], [5, 6, 4], [6, 5, 4], [6, 6, 3]]) add('Bearer eyJ' + q(a, 'b64') + '.' + q(b, 'b64') + '.' + q(c, 'b64') + ' ok'); |
| 58 | for (const n of [32, 31, 40]) add('relay tune-' + q(n, 'hex') + ' up'); |
| 59 | |
| 60 | // The pairs, where the name survives and the value goes. |
| 61 | for (const w of ['Bearer', 'Basic', 'Token']) add('Authorization: ' + w + ' ' + q(20, 'b64')); |
| 62 | add('Authorization: Bearer ' + q(7, 'b64')); |
| 63 | add('Authorization: Bearer ' + q(8, 'b64')); |
| 64 | for (const k of ['token', 'access_token', 'refresh_token', 'id_token', 'api_key', 'api-key', 'apikey', 'key', 'auth', 'secret', 'password', 'passwd', 'sig', 'signature']) |
| 65 | add('https://x.example/cb?a=1&' + k + '=' + q(12, 'alnum') + '&z=2'); |
| 66 | add('https://x.example/cb#access_token=' + q(30, 'b64') + '#end'); |
| 67 | add('https://x.example/cb?key=' + q(4, 'alnum')); |
| 68 | add('https://x.example/cb?key=' + q(3, 'alnum')); |
| 69 | add('https://x.example/cb?Token=' + q(12, 'alnum')); |
| 70 | for (const k of ['token', 'secret', 'apikey', 'api_key', 'api-key', 'auth_token', 'access-token', 'passphrase', 'password', 'master_key', 'private-key', 'mnemonic', 'seed_phrase', 'salt', 'sealed', 'wrapped']) |
| 71 | add('config ' + k + ': ' + q(16, 'alnum') + ' done'); |
| 72 | add('password="' + q(10, 'alnum') + '"'); |
| 73 | add("secret = '" + q(12, 'alnum') + "'"); |
| 74 | add('token=' + q(7, 'alnum')); |
| 75 | add('AKIA' + q(16, 'upper36') + ' and its secret ' + q(40, 'b64std') + ' here'); |
| 76 | add('aws_secret_access_key = ' + q(40, 'b64std')); |
| 77 | add('AWS_SECRET_ACCESS_KEY: "' + q(40, 'b64std') + '"'); |
| 78 | add('aws_secret_access_key = ' + q(39, 'b64std')); |
| 79 | |
| 80 | // The entropy catch, and what the feed is supposed to carry. |
| 81 | for (const n of [32, 33, 48, 64, 128]) add('blob ' + q(n, 'b64') + ' end'); |
| 82 | add(q(31, 'b64')); |
| 83 | for (const n of [32, 40, 64]) add('sha ' + q(n, 'hex') + ' ok'); |
| 84 | add(q(8, 'hex') + '-' + q(4, 'hex') + '-' + q(4, 'hex') + '-' + q(4, 'hex') + '-' + q(12, 'hex')); |
| 85 | add('call_' + q(24, 'hex') + '_' + q(12, 'hex')); |
| 86 | add(q(40, 'digit')); |
| 87 | add('worker_pool_seat_000412_alpha_beta_gamma_delta'); |
| 88 | add('build-6911be85c0f7-seq-312-stamp-20261002T120000Z-gamma'); |
| 89 | add(q(40, 'b64'), true); |
| 90 | add('id sk-' + q(20, 'b64'), true); |
| 91 | add('id eyJ' + q(8, 'b64') + '.' + q(8, 'b64') + '.' + q(8, 'b64'), true); |
| 92 | |
| 93 | // Together, and awkward. |
| 94 | add('first sk_test_' + q(24, 'alnum') + ' then eyJ' + q(10, 'b64') + '.' + q(10, 'b64') + '.' + q(10, 'b64') + ' then AKIA' + q(16, 'upper36')); |
| 95 | add('sk-' + q(20, 'b64') + ' sk-' + q(20, 'b64') + ' sk-' + q(20, 'b64')); |
| 96 | add('clé ' + 'sk_test_' + q(12, 'alnum') + ' 日本語 😀 ' + 'ghp_' + q(20, 'alnum')); |
| 97 | add('line one\nBearer ' + q(20, 'b64') + '\nline three ' + q(40, 'b64') + '\n'); |
| 98 | add('[redacted stripe #1f2e3d4c/32] already scrubbed'); |
| 99 | add('[redacted hi #deadbeef/40] and ?token=[redacted urlarg #12ab34cd/12]'); |
| 100 | add(''); |
| 101 | add('short'); |
| 102 | add(' '); |
| 103 | add('plain prose with no credential at all, only words and a number 12345 and a path /usr/local/bin'); |
| 104 | |
| 105 | // SCRUB-ONE (D-20261006-24): what the `named` and `bearer` rules once let through. A passphrase is |
| 106 | // words, a password may be short, a value may be quoted (with an escaped quote inside, or as a JSON |
| 107 | // member), and an `Authorization` scheme is case-free on the wire. Taken from Daimond's arms. Each |
| 108 | // is also a LEAK: `[template, what must not survive]`, which `gen_scrub_oracle.mjs` writes to |
| 109 | // `tests/data/lens_scrub_leaks.tsv` so that both languages hold the scrubber to the absence itself |
| 110 | // and not only to the JavaScript's own answer. |
| 111 | export const LEAKS = []; |
| 112 | const leak = (t, gone) => { add(t); LEAKS.push([t, gone]); }; |
| 113 | const PHRASE = 'correct horse battery staple'; |
| 114 | leak('passphrase: ' + PHRASE, 'horse'); |
| 115 | leak('[id] unlock with passphrase: ' + PHRASE, 'battery'); |
| 116 | leak('mnemonic = ' + PHRASE + ' again', 'battery'); |
| 117 | leak('PASSPHRASE=' + PHRASE, 'horse'); |
| 118 | leak('Password: hunter2', 'hunter2'); |
| 119 | leak('login password = tr0ub4dor', 'tr0ub4dor'); |
| 120 | leak('passwd=s3cr3t pa55', 's3cr3t'); |
| 121 | leak('PASSWD: s3cr3t pa55', 'pa55'); |
| 122 | leak('newPassword: abc', 'abc'); |
| 123 | leak('seed-phrase:\t' + PHRASE + ' \nnext line', 'staple'); |
| 124 | leak('password: "' + PHRASE + '"', 'horse'); |
| 125 | leak("passphrase='" + PHRASE + "'", 'horse'); |
| 126 | leak('password: "pa\\"ss wo rd"', 'wo rd'); |
| 127 | leak('config password="hunter2" set', 'hunter2'); |
| 128 | leak('{"user":"me","password":"' + PHRASE + '","port":993}', 'horse'); |
| 129 | leak('{"address":"me@x.org","host":"imap.x.org","user":"me","password":"abcd efgh ijkl mnop"}', 'efgh'); |
| 130 | leak("token = 'ab cd' and secret: \"xy\"", 'ab cd'); |
| 131 | leak("secret: \"xy\" after", '"xy"'); |
| 132 | leak('api_key: "' + q(6, 'alnum') + '"', '{{6:' + (seed - 1) + ':alnum}}'); |
| 133 | leak('authorization: bearer ' + q(20, 'b64'), '{{20:' + (seed - 1) + ':b64}}'); |
| 134 | leak('AUTHORIZATION: BEARER ' + q(20, 'b64'), '{{20:' + (seed - 1) + ':b64}}'); |
| 135 | leak('BeArEr ' + q(12, 'b64') + ' mixed', '{{12:' + (seed - 1) + ':b64}}'); |
| 136 | // And what must still read: prose about a password with no value, and a second pass. |
| 137 | for (const p of ['Forgot your password? Reset it from the sign-in screen.', 'the password was wrong', |
| 138 | 'Password must be at least 12 characters', 'Enter your passphrase:', 'passphrase: ', 'Change password', |
| 139 | 'Set a passphrase, then confirm it', 'the compass bypass passes', 'password: four spaces', |
| 140 | 'password: "unterminated value', 'password: [REDACTED by hand]', 'basic auth then token words']) |
| 141 | add(p); |
| 142 | add('passphrase: [redacted named #1a2b3c4d/28] and password="[redacted named #5e6f7a8b/7]"'); |