Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/lens/js/test/scrub_corpus.mjs

8.9 KiB, 2 runs

created by r1870400018:65630, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// scrub_corpus.mjs -- the inputs of `tests/data/lens_scrub.tsv`, as templates, and how to expand them.
2//
3// A template is built, never typed: a credential-shaped literal in a repository is what a scanner
4// is for, so the table holds `{{n:seed:alphabet}}` where a run of characters belongs, and the
5// scrubbed output, which holds markers and no credential. Whoever reads the table (the JavaScript
6// test, and the Rust `lens::Shapes`) expands a template with the same rule and compares:
7//
8// {{n:seed:alphabet}} n characters; state h = (seed * 2654435761) mod 2^32; for each character,
9// h = (h * 1103515245 + 12345) mod 2^32 (u32 wrapping arithmetic, exact) and
10// take alphabet[(h >> 16) mod len]
11// {{dash5}} five hyphens, which a PEM header is made of
12//
13// The alphabets are `alnum` (A-Z a-z 0-9), `hex` (0-9 a-f), `b64` (A-Z a-z 0-9 - _), `b64std`
14// (A-Z a-z 0-9 / +), `upper36` (A-Z 0-9) and `digit` (0-9), in those orders.
15export const ALPHABETS = {
16 alnum: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
17 hex: '0123456789abcdef',
18 b64: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_',
19 b64std: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/+',
20 upper36: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',
21 digit: '0123456789',
22};
23
24function run(n, seed, alpha) {
25 const a = ALPHABETS[alpha];
26 if (!a) throw new Error('no alphabet ' + alpha);
27 let s = '', h = (seed * 2654435761) >>> 0;
28 for (let i = 0; i < n; i++) { h = (Math.imul(h, 1103515245) + 12345) >>> 0; s += a[(h >>> 16) % a.length]; }
29 return s;
30}
31
32export function expand(t) {
33 return t.replace(/\{\{(\d+):(\d+):(\w+)\}\}/g, (_, n, seed, alpha) => run(Number(n), Number(seed), alpha))
34 .replace(/\{\{dash5\}\}/g, '-----');
35}
36
37// [template, isId]. Each credential shape at its minimum length, well above it, one short of it,
38// in a sentence, twice in a string; the pairs; the entropy catch and what it must leave alone.
39let seed = 100;
40const q = (n, alpha) => '{{' + n + ':' + (seed++) + ':' + alpha + '}}';
41export const TEMPLATES = [];
42const add = (t, id) => TEMPLATES.push([t, id ? 1 : 0]);
43
44// By shape.
45add('{{dash5}}BEGIN RSA PRIVATE KEY{{dash5}}\n' + q(64, 'b64std') + '\n' + q(64, 'b64std') + '\n{{dash5}}END RSA PRIVATE KEY{{dash5}}');
46add('log: {{dash5}}BEGIN PRIVATE KEY{{dash5}}\n' + q(40, 'b64std') + '… (clipped)');
47add('{{dash5}}BEGIN CERTIFICATE{{dash5}}\n' + q(40, 'b64std') + '\n{{dash5}}END CERTIFICATE{{dash5}}');
48for (const [pre, n] of [['ghp_', 16], ['gho_', 36], ['ghu_', 20], ['ghs_', 40], ['ghr_', 16], ['ghp_', 15]]) add('token ' + pre + q(n, 'alnum') + ' end');
49for (const n of [20, 22, 40, 19]) add('github_pat_' + q(n, 'alnum'));
50for (const [pre, n] of [['sk_test_', 10], ['sk_live_', 24], ['rk_live_', 24], ['rk_test_', 10], ['pk_test_', 24], ['sk_test_', 9]]) add('key=' + pre + q(n, 'alnum') + '&x=1');
51for (const n of [16, 32, 15]) add('hook whsec_' + q(n, 'alnum'));
52for (const [pre, n, a] of [['sk-', 16, 'b64'], ['sk-', 48, 'b64'], ['sk-or-v1-', 48, 'hex'], ['sk-ant-api03-', 80, 'b64'], ['sk-', 15, 'b64'], ['task-', 40, 'alnum']]) add(pre + q(n, a));
53for (const [pre, n] of [['AKIA', 16], ['ASIA', 12], ['ABIA', 20], ['ACCA', 16], ['AGPA', 16], ['AIDA', 16], ['AIPA', 16], ['ANPA', 16], ['ANVA', 16], ['AROA', 16], ['APKA', 16], ['AKIA', 11], ['AKIB', 16]]) add('id ' + pre + q(n, 'upper36') + ' used');
54for (const n of [35, 39, 29]) add('AIza' + q(n, 'b64'));
55for (const [a, b] of [[12, 12], [10, 10], [9, 12]]) add('xoxb-' + q(a, 'alnum') + '-' + q(b, 'alnum'));
56add('xoxp-' + q(12, 'alnum') + '-' + q(12, 'alnum') + '-' + q(24, 'alnum'));
57for (const [a, b, c] of [[6, 6, 4], [20, 40, 43], [36, 120, 43], [5, 6, 4], [6, 5, 4], [6, 6, 3]]) add('Bearer eyJ' + q(a, 'b64') + '.' + q(b, 'b64') + '.' + q(c, 'b64') + ' ok');
58for (const n of [32, 31, 40]) add('relay tune-' + q(n, 'hex') + ' up');
59
60// The pairs, where the name survives and the value goes.
61for (const w of ['Bearer', 'Basic', 'Token']) add('Authorization: ' + w + ' ' + q(20, 'b64'));
62add('Authorization: Bearer ' + q(7, 'b64'));
63add('Authorization: Bearer ' + q(8, 'b64'));
64for (const k of ['token', 'access_token', 'refresh_token', 'id_token', 'api_key', 'api-key', 'apikey', 'key', 'auth', 'secret', 'password', 'passwd', 'sig', 'signature'])
65 add('https://x.example/cb?a=1&' + k + '=' + q(12, 'alnum') + '&z=2');
66add('https://x.example/cb#access_token=' + q(30, 'b64') + '#end');
67add('https://x.example/cb?key=' + q(4, 'alnum'));
68add('https://x.example/cb?key=' + q(3, 'alnum'));
69add('https://x.example/cb?Token=' + q(12, 'alnum'));
70for (const k of ['token', 'secret', 'apikey', 'api_key', 'api-key', 'auth_token', 'access-token', 'passphrase', 'password', 'master_key', 'private-key', 'mnemonic', 'seed_phrase', 'salt', 'sealed', 'wrapped'])
71 add('config ' + k + ': ' + q(16, 'alnum') + ' done');
72add('password="' + q(10, 'alnum') + '"');
73add("secret = '" + q(12, 'alnum') + "'");
74add('token=' + q(7, 'alnum'));
75add('AKIA' + q(16, 'upper36') + ' and its secret ' + q(40, 'b64std') + ' here');
76add('aws_secret_access_key = ' + q(40, 'b64std'));
77add('AWS_SECRET_ACCESS_KEY: "' + q(40, 'b64std') + '"');
78add('aws_secret_access_key = ' + q(39, 'b64std'));
79
80// The entropy catch, and what the feed is supposed to carry.
81for (const n of [32, 33, 48, 64, 128]) add('blob ' + q(n, 'b64') + ' end');
82add(q(31, 'b64'));
83for (const n of [32, 40, 64]) add('sha ' + q(n, 'hex') + ' ok');
84add(q(8, 'hex') + '-' + q(4, 'hex') + '-' + q(4, 'hex') + '-' + q(4, 'hex') + '-' + q(12, 'hex'));
85add('call_' + q(24, 'hex') + '_' + q(12, 'hex'));
86add(q(40, 'digit'));
87add('worker_pool_seat_000412_alpha_beta_gamma_delta');
88add('build-6911be85c0f7-seq-312-stamp-20261002T120000Z-gamma');
89add(q(40, 'b64'), true);
90add('id sk-' + q(20, 'b64'), true);
91add('id eyJ' + q(8, 'b64') + '.' + q(8, 'b64') + '.' + q(8, 'b64'), true);
92
93// Together, and awkward.
94add('first sk_test_' + q(24, 'alnum') + ' then eyJ' + q(10, 'b64') + '.' + q(10, 'b64') + '.' + q(10, 'b64') + ' then AKIA' + q(16, 'upper36'));
95add('sk-' + q(20, 'b64') + ' sk-' + q(20, 'b64') + ' sk-' + q(20, 'b64'));
96add('clé ' + 'sk_test_' + q(12, 'alnum') + ' 日本語 😀 ' + 'ghp_' + q(20, 'alnum'));
97add('line one\nBearer ' + q(20, 'b64') + '\nline three ' + q(40, 'b64') + '\n');
98add('[redacted stripe #1f2e3d4c/32] already scrubbed');
99add('[redacted hi #deadbeef/40] and ?token=[redacted urlarg #12ab34cd/12]');
100add('');
101add('short');
102add(' ');
103add('plain prose with no credential at all, only words and a number 12345 and a path /usr/local/bin');
104
105// SCRUB-ONE (D-20261006-24): what the `named` and `bearer` rules once let through. A passphrase is
106// words, a password may be short, a value may be quoted (with an escaped quote inside, or as a JSON
107// member), and an `Authorization` scheme is case-free on the wire. Taken from Daimond's arms. Each
108// is also a LEAK: `[template, what must not survive]`, which `gen_scrub_oracle.mjs` writes to
109// `tests/data/lens_scrub_leaks.tsv` so that both languages hold the scrubber to the absence itself
110// and not only to the JavaScript's own answer.
111export const LEAKS = [];
112const leak = (t, gone) => { add(t); LEAKS.push([t, gone]); };
113const PHRASE = 'correct horse battery staple';
114leak('passphrase: ' + PHRASE, 'horse');
115leak('[id] unlock with passphrase: ' + PHRASE, 'battery');
116leak('mnemonic = ' + PHRASE + ' again', 'battery');
117leak('PASSPHRASE=' + PHRASE, 'horse');
118leak('Password: hunter2', 'hunter2');
119leak('login password = tr0ub4dor', 'tr0ub4dor');
120leak('passwd=s3cr3t pa55', 's3cr3t');
121leak('PASSWD: s3cr3t pa55', 'pa55');
122leak('newPassword: abc', 'abc');
123leak('seed-phrase:\t' + PHRASE + ' \nnext line', 'staple');
124leak('password: "' + PHRASE + '"', 'horse');
125leak("passphrase='" + PHRASE + "'", 'horse');
126leak('password: "pa\\"ss wo rd"', 'wo rd');
127leak('config password="hunter2" set', 'hunter2');
128leak('{"user":"me","password":"' + PHRASE + '","port":993}', 'horse');
129leak('{"address":"me@x.org","host":"imap.x.org","user":"me","password":"abcd efgh ijkl mnop"}', 'efgh');
130leak("token = 'ab cd' and secret: \"xy\"", 'ab cd');
131leak("secret: \"xy\" after", '"xy"');
132leak('api_key: "' + q(6, 'alnum') + '"', '{{6:' + (seed - 1) + ':alnum}}');
133leak('authorization: bearer ' + q(20, 'b64'), '{{20:' + (seed - 1) + ':b64}}');
134leak('AUTHORIZATION: BEARER ' + q(20, 'b64'), '{{20:' + (seed - 1) + ':b64}}');
135leak('BeArEr ' + q(12, 'b64') + ' mixed', '{{12:' + (seed - 1) + ':b64}}');
136// And what must still read: prose about a password with no value, and a second pass.
137for (const p of ['Forgot your password? Reset it from the sign-in screen.', 'the password was wrong',
138 'Password must be at least 12 characters', 'Enter your passphrase:', 'passphrase: ', 'Change password',
139 'Set a passphrase, then confirm it', 'the compass bypass passes', 'password: four spaces',
140 'password: "unterminated value', 'password: [REDACTED by hand]', 'basic auth then token words'])
141 add(p);
142add('passphrase: [redacted named #1a2b3c4d/28] and password="[redacted named #5e6f7a8b/7]"');