Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/tests/secret.rs

19.2 KiB, 34 runs

created by r1870400018:35197, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Every credential in this file is spelled in two pieces and joined at run time, so that the
2//! scanners which read this very file -- the git hook, and this crate's own scanner under a
3//! version control system that cannot forget -- find nothing in it to refuse.
4//!
5//! The DER fixtures are the one thing here written out whole, and they are not credentials. Each
6//! is the structural head of a key -- the outer length, the version, the algorithm's object
7//! identifier and the tag that opens the private bytes -- read off a key generated with `openssl
8//! genpkey` or `ring` for that purpose, and stopping exactly where the secret would begin. The
9//! body is filler put there at run time. That is enough to ask the detector the only question it
10//! asks, and it means no key was written into a file that is pushed to a public repository.
11
12use oxedyne_fe2o3_text::{
13 base2x,
14 secret::{
15 self,
16 Find,
17 Kind,
18 },
19};
20
21use oxedyne_fe2o3_core::{
22 prelude::*,
23 test::test_it,
24};
25
26
27// One credential of each shape, as an opening and the rest of it.
28const SHAPED: &[(&str, &str, Kind)] = &[
29 ("fw", "_3ZjKq81mAbCdEfGhIjKlMnOpQrSt", Kind::Fireworks),
30 ("sk-ant", "-api03-AbCdEfGhIjKlMnOpQrStUvWx", Kind::Anthropic),
31 ("sk-proj", "-AbCdEfGhIjKlMnOpQrStUvWxYz01", Kind::OpenAi),
32 ("sk-or", "-v1-0123456789abcdef0123456789abcdef", Kind::OpenAi),
33 ("sk-", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", Kind::OpenAiOld),
34 ("AKIA", "IOSFODNN7EXAMPLE", Kind::Aws),
35 ("ghp", "_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", Kind::GitHub),
36 ("github_pat", "_11ABCDEFG0AbCdEfGhIjKlMnOpQrStUvWxYz0123456789",
37 Kind::GitHubPat),
38 ("xoxb", "-1234567890-abcdefghij", Kind::Slack),
39 ("sk_live", "_AbCdEfGhIjKlMnOpQrStUv", Kind::Stripe),
40 ("AIza", "SyA0123456789abcdefghijklmnopqrstuv", Kind::Google),
41 ("-----BEGIN ", "OPENSSH PRIVATE KEY-----", Kind::PrivateKey),
42];
43
44// The value of a `Kind::Assigned` finding, in two pieces for the same reason.
45const LITERAL: (&str, &str) = ("9f3Bq7", "ZmR4tYuIoPkLjHgFdS");
46
47// The structural head of one private key of each form this catches, and the whole size of the key
48// it came off. Read off keys generated with `openssl genpkey`, `openssl ecparam -genkey` and
49// `ring`, in that order of appearance, and stopping before the private bytes: see the note at the
50// head of this file.
51const DER: &[(&str, &str, usize)] = &[
52 ("ed25519, PKCS#8", "302E020100300506032B657004220420", 48),
53 ("ed25519, with the public key",
54 "3051020101300506032B657004220420", 83),
55 ("X25519, PKCS#8", "302E020100300506032B656E04220420", 48),
56 ("RSA-2048, PKCS#8", "308204BD020100300D06092A864886F70D0101010500048204A7",
57 1217),
58 ("RSA-2048, PKCS#1", "308204A30201000282010100", 1191),
59 ("RSA-4096, PKCS#1", "308209290201000282020100", 2349),
60 ("P-256, PKCS#8", "308187020100301306072A8648CE3D020106082A8648CE3D030107",
61 138),
62 ("P-384, PKCS#8", "3081B6020100301006072A8648CE3D020106052B8104002204819E",
63 185),
64 ("P-521, PKCS#8", "3081EE020100301006072A8648CE3D020106052B81040023", 241),
65 ("P-256, SEC1", "30770201010420", 121),
66 ("P-384, SEC1", "3081A40201010430", 167),
67 ("P-521, SEC1", "3081DC0201010442", 223),
68];
69
70// The 83-byte shape the DKIM signing key was in, named on its own because it is the case this
71// rule was written for.
72const DKIM: (&str, usize) = (DER[1].1, DER[1].2);
73
74// A certificate, which is what a key is most often bundled with, and whose outer sequence opens
75// with another sequence where a key's opens with a version.
76const CERT: (&str, usize) = ("3082013B3081EEA003020102", 319);
77
78/// The key of that shape, its structure real and its body filler.
79fn der(head: &str, len: usize) -> Outcome<Vec<u8>> {
80 let mut out = res!(base2x::HEX.from_str(head));
81 // Whatever stands where the secret would is beside the point: the detector is being asked a
82 // question about the encoding, and it never looks at these bytes.
83 while out.len() < len {
84 out.push(0x5A);
85 }
86 out.truncate(len);
87 Ok(out)
88}
89
90
91pub fn test_secret(filter: &'static str) -> Outcome<()> {
92
93 res!(test_it(filter, &["Every shape is caught", "all", "secret", "shape"], || {
94 for (lead, rest, kind) in SHAPED {
95 let line = fmt!("let key = \"{}{}\";\n", lead, rest);
96 let found = secret::scan(line.as_bytes());
97 req!(found, vec![Find { line: 1, kind: *kind }], "for {:?}", lead);
98 }
99 Ok(())
100 }));
101
102 res!(test_it(filter, &["Nothing is caught in ordinary source", "all", "secret", "shape"], || {
103 let text = b"let key = res!(std::env::var(\"FIREWORKS_API_KEY\"),\n\
104 \t\"Set FIREWORKS_API_KEY before running this example.\");\n\
105 // A short one, sk-nope, and a field with nothing in it, api_key = \"\".\n";
106 req!(secret::scan(text), Vec::<Find>::new());
107 Ok(())
108 }));
109
110 res!(test_it(filter, &["The prefilter admits every opening", "all", "secret", "shape"], || {
111 // A shape whose opening the prefilter rejects would match nothing, and every other test
112 // here would still pass.
113 req!(secret::leads_are_covered(), true);
114 Ok(())
115 }));
116
117 res!(test_it(filter, &["A named field holding a long literal is caught", "all", "secret",
118 "assigned"], ||
119 {
120 for field in ["api_key", "API_KEY", "secret", "password", "access_token"] {
121 let line = fmt!("{} = \"{}{}\"\n", field, LITERAL.0, LITERAL.1);
122 req!(secret::scan(line.as_bytes()), vec![Find { line: 1, kind: Kind::Assigned }],
123 "for {:?}", field);
124 }
125 Ok(())
126 }));
127
128 res!(test_it(filter, &["A placeholder is not a credential", "all", "secret", "assigned"], || {
129 // The rule that decides whether the guard is left switched on. A documentation example
130 // refused is a guard somebody turns off, and then it protects nothing.
131 for value in [
132 "your-key-here",
133 "YOUR_API_KEY_GOES_HERE",
134 "xxxxxxxxxxxxxxxxxxxxxxxx",
135 "placeholder_value_here_ok",
136 "changeme_changeme_changeme",
137 "example_token_0123456789",
138 ] {
139 let line = fmt!("api_key = \"{}\"\n", value);
140 req!(secret::scan(line.as_bytes()), Vec::<Find>::new(), "for {:?}", value);
141 }
142 Ok(())
143 }));
144
145 res!(test_it(filter, &["A short literal is not a credential", "all", "secret", "assigned"],
146 ||
147 {
148 let line = fmt!("password: \"{}\"\n", "9f3Bq7ZmR4tYuIoPkLjH");
149 req!(secret::scan(line.as_bytes()), vec![Find { line: 1, kind: Kind::Assigned }]);
150 let line = fmt!("password: \"{}\"\n", "9f3Bq7ZmR4tYuIoPkLj");
151 req!(secret::scan(line.as_bytes()), Vec::<Find>::new());
152 Ok(())
153 }));
154
155 res!(test_it(filter, &["The marker excuses a line, and only while it is there", "all",
156 "secret", "marker"], ||
157 {
158 let bare = fmt!("let key = \"{}{}\";\n", SHAPED[0].0, SHAPED[0].1);
159 req!(secret::scan(bare.as_bytes()), vec![Find { line: 1, kind: Kind::Fireworks }]);
160 for marker in ["allowlist secret", "allowlist-secret", "ALLOWLIST SECRET",
161 "pragma: allowlist nextline"]
162 {
163 let line = fmt!("let key = \"{}{}\"; // {}\n", SHAPED[0].0, SHAPED[0].1, marker);
164 req!(secret::scan(line.as_bytes()), Vec::<Find>::new(), "for {:?}", marker);
165 let above = fmt!("// {}\nlet key = \"{}{}\";\n", marker, SHAPED[0].0, SHAPED[0].1);
166 req!(secret::scan(above.as_bytes()), Vec::<Find>::new(), "above, for {:?}", marker);
167 }
168 // One line above, and no further.
169 let far = fmt!("// {}\n\nlet key = \"{}{}\";\n", secret::MARKER, SHAPED[0].0, SHAPED[0].1);
170 req!(secret::scan(far.as_bytes()), vec![Find { line: 3, kind: Kind::Fireworks }]);
171 Ok(())
172 }));
173
174 res!(test_it(filter, &["A finding names the line it is on", "all", "secret", "marker"], || {
175 let text = fmt!("one\ntwo\nthree\nlet key = \"{}{}\";\nfive\n",
176 SHAPED[0].0, SHAPED[0].1);
177 req!(secret::scan(text.as_bytes()), vec![Find { line: 4, kind: Kind::Fireworks }]);
178 Ok(())
179 }));
180
181 res!(test_it(filter, &["A binary is left alone", "all", "secret", "binary"], || {
182 let mut data = fmt!("\0\u{1}\u{2}").into_bytes();
183 data.extend_from_slice(fmt!("key = \"{}{}\"\n", SHAPED[0].0, SHAPED[0].1).as_bytes());
184 req!(secret::scan(&data), Vec::<Find>::new());
185 Ok(())
186 }));
187
188 res!(test_it(filter, &["Lockfiles and vendored trees are not scanned", "all", "secret",
189 "path"], ||
190 {
191 for path in ["Cargo.lock", "web/package-lock.json", "go.sum", "node_modules/a/b.js",
192 "target/debug/build.rs", "a/vendor/b/c.go", "dist/app.js", ".venv/lib/x.py"]
193 {
194 req!(secret::skip_path(path.as_bytes()), true, "for {:?}", path);
195 }
196 for path in ["src/main.rs", "target.rs", "vendor.md", "a/build.rs", "notes/dist.txt"] {
197 req!(secret::skip_path(path.as_bytes()), false, "for {:?}", path);
198 }
199 Ok(())
200 }));
201
202 res!(test_it(filter, &["A vendored name below a source tree is not build output", "all",
203 "secret", "path"], ||
204 {
205 // `dist` earns its place on the list because a bundler writes one beside a source tree.
206 // Below a `src` the same name is a person's own, and reading it as build output left
207 // fourteen hand-written Rust files unscanned by this crate and by the git hook.
208 for path in ["fe2o3_o3db_sync/src/dist/cohort.rs", "src/dist/mod.rs", "a/b/src/build/x.rs",
209 "src/vendor/x.rs", "crate/src/target/y.rs", "src/node_modules/z.js", "src/.venv/w.py"]
210 {
211 req!(secret::skip_path(path.as_bytes()), false, "for {:?}", path);
212 }
213 // A source tree inside a vendored one is still somebody else's.
214 for path in ["node_modules/pkg/src/dist/bundle.js", "vendor/dep/src/lib.rs",
215 "target/debug/build/dep/src/main.rs"]
216 {
217 req!(secret::skip_path(path.as_bytes()), true, "for {:?}", path);
218 }
219 // The name of a lockfile still decides, wherever the file sits.
220 req!(secret::skip_path(b"src/dist/Cargo.lock"), true);
221 Ok(())
222 }));
223
224 res!(test_it(filter, &["A key in a source tree called dist is found", "all", "secret", "path"],
225 ||
226 {
227 // The two halves of the guard, put together the way a caller puts them: the path is
228 // scanned, and the scan refuses what is in it.
229 let path = b"fe2o3_o3db_sync/src/dist/transport.rs";
230 req!(secret::skip_path(path), false);
231 let line = fmt!("let key = \"{}{}\";\n", SHAPED[0].0, SHAPED[0].1);
232 req!(secret::scan(line.as_bytes()), vec![Find { line: 1, kind: Kind::Fireworks }]);
233 Ok(())
234 }));
235
236 res!(test_it(filter, &["A private key in DER form is caught", "all", "secret", "der"], || {
237 for (what, head, len) in DER {
238 let key = res!(der(head, *len));
239 req!(key.len(), *len, "for {:?}", what);
240 req!(secret::scan(&key), vec![Find { line: 1, kind: Kind::DerKey }], "for {:?}", what);
241 }
242 Ok(())
243 }));
244
245 res!(test_it(filter, &["The DKIM key's own shape is caught at 83 bytes", "all", "secret",
246 "der"], ||
247 {
248 // A raw PKCS#8 ed25519 key carrying its public half, which is what `ring` writes and what
249 // signed mail for four months from a folder that replicates. No armour, no vendor prefix
250 // and no field name beside it.
251 let key = res!(der(DKIM.0, DKIM.1));
252 req!(key.len(), 83);
253 req!(key[1], 0x51);
254 req!(secret::scan(&key), vec![Find { line: 1, kind: Kind::DerKey }]);
255 // The object identifier is the whole of what says so. One byte off it and this is 83 bytes
256 // that nothing else in the module can see -- no shape, no field name, no armour -- which is
257 // what the four months were.
258 let mut off = key.clone();
259 off[11] ^= 0x01;
260 req!(secret::scan(&off), Vec::<Find>::new());
261 // A real key's bytes are random, so a NUL stands somewhere in most of them, and the binary
262 // skip would then stop the scan before it began. This is asked first, and the order is what
263 // this line holds in place.
264 let mut nulled = key.clone();
265 nulled[20] = 0;
266 req!(secret::scan(&nulled), vec![Find { line: 1, kind: Kind::DerKey }]);
267 Ok(())
268 }));
269
270 res!(test_it(filter, &["A newline after the last byte does not hide a DER key", "all",
271 "secret", "der"], ||
272 {
273 for tail in ["\n", "\r\n", "\n\n"] {
274 let mut key = res!(der(DER[0].1, DER[0].2));
275 key.extend_from_slice(tail.as_bytes());
276 req!(secret::scan(&key), vec![Find { line: 1, kind: Kind::DerKey }], "for {:?}", tail);
277 }
278 Ok(())
279 }));
280
281 res!(test_it(filter, &["What is not a DER private key is left alone", "all", "secret",
282 "der"], ||
283 {
284 // A public key, which names the same algorithm and holds nothing worth refusing: the
285 // version integer this rule turns on is absent from it.
286 let public = res!(der("302A300506032B6570032100", 44));
287 req!(secret::scan(&public), Vec::<Find>::new(), "public key");
288 // A certificate, whose outer sequence opens with another sequence.
289 let cert = res!(der(CERT.0, CERT.1));
290 req!(secret::scan(&cert), Vec::<Find>::new(), "certificate");
291 // An algorithm nobody has, one object identifier byte away from ed25519.
292 let other = res!(der("302E020100300506032B657104220420", 48));
293 req!(secret::scan(&other), Vec::<Find>::new(), "unknown algorithm");
294 // Truncated: the outer length declares more than is there, so the sequence it names is not
295 // in the file. Nor is the key -- openssl reads nothing out of this one.
296 let short = res!(der(DER[0].1, 47));
297 req!(secret::scan(&short), Vec::<Find>::new(), "truncated");
298 // A three-byte sequence holding the version and stopping, with an ed25519 key's algorithm
299 // standing immediately after it. Every test below the length is asked inside the declared
300 // bytes, and this is the fixture that says so: unbounded, the sequence borrows the seven
301 // bytes after itself and this reads as a key.
302 let borrowed = res!(der("3003020100300506032B6570", 48));
303 req!(secret::scan(&borrowed), Vec::<Find>::new(), "borrowed algorithm");
304 // Nothing, and something far too small to be a key.
305 req!(secret::scan(b""), Vec::<Find>::new(), "empty");
306 req!(secret::scan(&[0x30, 0x02, 0x02, 0x01]), Vec::<Find>::new(), "tiny");
307 Ok(())
308 }));
309
310 res!(test_it(filter, &["Bytes written after a DER key do not hide it", "all", "secret",
311 "der"], ||
312 {
313 // What this rule asked until 2026-08-23 was that the outer sequence account for the input
314 // exactly, so that what it refused was a file that was a key and nothing else. One byte
315 // appended walked past the whole of it, and what walked past was a key openssl still read
316 // and signed with.
317 for tail in [&b"x"[..], b"\0", b" ", b"# the dkim signing key\n"] {
318 let mut key = res!(der(DKIM.0, DKIM.1));
319 key.extend_from_slice(tail);
320 req!(secret::scan(&key), vec![Find { line: 1, kind: Kind::DerKey }], "for {:?}", tail);
321 }
322 // The shape nobody has to tamper with to produce: a key and the certificate that goes with
323 // it in one file, which is what `cat key.der cert.der` writes.
324 let mut bundle = res!(der(DER[0].1, DER[0].2));
325 bundle.extend_from_slice(&res!(der(CERT.0, CERT.1)));
326 req!(bundle.len(), DER[0].2 + CERT.1);
327 req!(secret::scan(&bundle), vec![Find { line: 1, kind: Kind::DerKey }], "key and cert");
328 // And the marker is not a way out either, wherever it is written: this reads the key's own
329 // structure and never the bytes around it, so there is nowhere to put one that it looks at.
330 let mut marked = res!(der(DER[0].1, DER[0].2));
331 marked.extend_from_slice(fmt!("\n// {}\n", secret::MARKER).as_bytes());
332 req!(secret::scan(&marked), vec![Find { line: 1, kind: Kind::DerKey }], "marked");
333 Ok(())
334 }));
335
336 res!(test_it(filter, &["A DER key away from the front of the file is caught", "all", "secret",
337 "der"], ||
338 {
339 // `cat cert.der key.der`, which is the bundle a person writes without thinking about it:
340 // `openssl pkey -inform DER` reads the private key straight out of one, signs with it, and
341 // the signature verifies against the original key's public half. Both guards walked past it
342 // until 2026-08-23, because each only ever looked at byte 0.
343 for (what, head, len) in DER {
344 let mut bundle = res!(der(CERT.0, CERT.1));
345 bundle.extend_from_slice(&res!(der(head, *len)));
346 req!(bundle.len(), CERT.1 + *len, "for {:?}", what);
347 req!(secret::scan(&bundle), vec![Find { line: 1, kind: Kind::DerKey }], "for {:?}", what);
348 }
349 // One byte in front of it does the same, whatever the byte is, the SEQUENCE tag included.
350 for lead in [0x41u8, 0x00, 0x30, 0x02] {
351 let mut data = vec![lead];
352 data.extend_from_slice(&res!(der(DKIM.0, DKIM.1)));
353 req!(secret::scan(&data), vec![Find { line: 1, kind: Kind::DerKey }],
354 "for a leading {:#04x}", lead);
355 }
356 // And the finding names the line the key opens on, so that a key written into a file
357 // somebody reads is reported where they will find it rather than at the top.
358 let mut noted = fmt!("# the dkim signing key\n\n").into_bytes();
359 noted.extend_from_slice(&res!(der(DKIM.0, DKIM.1)));
360 req!(secret::scan(&noted), vec![Find { line: 3, kind: Kind::DerKey }]);
361 Ok(())
362 }));
363
364 res!(test_it(filter, &["Only a small file is read at every offset", "all", "secret", "der"],
365 ||
366 {
367 // `ring` holds the head of a PKCS#8 key as a `const` template -- these very bytes, which is
368 // why the fixture below is the DKIM shape -- and a compiler writes that template into the
369 // read-only data of whatever links it. A sweep of every file under ~/usr on 2026-08-23,
370 // 623,722 files and 202 GB with nothing skipped, found the structure at 1,729 offsets in 303
371 // files, and every one of the 303 was a compiled artefact carrying that template: not one was
372 // a key. Nothing separates a template from a key that is not a guess about the bytes around
373 // it, and refusing an ordinary build output is how a guard gets switched off. So the offsets
374 // are read only while the file is small enough to be a key and what a key is bundled with,
375 // and the smallest artefact in that sweep was 101,960 bytes, three times the span.
376 let key = res!(der(DKIM.0, DKIM.1));
377 let mut inside = vec![0x5A; secret::DER_SPAN - key.len()];
378 inside.extend_from_slice(&key);
379 req!(inside.len(), secret::DER_SPAN);
380 req!(secret::scan(&inside), vec![Find { line: 1, kind: Kind::DerKey }], "at the span");
381 // One byte wider and only the front is read, which is where this key is not.
382 let mut over = vec![0x5A; secret::DER_SPAN + 1 - key.len()];
383 over.extend_from_slice(&key);
384 req!(over.len(), secret::DER_SPAN + 1);
385 req!(secret::scan(&over), Vec::<Find>::new(), "past the span");
386 // The front of a file is still read whatever the file's size, which is the rule as it stood
387 // before offsets were looked at and is what catches `cat key.der cert.der`.
388 let mut wide = key.clone();
389 wide.resize(secret::DER_SPAN * 4, 0x5A);
390 req!(secret::scan(&wide), vec![Find { line: 1, kind: Kind::DerKey }], "at the front");
391 Ok(())
392 }));
393
394 res!(test_it(filter, &["A compiled artefact is not read for a DER key", "all", "secret",
395 "der"], ||
396 {
397 // Size is the whole of the gate, and it is asked of the length the sequence declares, which
398 // is read out of the first four bytes and nothing more. fe2o3 has a 22 MB binary in its
399 // history, and the estate has ONNX models and video beside it. What the ceiling costs is
400 // stated here rather than left to be found: this is a well formed ed25519 key declaring
401 // 8996 bytes, which is the size an RSA-16384 key would be, and it goes free.
402 let big = res!(der("30822324020100300506032B657004220420", 9000));
403 req!(big.len(), 9000);
404 req!(secret::scan(&big), Vec::<Find>::new(), "over the ceiling");
405 // One byte under the ceiling the same key is caught, so the gate and nothing else is what
406 // let the one above through.
407 let under = res!(der("30821F3C020100300506032B657004220420", 8000));
408 req!(under.len(), 8000);
409 req!(secret::scan(&under), vec![Find { line: 1, kind: Kind::DerKey }], "at the ceiling");
410 // And an ELF header opens with nothing this rule answers to.
411 let mut elf = fmt!("\u{7f}ELF").into_bytes();
412 elf.resize(200, 0);
413 req!(secret::scan(&elf), Vec::<Find>::new(), "ELF");
414 Ok(())
415 }));
416
417 Ok(())
418}