Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_graphics/tests/film_posters.rs

19.9 KiB, 27 runs

created by r1870400018:21195, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! How many of a real library's films can this crate draw a poster frame for?
2//!
3//! A film's first frame is an IDR coded with intra prediction only, which is the same thing a
4//! photograph is, and this crate carries an intra decoder for each of the two codecs a household's
5//! films are written in. So a poster costs the container's metadata, one sample and one decode --
6//! and the question worth asking of a real library is how many of its films that actually works for.
7//!
8//! Nothing here holds a film. The corpus runs to hundreds of gigabytes and single films in it are
9//! larger than any sensible buffer, so `mp4::moov_of` lifts the index out by walking the top-level
10//! box headers and `Film::read_sample` fetches the one sample that holds the first frame.
11//!
12//! Point `FILM_POSTERS` at a directory of films to run these:
13//!
14//! ```text
15//! FILM_POSTERS=/srv/nfs4/Gallery cargo test --release -p oxedyne_fe2o3_graphics \
16//! --test film_posters -- --nocapture
17//! ```
18//!
19//! `FILM_POSTERS_MAX` caps how many films are looked at. `FILM_POSTERS_ORACLE` says how many of
20//! them are held to FFmpeg, sample for sample, which costs an FFmpeg process each and is therefore
21//! a sample of the corpus rather than the whole of it.
22//!
23//! The first test **reports rather than asserts**: how many films can be drawn today is a
24//! measurement and not a promise, and a number that is asserted stops being measured. The tests
25//! after it assert, because agreeing with another decoder is not a matter of degree.
26//!
27//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
28//! Anthropic Claude
29
30use oxedyne_fe2o3_graphics::{
31 h264,
32 hevc,
33 jpeg,
34 mp4::{
35 self,
36 Film,
37 Kind,
38 },
39 pixmap::Pixmap,
40 yuv,
41};
42
43use oxedyne_fe2o3_core::prelude::*;
44
45use std::{
46 collections::BTreeMap,
47 env,
48 fs::File,
49 path::{
50 Path,
51 PathBuf,
52 },
53};
54
55/// Every film under a directory, in a stable order.
56fn films() -> Outcome<Option<Vec<PathBuf>>> {
57 let root = match env::var("FILM_POSTERS") {
58 Ok(d) if !d.is_empty() => PathBuf::from(d),
59 _ => {
60 println!(" skipped: set FILM_POSTERS to a directory of films");
61 return Ok(None);
62 },
63 };
64 let mut out = Vec::new();
65 let mut stack = vec![root];
66 while let Some(dir) = stack.pop() {
67 let entries = match std::fs::read_dir(&dir) {
68 Ok(e) => e,
69 Err(_) => continue,
70 };
71 for entry in entries.flatten() {
72 let path = entry.path();
73 if path.is_dir() {
74 stack.push(path);
75 } else {
76 let ext = path.extension()
77 .map(|e| e.to_string_lossy().to_lowercase())
78 .unwrap_or_default();
79 if matches!(ext.as_str(), "mp4" | "mov" | "m4v") {
80 out.push(path);
81 }
82 }
83 }
84 }
85 out.sort();
86 if let Ok(n) = env::var("FILM_POSTERS_MAX") {
87 if let Ok(n) = n.parse::<usize>() {
88 out.truncate(n);
89 }
90 }
91 Ok(Some(out))
92}
93
94/// A film's index and the bytes of its first frame, holding none of the film itself.
95fn first_frame(path: &Path) -> Outcome<(Film, Vec<u8>)> {
96 let mut f = res!(File::open(path));
97 let film = res!(Film::of(&mut f));
98 let sample = res!(film.read_first_sync(&mut f));
99 Ok((film, sample))
100}
101
102/// The picture a film's first frame holds, as its codec asks for it.
103///
104/// Both decoders answer the same planar picture -- `yuv::Frame` -- and each hands back the size the
105/// stream says is to be shown rather than the size it was coded at.
106fn decode(film: &Film, sample: &[u8]) -> Outcome<yuv::Frame> {
107 let pic = match film.kind() {
108 Kind::Hevc => res!(hevc::picture_shown(film.config(), sample)),
109 Kind::Avc => (&res!(h264::decode::picture(film.config(), sample))).into(),
110 Kind::Mjpeg => {
111 // A Motion JPEG sample is a whole JPEG, which is a different decoder and a picture
112 // that arrives already in red, green and blue. It has no place in a planar
113 // comparison, so this says so rather than pretending.
114 let _ = res!(jpeg::decode(sample));
115 return Err(err!("A Motion JPEG frame is not a planar picture."; Unimplemented));
116 },
117 Kind::Other(code) => return Err(err!(
118 "A film coded as {}, which there is no decoder for.", String::from_utf8_lossy(&code);
119 Unimplemented)),
120 };
121 // And then the window the container says is the picture, where it says so.
122 Ok(match film.aperture() {
123 Some((x, y, w, h)) => pic.window(x as usize, y as usize, w as usize, h as usize),
124 None => pic,
125 })
126}
127
128/// The first sentence of a refusal, which is the part that names the reason rather than the file.
129fn why(e: &Error<ErrTag>) -> String {
130 let text = e.plain();
131 match text.split_once(". ") {
132 Some((first, _)) => first.to_string(),
133 None => text,
134 }
135}
136
137#[test]
138fn test_how_many_films_get_a_poster_00() -> Outcome<()> {
139 let films = match res!(films()) {
140 Some(f) => f,
141 None => return Ok(()),
142 };
143 println!(" {} films", films.len());
144 let mut by_kind: BTreeMap<String, usize> = BTreeMap::new();
145 let mut drawn_by_kind: BTreeMap<String, usize> = BTreeMap::new();
146 let mut refused: BTreeMap<String, usize> = BTreeMap::new();
147 let mut named: BTreeMap<String, Vec<String>> = BTreeMap::new();
148 let (mut looked, mut drawn, mut turned) = (0usize, 0usize, 0usize);
149
150 let began = std::time::Instant::now();
151 for path in &films {
152 looked += 1;
153 // A run over thousands of films is minutes long, and a test that says nothing until it
154 // ends is a test nobody can tell from a hung one.
155 if looked % 250 == 0 {
156 println!(" {} of {} looked at, {} drawn, {:.1}/s", looked, films.len(), drawn,
157 looked as f64 / began.elapsed().as_secs_f64().max(0.001));
158 }
159 let (film, sample) = match first_frame(path) {
160 Ok(f) => f,
161 Err(e) => {
162 let head = why(&e);
163 *refused.entry(head.clone()).or_insert(0) += 1;
164 named.entry(head).or_default().push(name(path));
165 continue;
166 },
167 };
168 let kind = match film.kind() {
169 Kind::Hevc => "hevc".to_string(),
170 Kind::Avc => "avc".to_string(),
171 Kind::Mjpeg => "mjpeg".to_string(),
172 Kind::Other(code) => fmt!("{}", String::from_utf8_lossy(&code)),
173 };
174 *by_kind.entry(kind.clone()).or_insert(0) += 1;
175 if film.rotation() != 0 {
176 turned += 1;
177 }
178 // Motion JPEG is drawn by the JPEG decoder rather than by either video one, so it counts
179 // as a poster here: what is being measured is which films get a picture.
180 let made: Outcome<(usize, usize)> = if film.kind() == Kind::Mjpeg {
181 match jpeg::decode(&sample) {
182 Ok(pm) => Ok((pm.width(), pm.height())),
183 Err(e) => Err(e),
184 }
185 } else {
186 match decode(&film, &sample) {
187 Ok(pic) => Ok((pic.y.w, pic.y.h)),
188 Err(e) => Err(e),
189 }
190 };
191 match made {
192 Ok((w, h)) if w > 0 && h > 0 => {
193 drawn += 1;
194 *drawn_by_kind.entry(kind).or_insert(0) += 1;
195 },
196 Ok((w, h)) => {
197 let head = fmt!("A picture of {} by {} has nothing in it", w, h);
198 *refused.entry(head.clone()).or_insert(0) += 1;
199 named.entry(head).or_default().push(name(path));
200 },
201 Err(e) => {
202 let head = why(&e);
203 *refused.entry(head.clone()).or_insert(0) += 1;
204 named.entry(head).or_default().push(name(path));
205 },
206 }
207 }
208
209 println!(" {} films looked at, {} posters drawn", looked, drawn);
210 if looked > 0 {
211 println!(" that is {:.1}% of them", 100.0 * drawn as f64 / looked as f64);
212 }
213 println!(" {} of them carry a rotation in the track header", turned);
214 println!(" by codec:");
215 for (kind, n) in &by_kind {
216 println!(" {:>6} {:>5} films, {:>5} drawn", kind, n,
217 drawn_by_kind.get(kind).copied().unwrap_or(0));
218 }
219 println!(" refused:");
220 let mut sorted: Vec<(String, usize)> = refused.into_iter().collect();
221 sorted.sort_by(|a, b| b.1.cmp(&a.1));
222 for (head, n) in &sorted {
223 println!(" {:>5} {}", n, head);
224 // Named, because a count says how many and a name says which -- and the next fault is
225 // found by opening one of them.
226 if let Some(list) = named.get(head) {
227 for one in list.iter().take(4) {
228 println!(" {}", one);
229 }
230 if list.len() > 4 {
231 println!(" ... and {} more", list.len() - 4);
232 }
233 }
234 }
235 // The one thing asserted: that the chain works at all. A run where nothing was drawn has
236 // measured nothing, and a broken reader would otherwise report a tidy zero per cent.
237 if looked > 0 {
238 let any = drawn > 0;
239 req!(any, true, "not one film of {} gave up a frame", looked);
240 }
241 Ok(())
242}
243
244fn name(path: &Path) -> String {
245 path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default()
246}
247
248/// What FFmpeg makes of a film's first frame, in the planar form the decoders answer.
249///
250/// **`-noautorotate` is load-bearing.** A phone writes the angle it was held at into the track
251/// header, and FFmpeg turns the picture on the way out. A decoder answers the picture as it was
252/// *coded*, and the turn is the container's business, so without this the two are compared after
253/// one of them has been turned -- and at ninety degrees the byte count is unchanged, so nothing but
254/// the samples says so.
255fn ffmpeg_planes(path: &Path, rotate: bool) -> Outcome<Vec<u8>> {
256 let mut cmd = std::process::Command::new("ffmpeg");
257 cmd.args(["-v", "error"]);
258 if !rotate {
259 cmd.arg("-noautorotate");
260 }
261 cmd.arg("-i").arg(path)
262 .args(["-map", "0:v:0", "-frames:v", "1", "-pix_fmt", "yuv420p", "-f", "rawvideo", "-"]);
263 let out = res!(cmd.output());
264 if out.stdout.is_empty() {
265 return Err(err!(
266 "FFmpeg decoded no frame of {:?}: {}", path, String::from_utf8_lossy(&out.stderr);
267 Test, Missing));
268 }
269 Ok(out.stdout)
270}
271
272/// How many samples differ and by how much at worst.
273fn compare(mine: &[u16], theirs: &[u8]) -> (usize, i32) {
274 let mut wrong = 0usize;
275 let mut worst = 0i32;
276 for (a, b) in mine.iter().zip(theirs.iter()) {
277 let d = (*a as i32 - *b as i32).abs();
278 if d != 0 {
279 wrong += 1;
280 worst = worst.max(d);
281 }
282 }
283 (wrong, worst)
284}
285
286#[test]
287fn test_a_sample_of_posters_matches_ffmpeg_01() -> Outcome<()> {
288 // The check that matters. Every table, every prediction mode, every shift in an inverse
289 // transform and every threshold in a loop filter has to be right at once or some sample
290 // somewhere differs -- and a decoder that is nearly right produces a picture that looks
291 // decoded, so nothing short of sample-for-sample agreement is evidence of anything.
292 let films = match res!(films()) {
293 Some(f) => f,
294 None => return Ok(()),
295 };
296 let want: usize = match env::var("FILM_POSTERS_ORACLE") {
297 Ok(n) => res!(n.parse()),
298 Err(_) => 40,
299 };
300 if films.is_empty() || want == 0 {
301 return Ok(());
302 }
303 // Spread across the corpus rather than taken from its front: the front of this tree is one
304 // import from one phone, and a sample of that says nothing about the twenty years behind it.
305 let stride = (films.len() / want).max(1);
306 let mut exact = 0usize;
307 let mut compared = 0usize;
308 let mut differ: Vec<String> = Vec::new();
309 let mut refused: BTreeMap<String, usize> = BTreeMap::new();
310 let mut worst_seen = 0i32;
311 for path in films.iter().step_by(stride).take(want) {
312 let (film, sample) = match first_frame(path) {
313 Ok(f) => f,
314 Err(e) => {
315 *refused.entry(why(&e)).or_insert(0) += 1;
316 continue;
317 },
318 };
319 if matches!(film.kind(), Kind::Mjpeg | Kind::Other(_)) {
320 continue;
321 }
322 let mine = match decode(&film, &sample) {
323 Ok(p) => p,
324 Err(e) => {
325 *refused.entry(why(&e)).or_insert(0) += 1;
326 continue;
327 },
328 };
329 let theirs = res!(ffmpeg_planes(path, false));
330 let (w, h) = (mine.y.w, mine.y.h);
331 let luma = w * h;
332 let chroma = w.div_ceil(2) * h.div_ceil(2);
333 if theirs.len() != luma + 2 * chroma {
334 differ.push(fmt!(
335 "{}: this decoder made {} by {} and FFmpeg gave {} bytes, which is not {} plus \
336 two of {}", name(path), w, h, theirs.len(), luma, chroma));
337 continue;
338 }
339 compared += 1;
340 let (wy, dy) = compare(&mine.y.px, &theirs[..luma]);
341 let (wu, du) = compare(&mine.cb.px, &theirs[luma..luma + chroma]);
342 let (wv, dv) = compare(&mine.cr.px, &theirs[luma + chroma..]);
343 worst_seen = worst_seen.max(dy).max(du).max(dv);
344 if wy == 0 && wu == 0 && wv == 0 {
345 exact += 1;
346 } else {
347 differ.push(fmt!(
348 "{} {}x{} {:?}: luma {} wrong (worst {}), Cb {} (worst {}), Cr {} (worst {})",
349 name(path), w, h, film.kind(), wy, dy, wu, du, wv, dv));
350 }
351 }
352 println!(" {} posters held to FFmpeg, {} matched sample for sample", compared, exact);
353 println!(" worst difference anywhere in the sample: {}", worst_seen);
354 if !refused.is_empty() {
355 println!(" refused before any comparison:");
356 for (head, n) in &refused {
357 println!(" {:>5} {}", n, head);
358 }
359 }
360 for line in &differ {
361 println!(" DIFFERS {}", line);
362 }
363 if !differ.is_empty() {
364 return Err(err!(
365 "{} of {} posters decoded to something other than what FFmpeg decoded. A picture that \
366 is nearly right is a wrong picture.", differ.len(), compared; Test, Mismatch));
367 }
368 if compared == 0 {
369 println!(" nothing was comparable, so nothing was proved");
370 }
371 Ok(())
372}
373
374#[test]
375fn test_a_turned_film_says_so_and_ffmpeg_agrees_02() -> Outcome<()> {
376 // A rotation applied nowhere is invisible in a test of a film with a unity matrix, and at
377 // ninety degrees the turned picture has exactly as many samples as the untuned one -- so this
378 // finds a film the container says is turned and asks FFmpeg, twice, whether it agrees: once
379 // with the turn suppressed, where the frame must come out as it was coded, and once with the
380 // turn applied, where its two dimensions must be exchanged.
381 let films = match res!(films()) {
382 Some(f) => f,
383 None => return Ok(()),
384 };
385 let mut found = None;
386 let mut turns: BTreeMap<u16, usize> = BTreeMap::new();
387 for path in &films {
388 let (film, sample) = match first_frame(path) {
389 Ok(f) => f,
390 Err(_) => continue,
391 };
392 *turns.entry(film.rotation()).or_insert(0) += 1;
393 if film.rotation() == 90 && found.is_none() && !matches!(film.kind(), Kind::Other(_)) {
394 if let Ok(pic) = decode(&film, &sample) {
395 found = Some((path.clone(), film, pic));
396 }
397 }
398 }
399 println!(" rotations across the corpus: {:?}", turns);
400 let (path, film, pic) = match found {
401 Some(f) => f,
402 None => {
403 println!(" no film in this corpus is turned a quarter, so nothing was proved");
404 return Ok(());
405 },
406 };
407 println!(" {} is turned {} degrees, coded {} by {}",
408 name(&path), film.rotation(), pic.y.w, pic.y.h);
409
410 // Coded: the decoder's picture and FFmpeg's untuned one are the same shape and the same
411 // samples.
412 let coded = res!(ffmpeg_planes(&path, false));
413 let luma = pic.y.w * pic.y.h;
414 let chroma = pic.y.w.div_ceil(2) * pic.y.h.div_ceil(2);
415 req!(coded.len(), luma + 2 * chroma,
416 "FFmpeg's untuned frame is not the shape this decoder's is");
417 let (wrong, worst) = compare(&pic.y.px, &coded[..luma]);
418 req!(wrong, 0usize, "the coded frame differs from FFmpeg's by up to {}", worst);
419
420 // Shown: FFmpeg's turned frame has the two dimensions exchanged, which is what the turn a
421 // viewer applies has to do.
422 let shown = res!(ffmpeg_planes(&path, true));
423 let turned_luma = pic.y.h * pic.y.w;
424 let turned_chroma = pic.y.h.div_ceil(2) * pic.y.w.div_ceil(2);
425 req!(shown.len(), turned_luma + 2 * turned_chroma,
426 "FFmpeg's turned frame is not the size a quarter turn makes");
427 // And the turn is a transposition and not a copy: the sample at (x, y) of the coded frame is
428 // the one at (h - 1 - y, x) of the turned frame, which is what a quarter turn clockwise means.
429 let (w, h) = (pic.y.w, pic.y.h);
430 let mut checked = 0usize;
431 for y in (0..h).step_by((h / 16).max(1)) {
432 for x in (0..w).step_by((w / 16).max(1)) {
433 let mine = pic.y.px[y * w + x];
434 let theirs = shown[x * h + (h - 1 - y)];
435 req!(mine, theirs as u16,
436 "at ({}, {}) the turned frame is not the coded one transposed", x, y);
437 checked += 1;
438 }
439 }
440 println!(" {} sampled positions agree after the quarter turn", checked);
441 Ok(())
442}
443
444#[test]
445fn test_a_poster_can_be_drawn_in_colour_03() -> Outcome<()> {
446 // The last step of the chain, which the planar comparison above does not reach: the picture
447 // becomes red, green and blue, at the size the film says it is shown at.
448 let films = match res!(films()) {
449 Some(f) => f,
450 None => return Ok(()),
451 };
452 for path in films.iter().take(200) {
453 let (film, sample) = match first_frame(path) {
454 Ok(f) => f,
455 Err(_) => continue,
456 };
457 let pic = match decode(&film, &sample) {
458 Ok(p) => p,
459 Err(_) => continue,
460 };
461 let px: Pixmap = res!(yuv::rgb(&pic, yuv::Matrix::Hd, false));
462 req!(px.width(), pic.y.w);
463 req!(px.height(), pic.y.h);
464 // A picture, not a flat field: a decoder answering one colour everywhere would satisfy
465 // every dimension check there is.
466 let first = &px.data()[..4];
467 let differs = px.data().chunks(4).any(|p| p != first);
468 req!(differs, true, "{} came out one flat colour", name(path));
469 println!(" {} drew {} by {} in colour", name(path), px.width(), px.height());
470 return Ok(());
471 }
472 println!(" no film in this corpus gave up a frame, so nothing was proved");
473 Ok(())
474}
475
476#[test]
477fn test_zz_a_window_onto_one_film_05() -> Outcome<()> {
478 // Not a check but a window. `FILM_POSTERS_DUMP` names one film and this writes its first
479 // frame's brightness plane beside FFmpeg's, so that a disagreement can be looked at rather
480 // than counted: a picture decoded from the wrong sample, one shifted by a few rows and one
481 // quantised against the wrong weights are indistinguishable in a difference count and obvious
482 // side by side.
483 let one = match env::var("FILM_POSTERS_DUMP") {
484 Ok(p) => PathBuf::from(p),
485 Err(_) => return Ok(()),
486 };
487 let (film, sample) = res!(first_frame(&one));
488 // The parameter sets and the slice header as this crate reads them, to be held beside what
489 // FFmpeg's `trace_headers` prints for the same bytes.
490 if film.kind() == Kind::Hevc {
491 let cfg = res!(hevc::config(film.config()));
492 for unit in &cfg.sets {
493 match unit.kind {
494 hevc::nal::SPS => println!(" sps: {:?}", res!(hevc::sps(&unit.body))),
495 hevc::nal::PPS => println!(" pps: {:?}", res!(hevc::pps(&unit.body))),
496 _ => {},
497 }
498 }
499 let units = res!(hevc::split_lengthed(&sample, cfg.length_size));
500 let mut seqs = Vec::new();
501 let mut pics = Vec::new();
502 for unit in &cfg.sets {
503 match unit.kind {
504 hevc::nal::SPS => seqs.push(res!(hevc::sps(&unit.body))),
505 hevc::nal::PPS => pics.push(res!(hevc::pps(&unit.body))),
506 _ => {},
507 }
508 }
509 for unit in &units {
510 println!(" nal {} of {} bytes", unit.kind, unit.body.len());
511 if matches!(unit.kind, 19 | 20 | 21) {
512 let want = res!(hevc::slice_pps_id(&unit.body));
513 if let Some(pps) = pics.iter().find(|p| p.id == want) {
514 if let Some(sps) = seqs.iter().find(|s| s.id == pps.sps_id) {
515 println!(" slice: {:?}", res!(hevc::slice_of(unit.kind, &unit.body, sps, pps)));
516 }
517 }
518 }
519 }
520 }
521 let pic = res!(decode(&film, &sample));
522 println!(" {} decoded {} by {}", name(&one), pic.y.w, pic.y.h);
523 let mine: Vec<u8> = pic.y.px.iter().map(|v| *v as u8).collect();
524 let out = std::env::temp_dir().join("film_poster_luma.gray");
525 res!(std::fs::write(&out, &mine));
526 println!(" wrote {} bytes of luma to {:?}", mine.len(), out);
527 Ok(())
528}
529
530#[test]
531fn test_the_index_is_read_without_holding_the_film_04() -> Outcome<()> {
532 // The property the whole pass depends on: a film larger than any buffer a scan will allocate
533 // still gives up its metadata and one sample. Held to the file's own size, so a reader that
534 // quietly read the whole thing would have to have read more than this asserts.
535 let films = match res!(films()) {
536 Some(f) => f,
537 None => return Ok(()),
538 };
539 let mut biggest: Option<(PathBuf, u64)> = None;
540 for path in &films {
541 if let Ok(meta) = std::fs::metadata(path) {
542 let len = meta.len();
543 if biggest.as_ref().map(|(_, b)| len > *b).unwrap_or(true) {
544 biggest = Some((path.clone(), len));
545 }
546 }
547 }
548 let (path, len) = match biggest {
549 Some(b) => b,
550 None => return Ok(()),
551 };
552 println!(" the largest film here is {} at {} bytes", name(&path), len);
553 let mut f = res!(File::open(&path));
554 let moov = match res!(mp4::moov_of(&mut f)) {
555 Some(m) => m,
556 None => {
557 println!(" it carries no movie box, so nothing was proved");
558 return Ok(());
559 },
560 };
561 let film = res!(Film::from_moov(&moov));
562 let i = res!(film.first_sync());
563 let (_off, size) = res!(film.span(i));
564 let read = moov.len() as u64 + size as u64;
565 println!(" its index is {} bytes and its first sample {}, which is {:.2}% of the file",
566 moov.len(), size, 100.0 * read as f64 / len.max(1) as f64);
567 let part = read < len;
568 req!(part, true, "reading the index and one sample read the whole film");
569 Ok(())
570}