Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_graphics/src/h264/filter.rs

16.6 KiB, 54 runs

created by r1870400018:21099, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The deblocking filter (§8.7).
2//!
3//! Every block of the picture was predicted and transformed on its own, so the samples either side
4//! of a block boundary were arrived at by different routes and rarely meet smoothly. At a low
5//! quantisation the step is invisible; at a high one the picture is a grid. The filter smooths those
6//! steps -- and only those: it is told where the boundaries are, and it decides at each one whether
7//! the step across it is small enough to be an artefact of coding rather than an edge that was in
8//! the photograph.
9//!
10//! # It is not optional
11//!
12//! This is a normative in-loop filter, not a post-process. A decoder that leaves it out does not
13//! produce a slightly softer picture; it produces a **different** picture, and every later frame
14//! predicted from it diverges further. For a still frame drawn from the first picture of a film the
15//! divergence stops there, but the samples still differ from what every other decoder produces, so
16//! a decode that is held to FFmpeg sample for sample must run it.
17//!
18//! # How strongly, and where
19//!
20//! Two numbers govern each boundary. The **strength** `bS` says how much filtering the boundary may
21//! take; in an all-intra picture it is 4 at a macroblock edge and 3 inside one, which are the two
22//! strongest values, because there is no motion to weaken the case. The **thresholds** α and β come
23//! from the two macroblocks' quantisation parameters (Table 8-16): the coarser the quantisation, the
24//! larger a step the filter is willing to believe is an artefact. Where the step across the boundary
25//! is larger than α, or the steps just inside either side are larger than β, nothing is filtered --
26//! that is the test that keeps a real edge sharp.
27//!
28//! The order matters and is not obvious: **every vertical edge of a macroblock, left to right, then
29//! every horizontal one, top to bottom**, one macroblock at a time in raster order, each working on
30//! samples the macroblocks before it have already filtered. Filtering all the vertical edges of the
31//! picture and then all the horizontal ones gives a different answer.
32//!
33//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
34//! Anthropic Claude
35
36use crate::h264::decode::{
37 Filter,
38 View,
39};
40
41use oxedyne_fe2o3_core::prelude::*;
42
43// The first threshold, α′, indexed by indexA (Table 8-16).
44const ALPHA: [i32; 52] = [
45 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
46 4, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 17, 20, 22, 25, 28,
47 32, 36, 40, 45, 50, 56, 63, 71, 80, 90, 101, 113, 127, 144, 162, 182,
48 203, 226, 255, 255,
49];
50
51// The second threshold, β′, indexed by indexB (Table 8-16).
52const BETA: [i32; 52] = [
53 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
54 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 6, 6, 7, 7, 8, 8,
55 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
56 17, 17, 18, 18,
57];
58
59// The clipping limit t′C0, by boundary strength (1, 2 or 3) and indexA (Table 8-17).
60const TC0: [[i32; 52]; 3] = [
61 [
62 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
63 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,
64 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 7, 8,
65 9, 10, 11, 13,
66 ],
67 [
68 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
70 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 10, 11,
71 12, 13, 15, 17,
72 ],
73 [
74 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3,
76 3, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16,
77 18, 20, 23, 25,
78 ],
79];
80
81/// Every macroblock in this decoder is intra, which fixes the boundary strength: 4 at a macroblock
82/// edge and 3 within one. The strengths that depend on motion vectors and reference pictures do not
83/// arise, and are not implemented rather than being implemented and unreachable.
84pub fn deblock(v: &mut View) -> Outcome<()> {
85 for mb in 0..v.mbs_w * v.mbs_h {
86 let slice = match v.slice_of[mb] {
87 Some(s) => s,
88 None => continue,
89 };
90 // Each slice says for itself whether its macroblocks are filtered at all, and with what
91 // thresholds. A picture of one slice makes this look like a constant; a picture of two
92 // does not.
93 let ask = match v.filters.get(slice) {
94 Some(f) => *f,
95 None => continue,
96 };
97 if ask.idc == 1 {
98 continue;
99 }
100 let (mx, my) = ((mb % v.mbs_w) * 16, (mb / v.mbs_w) * 16);
101 let big = v.big[mb];
102 // Vertical edges, left to right. The leftmost is the macroblock's own edge; the rest are
103 // internal, and the eight-by-eight transform leaves out the ones at four and twelve
104 // because there is no transform boundary there.
105 let mut xs: Vec<usize> = vec![0];
106 if big {
107 xs.push(8);
108 } else {
109 xs.extend([4, 8, 12]);
110 }
111 for x in xs {
112 if x == 0 && mx == 0 {
113 continue;
114 }
115 res!(edge(v, mb, mx, my, x, 0, true, &ask));
116 }
117 // Then the horizontal ones, top to bottom.
118 let mut ys: Vec<usize> = vec![0];
119 if big {
120 ys.push(8);
121 } else {
122 ys.extend([4, 8, 12]);
123 }
124 for y in ys {
125 if y == 0 && my == 0 {
126 continue;
127 }
128 res!(edge(v, mb, mx, my, 0, y, false, &ask));
129 }
130 }
131 Ok(())
132}
133
134/// Filters one edge of one macroblock, luma and both colour differences.
135#[allow(clippy::too_many_arguments)]
136fn edge(v: &mut View, mb: usize, mx: usize, my: usize, ex: usize, ey: usize, vertical: bool,
137 ask: &Filter) -> Outcome<()>
138{
139 let mb_edge = if vertical { ex == 0 } else { ey == 0 };
140 // The macroblock on the other side of the edge.
141 let other = if !mb_edge {
142 mb
143 } else if vertical {
144 mb - 1
145 } else {
146 mb - v.mbs_w
147 };
148 if v.slice_of[other].is_none() {
149 return Ok(());
150 }
151 // A disposition of 2 filters everything within the slice and nothing across its boundary. The
152 // two films this was found by both use it, and the difference it makes is three rows of samples
153 // either side of one horizontal line across the picture -- small, and wrong.
154 if ask.idc == 2 && mb_edge && v.slice_of[other] != v.slice_of[mb] {
155 return Ok(());
156 }
157 // Every macroblock here is intra, so a macroblock edge takes the strongest filter the
158 // specification has and an internal one the next strongest (§8.7.2.1). The lower strengths are
159 // reached only through the coefficient and motion tests, and both are tested *after* the intra
160 // one, so an all-intra picture never sees them. They are absent rather than written and
161 // unreachable.
162 let bs = if mb_edge { 4 } else { 3 };
163
164 let qp_p = v.qp[other];
165 let qp_q = v.qp[mb];
166 // Luma: sixteen sets of samples across the edge.
167 for i in 0..16 {
168 let (px, py) = if vertical {
169 (mx + ex, my + i)
170 } else {
171 (mx + i, my + ey)
172 };
173 res!(one(v, px, py, vertical, bs, qp_p, qp_q, false, ask));
174 }
175 // Chroma: eight sets, at half the resolution, so only the edges at even luma positions have a
176 // chroma counterpart -- which for 4:2:0 is every edge this filter visits except the ones at
177 // four and twelve, whose chroma position is not on a transform boundary.
178 if (vertical && ex % 8 != 0) || (!vertical && ey % 8 != 0) {
179 return Ok(());
180 }
181 for c in 0..2usize {
182 let offset = if c == 0 { v.cb_qp_offset } else { v.cr_qp_offset };
183 let cp = crate::h264::transform::chroma_qp(qp_p, offset);
184 let cq = crate::h264::transform::chroma_qp(qp_q, offset);
185 for i in 0..8 {
186 let (px, py) = if vertical {
187 (mx / 2 + ex / 2, my / 2 + i)
188 } else {
189 (mx / 2 + i, my / 2 + ey / 2)
190 };
191 res!(one_chroma(v, c, px, py, vertical, bs, cp, cq, ask));
192 }
193 }
194 Ok(())
195}
196
197/// Reads the four samples either side of an edge out of a plane.
198fn take(px: &[u8], w: usize, h: usize, x: usize, y: usize, vertical: bool) -> Option<[i32; 8]> {
199 let mut out = [0i32; 8];
200 for k in 0..4usize {
201 // `p` runs away from the edge on the near side, `q` away from it on the far side. An edge
202 // with fewer than four samples behind it is at the picture's boundary, and there is nothing
203 // there to filter against.
204 let back = match if vertical { x.checked_sub(k + 1) } else { y.checked_sub(k + 1) } {
205 Some(v) => v,
206 None => return None,
207 };
208 let (pxx, pyy) = if vertical { (back, y) } else { (x, back) };
209 let (qxx, qyy) = if vertical { (x + k, y) } else { (x, y + k) };
210 if pxx >= w || pyy >= h || qxx >= w || qyy >= h {
211 return None;
212 }
213 out[k] = px[pyy * w + pxx] as i32;
214 out[4 + k] = px[qyy * w + qxx] as i32;
215 }
216 Some(out)
217}
218
219/// Writes the three samples either side of an edge back into a plane.
220fn give(px: &mut [u8], w: usize, h: usize, x: usize, y: usize, vertical: bool, s: &[i32; 8]) {
221 for k in 0..3usize {
222 let (pxx, pyy) = if vertical {
223 (match x.checked_sub(k + 1) { Some(v) => v, None => continue }, y)
224 } else {
225 (x, match y.checked_sub(k + 1) { Some(v) => v, None => continue })
226 };
227 let (qxx, qyy) = if vertical { (x + k, y) } else { (x, y + k) };
228 if pxx < w && pyy < h {
229 px[pyy * w + pxx] = s[k].clamp(0, 255) as u8;
230 }
231 if qxx < w && qyy < h {
232 px[qyy * w + qxx] = s[4 + k].clamp(0, 255) as u8;
233 }
234 }
235}
236
237/// Filters one set of luma samples across an edge (§8.7.2.3, §8.7.2.4).
238#[allow(clippy::too_many_arguments)]
239fn one(v: &mut View, x: usize, y: usize, vertical: bool, bs: i32, qp_p: i32, qp_q: i32,
240 chroma_style: bool, ask: &Filter) -> Outcome<()>
241{
242 let (w, h) = (v.pic.y.w, v.pic.y.h);
243 let mut s = match take(&v.pic.y.px, w, h, x, y, vertical) {
244 Some(s) => s,
245 None => return Ok(()),
246 };
247 if filter(&mut s, bs, qp_p, qp_q, ask.alpha, ask.beta, chroma_style) {
248 give(&mut v.pic.y.px, w, h, x, y, vertical, &s);
249 }
250 Ok(())
251}
252
253/// The same for one colour difference plane, which is always filtered in the chroma style.
254#[allow(clippy::too_many_arguments)]
255fn one_chroma(v: &mut View, c: usize, x: usize, y: usize, vertical: bool, bs: i32, qp_p: i32,
256 qp_q: i32, ask: &Filter) -> Outcome<()>
257{
258 let plane = if c == 0 { &mut v.pic.cb } else { &mut v.pic.cr };
259 let (w, h) = (plane.w, plane.h);
260 let mut s = match take(&plane.px, w, h, x, y, vertical) {
261 Some(s) => s,
262 None => return Ok(()),
263 };
264 if filter(&mut s, bs, qp_p, qp_q, ask.alpha, ask.beta, true) {
265 give(&mut plane.px, w, h, x, y, vertical, &s);
266 }
267 Ok(())
268}
269
270/// The filter itself, over one set of eight samples: `p3..p0` then `q0..q3`.
271///
272/// Whether anything changed, so that a caller need not write back a set the thresholds
273/// rejected.
274fn filter(s: &mut [i32; 8], bs: i32, qp_p: i32, qp_q: i32, off_a: i32, off_b: i32,
275 chroma_style: bool) -> bool
276{
277 let (p0, p1, p2, p3) = (s[0], s[1], s[2], s[3]);
278 let (q0, q1, q2, q3) = (s[4], s[5], s[6], s[7]);
279 let qp_av = (qp_p + qp_q + 1) >> 1;
280 let index_a = (qp_av + off_a).clamp(0, 51) as usize;
281 let index_b = (qp_av + off_b).clamp(0, 51) as usize;
282 let alpha = ALPHA[index_a];
283 let beta = BETA[index_b];
284 // The test that keeps a real edge sharp: a step larger than α across the boundary, or larger
285 // than β just inside either side, is a thing that was in the photograph.
286 if bs == 0 || (p0 - q0).abs() >= alpha || (p1 - p0).abs() >= beta || (q1 - q0).abs() >= beta {
287 return false;
288 }
289 let ap = (p2 - p0).abs();
290 let aq = (q2 - q0).abs();
291 if bs < 4 {
292 let tc0 = TC0[(bs - 1) as usize][index_a];
293 let tc = if chroma_style {
294 tc0 + 1
295 } else {
296 tc0 + i32::from(ap < beta) + i32::from(aq < beta)
297 };
298 let delta = ((((q0 - p0) << 2) + (p1 - q1) + 4) >> 3).clamp(-tc, tc);
299 s[0] = p0 + delta;
300 s[4] = q0 - delta;
301 if !chroma_style && ap < beta {
302 s[1] = p1 + ((p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1).clamp(-tc0, tc0);
303 }
304 if !chroma_style && aq < beta {
305 s[5] = q1 + ((q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1).clamp(-tc0, tc0);
306 }
307 return true;
308 }
309 // The strongest case, at a macroblock edge, which may move three samples either side.
310 let close = (p0 - q0).abs() < ((alpha >> 2) + 2);
311 if !chroma_style && ap < beta && close {
312 s[0] = (p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3;
313 s[1] = (p2 + p1 + p0 + q0 + 2) >> 2;
314 s[2] = (2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3;
315 } else {
316 s[0] = (2 * p1 + p0 + q1 + 2) >> 2;
317 }
318 if !chroma_style && aq < beta && close {
319 s[4] = (p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3;
320 s[5] = (p0 + q0 + q1 + q2 + 2) >> 2;
321 s[6] = (2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3;
322 } else {
323 s[4] = (2 * q1 + q0 + p1 + 2) >> 2;
324 }
325 true
326}
327
328#[cfg(test)]
329mod tests {
330 use super::*;
331
332 /// A set of samples with a step of `step` across the boundary and flat either side.
333 fn step(level: i32, step: i32) -> [i32; 8] {
334 [level, level, level, level, level + step, level + step, level + step, level + step]
335 }
336
337 #[test]
338 fn test_a_real_edge_is_left_alone_01() -> Outcome<()> {
339 // The whole point of the thresholds. A step larger than α is an edge that was in the
340 // photograph, and a filter that smoothed it would blur the picture wherever it is sharp.
341 // At a quantisation parameter of 30, α is 25.
342 let mut s = step(60, 200);
343 let changed = filter(&mut s, 4, 30, 30, 0, 0, false);
344 req!(changed, false, "a step of 200 was filtered at an alpha of {}", ALPHA[30]);
345 req!(s, step(60, 200), "the samples were altered anyway");
346 // And a small step at the same quantisation is filtered.
347 let mut s = step(60, 8);
348 let changed = filter(&mut s, 4, 30, 30, 0, 0, false);
349 req!(changed, true, "a step of 8 was left alone at an alpha of {}", ALPHA[30]);
350 let softened = s[0] > 60 && s[4] < 68;
351 req!(softened, true, "the step was not softened: {:?}", s);
352 Ok(())
353 }
354
355 #[test]
356 fn test_a_coarse_picture_is_filtered_harder_02() -> Outcome<()> {
357 // The thresholds climb with the quantisation parameter, because a coarsely quantised
358 // picture has larger coding steps in it and the filter must be willing to believe more of
359 // them are artefacts. At a parameter of 20 the same step is left alone that at 40 is
360 // filtered, and that difference is the whole of Table 8-16.
361 let mut fine = step(100, 20);
362 let mut coarse = step(100, 20);
363 let fine_changed = filter(&mut fine, 4, 20, 20, 0, 0, false);
364 let coarse_changed = filter(&mut coarse, 4, 40, 40, 0, 0, false);
365 req!(fine_changed, false, "a step of 20 was filtered at an alpha of {}", ALPHA[20]);
366 req!(coarse_changed, true, "a step of 20 was not filtered at an alpha of {}", ALPHA[40]);
367 // And below sixteen nothing is filtered at all, whatever the step.
368 let mut off = step(100, 1);
369 req!(filter(&mut off, 4, 15, 15, 0, 0, false), false,
370 "the filter ran at a quantisation parameter where alpha is nought");
371 Ok(())
372 }
373
374 #[test]
375 fn test_the_strongest_filter_moves_three_samples_03() -> Outcome<()> {
376 // At a macroblock edge the filter may reach three samples deep either side; inside a
377 // macroblock it reaches two. Confusing the two smooths a macroblock's interior more than
378 // the specification allows, which shows as a picture that is soft in patches.
379 let mut strong = step(100, 6);
380 let mut weak = step(100, 6);
381 req!(filter(&mut strong, 4, 30, 30, 0, 0, false), true);
382 req!(filter(&mut weak, 3, 30, 30, 0, 0, false), true);
383 let strong_deep = strong[2] != 100 || strong[6] != 106;
384 req!(strong_deep, true, "the macroblock-edge filter left the third sample alone");
385 let weak_deep = weak[2] != 100 || weak[6] != 106;
386 req!(weak_deep, false, "the internal filter reached the third sample: {:?}", weak);
387 Ok(())
388 }
389
390 #[test]
391 fn test_chroma_is_filtered_in_its_own_style_04() -> Outcome<()> {
392 // Chroma never moves more than the sample nearest the edge, whatever the strength. A
393 // decoder that filtered chroma as luma would soften colour two samples deep on every
394 // block boundary, which is visible as colour bleeding on a hard edge.
395 let mut s = step(100, 6);
396 req!(filter(&mut s, 4, 30, 30, 0, 0, true), true);
397 req!(s[1], 100, "chroma's second sample was moved");
398 req!(s[5], 106, "chroma's second sample on the far side was moved");
399 let near_moved = s[0] != 100 && s[4] != 106;
400 req!(near_moved, true, "chroma's nearest sample was not filtered at all");
401 Ok(())
402 }
403
404 #[test]
405 fn test_the_tables_are_monotone_and_the_right_length_05() -> Outcome<()> {
406 // Fifty-two entries each, three of them, and every one climbs. A transcription that
407 // dropped or duplicated an entry would shift the tail, and a shifted tail filters every
408 // coarsely quantised picture with the wrong threshold. Monotonicity is the cheapest
409 // property that a shift breaks.
410 for (name, table) in [("alpha", &ALPHA[..]), ("beta", &BETA[..]),
411 ("tc0 at 1", &TC0[0][..]), ("tc0 at 2", &TC0[1][..]),
412 ("tc0 at 3", &TC0[2][..])] {
413 req!(table.len(), 52, "{} holds the wrong number of entries", name);
414 for i in 1..52 {
415 let rising = table[i] >= table[i - 1];
416 req!(rising, true, "{} falls from {} to {} at {}",
417 name, table[i - 1], table[i], i);
418 }
419 // The first sixteen are nought, which is what turns the filter off at a fine
420 // quantisation.
421 for i in 0..16 {
422 req!(table[i], 0, "{} is not nought at {}", name, i);
423 }
424 }
425 // A stronger boundary never clips less than a weaker one.
426 for i in 0..52 {
427 let ordered = TC0[2][i] >= TC0[1][i] && TC0[1][i] >= TC0[0][i];
428 req!(ordered, true, "the strengths are out of order at indexA {}", i);
429 }
430 Ok(())
431 }
432}