Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_infer/src/face/mod.rs

5.5 KiB, 1 run

created by r1870400018:19753, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Face detection and face embedding: the two things this crate exists to do.
2//!
3//! # Order of operations
4//!
5//! 1. Fit the photograph into the detector's canvas with [`letterbox`], which
6//! answers the scale needed to put the results back in the original frame.
7//! 2. Run [`Detector::detect`], which gives a box, a score and five landmarks
8//! per face.
9//! 3. Run [`Embedder::embed`] on the original photograph and one detection's
10//! landmarks, which warps the face onto a fixed template and answers a
11//! hundred and twenty-eight dimensional unit vector.
12//! 4. Compare two of those with [`cosine`].
13//!
14//! # Channel order
15//!
16//! The two networks disagree, and neither says so. The detector was exported
17//! against blue-green-red input and the embedder against red-green-blue. Both
18//! entry points here take ordinary red-green-blue pixels and put the channels
19//! in the order each network was trained on, so a caller never has to know.
20
21pub mod align;
22pub mod detect;
23pub mod embed;
24
25pub use align::{
26 align_crop,
27 similarity,
28 Affine,
29 CROP,
30 TEMPLATE,
31};
32pub use detect::{
33 Detection,
34 Detector,
35 DetectorOptions,
36};
37pub use embed::{
38 cosine,
39 Embedder,
40 Embedding,
41};
42
43use oxedyne_fe2o3_core::prelude::*;
44
45/// A borrowed, interleaved, eight-bit image.
46#[derive(Clone, Copy, Debug)]
47pub struct Image<'a> {
48 /// Pixels, row-major, `channels` values per pixel.
49 pub pixels: &'a [u8],
50 /// Width in pixels.
51 pub width: usize,
52 /// Height in pixels.
53 pub height: usize,
54 /// Values per pixel, three for red-green-blue.
55 pub channels: usize,
56}
57
58impl<'a> Image<'a> {
59 /// Wraps a buffer, checking that it holds what the extents claim.
60 pub fn new(pixels: &'a [u8], width: usize, height: usize, channels: usize)
61 -> Outcome<Self>
62 {
63 let want = width * height * channels;
64 if pixels.len() != want {
65 return Err(err!(
66 "An image of {} by {} with {} channels wants {} bytes, but {} were given.",
67 width, height, channels, want, pixels.len();
68 Invalid, Input, Mismatch));
69 }
70 Ok(Self { pixels, width, height, channels })
71 }
72
73 /// Reads one channel of one pixel, answering zero outside the frame, which
74 /// is the constant border a warp needs.
75 #[inline]
76 pub fn sample(&self, x: f64, y: f64, c: usize) -> f64 {
77 if x < 0.0 || y < 0.0 || c >= self.channels {
78 return 0.0;
79 }
80 let (xi, yi) = (x as usize, y as usize);
81 if xi >= self.width || yi >= self.height {
82 return 0.0;
83 }
84 self.pixels[(yi * self.width + xi) * self.channels + c] as f64
85 }
86}
87
88/// What a letterbox did, so that a result can be put back in the original frame.
89#[derive(Clone, Copy, Debug, PartialEq)]
90pub struct Letterbox {
91 /// Multiplier applied to the original, at most one.
92 pub scale: f64,
93 /// Canvas width.
94 pub width: usize,
95 /// Canvas height.
96 pub height: usize,
97}
98
99impl Letterbox {
100 /// Maps a point on the canvas back to the original frame.
101 pub fn back(&self, x: f32, y: f32) -> (f32, f32) {
102 ((x as f64 / self.scale) as f32, (y as f64 / self.scale) as f32)
103 }
104}
105
106/// Fits an image into a canvas of the given size, keeping the aspect ratio and
107/// leaving the unused right and lower margin black.
108///
109/// Downscaling averages over the source footprint rather than taking one sample
110/// from it, because a face sixty pixels across in a four thousand pixel
111/// photograph is ten pixels across in a six hundred and forty pixel canvas, and
112/// a point sample of it is noise.
113pub fn letterbox(img: &Image<'_>, width: usize, height: usize)
114 -> Outcome<(Vec<u8>, Letterbox)>
115{
116 if width == 0 || height == 0 || img.width == 0 || img.height == 0 {
117 return Err(err!(
118 "A letterbox of {} by {} from {} by {} has no area.",
119 width, height, img.width, img.height;
120 Invalid, Input, Range));
121 }
122 let ch = img.channels;
123 let scale = (width as f64 / img.width as f64).min(height as f64 / img.height as f64);
124 let dw = ((img.width as f64 * scale).round() as usize).clamp(1, width);
125 let dh = ((img.height as f64 * scale).round() as usize).clamp(1, height);
126 let mut out = vec![0u8; width * height * ch];
127
128 if scale <= 1.0 {
129 // Area average: each destination pixel is the mean of the source
130 // rectangle that maps onto it.
131 for y in 0..dh {
132 let y0 = (y as f64 * img.height as f64 / dh as f64).floor() as usize;
133 let y1 = (((y + 1) as f64 * img.height as f64 / dh as f64).ceil() as usize)
134 .clamp(y0 + 1, img.height);
135 for x in 0..dw {
136 let x0 = (x as f64 * img.width as f64 / dw as f64).floor() as usize;
137 let x1 = (((x + 1) as f64 * img.width as f64 / dw as f64).ceil() as usize)
138 .clamp(x0 + 1, img.width);
139 let n = ((y1 - y0) * (x1 - x0)) as f64;
140 for c in 0..ch {
141 let mut s = 0.0f64;
142 for sy in y0..y1 {
143 for sx in x0..x1 {
144 s += img.pixels[(sy * img.width + sx) * ch + c] as f64;
145 }
146 }
147 out[(y * width + x) * ch + c] = (s / n).round().clamp(0.0, 255.0) as u8;
148 }
149 }
150 }
151 } else {
152 // Bilinear, which is what an enlargement wants.
153 for y in 0..dh {
154 let sy = (y as f64 + 0.5) / scale - 0.5;
155 let yb = sy.floor();
156 let fy = sy - yb;
157 for x in 0..dw {
158 let sx = (x as f64 + 0.5) / scale - 0.5;
159 let xb = sx.floor();
160 let fx = sx - xb;
161 for c in 0..ch {
162 let p00 = img.sample(xb.max(0.0), yb.max(0.0), c);
163 let p10 = img.sample((xb + 1.0).max(0.0), yb.max(0.0), c);
164 let p01 = img.sample(xb.max(0.0), (yb + 1.0).max(0.0), c);
165 let p11 = img.sample((xb + 1.0).max(0.0), (yb + 1.0).max(0.0), c);
166 let top = p00 + (p10 - p00) * fx;
167 let bot = p01 + (p11 - p01) * fx;
168 let v = top + (bot - top) * fy;
169 out[(y * width + x) * ch + c] = v.round().clamp(0.0, 255.0) as u8;
170 }
171 }
172 }
173 }
174 Ok((out, Letterbox { scale, width, height }))
175}