Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_mail/tests/mail_plan.rs

17.7 KiB, 422 runs

created by r1555483153:78613, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `search::plan` and `SearchKey::wire`: each planned key, sent through `wire`, read back by the
2//! server's own `parse` and evaluated by its `matches` over a corpus, holds every mail the one
3//! matcher answers a hit (or unknown) for (S2). Cases QP1-QP7, QS1-QS15 of MAIL-q-plan's CASES.md.
4
5use oxedyne_fe2o3_core::prelude::*;
6use oxedyne_fe2o3_mail::search::{plan, Day, Limits, Mail, Matcher, Mode, Needs, Plan, Query, Scope, Server, Term};
7use oxedyne_fe2o3_net::imap::search::{civil_from_days, days_from_civil, parse, Content, Flag, SearchKey, Segment};
8use oxedyne_fe2o3_net::mail::store::{MessageFlags, MessageMeta, MessageUid};
9use oxedyne_fe2o3_text::{regex::Regex, unicode::norm::nfc};
10
11use std::time::{Duration, UNIX_EPOCH};
12
13const LIM: Limits = Limits { search_max_terms: 2000, search_regex_min_literal: 2, search_key_max_bytes: 4096, ..Limits::DEFAULT };
14const TODAY: Day = Day { year: 2026, month: 3, day: 4 };
15const PLAIN: Server = Server { gmail: false, names: true };
16const GMAIL: Server = Server { gmail: true, names: true };
17
18fn mime(h: &str, ct: &str, body: &str) -> String {
19 fmt!("From: Ada <ada@x.example>\r\nTo: Bob <bob@x.example>\r\n{}Content-Type: {}\r\n\r\n{}\r\n", h, ct, body)
20}
21fn txt(body: &str) -> String { mime("Subject: note\r\n", "text/plain; charset=utf-8", body) }
22fn html(body: &str) -> String { mime("Subject: note\r\n", "text/html; charset=utf-8", body) }
23fn att(part: &str) -> String {
24 mime("Subject: files\r\n", "multipart/mixed; boundary=b", &fmt!("--b\r\nContent-Type: text/plain\r\n\r\nsee attached\r\n\
25 --b\r\n{}\r\n--b--", part))
26}
27fn pdf(disp: &str) -> String { att(&fmt!("Content-Type: application/pdf\r\nContent-Disposition: attachment{}\r\n\r\nJVBERi0=", disp)) }
28
29/// A corpus mail: its bytes, its INTERNALDATE (year, month, day, hour, minute, zone in minutes),
30/// its flags (`s` seen, `f` flagged) and its size (0: its length).
31struct M { raw: String, at: (i32, u32, u32, i64, i64, i64), flags: &'static str, size: u64 }
32
33fn corpus() -> Vec<M> {
34 let mut v: Vec<M> = [
35 txt("Please pay invoice INV-2024 by Friday."), txt("the concat function"), txt("a cat."),
36 mime("Subject: Cat pictures\r\n", "text/plain", "meow"), txt("concatenate them"),
37 txt("\u{212A}elvin scale"), txt("cla\u{017F}s act"),
38 mime("Subject: q\r\nContent-Transfer-Encoding: quoted-printable\r\n", "text/plain", "net =\r\nprofit up"),
39 txt("net\r\n profit down"), html("<p>net <b>profit</b> AT&amp;T</p>"),
40 mime("Subject: =?UTF-8?B?R3LDvMOfZQ==?=\r\n", "text/plain", "hallo"),
41 pdf("; filename*=UTF-8''re%CC%81sume%CC%81.pdf"), pdf("; filename=\"report.pdf\""),
42 pdf("; filename=\"Q3-report-final.xlsx\""), mime("Subject: zeta news\r\nCc: Carol <carol@x.example>\r\n", "text/plain", "zeta"),
43 mime("Subject: hi\r\nBcc: Eve <eve@x.example>\r\n", "text/plain", "alpha"), txt("alpha and zeta"),
44 txt("inventory-2024 count"), txt("invoice-77 due"), txt("abcx and abdx"), txt("aaab"), txt("xyz"), txt("yz"),
45 html("<a href=\"https://concat.example\">link</a>"), txt("\u{65E5}\u{672C}\u{8A9E} text"), txt("\u{E9}cole invoice"),
46 txt("a\u{37E}b"), mime("Subject: alt\r\n", "multipart/alternative; boundary=a",
47 "--a\r\nContent-Type: text/plain\r\n\r\nhello\r\n--a\r\nContent-Type: text/html\r\n\r\n<p>hello secretword</p>\r\n--a--"),
48 txt("Cat and cat"), txt(&"lorem ".repeat(50)), pdf("; filename=\"cat.png\""), txt("AT&T merger"),
49 mime("Subject: cat\u{30C}\r\n", "text/plain", "caron"),
50 mime("Subject: m\r\n", "multipart/mixed", "invoice here"), // a multipart left a leaf: no boundary
51 pdf(""), // named "attachment" by the matcher
52 att("Content-Type: message/rfc822\r\n\r\nSubject: Quarterly figures\r\n\r\nhello"),
53 ].into_iter().map(|raw| M { raw, at: (2026, 3, 4, 12, 0, 0), flags: "", size: 0 }).collect();
54 for (at, flags, size) in [((2026, 1, 30, 23, 30, -420), "", 9), ((2026, 1, 31, 0, 10, 660), "s", 99),
55 ((2026, 2, 1, 12, 0, 0), "f", 2_000_000), ((2024, 2, 29, 8, 0, 0), "sf", 10), ((2024, 2, 22, 23, 59, 0), "", 10),
56 ((2024, 2, 23, 0, 1, 0), "s", 10), ((2024, 2, 21, 10, 0, 0), "", 10)]
57 {
58 v.push(M { raw: txt("dated"), at, flags, size });
59 }
60 v
61}
62
63impl M {
64 /// The server's metadata (a UTC instant) and the matcher's view (the calendar day as written).
65 fn meta(&self, i: usize) -> (MessageMeta, Mail<'_>) {
66 let (y, m, d, h, mi, z) = self.at;
67 let secs = days_from_civil(y, m, d) * 86_400 + h * 3600 + mi * 60 - z * 60;
68 let flags = MessageFlags { seen: self.flags.contains('s'), flagged: self.flags.contains('f'), ..MessageFlags::default() };
69 let size = if self.size == 0 { self.raw.len() as u64 } else { self.size };
70 let day = Day { year: y as u16, month: m as u8, day: d as u8 };
71 (MessageMeta { uid: MessageUid(i as u32 + 1), size, internal: UNIX_EPOCH + Duration::from_secs(secs as u64), flags },
72 Mail { raw: self.raw.as_bytes(), seen: flags.seen, flagged: flags.flagged, size, day: Some(day) })
73 }
74}
75
76/// The key as sent: text as is, a literal as `{n}` and its contents.
77fn show(k: &SearchKey) -> Outcome<String> {
78 Ok(res!(k.wire()).iter().map(|s| match s {
79 Segment::Text(t) => t.clone(),
80 Segment::Literal(l) => fmt!("{{{}}}{}", l.len(), l),
81 }).collect())
82}
83
84/// Gmail's `X-GM-RAW`, which Steel refuses, read as every mail; the rest of the key is kept, so
85/// that a Gmail plan is held to S2 as far as Steel can read it.
86fn steel(k: &SearchKey) -> SearchKey {
87 match k {
88 SearchKey::GmRaw(_) => SearchKey::All,
89 SearchKey::Not(x) => SearchKey::Not(Box::new(steel(x))),
90 SearchKey::Or(a, b) => SearchKey::Or(Box::new(steel(a)), Box::new(steel(b))),
91 SearchKey::And(v) => SearchKey::And(v.iter().map(steel).collect()),
92 k => k.clone(),
93 }
94}
95
96fn q(s: &str) -> Outcome<Query> { Query::parse(s, &LIM) }
97
98/// Plans `qy` and holds S2 over `mails` with the server's own reading of the key.
99fn hold(qy: &Query, srv: &Server, today: Day, mails: &[M]) -> Outcome<Plan> {
100 let p = res!(plan(qy, srv, &LIM, today));
101 let m = res!(Matcher::new(qy, &LIM, today));
102 let key = match &p.key {
103 Some(k) => Some(res!(parse(&res!(steel(k).wire())).map_err(|e| err!("Steel refused {:?}: {:?}", k, e; Invalid)))),
104 None => None,
105 };
106 let metas: Vec<MessageMeta> = mails.iter().enumerate().map(|(i, x)| x.meta(i).0).collect();
107 for (i, x) in mails.iter().enumerate() {
108 let hit = !matches!(m.matches(&x.meta(i).1), Ok(None));
109 let sel = key.as_ref().is_some_and(|k| k.matches(i, &metas, Some(&Content::new(x.raw.as_bytes()))));
110 req!(true, sel || !hit, "S2: {} (Gmail {}): mail {} {:?} is a hit the key {:?} drops", qy, srv.gmail, i, x.raw, p.key);
111 }
112 Ok(p)
113}
114
115fn check(qy: &Query, srv: &Server, today: Day) -> Outcome<Plan> { hold(qy, srv, today, &corpus()) }
116
117fn key(qy: &str, srv: &Server) -> Outcome<(String, bool)> {
118 let p = res!(check(&res!(q(qy)), srv, TODAY));
119 Ok((match &p.key { Some(k) => res!(show(k)), None => "none".into() }, p.complete))
120}
121
122fn pair(k: &str, complete: bool) -> (String, bool) { (k.to_string(), complete) }
123
124#[test]
125fn qp1_negation_on_either_server() -> Outcome<()> {
126 for s in [PLAIN, GMAIL] { req!(false, res!(key("-cat", &s)).0.contains("NOT"), "Gmail {}", s.gmail); }
127 Ok(())
128}
129
130#[test]
131fn qp2_no_charset_needed() -> Outcome<()> {
132 req!(res!(key("scope:body école invoice", &PLAIN)), pair("BODY \"cole\" BODY \"invoice\"", true));
133 req!(res!(key("body:\u{65E5}\u{672C}\u{8A9E}", &PLAIN)), pair("ALL", false));
134 Ok(())
135}
136
137#[test]
138fn qp3_wire_reads_back() -> Outcome<()> {
139 let b = |k| Box::new(k);
140 let day = days_from_civil;
141 let keys = [
142 SearchKey::Not(b(SearchKey::Or(b(SearchKey::Or(b(SearchKey::Or(b(SearchKey::Text("a".into())), b(SearchKey::Body("b".into())))),
143 b(SearchKey::Header("x-spam".into(), "yes".into())))), b(SearchKey::And(vec![SearchKey::Since(day(2024, 2, 29)), SearchKey::Before(day(1999, 12, 31))]))))),
144 SearchKey::And(["from", "to", "cc", "bcc", "subject"].iter().map(|h| SearchKey::Header(h.to_string(), "x y".into())).collect()),
145 SearchKey::Since(0), SearchKey::On(day(0, 1, 1)), SearchKey::SentBefore(day(9999, 12, 31)), SearchKey::SentOn(1), SearchKey::SentSince(-1),
146 SearchKey::Larger(u64::MAX), SearchKey::Smaller(0), SearchKey::Uid("1:5,9,*".into()), SearchKey::Seq("3,1:2".into()),
147 SearchKey::All, SearchKey::New, SearchKey::Old, SearchKey::And(vec![SearchKey::Flag(Flag::Seen, false)]),
148 SearchKey::Flag(Flag::Answered, true), SearchKey::Flag(Flag::Draft, false), SearchKey::Flag(Flag::Recent, true),
149 ];
150 for k in keys { req!(parse(&res!(k.wire())).ok(), Some(k.clone()), "{}", res!(show(&k))); }
151 for n in day(1600, 1, 1)..day(2400, 12, 31) {
152 let (y, m, d) = civil_from_days(n);
153 req!(days_from_civil(y, m, d), n);
154 }
155 req!(res!(show(&SearchKey::Since(day(0, 1, 1)))), "SINCE 1-Jan-0000".to_string()); // F5
156 for bad in [SearchKey::Since(day(-1, 12, 31)), SearchKey::Before(day(10000, 1, 1)), SearchKey::Uid("1\r\nA1 DELETE INBOX".into()),
157 SearchKey::Seq("0".into()), SearchKey::Uid("1:\u{e9}".into())]
158 {
159 req!(true, bad.wire().is_err(), "{:?}", bad); // F6
160 }
161 Ok(())
162}
163
164#[test]
165fn qp4_hostile_text() -> Outcome<()> {
166 let long = "a".repeat(4096);
167 for s in ["a\"b", "a\\b", "x\r\ny", "x\r\nA1 DELETE INBOX", "{5}", ")", "*lead", "grüße", &long] {
168 let k = SearchKey::Body(s.to_lowercase());
169 let w = res!(k.wire());
170 for g in &w {
171 if let Segment::Text(t) = g { req!(true, t.is_ascii() && !t.contains(['\r', '\n', '\0']), "{:?}", t); }
172 }
173 req!(parse(&w).ok(), Some(k.clone()), "{:?}", s);
174 req!(matches!(w.first(), Some(Segment::Text(t)) if t.starts_with("CHARSET UTF-8")), !s.is_ascii());
175 }
176 req!(true, SearchKey::Body("nul\0".into()).wire().is_err());
177 Ok(())
178}
179
180#[test]
181fn qp5_deep_tree_refused() -> Outcome<()> {
182 for n in [10_000, LIM.search_max_depth + 1] {
183 let mut t = Term::Text { scope: Some(Scope::Body), mode: Mode::Word, text: "x".into() };
184 for _ in 0..n { t = Term::Not(Box::new(t)); }
185 let e = plan(&Query { terms: t, ..Query::default() }, &PLAIN, &LIM, TODAY).err().map(|e| e.to_string());
186 req!(true, e.as_ref().is_some_and(|e| e.contains("search_max_depth")), "{:?}", e);
187 }
188 Ok(())
189}
190
191#[test]
192fn qp6_wide_any_widens() -> Outcome<()> {
193 let any = |n: usize| Query { terms: Term::Any((0..n).map(|i| Term::Text { scope: Some(Scope::Body), mode: Mode::Word,
194 text: fmt!("word{}", i) }).collect()), ..Query::default() };
195 let p = res!(check(&any(1000), &PLAIN, TODAY));
196 req!((p.key.clone(), p.complete), (Some(SearchKey::All), false));
197 let p = res!(check(&any(10), &PLAIN, TODAY));
198 let k = match &p.key { Some(k) => res!(show(k)), None => String::new() };
199 req!(true, p.complete && (0..10).all(|i| k.contains(&fmt!("\"word{}\"", i))), "{}", k);
200 Ok(())
201}
202
203#[test]
204fn qp7_relative_days_from_the_callers_day() -> Outcome<()> {
205 for (d, k) in [(28, "SINCE 20-Feb-2024"), (1, "SINCE 22-Feb-2024")] {
206 let today = Day { year: 2024, month: if d == 1 { 3 } else { 2 }, day: d };
207 let p = res!(check(&res!(q("newer_than:7d")), &PLAIN, today));
208 req!(match &p.key { Some(x) => res!(show(x)), None => String::new() }, k.to_string());
209 }
210 Ok(())
211}
212
213#[test]
214fn qs1_every_row_on_both_servers() -> Outcome<()> {
215 for s in ["from:ada", "to:bob", "cc:carol", "bcc:eve", "subject:invoice", "body:invoice", "invoice", "filename:report",
216 "match:word cat", "match:case Cat", "/inv(oice|entory)-\\d+/", "alpha zeta", "alpha OR zeta", "\"net profit\"",
217 "-alpha", "-cat", "after:2026/01/30 before:2026/02/02", "has:attachment", "is:read", "is:unread", "is:starred",
218 "larger:1000", "smaller:1000", "match:word scope:body cat", "match:word scope:subject,body invoice",
219 "match:word filename:attachment", "match:word filename:quarterly", "match:word quarterly",
220 "newer_than:999999d", "after:0000/01/01", "before:9999/12/31", "after:20000/01/01", "-after:9999/12/31"]
221 {
222 for v in [PLAIN, GMAIL] { res!(check(&res!(q(s)), &v, TODAY)); }
223 }
224 Ok(())
225}
226
227#[test]
228fn qs2_to_qs8_text_and_polarity() -> Outcome<()> {
229 req!(res!(key("match:word scope:body cat", &PLAIN)), pair("BODY \"cat\"", true)); // QS2
230 req!(res!(key("match:case scope:body Cat", &PLAIN)), pair("BODY \"cat\"", true)); // QS3
231 req!(res!(key("match:word -cat", &PLAIN)), pair("ALL", false)); // QS4
232 req!(res!(key("scope:subject -cat", &PLAIN)), pair("ALL", false)); // QS5, strict
233 req!(res!(key("match:case -Cat", &PLAIN)), pair("ALL", false)); // QS6
234 req!(res!(key("scope:body \"net profit\"", &PLAIN)), pair("BODY \"net\" BODY \"profit\"", true)); // QS7
235 req!(res!(key("\"net profit\"", &PLAIN)), pair("ALL", false)); // QS7, unscoped
236 req!(res!(key("-(is:unread OR /\\d+/)", &PLAIN)), pair("NOT UNSEEN", false)); // QS8
237 Ok(())
238}
239
240#[test]
241fn qs9_regex_literal() -> Outcome<()> {
242 for (re, lit) in [("inv(oice|entory)-\\d+", Some("inv")), ("\\d{4}-\\d{2}", Some("-")), ("(abc|abd)x", Some("ab")),
243 ("a+b", Some("ab")), ("(?i)kel", Some("kel")), ("x?yz", Some("yz")), ("a|b", None), ("\\w+", None)]
244 {
245 req!(res!(Regex::required_literal(re, true)).as_deref(), lit, "{}", re);
246 res!(check(&res!(q(&fmt!("scope:body /{}/", re))), &PLAIN, TODAY));
247 }
248 req!(res!(key("scope:body /\\d{4}-\\d{2}/", &PLAIN)), pair("ALL", false));
249 Ok(())
250}
251
252#[test]
253fn qs10_fold_and_nfc_partners_cut() -> Outcome<()> {
254 req!(res!(key("scope:body kelvin", &PLAIN)), pair("BODY \"elvin\"", true));
255 req!(res!(key("scope:body class", &PLAIN)), pair("BODY \"cla\"", true));
256 // Every ASCII character a non-ASCII one folds to (the matcher's case) or composes to (its NFC).
257 let ci = res!(Regex::with_case("[a-z]", true));
258 let mut odd: Vec<char> = Vec::new();
259 for c in (0x80..0x11_0000).filter_map(char::from_u32) {
260 let s = c.to_string();
261 if res!(ci.is_match(&s)) {
262 for a in 'a'..='z' { if res!(res!(Regex::with_case(&a.to_string(), true)).is_match(&s)) { odd.push(a); } }
263 }
264 odd.extend(nfc(&s).chars().filter(|x| x.is_ascii()));
265 }
266 odd.sort();
267 odd.dedup();
268 req!(true, odd.len() >= 3, "{:?}", odd);
269 for a in odd {
270 for x in [a, a.to_ascii_uppercase()] { req!(res!(key(&fmt!("scope:body zz{}zz", x), &PLAIN)).0, "BODY \"zz\"".to_string(), "{:?}", x); }
271 }
272 Ok(())
273}
274
275#[test]
276fn qs11_qs14_unicode_and_attachments() -> Outcome<()> {
277 req!(res!(key("subject:Grüße", &PLAIN)), pair("SUBJECT \"gr\"", true));
278 for s in [PLAIN, GMAIL] {
279 for x in ["filename:résumé", "match:word filename:report", "match:word filename:attachment", "match:word quarterly"] {
280 req!(res!(key(x, &s)), pair("ALL", false), "{} Gmail {}", x, s.gmail); // F3: no name key
281 }
282 }
283 req!(res!(key("has:attachment", &PLAIN)), pair("ALL", false));
284 req!(res!(key("has:attachment", &GMAIL)), pair("X-GM-RAW \"has:attachment\"", true));
285 req!(res!(key("match:word scope:body cat", &GMAIL)), pair("BODY \"cat\"", true));
286 req!(res!(key("scope:body cat", &GMAIL)), pair("ALL", false));
287 req!(true, parse(&res!(SearchKey::GmRaw("has:attachment".into()).wire())).is_err());
288 Ok(())
289}
290
291#[test]
292fn qs12_server_days() -> Outcome<()> {
293 for (s, k) in [("after:2026/01/31", "SINCE 30-Jan-2026"), ("before:2026/01/31", "BEFORE 1-Feb-2026"),
294 ("-after:2026/01/31", "NOT SINCE 1-Feb-2026"), ("-before:2026/01/31", "NOT BEFORE 30-Jan-2026"),
295 ("is:unread -before:0000/01/02", "UNSEEN NOT BEFORE 1-Jan-0000")]
296 {
297 req!(res!(key(s, &PLAIN)), pair(k, true));
298 }
299 for (s, k) in [("after:0000/01/01", "ALL"), ("before:9999/12/31", "ALL"), ("-after:9999/12/31", "ALL"), ("newer_than:999999d", "ALL")] {
300 req!(res!(key(s, &PLAIN)), pair(k, false), "{}", s); // F4: widened, never refused
301 }
302 Ok(())
303}
304
305#[test]
306fn qs13_empty() -> Outcome<()> {
307 let p = res!(check(&Query { terms: Term::Any(Vec::new()), ..Query::default() }, &PLAIN, TODAY));
308 req!((p.key.clone(), p.complete), (None::<SearchKey>, true));
309 let p = res!(check(&Query::default(), &PLAIN, TODAY));
310 req!((p.key.clone(), p.complete), (Some(SearchKey::All), true));
311 Ok(())
312}
313
314#[test]
315fn qs15_needs() -> Outcome<()> {
316 for (s, n) in [("is:unread", Needs::Meta), ("from:x", Needs::Headers), ("body:x", Needs::Text), ("x", Needs::Text),
317 ("filename:x", Needs::Structure), ("larger:1M", Needs::Meta), ("has:attachment", Needs::Structure)]
318 {
319 req!(res!(plan(&res!(q(s)), &PLAIN, &LIM, TODAY)).needs, n, "{}", s);
320 }
321 Ok(())
322}
323
324/// The stated limit: a word split by a bidirectional control or an HTML tag is one word to the
325/// matcher and none to a server, so a keyed search (complete) misses the mail.
326#[test]
327fn limit_split_words_missed_by_a_server_search() -> Outcome<()> {
328 for (qs, raw) in [("subject:invoice", mime("Subject: in\u{200E}voice\r\n", "text/plain", "x")),
329 ("from:lovelace", "From: Ada Love\u{200F}lace <a@x>\r\nContent-Type: text/plain\r\n\r\nx\r\n".to_string()),
330 ("body:invoice", txt("in\u{200E}voice hidden")), ("body:invoice", html("<p>inv<b>oice</b> split</p>"))]
331 {
332 let mails = [M { raw, at: (2026, 3, 4, 12, 0, 0), flags: "", size: 0 }];
333 let qy = res!(q(qs));
334 let missed = hold(&qy, &PLAIN, TODAY, &mails).is_err();
335 let p = res!(plan(&qy, &PLAIN, &LIM, TODAY));
336 req!((missed, p.complete), (true, true), "{}", qs);
337 }
338 Ok(())
339}