Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_mail/src/sync/query.rs

37.9 KiB, 280 runs

created by r1870400018:76323, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Reading one account's mail on the server, for an agent that asks rather than syncs: the
2//! newest messages of a folder, a search over its folders a page at a time, one message read by
3//! part, one message raw, and the folders by role.
4//!
5//! - Nothing here changes the mailbox: every folder is opened with `EXAMINE`, so the session can
6//! write no flag and `\Recent` is left for the next `SELECT` (UNBLOCK-63), and every body is
7//! fetched with `BODY.PEEK`, so a daimon's read leaves `\Seen` as it found it.
8//! - A folder is named by its role ([`Target`]), resolved by its RFC 6154 attribute, so Gmail's
9//! "[Gmail]/All Mail" and a localised "[Gmail]/Gesendet" are found without their names.
10//! - [`search`] trusts the server's `SEARCH` for candidates only, since a server may match more
11//! than asked (Gmail's search is fuzzy): `search::Matcher` decides each, and a page holds no
12//! state but its [`Cursor`].
13//! - [`read`] fetches `BODYSTRUCTURE` first, then the text parts within the byte cap, and lists
14//! attachments by name and size without their bytes, so a large attachment never cuts off the
15//! body it came with.
16//! - Every size and count is the caller's, from its config ([`Limits`]).
17
18use crate::{
19 message,
20 search::{
21 self,
22 Fetched,
23 Folders,
24 Hit,
25 Mail,
26 Matcher,
27 Needs,
28 Query,
29 Scope,
30 Server,
31 },
32 sync::{
33 act::Target,
34 BodyPart,
35 Caps,
36 FetchWhat,
37 FetchedMessage,
38 MailSession,
39 Roles,
40 SelectWith,
41 SpecialUse,
42 UidSet,
43 },
44};
45
46use oxedyne_fe2o3_core::prelude::*;
47use oxedyne_fe2o3_net::imap::search::{
48 civil_from_days,
49 parse_search_date,
50 SearchKey,
51};
52use oxedyne_fe2o3_text::mime;
53
54use std::fmt;
55
56
57/// The bounds on what one query returns, from the caller's config.
58#[derive(Clone, Copy, Debug, Eq, PartialEq)]
59pub struct Limits {
60 pub max_rows: usize, // rows a listing or a search returns
61 pub max_read_bytes: u32, // of text parts one read fetches, across its parts
62 pub max_folders: usize, // folders counted by `folders`; the rest are listed uncounted
63}
64
65/// A calendar day, as `SEARCH SINCE` and `BEFORE` take it.
66#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
67pub struct Day {
68 pub year: u16,
69 pub month: u8,
70 pub day: u8,
71}
72
73impl Day {
74 /// `2024/01/31` or `2024-01-31`.
75 pub fn parse(s: &str) -> Outcome<Self> {
76 let n: Vec<Option<u16>> = s.split(['/', '-']).map(|x| x.parse().ok()).collect();
77 if let [Some(year), Some(m), Some(d)] = n[..] {
78 if (1..=12).contains(&m) && (1..=Self::last(year, m as u8) as u16).contains(&d) {
79 return Ok(Self { year, month: m as u8, day: d as u8 });
80 }
81 }
82 Err(err!("'{}' is not a date; write it as 2024/01/31.", s; Invalid, Input))
83 }
84
85 /// INTERNALDATE's own calendar day (`30-Jan-2026 23:30:00 -0700` is 30 January), in the zone
86 /// the server wrote it in, never converted.
87 pub fn internal(s: &str) -> Option<Self> {
88 s.split_whitespace().next().and_then(parse_search_date).map(civil_from_days)
89 .and_then(|(y, m, d)| u16::try_from(y).ok().map(|year| Self { year, month: m as u8, day: d as u8 }))
90 }
91
92 /// The month's last day.
93 pub fn last(year: u16, month: u8) -> u8 {
94 let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
95 match month { 2 => 28 + leap as u8, 4 | 6 | 9 | 11 => 30, _ => 31 } // fixed: Gregorian
96 }
97
98 /// `1-Feb-2026` (RFC 3501 `date`).
99 pub fn imap(&self) -> Outcome<String> {
100 // fixed: the month names of RFC 3501's date-month.
101 const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
102 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
103 if self.month < 1 || self.month > 12 || self.day < 1 || self.day > 31 {
104 return Err(err!("The day {}-{}-{} is not a calendar day.",
105 self.year, self.month, self.day; Invalid, Input));
106 }
107 Ok(fmt!("{}-{}-{}", self.day, MONTHS[(self.month - 1) as usize], self.year))
108 }
109}
110
111impl std::fmt::Display for Day {
112 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
113 write!(f, "{:04}/{:02}/{:02}", self.year, self.month, self.day)
114 }
115}
116
117/// The dates a query is bounded by, by the server's internal date: on or after `since`, and
118/// before `before`.
119#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
120pub struct Range {
121 pub since: Option<Day>,
122 pub before: Option<Day>,
123}
124
125impl Range {
126 fn criteria(&self) -> Outcome<String> {
127 let mut c = Vec::new();
128 if let Some(d) = &self.since {
129 c.push(fmt!("SINCE {}", res!(d.imap())));
130 }
131 if let Some(d) = &self.before {
132 c.push(fmt!("BEFORE {}", res!(d.imap())));
133 }
134 Ok(c.join(" "))
135 }
136}
137
138/// One message as a listing shows it, its fields with their encoded words decoded.
139#[derive(Clone, Debug, Default, Eq, PartialEq)]
140pub struct Header {
141 pub uid: u32,
142 pub flags: Vec<String>,
143 pub internal_date: String, // as the server gave it
144 pub size: u32,
145 pub date: String, // the Date field
146 pub from: String,
147 pub to: String,
148 pub cc: String,
149 pub subject: String,
150 pub message_id: String, // without its angle brackets
151}
152
153impl Header {
154 fn of(m: &FetchedMessage) -> Self {
155 let hs = message::headers(&m.body);
156 let get = |n: &str| hs.iter()
157 .find(|(k, _)| k == n)
158 .map(|(_, v)| mime::decode_words(v).trim().to_string())
159 .unwrap_or_default();
160 Self {
161 uid: m.uid,
162 flags: m.flags.clone(),
163 internal_date: m.internal_date.clone(),
164 size: m.size,
165 date: get("date"),
166 from: get("from"),
167 to: get("to"),
168 cc: get("cc"),
169 subject: get("subject"),
170 message_id: crate::sync::bare_message_id(&get("message-id")).to_string(),
171 }
172 }
173
174 pub fn seen(&self) -> bool {
175 self.flags.iter().any(|f| f.eq_ignore_ascii_case("\\Seen"))
176 }
177}
178
179/// What a listing or a search found in one folder, newest first.
180#[derive(Clone, Debug, Default, Eq, PartialEq)]
181pub struct Listing {
182 pub mailbox: String,
183 pub total: u32, // messages in the folder
184 pub rows: Vec<Header>,
185 pub more: bool, // more matched than `max_rows`
186}
187
188/// One message read by part.
189#[derive(Clone, Debug, Default, Eq, PartialEq)]
190pub struct Read {
191 pub header: Header,
192 pub texts: Vec<TextPart>,
193 pub attachments: Vec<Attachment>,
194}
195
196impl Read {
197 /// Was any text part cut at the byte cap, or left unread?
198 pub fn cut(&self) -> bool {
199 self.texts.iter().any(|t| t.cut)
200 }
201}
202
203#[derive(Clone, Debug, Default, Eq, PartialEq)]
204pub struct TextPart {
205 pub section: String, // `1.1`
206 pub media: String, // `text/plain` or `text/html`
207 pub text: String, // decoded to UTF-8, as far as it was read
208 pub cut: bool, // the cap stopped it short
209}
210
211#[derive(Clone, Debug, Default, Eq, PartialEq)]
212pub struct Attachment {
213 pub section: String,
214 pub media: String,
215 pub name: Option<String>,
216 pub size: u32, // octets in its transfer encoding
217}
218
219/// One message's bytes, read without marking it read.
220#[derive(Clone, Debug, Default, Eq, PartialEq)]
221pub struct Raw {
222 pub bytes: Vec<u8>,
223 pub size: u32, // the whole message's
224 pub cut: bool, // `max` stopped it short
225}
226
227#[derive(Clone, Copy, Debug, Eq, PartialEq)]
228pub enum Role {
229 Inbox,
230 Use(SpecialUse),
231}
232
233/// One folder, by role where it has one.
234#[derive(Clone, Debug, Eq, PartialEq)]
235pub struct Folder {
236 pub name: String,
237 pub role: Option<Role>,
238 pub count: Option<u32>, // `None` past `max_folders`
239}
240
241/// One page of a search over the server's folders.
242#[derive(Clone, Debug, Default)]
243pub struct Page {
244 pub hits: Vec<Found>,
245 pub coverage: Coverage,
246 pub cursor: Option<Cursor>, // `None`: the run is done
247}
248
249/// One message found, once a page, naming each folder of the page it was found in.
250#[derive(Clone, Debug)]
251pub struct Found {
252 pub folders: Vec<String>, // where found, then Gmail's labels where fetched
253 pub uid: u32, // in the first of them
254 pub uid_validity: u32,
255 pub header: Header,
256 pub hit: Hit,
257 key: String, // X-GM-MSGID, else Message-ID; empty matches nothing
258}
259
260/// What one page looked at. A message is `checked` only where the matcher decided it.
261#[derive(Clone, Debug, Default, Eq, PartialEq)]
262pub struct Coverage {
263 pub checked: usize,
264 pub undecided: usize, // a text past the byte bound, an unreadable structure
265 pub total: u32, // messages in the folders the page examined
266 pub top_uid: u32, // the run's top in the last folder examined
267 pub restarted: Vec<String>, // renumbered since the cursor, searched again from their newest
268 pub skipped: Vec<(String, String)>, // a folder not searched, and why
269 pub complete: bool, // the server's key was complete; else each page is a bounded scan
270 pub stopped: Option<String>, // why the page ended early, its hits kept
271}
272
273/// The whole state of a search between its pages, carried as text (`Display`, `parse`): for each
274/// folder of the run, in order, where it stands. A digest binds it to the query, the folders and
275/// itself, so that one which does not fit is refused.
276#[derive(Clone, Debug, Eq, PartialEq)]
277pub struct Cursor {
278 digest: u64,
279 at: Vec<Place>,
280}
281
282#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
283struct Place {
284 uv: u32, // UIDVALIDITY; 0 before the folder is begun
285 top: u32, // the run's newest UID in it, so that later mail is not searched
286 below: u32, // the next page searches under this UID; 1 once done
287}
288
289impl Cursor {
290 fn new(q: &Query, folders: &[String], at: Vec<Place>) -> Self {
291 Self { digest: Self::digest(q, folders, &at), at }
292 }
293
294 fn digest(q: &Query, folders: &[String], at: &[Place]) -> u64 {
295 message::fnv1a(fmt!("{}\n{}\n{}", q, folders.join("\n"), places(at)).as_bytes())
296 }
297
298 pub fn parse(s: &str) -> Outcome<Self> {
299 let bad = || err!("{:?} is not a search cursor.", s; Invalid, Input);
300 let mut it = s.split(':');
301 let digest = match (it.next(), it.next().map(|d| u64::from_str_radix(d, 16))) {
302 (Some("q1"), Some(Ok(d))) => d,
303 _ => return Err(bad()),
304 };
305 let mut at = Vec::new();
306 for p in it {
307 match p.split(',').map(|n| n.parse::<u32>().ok()).collect::<Vec<_>>()[..] {
308 [Some(uv), Some(top), Some(below)] => at.push(Place { uv, top, below }),
309 _ => return Err(bad()),
310 }
311 }
312 Ok(Self { digest, at })
313 }
314}
315
316fn places(at: &[Place]) -> String {
317 at.iter().map(|p| fmt!(":{},{},{}", p.uv, p.top, p.below)).collect()
318}
319
320impl fmt::Display for Cursor {
321 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
322 write!(f, "q1:{:016x}{}", self.digest, places(&self.at))
323 }
324}
325
326
327// ┌───────────────────────────────────────────────────────────────────────────┐
328// │ THE QUERIES │
329// └───────────────────────────────────────────────────────────────────────────┘
330
331/// The mailbox a target names, by its role's attribute.
332pub fn resolve(roles: &Roles, target: &Target) -> Outcome<String> {
333 match target {
334 Target::Inbox => Ok(roles.inbox.clone()),
335 Target::Role(r) => Ok(res!(roles.role(*r)
336 .ok_or_else(|| err!("The account has no folder with the {:?} role.", r; Missing)))
337 .to_string()),
338 Target::Path(p) => if roles.exists(p) {
339 Ok(p.clone())
340 } else {
341 Err(err!("The account has no folder {:?}.", p; Missing))
342 },
343 }
344}
345
346/// The newest messages of a folder within `range`.
347pub async fn list<S: MailSession>(
348 s: &mut S,
349 roles: &Roles,
350 target: &Target,
351 range: &Range,
352 lim: &Limits,
353)
354 -> Outcome<Listing>
355{
356 let mailbox = res!(resolve(roles, target));
357 let st = res!(s.examine(&mailbox, &SelectWith::Plain).await);
358 let crit = res!(range.criteria());
359 let mut uids = res!(s.uid_search(if crit.is_empty() { "ALL" } else { &crit }).await);
360 uids.sort_unstable();
361 let more = uids.len() > lim.max_rows;
362 let newest: Vec<u32> = uids.iter().rev().take(lim.max_rows).copied().collect();
363 let rows = res!(summaries(s, &newest).await);
364 Ok(Listing { mailbox, total: st.exists, rows, more })
365}
366
367/// One message by part, `None` where the folder does not hold `uid`: its structure first, then
368/// each text body within `max_read_bytes` across them, the plain form of an alternative over
369/// its HTML, and every other part listed by name.
370pub async fn read<S: MailSession>(
371 s: &mut S,
372 roles: &Roles,
373 target: &Target,
374 uid: u32,
375 lim: &Limits,
376)
377 -> Outcome<Option<Read>>
378{
379 let mailbox = res!(resolve(roles, target));
380 res!(s.examine(&mailbox, &SelectWith::Plain).await);
381 let one = UidSet::from([uid]);
382 let m = match res!(s.uid_fetch(&one, FetchWhat::Structure).await).into_iter()
383 .find(|m| m.uid == uid)
384 {
385 Some(m) => m,
386 None => return Ok(None),
387 };
388 let mut out = Read { header: Header::of(&m), ..Default::default() };
389 let tree = match &m.structure {
390 Some(t) => t.clone(),
391 // A structure the server gave malformed: the whole message as one part, within the cap.
392 None => BodyPart {
393 section: String::new(),
394 media: "message/rfc822".to_string(),
395 size: m.size,
396 ..Default::default()
397 },
398 };
399 let mut texts: Vec<&BodyPart> = Vec::new();
400 bodies(&tree, &mut texts, &mut out.attachments);
401 if texts.is_empty() && tree.parts.is_empty() && tree.media == "message/rfc822" {
402 texts.push(&tree);
403 }
404 let mut left = lim.max_read_bytes;
405 for p in texts {
406 let ask = p.size.min(left);
407 let mut t = TextPart {
408 section: p.section.clone(),
409 media: p.media.clone(),
410 cut: ask < p.size,
411 ..Default::default()
412 };
413 if ask > 0 {
414 if let Some(f) = res!(s.uid_fetch_section(uid, &p.section, ask).await) {
415 left = left.saturating_sub(f.body.len() as u32);
416 t.text = if p.section.is_empty() {
417 message::readable_text(&f.body)
418 } else {
419 mime::decode_body(&f.body, &p.encoding, p.param("charset"))
420 };
421 }
422 }
423 out.texts.push(t);
424 }
425 Ok(Some(out))
426}
427
428/// The text bodies to read and the attachments to list: an alternative gives its first child
429/// that holds plain text, else its last, its richest (RFC 2046 §5.1.4).
430fn bodies<'a>(p: &'a BodyPart, texts: &mut Vec<&'a BodyPart>, atts: &mut Vec<Attachment>) {
431 if p.parts.is_empty() {
432 if p.is_text_body() {
433 texts.push(p);
434 } else if !p.section.is_empty() {
435 atts.push(Attachment {
436 section: p.section.clone(),
437 media: p.media.clone(),
438 name: p.filename.clone(),
439 size: p.size,
440 });
441 }
442 return;
443 }
444 if p.media == "multipart/alternative" {
445 let plain = p.parts.iter()
446 .find(|c| c.leaves().iter().any(|l| l.media == "text/plain" && l.is_text_body()));
447 if let Some(c) = plain.or(p.parts.last()) {
448 bodies(c, texts, atts);
449 }
450 return;
451 }
452 for c in &p.parts {
453 bodies(c, texts, atts);
454 }
455}
456
457/// One whole message, at most `max` bytes of it, leaving `\Seen` as it was.
458pub async fn raw<S: MailSession>(
459 s: &mut S,
460 roles: &Roles,
461 target: &Target,
462 uid: u32,
463 max: u32,
464)
465 -> Outcome<Option<Raw>>
466{
467 let mailbox = res!(resolve(roles, target));
468 res!(s.examine(&mailbox, &SelectWith::Plain).await);
469 Ok(res!(s.uid_fetch_section(uid, "", max).await).map(|m| Raw {
470 cut: (m.body.len() as u32) < m.size,
471 size: m.size,
472 bytes: m.body,
473 }))
474}
475
476/// Every folder, the inbox and the roles first, each with its role found by attribute and, for
477/// the first `max_folders`, its message count.
478pub async fn folders<S: MailSession>(s: &mut S, lim: &Limits) -> Outcome<(Roles, Vec<Folder>)> {
479 let roles = Roles::from_list(&res!(s.list().await));
480 let mut out = Vec::new();
481 for (i, name) in roles.synced().into_iter().enumerate() {
482 let role = if name == roles.inbox {
483 Some(Role::Inbox)
484 } else {
485 roles.mailboxes.iter().find(|m| m.name == name)
486 .and_then(|m| m.special_use())
487 .filter(|r| roles.role(*r) == Some(name.as_str()))
488 .map(Role::Use)
489 };
490 let count = if i < lim.max_folders { Some(res!(s.count(&name).await)) } else { None };
491 out.push(Folder { name, role, count });
492 }
493 Ok((roles, out))
494}
495
496/// The listing rows of `uids`, newest first.
497async fn summaries<S: MailSession>(s: &mut S, uids: &[u32]) -> Outcome<Vec<Header>> {
498 if uids.is_empty() {
499 return Ok(Vec::new());
500 }
501 let asked = UidSet::from(uids);
502 let mut rows: Vec<Header> = res!(s.uid_fetch(&asked, FetchWhat::Summary).await).iter()
503 .filter(|m| asked.contains(m.uid))
504 .map(Header::of)
505 .collect();
506 rows.sort_by(|a, b| b.uid.cmp(&a.uid));
507 rows.dedup_by_key(|h| h.uid);
508 Ok(rows)
509}
510
511
512// ┌───────────────────────────────────────────────────────────────────────────┐
513// │ THE SEARCH │
514// └───────────────────────────────────────────────────────────────────────────┘
515
516/// One page of `q` over the server's folders from `cur` (`None` begins the run). The folders are
517/// those `q.folders` names: the inbox where it names none, a list, or all, Junk and Trash only
518/// where asked, and on Gmail its All Mail in place of the rest. Each is examined, then searched
519/// newest first below the cursor by `search::plan`'s key, a window of UIDs at a time where that
520/// key is not complete; each candidate is decided by the matcher from no more than the plan needs,
521/// within the page's bounds in `lim`. The byte bound is charged with what each fetch reads from the
522/// wire, and no chunk of `search_page_rows` UIDs is fetched once it is spent and something is
523/// decided; a page's first candidate is read within the whole bound, so a page reads at most twice
524/// the bound and one chunk's structures and header blocks. A complete key's SEARCH
525/// names every candidate below the cursor, so a page holds one folder's list of them. A message
526/// found in several folders of a page is one hit. A message the server refuses is undecided and
527/// the page goes on. A call that fails otherwise, once something is decided, ends the page there,
528/// `stopped` saying why; before, it is an error and the cursor given in stands.
529pub async fn search<S: MailSession>(
530 s: &mut S,
531 caps: &Caps,
532 roles: &Roles,
533 q: &Query,
534 cur: Option<&Cursor>,
535 lim: &search::Limits,
536 today: Day,
537)
538 -> Outcome<Page>
539{
540 let (names, skipped) = run_folders(caps, roles, q, lim);
541 let mut at = match cur {
542 None => vec![Place::default(); names.len()],
543 Some(c) if c.at.len() == names.len() && c.digest == Cursor::digest(q, &names, &c.at) => c.at.clone(),
544 Some(_) => return Err(err!("The cursor is not one this search gave over these folders; begin \
545 the search again."; Invalid, Input)),
546 };
547 let srv = |names| Server { gmail: caps.gmail, names };
548 let plan = res!(search::plan(q, &srv(true), lim, today));
549 // Where the key is not complete, the key with each attachment fact false narrows a window's
550 // answer, and of the rest those whose structures' names may hold are candidates: a superset.
551 let narrow = match plan.complete {
552 true => None,
553 false => Some(res!(search::plan(q, &srv(false), lim, today)).key).filter(|k| *k != plan.key),
554 };
555 let m = res!(Matcher::new(q, lim, today));
556 let mut pg = Page {
557 coverage: Coverage { skipped, complete: plan.complete, ..Default::default() },
558 ..Default::default()
559 };
560 let key = match plan.key {
561 Some(k) => k,
562 None => return Ok(pg), // nothing can match
563 };
564 let what = if plan.needs >= Needs::Structure { FetchWhat::Structure } else { FetchWhat::Headers };
565 let rows = lim.search_page_rows.max(1);
566 let (cap, mut scan) = (lim.search_confirm_max_bytes, lim.search_scan_max_messages.max(1));
567 let (mut bytes, mut fresh, mut refusals, mut why) = (cap, true, 0, None);
568 'folders: for (i, name) in names.iter().enumerate() {
569 if at[i].below == 1 { continue; }
570 let st = match s.examine(name, &SelectWith::Plain).await {
571 Ok(st) => st,
572 Err(e) if e.tags().contains(&ErrTag::NotFound) => {
573 pg.coverage.skipped.push((name.clone(), e.plain()));
574 at[i].below = 1;
575 continue;
576 }
577 Err(e) => return stop(pg, e, q, &names, at),
578 };
579 if st.uid_validity == 0 || st.uid_next == 0 {
580 return stop(pg, err!("The server examined {:?} without its UIDVALIDITY or UIDNEXT, so its \
581 UIDs name nothing certain.", name; Invalid, Input), q, &names, at);
582 }
583 if at[i].uv != st.uid_validity {
584 if at[i].uv != 0 { pg.coverage.restarted.push(name.clone()); }
585 at[i] = Place { uv: st.uid_validity, top: st.uid_next - 1, below: st.uid_next };
586 }
587 pg.coverage.total += st.exists;
588 pg.coverage.top_uid = at[i].top;
589 while at[i].below > 1 {
590 let spent = bytes == 0 && pg.coverage.checked + pg.coverage.undecided > 0;
591 if pg.hits.len() >= rows || spent || scan == 0 { break 'folders; }
592 let below = at[i].below;
593 let lo = match plan.complete {
594 true => 1,
595 false => below.saturating_sub(u32::try_from(scan).unwrap_or(u32::MAX)).max(1),
596 };
597 let win = |k: &SearchKey| SearchKey::And(vec![k.clone(), SearchKey::Uid(fmt!("{}:{}", lo, below - 1))]);
598 let mut ids = match s.uid_search_key(&res!(win(&key).wire())).await {
599 Ok(u) => u,
600 Err(e) => return stop(pg, e, q, &names, at),
601 };
602 let mut near = match &narrow {
603 Some(Some(n)) => match s.uid_search_key(&res!(win(n).wire())).await {
604 Ok(u) => u,
605 Err(e) => return stop(pg, e, q, &names, at),
606 },
607 _ => Vec::new(),
608 };
609 ids.retain(|u| (lo..below).contains(u)); // a wider answer costs time only
610 ids.sort_unstable_by(|a, b| b.cmp(a));
611 ids.dedup();
612 near.sort_unstable();
613 // A candidate: any of the key's answer, or where narrowed, one the narrowed key names too.
614 let near = |u: &u32| narrow.is_none() || near.binary_search(u).is_ok();
615 if !plan.complete { scan -= (below - lo) as usize; }
616 // The window in chunks of `rows`, newest first: the structures of the chunk's rest, then
617 // its candidates, each fetched once.
618 for chunk in ids.chunks(rows) {
619 let spent = bytes == 0 && pg.coverage.checked + pg.coverage.undecided > 0;
620 if pg.hits.len() >= rows || spent || (plan.complete && scan == 0) { break 'folders; }
621 let rest: Vec<u32> = chunk.iter().copied().filter(|u| !near(u)).collect();
622 let (shape, shape_no, why_s) = match rest.is_empty() {
623 true => Default::default(),
624 false => match fetch(s, &rest, FetchWhat::BodyStructure, &mut bytes).await {
625 Ok(v) => v,
626 Err(e) => return stop(pg, e, q, &names, at),
627 },
628 };
629 let cands: Vec<u32> = chunk.iter().copied().filter(|u| near(u) || may(&m, &shape, &shape_no, *u)).collect();
630 let (got, got_no, why_c) = match cands.is_empty() {
631 true => Default::default(),
632 false => match fetch(s, &cands, what, &mut bytes).await {
633 Ok(v) => v,
634 Err(e) => return stop(pg, e, q, &names, at),
635 },
636 };
637 why = why_c.or(why_s).or(why);
638 for &uid in chunk {
639 if !cands.contains(&uid) {
640 if shape.iter().any(|f| f.uid == uid) { pg.coverage.checked += 1; }
641 at[i].below = uid;
642 continue;
643 }
644 if pg.hits.len() >= rows || (bytes == 0 && !fresh) || (plan.complete && scan == 0) { break 'folders; }
645 let fm = got.iter().find(|f| f.uid == uid);
646 let v = match fm {
647 Some(f) => match confirm(s, &m, plan.needs, f, &mut bytes, fresh.then_some(cap)).await {
648 Ok(Verdict::Wait) => break 'folders,
649 Ok(v) => v,
650 Err(e) => return stop(pg, e, q, &names, at),
651 },
652 None if got_no.contains(&uid) => Verdict::Refused(None),
653 None => Verdict::Gone, // expunged meanwhile
654 };
655 at[i].below = uid;
656 fresh = false;
657 if plan.complete { scan -= 1; }
658 match (v, fm) {
659 (Verdict::Hit(h), Some(f)) => {
660 pg.coverage.checked += 1;
661 found(&mut pg, name, st.uid_validity, f, h);
662 }
663 (Verdict::Miss, _) => pg.coverage.checked += 1,
664 (Verdict::Undecided, _) => pg.coverage.undecided += 1,
665 (Verdict::Refused(e), _) => {
666 // A server that refuses a page's worth ends the page in its own words.
667 (pg.coverage.undecided, refusals, why) = (pg.coverage.undecided + 1, refusals + 1, e.or(why));
668 if refusals >= rows {
669 if let Some(e) = why.take() { return stop(pg, e, q, &names, at); }
670 }
671 }
672 _ => {}
673 }
674 }
675 }
676 at[i].below = lo;
677 }
678 }
679 if at.iter().any(|p| p.below != 1) { pg.cursor = Some(Cursor::new(q, &names, at)); }
680 Ok(pg)
681}
682
683/// The folders `q` runs over, at most `search_max_folders`, and those it leaves, with why.
684fn run_folders(caps: &Caps, roles: &Roles, q: &Query, lim: &search::Limits) -> (Vec<String>, Vec<(String, String)>) {
685 let (mut names, mut skipped) = (Vec::new(), Vec::new());
686 match &q.folders {
687 Folders::Selected => names.push(roles.inbox.clone()),
688 Folders::List(l) => for f in l {
689 match resolve(roles, &Target::Path(f.clone())) {
690 Ok(n) if !names.contains(&n) => names.push(n),
691 Ok(_) => {}
692 Err(e) => skipped.push((f.clone(), e.plain())),
693 }
694 },
695 Folders::All { junk_trash } => {
696 let bin = [roles.role(SpecialUse::Junk), roles.role(SpecialUse::Trash)];
697 let all = roles.role(SpecialUse::All).filter(|_| caps.gmail); // Gmail's every label but those two
698 for n in roles.synced() {
699 let b = bin.contains(&Some(n.as_str()));
700 if (b && *junk_trash) || (!b && all.map_or(true, |a| a == n)) { names.push(n); }
701 }
702 }
703 }
704 if names.len() > lim.search_max_folders {
705 let why = fmt!("past the {} folders one search takes", lim.search_max_folders);
706 skipped.extend(names.split_off(lim.search_max_folders).into_iter().map(|n| (n, why.clone())));
707 }
708 (names, skipped)
709}
710
711/// What the matcher made of one candidate.
712enum Verdict {
713 Hit(Hit),
714 Miss,
715 Undecided, // a text past the byte bound, an unreadable structure
716 Refused(Option<Error<ErrTag>>), // by the server, in its words where they are in hand
717 Gone, // expunged while it was read
718 Wait, // its text does not fit what is left of the page, so it waits for a page of its own
719}
720
721/// The matcher's verdict on one candidate, read no further than `needs`. The first candidate of
722/// a page (`first`, the page's bound) is read as far as that bound goes, whatever is spent, so that
723/// each page decides something; another waits where its text does not fit `bytes`.
724async fn confirm<S: MailSession>(
725 s: &mut S,
726 m: &Matcher,
727 needs: Needs,
728 fm: &FetchedMessage,
729 bytes: &mut u32,
730 first: Option<u32>,
731)
732 -> Outcome<Verdict>
733{
734 let has = |f: &str| fm.flags.iter().any(|x| x.eq_ignore_ascii_case(f));
735 let mail = Mail { raw: &fm.body, seen: has("\\Seen"), flagged: has("\\Flagged"),
736 size: u64::from(fm.size), day: Day::internal(&fm.internal_date) };
737 let mut f = Fetched::default();
738 if needs >= Needs::Structure {
739 let tree = match &fm.structure {
740 Some(t) => t,
741 None => return Ok(Verdict::Undecided),
742 };
743 names_of(tree, &mut f);
744 // A body that decodes blank shows nothing, so the reader passes over it, as here.
745 let (mut blank, mut read): (Vec<&str>, Vec<(&str, String)>) = (Vec::new(), Vec::new());
746 while needs == Needs::Text {
747 let mut texts = Vec::new();
748 shown(tree, &blank, &mut texts);
749 let todo: Vec<&BodyPart> = texts.iter().copied()
750 .filter(|p| !read.iter().any(|(x, _)| *x == p.section)).collect();
751 if first.is_none() && blank.is_empty() && todo.iter().map(|p| u64::from(p.size)).sum::<u64>() > u64::from(*bytes) {
752 return Ok(Verdict::Wait);
753 }
754 let mut again = false;
755 for p in todo {
756 let ask = p.size.min(first.map_or(*bytes, |c| c.max(*bytes)));
757 if ask < p.size && !f.unknown.contains(&Scope::Body) { f.unknown.push(Scope::Body); }
758 let t = match ask {
759 0 => String::new(),
760 _ => {
761 let from = s.read_bytes();
762 let g = match s.uid_fetch_section(fm.uid, &p.section, ask).await {
763 Ok(Some(g)) => g,
764 Ok(None) => return Ok(Verdict::Gone),
765 Err(e) if refused(&e) => return Ok(Verdict::Refused(Some(e))),
766 Err(e) => return Err(e),
767 };
768 charge(bytes, s.read_bytes().saturating_sub(from));
769 mime::decode_body(&g.body, &p.encoding, p.param("charset"))
770 }
771 };
772 if ask == p.size && t.trim().is_empty() {
773 blank.push(&p.section);
774 again = true;
775 }
776 read.push((&p.section, t));
777 }
778 if !again {
779 f.body = message::readable(texts.iter().filter_map(|p| read.iter()
780 .find(|(x, _)| *x == p.section).map(|(_, t)| (p.media.clone(), t.clone()))));
781 break;
782 }
783 }
784 }
785 Ok(match m.matches_fetched(&mail, &f) {
786 Ok(Some(h)) => Verdict::Hit(h),
787 Ok(None) => Verdict::Miss,
788 Err(_) => Verdict::Undecided,
789 })
790}
791
792/// The attachments' names of a structure, as `mime::Part::attachment_name` gives them.
793fn names_of(tree: &BodyPart, f: &mut Fetched) {
794 for l in tree.leaves() {
795 let msg = l.media == "message/rfc822" || l.media == "message/global";
796 match (&l.filename, msg, l.disposition.as_deref()) {
797 (Some(n), _, _) => f.names.push(n.clone()),
798 (None, true, _) => f.unknown.push(Scope::Attachment), // named by its Subject
799 (None, false, Some("attachment")) => f.names.push(fmt!("attachment")),
800 _ => {}
801 }
802 }
803}
804
805/// The text bodies a reader is shown, chosen from the structure as `mime::Part::display_parts`
806/// chooses them from the bytes, a body in `blank` showing nothing: the plain form of an
807/// alternative, else its last that shows anything; the first part of a related; every part of
808/// the rest.
809fn shown<'a>(p: &'a BodyPart, blank: &[&str], out: &mut Vec<&'a BodyPart>) {
810 let of = |c: &'a BodyPart| { let mut v = Vec::new(); shown(c, blank, &mut v); v };
811 match (p.parts.is_empty(), p.media.as_str()) {
812 (true, m) => {
813 let filed = match p.disposition.as_deref() {
814 Some("attachment") => true,
815 Some("inline") | None => false,
816 Some(_) => p.filename.is_some(),
817 };
818 let text = m == "text/plain" || m == "text/html";
819 if text && !filed && p.size > 0 && !blank.contains(&p.section.as_str()) { out.push(p); }
820 }
821 (false, "multipart/alternative") => {
822 let plain = p.parts.iter().filter(|c| c.parts.is_empty() && c.media == "text/plain")
823 .map(of).find(|v| !v.is_empty());
824 if let Some(v) = plain.or_else(|| p.parts.iter().rev().map(of).find(|v| !v.is_empty())) {
825 out.extend(v);
826 }
827 }
828 (false, "multipart/related") => if let Some(c) = p.parts.first() { shown(c, blank, out); },
829 (false, _) => for c in &p.parts { shown(c, blank, out); },
830 }
831}
832
833/// `uids` fetched for `what`, the bytes read charged, those the server refused and its last
834/// words: where it refuses a set, each is asked for alone. Any other failure is the error.
835async fn fetch<S: MailSession>(s: &mut S, uids: &[u32], what: FetchWhat, bytes: &mut u32)
836 -> Outcome<(Vec<FetchedMessage>, Vec<u32>, Option<Error<ErrTag>>)>
837{
838 let (mut got, mut no, mut why, mut sets, from) = (Vec::new(), Vec::new(), None, vec![uids], s.read_bytes());
839 while let Some(set) = sets.pop() {
840 match s.uid_fetch(&UidSet::from(set), what).await {
841 Ok(v) => got.extend(v),
842 Err(e) if !refused(&e) => return Err(e),
843 Err(e) if set.len() == 1 => { no.extend_from_slice(set); why = Some(e); }
844 Err(_) => sets.extend(set.chunks(1)),
845 }
846 }
847 charge(bytes, s.read_bytes().saturating_sub(from));
848 Ok((got, no, why))
849}
850
851/// Did the server refuse the command, the connection standing (a NO, a BAD, an answer past a
852/// bound)? A drop, a renumbering or a fault of ours is not a refusal.
853fn refused(e: &Error<ErrTag>) -> bool {
854 let t = e.tags();
855 t.contains(&ErrTag::Network) && !t.contains(&ErrTag::Interrupted)
856 && t.iter().any(|x| matches!(x, ErrTag::Invalid | ErrTag::Wire | ErrTag::Excessive))
857}
858
859/// Of a message the narrowed key left, fetched for its structure in `shape` or refused in `no`:
860/// may its attachments' names make it a hit? False for one the server no longer holds.
861fn may(m: &Matcher, shape: &[FetchedMessage], no: &[u32], uid: u32) -> bool {
862 no.contains(&uid) || shape.iter().find(|f| f.uid == uid).map_or(false, |f| f.structure.as_ref()
863 .map_or(true, |t| { let mut x = Fetched::default(); names_of(t, &mut x); m.names_may_hold(&x) }))
864}
865
866fn charge(bytes: &mut u32, n: u64) {
867 *bytes = bytes.saturating_sub(u32::try_from(n).unwrap_or(u32::MAX));
868}
869
870/// A hit added to the page, or its folders to the hit of the same message there.
871fn found(pg: &mut Page, name: &str, uv: u32, fm: &FetchedMessage, hit: Hit) {
872 let header = Header::of(fm);
873 let key = fm.gm_msgid.map_or_else(|| header.message_id.clone(), |g| fmt!("gm:{}", g));
874 let mut folders = vec![name.to_string()];
875 for l in &fm.gm_labels {
876 if !folders.contains(l) { folders.push(l.clone()); }
877 }
878 match pg.hits.iter_mut().find(|f| !key.is_empty() && f.key == key) {
879 Some(f) => for x in folders {
880 if !f.folders.contains(&x) { f.folders.push(x); }
881 },
882 None => pg.hits.push(Found { folders, uid: fm.uid, uid_validity: uv, header, hit, key }),
883 }
884}
885
886/// A failed call: an error where the page has decided nothing, so that the cursor given in stands;
887/// else the page so far, its cursor at the last message decided.
888fn stop(mut pg: Page, e: Error<ErrTag>, q: &Query, names: &[String], at: Vec<Place>) -> Outcome<Page> {
889 if pg.coverage.checked + pg.coverage.undecided == 0 {
890 return Err(e);
891 }
892 pg.coverage.stopped = Some(e.plain());
893 pg.cursor = Some(Cursor::new(q, names, at));
894 Ok(pg)
895}