oxedyne/fe2o3/fe2o3_mail/src/sync/query.rs
37.9 KiB, 280 runs
created by r1870400018:76323, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Reading one account's mail on the server, for an agent that asks rather than syncs: the |
| 2 | //! newest messages of a folder, a search over its folders a page at a time, one message read by |
| 3 | //! part, one message raw, and the folders by role. |
| 4 | //! |
| 5 | //! - Nothing here changes the mailbox: every folder is opened with `EXAMINE`, so the session can |
| 6 | //! write no flag and `\Recent` is left for the next `SELECT` (UNBLOCK-63), and every body is |
| 7 | //! fetched with `BODY.PEEK`, so a daimon's read leaves `\Seen` as it found it. |
| 8 | //! - A folder is named by its role ([`Target`]), resolved by its RFC 6154 attribute, so Gmail's |
| 9 | //! "[Gmail]/All Mail" and a localised "[Gmail]/Gesendet" are found without their names. |
| 10 | //! - [`search`] trusts the server's `SEARCH` for candidates only, since a server may match more |
| 11 | //! than asked (Gmail's search is fuzzy): `search::Matcher` decides each, and a page holds no |
| 12 | //! state but its [`Cursor`]. |
| 13 | //! - [`read`] fetches `BODYSTRUCTURE` first, then the text parts within the byte cap, and lists |
| 14 | //! attachments by name and size without their bytes, so a large attachment never cuts off the |
| 15 | //! body it came with. |
| 16 | //! - Every size and count is the caller's, from its config ([`Limits`]). |
| 17 | |
| 18 | use crate::{ |
| 19 | message, |
| 20 | search::{ |
| 21 | self, |
| 22 | Fetched, |
| 23 | Folders, |
| 24 | Hit, |
| 25 | Mail, |
| 26 | Matcher, |
| 27 | Needs, |
| 28 | Query, |
| 29 | Scope, |
| 30 | Server, |
| 31 | }, |
| 32 | sync::{ |
| 33 | act::Target, |
| 34 | BodyPart, |
| 35 | Caps, |
| 36 | FetchWhat, |
| 37 | FetchedMessage, |
| 38 | MailSession, |
| 39 | Roles, |
| 40 | SelectWith, |
| 41 | SpecialUse, |
| 42 | UidSet, |
| 43 | }, |
| 44 | }; |
| 45 | |
| 46 | use oxedyne_fe2o3_core::prelude::*; |
| 47 | use oxedyne_fe2o3_net::imap::search::{ |
| 48 | civil_from_days, |
| 49 | parse_search_date, |
| 50 | SearchKey, |
| 51 | }; |
| 52 | use oxedyne_fe2o3_text::mime; |
| 53 | |
| 54 | use std::fmt; |
| 55 | |
| 56 | |
| 57 | /// The bounds on what one query returns, from the caller's config. |
| 58 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 59 | pub struct Limits { |
| 60 | pub max_rows: usize, // rows a listing or a search returns |
| 61 | pub max_read_bytes: u32, // of text parts one read fetches, across its parts |
| 62 | pub max_folders: usize, // folders counted by `folders`; the rest are listed uncounted |
| 63 | } |
| 64 | |
| 65 | /// A calendar day, as `SEARCH SINCE` and `BEFORE` take it. |
| 66 | #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] |
| 67 | pub struct Day { |
| 68 | pub year: u16, |
| 69 | pub month: u8, |
| 70 | pub day: u8, |
| 71 | } |
| 72 | |
| 73 | impl Day { |
| 74 | /// `2024/01/31` or `2024-01-31`. |
| 75 | pub fn parse(s: &str) -> Outcome<Self> { |
| 76 | let n: Vec<Option<u16>> = s.split(['/', '-']).map(|x| x.parse().ok()).collect(); |
| 77 | if let [Some(year), Some(m), Some(d)] = n[..] { |
| 78 | if (1..=12).contains(&m) && (1..=Self::last(year, m as u8) as u16).contains(&d) { |
| 79 | return Ok(Self { year, month: m as u8, day: d as u8 }); |
| 80 | } |
| 81 | } |
| 82 | Err(err!("'{}' is not a date; write it as 2024/01/31.", s; Invalid, Input)) |
| 83 | } |
| 84 | |
| 85 | /// INTERNALDATE's own calendar day (`30-Jan-2026 23:30:00 -0700` is 30 January), in the zone |
| 86 | /// the server wrote it in, never converted. |
| 87 | pub fn internal(s: &str) -> Option<Self> { |
| 88 | s.split_whitespace().next().and_then(parse_search_date).map(civil_from_days) |
| 89 | .and_then(|(y, m, d)| u16::try_from(y).ok().map(|year| Self { year, month: m as u8, day: d as u8 })) |
| 90 | } |
| 91 | |
| 92 | /// The month's last day. |
| 93 | pub fn last(year: u16, month: u8) -> u8 { |
| 94 | let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); |
| 95 | match month { 2 => 28 + leap as u8, 4 | 6 | 9 | 11 => 30, _ => 31 } // fixed: Gregorian |
| 96 | } |
| 97 | |
| 98 | /// `1-Feb-2026` (RFC 3501 `date`). |
| 99 | pub fn imap(&self) -> Outcome<String> { |
| 100 | // fixed: the month names of RFC 3501's date-month. |
| 101 | const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", |
| 102 | "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; |
| 103 | if self.month < 1 || self.month > 12 || self.day < 1 || self.day > 31 { |
| 104 | return Err(err!("The day {}-{}-{} is not a calendar day.", |
| 105 | self.year, self.month, self.day; Invalid, Input)); |
| 106 | } |
| 107 | Ok(fmt!("{}-{}-{}", self.day, MONTHS[(self.month - 1) as usize], self.year)) |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | impl std::fmt::Display for Day { |
| 112 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { |
| 113 | write!(f, "{:04}/{:02}/{:02}", self.year, self.month, self.day) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | /// The dates a query is bounded by, by the server's internal date: on or after `since`, and |
| 118 | /// before `before`. |
| 119 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] |
| 120 | pub struct Range { |
| 121 | pub since: Option<Day>, |
| 122 | pub before: Option<Day>, |
| 123 | } |
| 124 | |
| 125 | impl Range { |
| 126 | fn criteria(&self) -> Outcome<String> { |
| 127 | let mut c = Vec::new(); |
| 128 | if let Some(d) = &self.since { |
| 129 | c.push(fmt!("SINCE {}", res!(d.imap()))); |
| 130 | } |
| 131 | if let Some(d) = &self.before { |
| 132 | c.push(fmt!("BEFORE {}", res!(d.imap()))); |
| 133 | } |
| 134 | Ok(c.join(" ")) |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | /// One message as a listing shows it, its fields with their encoded words decoded. |
| 139 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 140 | pub struct Header { |
| 141 | pub uid: u32, |
| 142 | pub flags: Vec<String>, |
| 143 | pub internal_date: String, // as the server gave it |
| 144 | pub size: u32, |
| 145 | pub date: String, // the Date field |
| 146 | pub from: String, |
| 147 | pub to: String, |
| 148 | pub cc: String, |
| 149 | pub subject: String, |
| 150 | pub message_id: String, // without its angle brackets |
| 151 | } |
| 152 | |
| 153 | impl Header { |
| 154 | fn of(m: &FetchedMessage) -> Self { |
| 155 | let hs = message::headers(&m.body); |
| 156 | let get = |n: &str| hs.iter() |
| 157 | .find(|(k, _)| k == n) |
| 158 | .map(|(_, v)| mime::decode_words(v).trim().to_string()) |
| 159 | .unwrap_or_default(); |
| 160 | Self { |
| 161 | uid: m.uid, |
| 162 | flags: m.flags.clone(), |
| 163 | internal_date: m.internal_date.clone(), |
| 164 | size: m.size, |
| 165 | date: get("date"), |
| 166 | from: get("from"), |
| 167 | to: get("to"), |
| 168 | cc: get("cc"), |
| 169 | subject: get("subject"), |
| 170 | message_id: crate::sync::bare_message_id(&get("message-id")).to_string(), |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | pub fn seen(&self) -> bool { |
| 175 | self.flags.iter().any(|f| f.eq_ignore_ascii_case("\\Seen")) |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | /// What a listing or a search found in one folder, newest first. |
| 180 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 181 | pub struct Listing { |
| 182 | pub mailbox: String, |
| 183 | pub total: u32, // messages in the folder |
| 184 | pub rows: Vec<Header>, |
| 185 | pub more: bool, // more matched than `max_rows` |
| 186 | } |
| 187 | |
| 188 | /// One message read by part. |
| 189 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 190 | pub struct Read { |
| 191 | pub header: Header, |
| 192 | pub texts: Vec<TextPart>, |
| 193 | pub attachments: Vec<Attachment>, |
| 194 | } |
| 195 | |
| 196 | impl Read { |
| 197 | /// Was any text part cut at the byte cap, or left unread? |
| 198 | pub fn cut(&self) -> bool { |
| 199 | self.texts.iter().any(|t| t.cut) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 204 | pub struct TextPart { |
| 205 | pub section: String, // `1.1` |
| 206 | pub media: String, // `text/plain` or `text/html` |
| 207 | pub text: String, // decoded to UTF-8, as far as it was read |
| 208 | pub cut: bool, // the cap stopped it short |
| 209 | } |
| 210 | |
| 211 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 212 | pub struct Attachment { |
| 213 | pub section: String, |
| 214 | pub media: String, |
| 215 | pub name: Option<String>, |
| 216 | pub size: u32, // octets in its transfer encoding |
| 217 | } |
| 218 | |
| 219 | /// One message's bytes, read without marking it read. |
| 220 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 221 | pub struct Raw { |
| 222 | pub bytes: Vec<u8>, |
| 223 | pub size: u32, // the whole message's |
| 224 | pub cut: bool, // `max` stopped it short |
| 225 | } |
| 226 | |
| 227 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 228 | pub enum Role { |
| 229 | Inbox, |
| 230 | Use(SpecialUse), |
| 231 | } |
| 232 | |
| 233 | /// One folder, by role where it has one. |
| 234 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 235 | pub struct Folder { |
| 236 | pub name: String, |
| 237 | pub role: Option<Role>, |
| 238 | pub count: Option<u32>, // `None` past `max_folders` |
| 239 | } |
| 240 | |
| 241 | /// One page of a search over the server's folders. |
| 242 | #[derive(Clone, Debug, Default)] |
| 243 | pub struct Page { |
| 244 | pub hits: Vec<Found>, |
| 245 | pub coverage: Coverage, |
| 246 | pub cursor: Option<Cursor>, // `None`: the run is done |
| 247 | } |
| 248 | |
| 249 | /// One message found, once a page, naming each folder of the page it was found in. |
| 250 | #[derive(Clone, Debug)] |
| 251 | pub struct Found { |
| 252 | pub folders: Vec<String>, // where found, then Gmail's labels where fetched |
| 253 | pub uid: u32, // in the first of them |
| 254 | pub uid_validity: u32, |
| 255 | pub header: Header, |
| 256 | pub hit: Hit, |
| 257 | key: String, // X-GM-MSGID, else Message-ID; empty matches nothing |
| 258 | } |
| 259 | |
| 260 | /// What one page looked at. A message is `checked` only where the matcher decided it. |
| 261 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 262 | pub struct Coverage { |
| 263 | pub checked: usize, |
| 264 | pub undecided: usize, // a text past the byte bound, an unreadable structure |
| 265 | pub total: u32, // messages in the folders the page examined |
| 266 | pub top_uid: u32, // the run's top in the last folder examined |
| 267 | pub restarted: Vec<String>, // renumbered since the cursor, searched again from their newest |
| 268 | pub skipped: Vec<(String, String)>, // a folder not searched, and why |
| 269 | pub complete: bool, // the server's key was complete; else each page is a bounded scan |
| 270 | pub stopped: Option<String>, // why the page ended early, its hits kept |
| 271 | } |
| 272 | |
| 273 | /// The whole state of a search between its pages, carried as text (`Display`, `parse`): for each |
| 274 | /// folder of the run, in order, where it stands. A digest binds it to the query, the folders and |
| 275 | /// itself, so that one which does not fit is refused. |
| 276 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 277 | pub struct Cursor { |
| 278 | digest: u64, |
| 279 | at: Vec<Place>, |
| 280 | } |
| 281 | |
| 282 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] |
| 283 | struct Place { |
| 284 | uv: u32, // UIDVALIDITY; 0 before the folder is begun |
| 285 | top: u32, // the run's newest UID in it, so that later mail is not searched |
| 286 | below: u32, // the next page searches under this UID; 1 once done |
| 287 | } |
| 288 | |
| 289 | impl Cursor { |
| 290 | fn new(q: &Query, folders: &[String], at: Vec<Place>) -> Self { |
| 291 | Self { digest: Self::digest(q, folders, &at), at } |
| 292 | } |
| 293 | |
| 294 | fn digest(q: &Query, folders: &[String], at: &[Place]) -> u64 { |
| 295 | message::fnv1a(fmt!("{}\n{}\n{}", q, folders.join("\n"), places(at)).as_bytes()) |
| 296 | } |
| 297 | |
| 298 | pub fn parse(s: &str) -> Outcome<Self> { |
| 299 | let bad = || err!("{:?} is not a search cursor.", s; Invalid, Input); |
| 300 | let mut it = s.split(':'); |
| 301 | let digest = match (it.next(), it.next().map(|d| u64::from_str_radix(d, 16))) { |
| 302 | (Some("q1"), Some(Ok(d))) => d, |
| 303 | _ => return Err(bad()), |
| 304 | }; |
| 305 | let mut at = Vec::new(); |
| 306 | for p in it { |
| 307 | match p.split(',').map(|n| n.parse::<u32>().ok()).collect::<Vec<_>>()[..] { |
| 308 | [Some(uv), Some(top), Some(below)] => at.push(Place { uv, top, below }), |
| 309 | _ => return Err(bad()), |
| 310 | } |
| 311 | } |
| 312 | Ok(Self { digest, at }) |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | fn places(at: &[Place]) -> String { |
| 317 | at.iter().map(|p| fmt!(":{},{},{}", p.uv, p.top, p.below)).collect() |
| 318 | } |
| 319 | |
| 320 | impl fmt::Display for Cursor { |
| 321 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| 322 | write!(f, "q1:{:016x}{}", self.digest, places(&self.at)) |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | |
| 327 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 328 | // │ THE QUERIES │ |
| 329 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 330 | |
| 331 | /// The mailbox a target names, by its role's attribute. |
| 332 | pub fn resolve(roles: &Roles, target: &Target) -> Outcome<String> { |
| 333 | match target { |
| 334 | Target::Inbox => Ok(roles.inbox.clone()), |
| 335 | Target::Role(r) => Ok(res!(roles.role(*r) |
| 336 | .ok_or_else(|| err!("The account has no folder with the {:?} role.", r; Missing))) |
| 337 | .to_string()), |
| 338 | Target::Path(p) => if roles.exists(p) { |
| 339 | Ok(p.clone()) |
| 340 | } else { |
| 341 | Err(err!("The account has no folder {:?}.", p; Missing)) |
| 342 | }, |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | /// The newest messages of a folder within `range`. |
| 347 | pub async fn list<S: MailSession>( |
| 348 | s: &mut S, |
| 349 | roles: &Roles, |
| 350 | target: &Target, |
| 351 | range: &Range, |
| 352 | lim: &Limits, |
| 353 | ) |
| 354 | -> Outcome<Listing> |
| 355 | { |
| 356 | let mailbox = res!(resolve(roles, target)); |
| 357 | let st = res!(s.examine(&mailbox, &SelectWith::Plain).await); |
| 358 | let crit = res!(range.criteria()); |
| 359 | let mut uids = res!(s.uid_search(if crit.is_empty() { "ALL" } else { &crit }).await); |
| 360 | uids.sort_unstable(); |
| 361 | let more = uids.len() > lim.max_rows; |
| 362 | let newest: Vec<u32> = uids.iter().rev().take(lim.max_rows).copied().collect(); |
| 363 | let rows = res!(summaries(s, &newest).await); |
| 364 | Ok(Listing { mailbox, total: st.exists, rows, more }) |
| 365 | } |
| 366 | |
| 367 | /// One message by part, `None` where the folder does not hold `uid`: its structure first, then |
| 368 | /// each text body within `max_read_bytes` across them, the plain form of an alternative over |
| 369 | /// its HTML, and every other part listed by name. |
| 370 | pub async fn read<S: MailSession>( |
| 371 | s: &mut S, |
| 372 | roles: &Roles, |
| 373 | target: &Target, |
| 374 | uid: u32, |
| 375 | lim: &Limits, |
| 376 | ) |
| 377 | -> Outcome<Option<Read>> |
| 378 | { |
| 379 | let mailbox = res!(resolve(roles, target)); |
| 380 | res!(s.examine(&mailbox, &SelectWith::Plain).await); |
| 381 | let one = UidSet::from([uid]); |
| 382 | let m = match res!(s.uid_fetch(&one, FetchWhat::Structure).await).into_iter() |
| 383 | .find(|m| m.uid == uid) |
| 384 | { |
| 385 | Some(m) => m, |
| 386 | None => return Ok(None), |
| 387 | }; |
| 388 | let mut out = Read { header: Header::of(&m), ..Default::default() }; |
| 389 | let tree = match &m.structure { |
| 390 | Some(t) => t.clone(), |
| 391 | // A structure the server gave malformed: the whole message as one part, within the cap. |
| 392 | None => BodyPart { |
| 393 | section: String::new(), |
| 394 | media: "message/rfc822".to_string(), |
| 395 | size: m.size, |
| 396 | ..Default::default() |
| 397 | }, |
| 398 | }; |
| 399 | let mut texts: Vec<&BodyPart> = Vec::new(); |
| 400 | bodies(&tree, &mut texts, &mut out.attachments); |
| 401 | if texts.is_empty() && tree.parts.is_empty() && tree.media == "message/rfc822" { |
| 402 | texts.push(&tree); |
| 403 | } |
| 404 | let mut left = lim.max_read_bytes; |
| 405 | for p in texts { |
| 406 | let ask = p.size.min(left); |
| 407 | let mut t = TextPart { |
| 408 | section: p.section.clone(), |
| 409 | media: p.media.clone(), |
| 410 | cut: ask < p.size, |
| 411 | ..Default::default() |
| 412 | }; |
| 413 | if ask > 0 { |
| 414 | if let Some(f) = res!(s.uid_fetch_section(uid, &p.section, ask).await) { |
| 415 | left = left.saturating_sub(f.body.len() as u32); |
| 416 | t.text = if p.section.is_empty() { |
| 417 | message::readable_text(&f.body) |
| 418 | } else { |
| 419 | mime::decode_body(&f.body, &p.encoding, p.param("charset")) |
| 420 | }; |
| 421 | } |
| 422 | } |
| 423 | out.texts.push(t); |
| 424 | } |
| 425 | Ok(Some(out)) |
| 426 | } |
| 427 | |
| 428 | /// The text bodies to read and the attachments to list: an alternative gives its first child |
| 429 | /// that holds plain text, else its last, its richest (RFC 2046 §5.1.4). |
| 430 | fn bodies<'a>(p: &'a BodyPart, texts: &mut Vec<&'a BodyPart>, atts: &mut Vec<Attachment>) { |
| 431 | if p.parts.is_empty() { |
| 432 | if p.is_text_body() { |
| 433 | texts.push(p); |
| 434 | } else if !p.section.is_empty() { |
| 435 | atts.push(Attachment { |
| 436 | section: p.section.clone(), |
| 437 | media: p.media.clone(), |
| 438 | name: p.filename.clone(), |
| 439 | size: p.size, |
| 440 | }); |
| 441 | } |
| 442 | return; |
| 443 | } |
| 444 | if p.media == "multipart/alternative" { |
| 445 | let plain = p.parts.iter() |
| 446 | .find(|c| c.leaves().iter().any(|l| l.media == "text/plain" && l.is_text_body())); |
| 447 | if let Some(c) = plain.or(p.parts.last()) { |
| 448 | bodies(c, texts, atts); |
| 449 | } |
| 450 | return; |
| 451 | } |
| 452 | for c in &p.parts { |
| 453 | bodies(c, texts, atts); |
| 454 | } |
| 455 | } |
| 456 | |
| 457 | /// One whole message, at most `max` bytes of it, leaving `\Seen` as it was. |
| 458 | pub async fn raw<S: MailSession>( |
| 459 | s: &mut S, |
| 460 | roles: &Roles, |
| 461 | target: &Target, |
| 462 | uid: u32, |
| 463 | max: u32, |
| 464 | ) |
| 465 | -> Outcome<Option<Raw>> |
| 466 | { |
| 467 | let mailbox = res!(resolve(roles, target)); |
| 468 | res!(s.examine(&mailbox, &SelectWith::Plain).await); |
| 469 | Ok(res!(s.uid_fetch_section(uid, "", max).await).map(|m| Raw { |
| 470 | cut: (m.body.len() as u32) < m.size, |
| 471 | size: m.size, |
| 472 | bytes: m.body, |
| 473 | })) |
| 474 | } |
| 475 | |
| 476 | /// Every folder, the inbox and the roles first, each with its role found by attribute and, for |
| 477 | /// the first `max_folders`, its message count. |
| 478 | pub async fn folders<S: MailSession>(s: &mut S, lim: &Limits) -> Outcome<(Roles, Vec<Folder>)> { |
| 479 | let roles = Roles::from_list(&res!(s.list().await)); |
| 480 | let mut out = Vec::new(); |
| 481 | for (i, name) in roles.synced().into_iter().enumerate() { |
| 482 | let role = if name == roles.inbox { |
| 483 | Some(Role::Inbox) |
| 484 | } else { |
| 485 | roles.mailboxes.iter().find(|m| m.name == name) |
| 486 | .and_then(|m| m.special_use()) |
| 487 | .filter(|r| roles.role(*r) == Some(name.as_str())) |
| 488 | .map(Role::Use) |
| 489 | }; |
| 490 | let count = if i < lim.max_folders { Some(res!(s.count(&name).await)) } else { None }; |
| 491 | out.push(Folder { name, role, count }); |
| 492 | } |
| 493 | Ok((roles, out)) |
| 494 | } |
| 495 | |
| 496 | /// The listing rows of `uids`, newest first. |
| 497 | async fn summaries<S: MailSession>(s: &mut S, uids: &[u32]) -> Outcome<Vec<Header>> { |
| 498 | if uids.is_empty() { |
| 499 | return Ok(Vec::new()); |
| 500 | } |
| 501 | let asked = UidSet::from(uids); |
| 502 | let mut rows: Vec<Header> = res!(s.uid_fetch(&asked, FetchWhat::Summary).await).iter() |
| 503 | .filter(|m| asked.contains(m.uid)) |
| 504 | .map(Header::of) |
| 505 | .collect(); |
| 506 | rows.sort_by(|a, b| b.uid.cmp(&a.uid)); |
| 507 | rows.dedup_by_key(|h| h.uid); |
| 508 | Ok(rows) |
| 509 | } |
| 510 | |
| 511 | |
| 512 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 513 | // │ THE SEARCH │ |
| 514 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 515 | |
| 516 | /// One page of `q` over the server's folders from `cur` (`None` begins the run). The folders are |
| 517 | /// those `q.folders` names: the inbox where it names none, a list, or all, Junk and Trash only |
| 518 | /// where asked, and on Gmail its All Mail in place of the rest. Each is examined, then searched |
| 519 | /// newest first below the cursor by `search::plan`'s key, a window of UIDs at a time where that |
| 520 | /// key is not complete; each candidate is decided by the matcher from no more than the plan needs, |
| 521 | /// within the page's bounds in `lim`. The byte bound is charged with what each fetch reads from the |
| 522 | /// wire, and no chunk of `search_page_rows` UIDs is fetched once it is spent and something is |
| 523 | /// decided; a page's first candidate is read within the whole bound, so a page reads at most twice |
| 524 | /// the bound and one chunk's structures and header blocks. A complete key's SEARCH |
| 525 | /// names every candidate below the cursor, so a page holds one folder's list of them. A message |
| 526 | /// found in several folders of a page is one hit. A message the server refuses is undecided and |
| 527 | /// the page goes on. A call that fails otherwise, once something is decided, ends the page there, |
| 528 | /// `stopped` saying why; before, it is an error and the cursor given in stands. |
| 529 | pub async fn search<S: MailSession>( |
| 530 | s: &mut S, |
| 531 | caps: &Caps, |
| 532 | roles: &Roles, |
| 533 | q: &Query, |
| 534 | cur: Option<&Cursor>, |
| 535 | lim: &search::Limits, |
| 536 | today: Day, |
| 537 | ) |
| 538 | -> Outcome<Page> |
| 539 | { |
| 540 | let (names, skipped) = run_folders(caps, roles, q, lim); |
| 541 | let mut at = match cur { |
| 542 | None => vec![Place::default(); names.len()], |
| 543 | Some(c) if c.at.len() == names.len() && c.digest == Cursor::digest(q, &names, &c.at) => c.at.clone(), |
| 544 | Some(_) => return Err(err!("The cursor is not one this search gave over these folders; begin \ |
| 545 | the search again."; Invalid, Input)), |
| 546 | }; |
| 547 | let srv = |names| Server { gmail: caps.gmail, names }; |
| 548 | let plan = res!(search::plan(q, &srv(true), lim, today)); |
| 549 | // Where the key is not complete, the key with each attachment fact false narrows a window's |
| 550 | // answer, and of the rest those whose structures' names may hold are candidates: a superset. |
| 551 | let narrow = match plan.complete { |
| 552 | true => None, |
| 553 | false => Some(res!(search::plan(q, &srv(false), lim, today)).key).filter(|k| *k != plan.key), |
| 554 | }; |
| 555 | let m = res!(Matcher::new(q, lim, today)); |
| 556 | let mut pg = Page { |
| 557 | coverage: Coverage { skipped, complete: plan.complete, ..Default::default() }, |
| 558 | ..Default::default() |
| 559 | }; |
| 560 | let key = match plan.key { |
| 561 | Some(k) => k, |
| 562 | None => return Ok(pg), // nothing can match |
| 563 | }; |
| 564 | let what = if plan.needs >= Needs::Structure { FetchWhat::Structure } else { FetchWhat::Headers }; |
| 565 | let rows = lim.search_page_rows.max(1); |
| 566 | let (cap, mut scan) = (lim.search_confirm_max_bytes, lim.search_scan_max_messages.max(1)); |
| 567 | let (mut bytes, mut fresh, mut refusals, mut why) = (cap, true, 0, None); |
| 568 | 'folders: for (i, name) in names.iter().enumerate() { |
| 569 | if at[i].below == 1 { continue; } |
| 570 | let st = match s.examine(name, &SelectWith::Plain).await { |
| 571 | Ok(st) => st, |
| 572 | Err(e) if e.tags().contains(&ErrTag::NotFound) => { |
| 573 | pg.coverage.skipped.push((name.clone(), e.plain())); |
| 574 | at[i].below = 1; |
| 575 | continue; |
| 576 | } |
| 577 | Err(e) => return stop(pg, e, q, &names, at), |
| 578 | }; |
| 579 | if st.uid_validity == 0 || st.uid_next == 0 { |
| 580 | return stop(pg, err!("The server examined {:?} without its UIDVALIDITY or UIDNEXT, so its \ |
| 581 | UIDs name nothing certain.", name; Invalid, Input), q, &names, at); |
| 582 | } |
| 583 | if at[i].uv != st.uid_validity { |
| 584 | if at[i].uv != 0 { pg.coverage.restarted.push(name.clone()); } |
| 585 | at[i] = Place { uv: st.uid_validity, top: st.uid_next - 1, below: st.uid_next }; |
| 586 | } |
| 587 | pg.coverage.total += st.exists; |
| 588 | pg.coverage.top_uid = at[i].top; |
| 589 | while at[i].below > 1 { |
| 590 | let spent = bytes == 0 && pg.coverage.checked + pg.coverage.undecided > 0; |
| 591 | if pg.hits.len() >= rows || spent || scan == 0 { break 'folders; } |
| 592 | let below = at[i].below; |
| 593 | let lo = match plan.complete { |
| 594 | true => 1, |
| 595 | false => below.saturating_sub(u32::try_from(scan).unwrap_or(u32::MAX)).max(1), |
| 596 | }; |
| 597 | let win = |k: &SearchKey| SearchKey::And(vec![k.clone(), SearchKey::Uid(fmt!("{}:{}", lo, below - 1))]); |
| 598 | let mut ids = match s.uid_search_key(&res!(win(&key).wire())).await { |
| 599 | Ok(u) => u, |
| 600 | Err(e) => return stop(pg, e, q, &names, at), |
| 601 | }; |
| 602 | let mut near = match &narrow { |
| 603 | Some(Some(n)) => match s.uid_search_key(&res!(win(n).wire())).await { |
| 604 | Ok(u) => u, |
| 605 | Err(e) => return stop(pg, e, q, &names, at), |
| 606 | }, |
| 607 | _ => Vec::new(), |
| 608 | }; |
| 609 | ids.retain(|u| (lo..below).contains(u)); // a wider answer costs time only |
| 610 | ids.sort_unstable_by(|a, b| b.cmp(a)); |
| 611 | ids.dedup(); |
| 612 | near.sort_unstable(); |
| 613 | // A candidate: any of the key's answer, or where narrowed, one the narrowed key names too. |
| 614 | let near = |u: &u32| narrow.is_none() || near.binary_search(u).is_ok(); |
| 615 | if !plan.complete { scan -= (below - lo) as usize; } |
| 616 | // The window in chunks of `rows`, newest first: the structures of the chunk's rest, then |
| 617 | // its candidates, each fetched once. |
| 618 | for chunk in ids.chunks(rows) { |
| 619 | let spent = bytes == 0 && pg.coverage.checked + pg.coverage.undecided > 0; |
| 620 | if pg.hits.len() >= rows || spent || (plan.complete && scan == 0) { break 'folders; } |
| 621 | let rest: Vec<u32> = chunk.iter().copied().filter(|u| !near(u)).collect(); |
| 622 | let (shape, shape_no, why_s) = match rest.is_empty() { |
| 623 | true => Default::default(), |
| 624 | false => match fetch(s, &rest, FetchWhat::BodyStructure, &mut bytes).await { |
| 625 | Ok(v) => v, |
| 626 | Err(e) => return stop(pg, e, q, &names, at), |
| 627 | }, |
| 628 | }; |
| 629 | let cands: Vec<u32> = chunk.iter().copied().filter(|u| near(u) || may(&m, &shape, &shape_no, *u)).collect(); |
| 630 | let (got, got_no, why_c) = match cands.is_empty() { |
| 631 | true => Default::default(), |
| 632 | false => match fetch(s, &cands, what, &mut bytes).await { |
| 633 | Ok(v) => v, |
| 634 | Err(e) => return stop(pg, e, q, &names, at), |
| 635 | }, |
| 636 | }; |
| 637 | why = why_c.or(why_s).or(why); |
| 638 | for &uid in chunk { |
| 639 | if !cands.contains(&uid) { |
| 640 | if shape.iter().any(|f| f.uid == uid) { pg.coverage.checked += 1; } |
| 641 | at[i].below = uid; |
| 642 | continue; |
| 643 | } |
| 644 | if pg.hits.len() >= rows || (bytes == 0 && !fresh) || (plan.complete && scan == 0) { break 'folders; } |
| 645 | let fm = got.iter().find(|f| f.uid == uid); |
| 646 | let v = match fm { |
| 647 | Some(f) => match confirm(s, &m, plan.needs, f, &mut bytes, fresh.then_some(cap)).await { |
| 648 | Ok(Verdict::Wait) => break 'folders, |
| 649 | Ok(v) => v, |
| 650 | Err(e) => return stop(pg, e, q, &names, at), |
| 651 | }, |
| 652 | None if got_no.contains(&uid) => Verdict::Refused(None), |
| 653 | None => Verdict::Gone, // expunged meanwhile |
| 654 | }; |
| 655 | at[i].below = uid; |
| 656 | fresh = false; |
| 657 | if plan.complete { scan -= 1; } |
| 658 | match (v, fm) { |
| 659 | (Verdict::Hit(h), Some(f)) => { |
| 660 | pg.coverage.checked += 1; |
| 661 | found(&mut pg, name, st.uid_validity, f, h); |
| 662 | } |
| 663 | (Verdict::Miss, _) => pg.coverage.checked += 1, |
| 664 | (Verdict::Undecided, _) => pg.coverage.undecided += 1, |
| 665 | (Verdict::Refused(e), _) => { |
| 666 | // A server that refuses a page's worth ends the page in its own words. |
| 667 | (pg.coverage.undecided, refusals, why) = (pg.coverage.undecided + 1, refusals + 1, e.or(why)); |
| 668 | if refusals >= rows { |
| 669 | if let Some(e) = why.take() { return stop(pg, e, q, &names, at); } |
| 670 | } |
| 671 | } |
| 672 | _ => {} |
| 673 | } |
| 674 | } |
| 675 | } |
| 676 | at[i].below = lo; |
| 677 | } |
| 678 | } |
| 679 | if at.iter().any(|p| p.below != 1) { pg.cursor = Some(Cursor::new(q, &names, at)); } |
| 680 | Ok(pg) |
| 681 | } |
| 682 | |
| 683 | /// The folders `q` runs over, at most `search_max_folders`, and those it leaves, with why. |
| 684 | fn run_folders(caps: &Caps, roles: &Roles, q: &Query, lim: &search::Limits) -> (Vec<String>, Vec<(String, String)>) { |
| 685 | let (mut names, mut skipped) = (Vec::new(), Vec::new()); |
| 686 | match &q.folders { |
| 687 | Folders::Selected => names.push(roles.inbox.clone()), |
| 688 | Folders::List(l) => for f in l { |
| 689 | match resolve(roles, &Target::Path(f.clone())) { |
| 690 | Ok(n) if !names.contains(&n) => names.push(n), |
| 691 | Ok(_) => {} |
| 692 | Err(e) => skipped.push((f.clone(), e.plain())), |
| 693 | } |
| 694 | }, |
| 695 | Folders::All { junk_trash } => { |
| 696 | let bin = [roles.role(SpecialUse::Junk), roles.role(SpecialUse::Trash)]; |
| 697 | let all = roles.role(SpecialUse::All).filter(|_| caps.gmail); // Gmail's every label but those two |
| 698 | for n in roles.synced() { |
| 699 | let b = bin.contains(&Some(n.as_str())); |
| 700 | if (b && *junk_trash) || (!b && all.map_or(true, |a| a == n)) { names.push(n); } |
| 701 | } |
| 702 | } |
| 703 | } |
| 704 | if names.len() > lim.search_max_folders { |
| 705 | let why = fmt!("past the {} folders one search takes", lim.search_max_folders); |
| 706 | skipped.extend(names.split_off(lim.search_max_folders).into_iter().map(|n| (n, why.clone()))); |
| 707 | } |
| 708 | (names, skipped) |
| 709 | } |
| 710 | |
| 711 | /// What the matcher made of one candidate. |
| 712 | enum Verdict { |
| 713 | Hit(Hit), |
| 714 | Miss, |
| 715 | Undecided, // a text past the byte bound, an unreadable structure |
| 716 | Refused(Option<Error<ErrTag>>), // by the server, in its words where they are in hand |
| 717 | Gone, // expunged while it was read |
| 718 | Wait, // its text does not fit what is left of the page, so it waits for a page of its own |
| 719 | } |
| 720 | |
| 721 | /// The matcher's verdict on one candidate, read no further than `needs`. The first candidate of |
| 722 | /// a page (`first`, the page's bound) is read as far as that bound goes, whatever is spent, so that |
| 723 | /// each page decides something; another waits where its text does not fit `bytes`. |
| 724 | async fn confirm<S: MailSession>( |
| 725 | s: &mut S, |
| 726 | m: &Matcher, |
| 727 | needs: Needs, |
| 728 | fm: &FetchedMessage, |
| 729 | bytes: &mut u32, |
| 730 | first: Option<u32>, |
| 731 | ) |
| 732 | -> Outcome<Verdict> |
| 733 | { |
| 734 | let has = |f: &str| fm.flags.iter().any(|x| x.eq_ignore_ascii_case(f)); |
| 735 | let mail = Mail { raw: &fm.body, seen: has("\\Seen"), flagged: has("\\Flagged"), |
| 736 | size: u64::from(fm.size), day: Day::internal(&fm.internal_date) }; |
| 737 | let mut f = Fetched::default(); |
| 738 | if needs >= Needs::Structure { |
| 739 | let tree = match &fm.structure { |
| 740 | Some(t) => t, |
| 741 | None => return Ok(Verdict::Undecided), |
| 742 | }; |
| 743 | names_of(tree, &mut f); |
| 744 | // A body that decodes blank shows nothing, so the reader passes over it, as here. |
| 745 | let (mut blank, mut read): (Vec<&str>, Vec<(&str, String)>) = (Vec::new(), Vec::new()); |
| 746 | while needs == Needs::Text { |
| 747 | let mut texts = Vec::new(); |
| 748 | shown(tree, &blank, &mut texts); |
| 749 | let todo: Vec<&BodyPart> = texts.iter().copied() |
| 750 | .filter(|p| !read.iter().any(|(x, _)| *x == p.section)).collect(); |
| 751 | if first.is_none() && blank.is_empty() && todo.iter().map(|p| u64::from(p.size)).sum::<u64>() > u64::from(*bytes) { |
| 752 | return Ok(Verdict::Wait); |
| 753 | } |
| 754 | let mut again = false; |
| 755 | for p in todo { |
| 756 | let ask = p.size.min(first.map_or(*bytes, |c| c.max(*bytes))); |
| 757 | if ask < p.size && !f.unknown.contains(&Scope::Body) { f.unknown.push(Scope::Body); } |
| 758 | let t = match ask { |
| 759 | 0 => String::new(), |
| 760 | _ => { |
| 761 | let from = s.read_bytes(); |
| 762 | let g = match s.uid_fetch_section(fm.uid, &p.section, ask).await { |
| 763 | Ok(Some(g)) => g, |
| 764 | Ok(None) => return Ok(Verdict::Gone), |
| 765 | Err(e) if refused(&e) => return Ok(Verdict::Refused(Some(e))), |
| 766 | Err(e) => return Err(e), |
| 767 | }; |
| 768 | charge(bytes, s.read_bytes().saturating_sub(from)); |
| 769 | mime::decode_body(&g.body, &p.encoding, p.param("charset")) |
| 770 | } |
| 771 | }; |
| 772 | if ask == p.size && t.trim().is_empty() { |
| 773 | blank.push(&p.section); |
| 774 | again = true; |
| 775 | } |
| 776 | read.push((&p.section, t)); |
| 777 | } |
| 778 | if !again { |
| 779 | f.body = message::readable(texts.iter().filter_map(|p| read.iter() |
| 780 | .find(|(x, _)| *x == p.section).map(|(_, t)| (p.media.clone(), t.clone())))); |
| 781 | break; |
| 782 | } |
| 783 | } |
| 784 | } |
| 785 | Ok(match m.matches_fetched(&mail, &f) { |
| 786 | Ok(Some(h)) => Verdict::Hit(h), |
| 787 | Ok(None) => Verdict::Miss, |
| 788 | Err(_) => Verdict::Undecided, |
| 789 | }) |
| 790 | } |
| 791 | |
| 792 | /// The attachments' names of a structure, as `mime::Part::attachment_name` gives them. |
| 793 | fn names_of(tree: &BodyPart, f: &mut Fetched) { |
| 794 | for l in tree.leaves() { |
| 795 | let msg = l.media == "message/rfc822" || l.media == "message/global"; |
| 796 | match (&l.filename, msg, l.disposition.as_deref()) { |
| 797 | (Some(n), _, _) => f.names.push(n.clone()), |
| 798 | (None, true, _) => f.unknown.push(Scope::Attachment), // named by its Subject |
| 799 | (None, false, Some("attachment")) => f.names.push(fmt!("attachment")), |
| 800 | _ => {} |
| 801 | } |
| 802 | } |
| 803 | } |
| 804 | |
| 805 | /// The text bodies a reader is shown, chosen from the structure as `mime::Part::display_parts` |
| 806 | /// chooses them from the bytes, a body in `blank` showing nothing: the plain form of an |
| 807 | /// alternative, else its last that shows anything; the first part of a related; every part of |
| 808 | /// the rest. |
| 809 | fn shown<'a>(p: &'a BodyPart, blank: &[&str], out: &mut Vec<&'a BodyPart>) { |
| 810 | let of = |c: &'a BodyPart| { let mut v = Vec::new(); shown(c, blank, &mut v); v }; |
| 811 | match (p.parts.is_empty(), p.media.as_str()) { |
| 812 | (true, m) => { |
| 813 | let filed = match p.disposition.as_deref() { |
| 814 | Some("attachment") => true, |
| 815 | Some("inline") | None => false, |
| 816 | Some(_) => p.filename.is_some(), |
| 817 | }; |
| 818 | let text = m == "text/plain" || m == "text/html"; |
| 819 | if text && !filed && p.size > 0 && !blank.contains(&p.section.as_str()) { out.push(p); } |
| 820 | } |
| 821 | (false, "multipart/alternative") => { |
| 822 | let plain = p.parts.iter().filter(|c| c.parts.is_empty() && c.media == "text/plain") |
| 823 | .map(of).find(|v| !v.is_empty()); |
| 824 | if let Some(v) = plain.or_else(|| p.parts.iter().rev().map(of).find(|v| !v.is_empty())) { |
| 825 | out.extend(v); |
| 826 | } |
| 827 | } |
| 828 | (false, "multipart/related") => if let Some(c) = p.parts.first() { shown(c, blank, out); }, |
| 829 | (false, _) => for c in &p.parts { shown(c, blank, out); }, |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | /// `uids` fetched for `what`, the bytes read charged, those the server refused and its last |
| 834 | /// words: where it refuses a set, each is asked for alone. Any other failure is the error. |
| 835 | async fn fetch<S: MailSession>(s: &mut S, uids: &[u32], what: FetchWhat, bytes: &mut u32) |
| 836 | -> Outcome<(Vec<FetchedMessage>, Vec<u32>, Option<Error<ErrTag>>)> |
| 837 | { |
| 838 | let (mut got, mut no, mut why, mut sets, from) = (Vec::new(), Vec::new(), None, vec![uids], s.read_bytes()); |
| 839 | while let Some(set) = sets.pop() { |
| 840 | match s.uid_fetch(&UidSet::from(set), what).await { |
| 841 | Ok(v) => got.extend(v), |
| 842 | Err(e) if !refused(&e) => return Err(e), |
| 843 | Err(e) if set.len() == 1 => { no.extend_from_slice(set); why = Some(e); } |
| 844 | Err(_) => sets.extend(set.chunks(1)), |
| 845 | } |
| 846 | } |
| 847 | charge(bytes, s.read_bytes().saturating_sub(from)); |
| 848 | Ok((got, no, why)) |
| 849 | } |
| 850 | |
| 851 | /// Did the server refuse the command, the connection standing (a NO, a BAD, an answer past a |
| 852 | /// bound)? A drop, a renumbering or a fault of ours is not a refusal. |
| 853 | fn refused(e: &Error<ErrTag>) -> bool { |
| 854 | let t = e.tags(); |
| 855 | t.contains(&ErrTag::Network) && !t.contains(&ErrTag::Interrupted) |
| 856 | && t.iter().any(|x| matches!(x, ErrTag::Invalid | ErrTag::Wire | ErrTag::Excessive)) |
| 857 | } |
| 858 | |
| 859 | /// Of a message the narrowed key left, fetched for its structure in `shape` or refused in `no`: |
| 860 | /// may its attachments' names make it a hit? False for one the server no longer holds. |
| 861 | fn may(m: &Matcher, shape: &[FetchedMessage], no: &[u32], uid: u32) -> bool { |
| 862 | no.contains(&uid) || shape.iter().find(|f| f.uid == uid).map_or(false, |f| f.structure.as_ref() |
| 863 | .map_or(true, |t| { let mut x = Fetched::default(); names_of(t, &mut x); m.names_may_hold(&x) })) |
| 864 | } |
| 865 | |
| 866 | fn charge(bytes: &mut u32, n: u64) { |
| 867 | *bytes = bytes.saturating_sub(u32::try_from(n).unwrap_or(u32::MAX)); |
| 868 | } |
| 869 | |
| 870 | /// A hit added to the page, or its folders to the hit of the same message there. |
| 871 | fn found(pg: &mut Page, name: &str, uv: u32, fm: &FetchedMessage, hit: Hit) { |
| 872 | let header = Header::of(fm); |
| 873 | let key = fm.gm_msgid.map_or_else(|| header.message_id.clone(), |g| fmt!("gm:{}", g)); |
| 874 | let mut folders = vec![name.to_string()]; |
| 875 | for l in &fm.gm_labels { |
| 876 | if !folders.contains(l) { folders.push(l.clone()); } |
| 877 | } |
| 878 | match pg.hits.iter_mut().find(|f| !key.is_empty() && f.key == key) { |
| 879 | Some(f) => for x in folders { |
| 880 | if !f.folders.contains(&x) { f.folders.push(x); } |
| 881 | }, |
| 882 | None => pg.hits.push(Found { folders, uid: fm.uid, uid_validity: uv, header, hit, key }), |
| 883 | } |
| 884 | } |
| 885 | |
| 886 | /// A failed call: an error where the page has decided nothing, so that the cursor given in stands; |
| 887 | /// else the page so far, its cursor at the last message decided. |
| 888 | fn stop(mut pg: Page, e: Error<ErrTag>, q: &Query, names: &[String], at: Vec<Place>) -> Outcome<Page> { |
| 889 | if pg.coverage.checked + pg.coverage.undecided == 0 { |
| 890 | return Err(e); |
| 891 | } |
| 892 | pg.coverage.stopped = Some(e.plain()); |
| 893 | pg.cursor = Some(Cursor::new(q, names, at)); |
| 894 | Ok(pg) |
| 895 | } |