oxedyne/fe2o3/fe2o3_mail/src/sync/mod.rs
24.0 KiB, 22 runs
created by r1870400018:75207, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The folder sync engine: what changed in one IMAP folder since the device last looked, as a |
| 2 | //! bounded [`Delta`] the device applies to its cache. |
| 3 | //! |
| 4 | //! The engine is written once, against [`MailSession`], so the gateway's `ImapClient` and a |
| 5 | //! browser's tunnel drive the same algorithm. It owns no socket, no clock and no runtime, and |
| 6 | //! builds for `wasm32-unknown-unknown` with the crate's default features off. |
| 7 | //! |
| 8 | //! A sync works by capability tier ([`Tier`]): |
| 9 | //! - **QRESYNC** (RFC 7162): `SELECT (QRESYNC ...)` returns the changed flags and the `VANISHED` |
| 10 | //! UIDs directly; |
| 11 | //! - **CONDSTORE**: `UID FETCH held (FLAGS) (CHANGEDSINCE m)`, then `UID SEARCH UID held` for |
| 12 | //! what vanished; |
| 13 | //! - **full reconcile** (Steel, Office 365): `UID FETCH (FLAGS)` over the window on every poll and |
| 14 | //! over the whole held set every `sweep_every`th, so a message read elsewhere long ago does not |
| 15 | //! keep a stale state here; `UID SEARCH ALL` against the held set for what vanished. |
| 16 | //! |
| 17 | //! Each new message says whether it is an arrival ([`NewMessage::arrival`]): only what came after |
| 18 | //! the folder became known to the device is, never what a capped first read or a rebuild left in |
| 19 | //! the backlog, so a trigger that fires on arrivals does not fire on old mail. |
| 20 | //! |
| 21 | //! The device advances its [`Cursor`] only once the delta's writes have landed ([`apply`]), so a |
| 22 | //! crash replays a delta and never skips one, and a replayed apply changes nothing. A UIDVALIDITY |
| 23 | //! change makes the delta a [`Reset`]: the device sets its one old generation aside, rebuilds, |
| 24 | //! and drops the old one when the rebuild completes. |
| 25 | //! |
| 26 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 27 | //! Anthropic Claude |
| 28 | |
| 29 | pub mod act; |
| 30 | pub mod apply; |
| 31 | pub mod query; |
| 32 | pub mod imap; |
| 33 | #[cfg(feature = "jdat")] |
| 34 | pub mod wire; |
| 35 | |
| 36 | pub use act::{ |
| 37 | act, |
| 38 | date_year, |
| 39 | file_sent, |
| 40 | find_message_id, |
| 41 | save_draft, |
| 42 | ActOutcome, |
| 43 | Action, |
| 44 | ArchiveRule, |
| 45 | Entry, |
| 46 | Filed, |
| 47 | Provider, |
| 48 | Saved, |
| 49 | SentFiling, |
| 50 | Target, |
| 51 | }; |
| 52 | pub use apply::{ |
| 53 | apply, |
| 54 | maildir_name, |
| 55 | parse_maildir_name, |
| 56 | rebase_flags, |
| 57 | Applied, |
| 58 | MsgKey, |
| 59 | SyncStore, |
| 60 | }; |
| 61 | |
| 62 | use crate::message; |
| 63 | |
| 64 | use oxedyne_fe2o3_core::prelude::*; |
| 65 | pub use oxedyne_fe2o3_net::imap::{ |
| 66 | types::{ |
| 67 | AppendUid, |
| 68 | BodyPart, |
| 69 | CopyUid, |
| 70 | FetchChanged, |
| 71 | FetchWhat, |
| 72 | FetchedMessage, |
| 73 | MailboxInfo, |
| 74 | MailboxStatus, |
| 75 | SelectWith, |
| 76 | SpecialUse, |
| 77 | }, |
| 78 | search::Segment, |
| 79 | uid_set::UidSet, |
| 80 | }; |
| 81 | |
| 82 | use std::collections::BTreeMap; |
| 83 | |
| 84 | |
| 85 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 86 | // │ THE SESSION │ |
| 87 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 88 | |
| 89 | /// One logged-in IMAP conversation, as the engine needs it: the wire verbs and nothing of policy, |
| 90 | /// so that every rule (tiers, the Message-ID confirmation, the outcomes of an action) lives in |
| 91 | /// the engine, once, whichever transport carries it. |
| 92 | /// |
| 93 | /// `async fn` without a `Send` bound on purpose: a browser's tunnel is single-threaded, and the |
| 94 | /// engine is only ever driven by one task. A UID the server does not hold is passed over in a |
| 95 | /// fetch, a store or an expunge, never an error. |
| 96 | #[allow(async_fn_in_trait)] |
| 97 | pub trait MailSession { |
| 98 | async fn capabilities(&mut self) -> Outcome<Vec<String>>; |
| 99 | async fn list(&mut self) -> Outcome<Vec<MailboxInfo>>; |
| 100 | async fn create(&mut self, mailbox: &str) -> Outcome<()>; |
| 101 | /// `SELECT`; a QRESYNC select must have QRESYNC enabled first, which the session does. |
| 102 | async fn select(&mut self, mailbox: &str, with: &SelectWith) -> Outcome<MailboxStatus>; |
| 103 | /// `EXAMINE`: the mailbox opened read-only, so the session can write no flag and `\Recent` |
| 104 | /// is left for the next `SELECT`. |
| 105 | async fn examine(&mut self, mailbox: &str, with: &SelectWith) -> Outcome<MailboxStatus>; |
| 106 | async fn uid_fetch(&mut self, uids: &UidSet, what: FetchWhat) -> Outcome<Vec<FetchedMessage>>; |
| 107 | async fn uid_fetch_changed(&mut self, uids: &UidSet, what: FetchWhat, modseq: u64) |
| 108 | -> Outcome<FetchChanged>; |
| 109 | async fn uid_search(&mut self, criteria: &str) -> Outcome<Vec<u32>>; |
| 110 | /// `UID SEARCH` with the arguments `SearchKey::wire` writes. A session that cannot send a |
| 111 | /// literal takes 7-bit text only and refuses the rest. |
| 112 | async fn uid_search_key(&mut self, args: &[Segment]) -> Outcome<Vec<u32>> { |
| 113 | let mut t = String::new(); |
| 114 | for a in args { |
| 115 | match a { |
| 116 | Segment::Text(x) if x.is_ascii() => t.push_str(x), |
| 117 | _ => return Err(err!( |
| 118 | "This session cannot send the search text {:?}, which needs a literal.", a; Unimplemented, Input)), |
| 119 | } |
| 120 | } |
| 121 | self.uid_search(&t).await |
| 122 | } |
| 123 | /// `BODY.PEEK[section]<0.max>`: `""` the whole message, `1.2` a part; `None` for a UID the |
| 124 | /// server does not hold. |
| 125 | async fn uid_fetch_section(&mut self, uid: u32, section: &str, max: u32) |
| 126 | -> Outcome<Option<FetchedMessage>>; |
| 127 | /// The bytes read from the server so far, which a search's byte bound charges. |
| 128 | fn read_bytes(&self) -> u64; |
| 129 | /// `STATUS (MESSAGES)`, without selecting the mailbox. |
| 130 | async fn count(&mut self, mailbox: &str) -> Outcome<u32>; |
| 131 | /// `+FLAGS` and `-FLAGS` only; there is no absolute set. |
| 132 | async fn uid_store_flags(&mut self, uids: &UidSet, add: &[&str], remove: &[&str]) -> Outcome<()>; |
| 133 | async fn uid_copy(&mut self, uids: &UidSet, to: &str) -> Outcome<Option<CopyUid>>; |
| 134 | async fn uid_move(&mut self, uids: &UidSet, to: &str) -> Outcome<Option<CopyUid>>; |
| 135 | async fn uid_expunge(&mut self, uids: &UidSet) -> Outcome<()>; |
| 136 | async fn append(&mut self, mailbox: &str, flags: &[&str], raw: &[u8]) -> Outcome<Option<AppendUid>>; |
| 137 | } |
| 138 | |
| 139 | |
| 140 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 141 | // │ CAPABILITIES AND ROLES │ |
| 142 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 143 | |
| 144 | /// What the engine needs to know of a server's `CAPABILITY` list. |
| 145 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] |
| 146 | pub struct Caps { |
| 147 | pub qresync: bool, |
| 148 | pub condstore: bool, // true wherever `qresync` is |
| 149 | pub mov: bool, // RFC 6851 `MOVE` |
| 150 | pub uidplus: bool, // RFC 4315, for `UID EXPUNGE` |
| 151 | pub gmail: bool, // `X-GM-EXT-1`, so `X-GM-MSGID` is fetched |
| 152 | } |
| 153 | |
| 154 | impl Caps { |
| 155 | pub fn from_list(caps: &[String]) -> Self { |
| 156 | let has = |n: &str| caps.iter().any(|c| c.eq_ignore_ascii_case(n)); |
| 157 | let qresync = has("QRESYNC"); |
| 158 | Self { |
| 159 | qresync, |
| 160 | condstore: qresync || has("CONDSTORE"), |
| 161 | mov: has("MOVE"), |
| 162 | uidplus: has("UIDPLUS"), |
| 163 | gmail: has("X-GM-EXT-1"), |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | pub fn tier(&self) -> Tier { |
| 168 | match (self.qresync, self.condstore) { |
| 169 | (true, _) => Tier::QResync, |
| 170 | (false, true) => Tier::CondStore, |
| 171 | (false, false) => Tier::Full, |
| 172 | } |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 177 | pub enum Tier { |
| 178 | QResync, |
| 179 | CondStore, |
| 180 | Full, |
| 181 | } |
| 182 | |
| 183 | /// The folders of one account by RFC 6154 role. A role is found by its attribute only, never by |
| 184 | /// a folder's name, which is localised ("[Gmail]/Gesendet") and which a user may reuse. |
| 185 | #[derive(Clone, Debug, Default)] |
| 186 | pub struct Roles { |
| 187 | pub inbox: String, // as the server spells it |
| 188 | pub roles: BTreeMap<RoleKey, String>, // role to mailbox name |
| 189 | pub delimiter: Option<char>, // the hierarchy delimiter of the first entry |
| 190 | pub mailboxes: Vec<MailboxInfo>, // every mailbox, as listed |
| 191 | pub archive: Option<String>, // where no folder has the \Archive role: the caller's, from config |
| 192 | } |
| 193 | |
| 194 | /// `SpecialUse` as a map key. |
| 195 | #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] |
| 196 | pub struct RoleKey(u8); |
| 197 | |
| 198 | impl From<SpecialUse> for RoleKey { |
| 199 | fn from(r: SpecialUse) -> Self { |
| 200 | RoleKey(match r { |
| 201 | SpecialUse::Sent => 0, |
| 202 | SpecialUse::Drafts => 1, |
| 203 | SpecialUse::Trash => 2, |
| 204 | SpecialUse::Junk => 3, |
| 205 | SpecialUse::Archive => 4, |
| 206 | SpecialUse::All => 5, |
| 207 | SpecialUse::Flagged => 6, |
| 208 | }) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | impl Roles { |
| 213 | /// The first mailbox to declare a role keeps it. |
| 214 | pub fn from_list(list: &[MailboxInfo]) -> Self { |
| 215 | let mut r = Self { |
| 216 | inbox: "INBOX".to_string(), |
| 217 | delimiter: list.iter().find_map(|m| m.delimiter), |
| 218 | mailboxes: list.to_vec(), |
| 219 | ..Default::default() |
| 220 | }; |
| 221 | for m in list { |
| 222 | if m.name.eq_ignore_ascii_case("INBOX") { |
| 223 | r.inbox = m.name.clone(); |
| 224 | } |
| 225 | if let Some(role) = m.special_use() { |
| 226 | r.roles.entry(role.into()).or_insert_with(|| m.name.clone()); |
| 227 | } |
| 228 | } |
| 229 | r |
| 230 | } |
| 231 | |
| 232 | pub fn role(&self, role: SpecialUse) -> Option<&str> { |
| 233 | self.roles.get(&role.into()).map(|s| s.as_str()) |
| 234 | } |
| 235 | |
| 236 | pub fn exists(&self, mailbox: &str) -> bool { |
| 237 | self.mailboxes.iter().any(|m| m.name == mailbox) |
| 238 | } |
| 239 | |
| 240 | /// Every mailbox that can be selected, the order the server listed them in. |
| 241 | pub fn selectable(&self) -> impl Iterator<Item = &str> { |
| 242 | self.mailboxes.iter().filter(|m| m.selectable()).map(|m| m.name.as_str()) |
| 243 | } |
| 244 | |
| 245 | /// Every folder a sync follows: the inbox, each role, and each subscribed-or-listed |
| 246 | /// selectable folder, each named once. |
| 247 | pub fn synced(&self) -> Vec<String> { |
| 248 | let mut out = vec![self.inbox.clone()]; |
| 249 | for name in self.roles.values().map(|s| s.as_str()).chain(self.selectable()) { |
| 250 | if !out.iter().any(|o| o == name) { |
| 251 | out.push(name.to_string()); |
| 252 | } |
| 253 | } |
| 254 | out |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | |
| 259 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 260 | // │ THE REQUEST AND THE DELTA │ |
| 261 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 262 | |
| 263 | /// Where a device's copy of one folder stands. It is the engine's to compute and the device's to |
| 264 | /// keep, and it is stored only once the delta that carried it has landed. |
| 265 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 266 | pub struct Cursor { |
| 267 | pub uid_validity: u32, // 0 for a folder never synced |
| 268 | pub highest_uid: u32, // every UID up to here is held, or in `backlog` |
| 269 | pub highest_modseq: Option<u64>, // CONDSTORE and QRESYNC only |
| 270 | pub polls: u32, // counts full-reconcile polls, for the sweep |
| 271 | pub backlog: UidSet, // seen arriving, not yet fetched; newest are fetched first |
| 272 | pub floor: u32, // the folder's last UID when it became known: none up to here arrived |
| 273 | } |
| 274 | |
| 275 | /// What a device holds of one folder, sent with each sync. |
| 276 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 277 | pub struct Snapshot { |
| 278 | pub mailbox: String, |
| 279 | pub cursor: Cursor, |
| 280 | pub held: UidSet, // every UID held under `cursor.uid_validity` |
| 281 | pub window: UidSet, // the held UIDs a full-reconcile poll re-reads every time |
| 282 | } |
| 283 | |
| 284 | /// The bound on one request, so a mailbox of 10^5 messages costs many small requests. |
| 285 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 286 | pub struct Budget { |
| 287 | pub max_count: usize, // new messages per delta |
| 288 | pub max_bytes: u64, // of bodies and headers per delta; a body past it comes as its header |
| 289 | pub bodies: bool, // fetch whole new messages, where they fit |
| 290 | pub sweep_every: u32, // a full-reconcile poll re-reads every held flag this often |
| 291 | } |
| 292 | |
| 293 | impl Default for Budget { |
| 294 | fn default() -> Self { |
| 295 | Self { |
| 296 | max_count: 200, |
| 297 | max_bytes: 4 * 1024 * 1024, |
| 298 | bodies: false, |
| 299 | sweep_every: 10, |
| 300 | } |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | /// The UIDVALIDITY changed: everything held under `old` is void. |
| 305 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 306 | pub struct Reset { |
| 307 | pub old: u32, |
| 308 | pub new: u32, |
| 309 | } |
| 310 | |
| 311 | /// How copies of one message are linked across folders, Gmail's All Mail above all. |
| 312 | #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] |
| 313 | pub enum LinkKey { |
| 314 | GmMsgId(u64), // only where the server advertises `X-GM-EXT-1` |
| 315 | MessageId(String), // the general rule, without angle brackets |
| 316 | } |
| 317 | |
| 318 | /// A message that arrived since the cursor. |
| 319 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 320 | pub struct NewMessage { |
| 321 | pub uid: u32, |
| 322 | pub flags: Vec<String>, |
| 323 | pub internal_date: String, |
| 324 | pub size: u32, |
| 325 | pub raw: Vec<u8>, // the whole message where `whole`, else its header block |
| 326 | pub whole: bool, |
| 327 | pub link: Option<LinkKey>, |
| 328 | pub arrival: bool, // came after the folder became known, so a trigger may fire on it |
| 329 | } |
| 330 | |
| 331 | /// A held message's flags as the server now has them. [`apply`] turns them into `+`/`-` changes |
| 332 | /// against the device's copy, rebased under its pending journal. |
| 333 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 334 | pub struct FlagState { |
| 335 | pub uid: u32, |
| 336 | pub flags: Vec<String>, |
| 337 | } |
| 338 | |
| 339 | #[derive(Clone, Debug, Default, Eq, PartialEq)] |
| 340 | pub struct Delta { |
| 341 | pub mailbox: String, |
| 342 | pub tier: Option<Tier>, // the tier this delta was read by |
| 343 | pub reset: Option<Reset>, |
| 344 | pub new: Vec<NewMessage>, // newest first |
| 345 | pub flags: Vec<FlagState>, |
| 346 | pub vanished: UidSet, // held UIDs the server no longer has |
| 347 | pub cursor: Cursor, // to be stored only once this delta has landed |
| 348 | pub more: bool, // call again with `cursor` for the rest |
| 349 | } |
| 350 | |
| 351 | |
| 352 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 353 | // │ THE ENGINE │ |
| 354 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 355 | |
| 356 | /// One bounded sync of one folder. |
| 357 | pub async fn sync_folder<S: MailSession>( |
| 358 | s: &mut S, |
| 359 | caps: &Caps, |
| 360 | snap: &Snapshot, |
| 361 | budget: &Budget, |
| 362 | ) |
| 363 | -> Outcome<Delta> |
| 364 | { |
| 365 | let cur = &snap.cursor; |
| 366 | let fresh = cur.uid_validity == 0; |
| 367 | let tier = caps.tier(); |
| 368 | let with = match (tier, cur.highest_modseq) { |
| 369 | (Tier::QResync, Some(m)) if !fresh => SelectWith::QResync { |
| 370 | uid_validity: cur.uid_validity, |
| 371 | modseq: m, |
| 372 | known: if snap.held.is_empty() { None } else { Some(snap.held.clone()) }, |
| 373 | }, |
| 374 | (Tier::QResync, _) | (Tier::CondStore, _) => SelectWith::CondStore, |
| 375 | (Tier::Full, _) => SelectWith::Plain, |
| 376 | }; |
| 377 | let st = res!(s.select(&snap.mailbox, &with).await); |
| 378 | if st.uid_validity == 0 { |
| 379 | return Err(err!("The server gave {} no UIDVALIDITY, so nothing held from it could be \ |
| 380 | trusted.", snap.mailbox; Invalid, Input, Network)); |
| 381 | } |
| 382 | |
| 383 | let mut d = Delta { |
| 384 | mailbox: snap.mailbox.clone(), |
| 385 | ..Default::default() |
| 386 | }; |
| 387 | // A changed UIDVALIDITY voids every UID held: start again from nothing under the new one. |
| 388 | let (held, base) = if fresh || st.uid_validity != cur.uid_validity { |
| 389 | if !fresh { |
| 390 | d.reset = Some(Reset { old: cur.uid_validity, new: st.uid_validity }); |
| 391 | } |
| 392 | (UidSet::new(), Cursor { uid_validity: st.uid_validity, ..Default::default() }) |
| 393 | } else { |
| 394 | (snap.held.clone(), cur.clone()) |
| 395 | }; |
| 396 | |
| 397 | // Changed flags and vanished UIDs, by tier. |
| 398 | let mut polls = base.polls; |
| 399 | if !held.is_empty() { |
| 400 | let (used, flags, gone) = match (&with, cur.highest_modseq, st.highest_modseq) { |
| 401 | (SelectWith::QResync { .. }, _, Some(_)) => { |
| 402 | (Tier::QResync, st.changed.clone(), st.vanished.intersection(&held)) |
| 403 | } |
| 404 | (_, Some(was), Some(now)) if tier != Tier::Full => { |
| 405 | if was == now { |
| 406 | (Tier::CondStore, Vec::new(), UidSet::new()) |
| 407 | } else { |
| 408 | let fc = res!(s.uid_fetch_changed(&held, FetchWhat::Flags, was).await); |
| 409 | let present = res!(s.uid_search(&fmt!("UID {}", held)).await); |
| 410 | let gone = held.difference(&UidSet::from(&present[..])); |
| 411 | (Tier::CondStore, fc.messages, gone) |
| 412 | } |
| 413 | } |
| 414 | // No mod-sequences to go by, from the server or yet from this device. |
| 415 | _ => { |
| 416 | let sweep = budget.sweep_every <= 1 || polls % budget.sweep_every == 0; |
| 417 | let set = if sweep { held.clone() } else { snap.window.intersection(&held) }; |
| 418 | polls = polls.wrapping_add(1); |
| 419 | let flags = if set.is_empty() { |
| 420 | Vec::new() |
| 421 | } else { |
| 422 | res!(s.uid_fetch(&set, FetchWhat::Flags).await) |
| 423 | }; |
| 424 | let present = res!(s.uid_search("ALL").await); |
| 425 | let gone = held.difference(&UidSet::from(&present[..])); |
| 426 | (Tier::Full, flags, gone) |
| 427 | } |
| 428 | }; |
| 429 | d.tier = Some(used); |
| 430 | d.flags = flags.into_iter() |
| 431 | .filter(|m| held.contains(m.uid) && !gone.contains(m.uid)) |
| 432 | .map(|m| FlagState { uid: m.uid, flags: m.flags }) |
| 433 | .collect(); |
| 434 | d.vanished = gone; |
| 435 | } else { |
| 436 | d.tier = Some(match &with { |
| 437 | SelectWith::QResync { .. } => Tier::QResync, |
| 438 | SelectWith::CondStore => Tier::CondStore, |
| 439 | SelectWith::Plain => Tier::Full, |
| 440 | }); |
| 441 | // A first read counts as a sweep: every flag it holds is fresh. |
| 442 | if with == SelectWith::Plain { |
| 443 | polls = polls.wrapping_add(1); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | // New messages: what arrived above the cursor, and what an earlier delta left in the backlog. |
| 448 | let backlog = base.backlog.clone(); |
| 449 | let mut ask = backlog.clone(); |
| 450 | if st.uid_next == 0 || st.uid_next > base.highest_uid.saturating_add(1) { |
| 451 | ask = ask.union(&UidSet::from_open(base.highest_uid.saturating_add(1))); |
| 452 | } |
| 453 | let mut sizes: Vec<(u32, u32)> = Vec::new(); |
| 454 | if !ask.is_empty() { |
| 455 | // `n:*` past the last UID answers with the last message (RFC 3501 6.4.8): drop it. |
| 456 | for m in res!(s.uid_fetch(&ask, FetchWhat::Meta).await) { |
| 457 | if m.uid > base.highest_uid || backlog.contains(m.uid) { |
| 458 | sizes.push((m.uid, m.size)); |
| 459 | } |
| 460 | } |
| 461 | } |
| 462 | sizes.sort(); |
| 463 | sizes.dedup_by_key(|p| p.0); |
| 464 | let highest = sizes.last().map(|p| p.0.max(base.highest_uid)).unwrap_or(base.highest_uid); |
| 465 | // Arrivals are told from the floor: the folder's last UID when this device first read it, |
| 466 | // or rebuilt it under a new UIDVALIDITY. What was already there is fetched, a capped first |
| 467 | // read's remainder later from the backlog, without being an arrival; what came after is one, |
| 468 | // however long it waits in the backlog. UIDNEXT is read at the select, so a message delivered |
| 469 | // between the select and the fetch counts as arriving. |
| 470 | let floor = if fresh || d.reset.is_some() { |
| 471 | if st.uid_next > 0 { st.uid_next - 1 } else { highest } |
| 472 | } else { |
| 473 | base.floor |
| 474 | }; |
| 475 | // A backlog UID the server no longer returns was expunged before it was fetched. |
| 476 | let mut backlog = UidSet::from(&sizes.iter().map(|p| p.0).collect::<Vec<_>>()[..]); |
| 477 | |
| 478 | // Newest first, within the count; whole bodies within the byte budget, else headers. |
| 479 | let mut whole = UidSet::new(); |
| 480 | let mut heads: Vec<(u32, u32)> = Vec::new(); |
| 481 | let mut bytes = 0u64; |
| 482 | for (uid, size) in sizes.iter().rev().take(budget.max_count) { |
| 483 | if budget.bodies && bytes + (*size as u64) <= budget.max_bytes { |
| 484 | bytes += *size as u64; |
| 485 | whole.insert(*uid); |
| 486 | } else { |
| 487 | heads.push((*uid, *size)); |
| 488 | } |
| 489 | } |
| 490 | let mut got = Vec::new(); |
| 491 | if !whole.is_empty() { |
| 492 | got.extend(res!(s.uid_fetch(&whole, FetchWhat::Full).await).into_iter() |
| 493 | .filter(|m| whole.contains(m.uid)) |
| 494 | .map(|m| (m, true))); |
| 495 | } |
| 496 | // Headers count against the same budget. A header's length is unknown until it arrives but |
| 497 | // never exceeds its message's size, so each batch is sized by that bound against what is |
| 498 | // left, and what is left is then charged with what actually came. The first message of a |
| 499 | // request moves however big its header, so no request is empty while any is due. |
| 500 | let mut asked = whole.clone(); |
| 501 | let mut i = 0; |
| 502 | while i < heads.len() { |
| 503 | let left = budget.max_bytes.saturating_sub(bytes); |
| 504 | let mut batch = UidSet::new(); |
| 505 | let mut bound = 0u64; |
| 506 | while i < heads.len() { |
| 507 | let size = heads[i].1 as u64; |
| 508 | if bound + size > left && !(batch.is_empty() && asked.is_empty()) { |
| 509 | break; |
| 510 | } |
| 511 | bound += size; |
| 512 | batch.insert(heads[i].0); |
| 513 | i += 1; |
| 514 | } |
| 515 | if batch.is_empty() { |
| 516 | break; |
| 517 | } |
| 518 | for m in res!(s.uid_fetch(&batch, FetchWhat::Headers).await) { |
| 519 | if batch.contains(m.uid) { |
| 520 | bytes += m.body.len() as u64; |
| 521 | got.push((m, false)); |
| 522 | } |
| 523 | } |
| 524 | asked = asked.union(&batch); |
| 525 | } |
| 526 | got.sort_by(|a, b| b.0.uid.cmp(&a.0.uid)); |
| 527 | got.dedup_by(|a, b| a.0.uid == b.0.uid); |
| 528 | for (m, w) in got { |
| 529 | d.new.push(NewMessage { |
| 530 | uid: m.uid, |
| 531 | link: link_key(&m, caps.gmail), |
| 532 | flags: m.flags, |
| 533 | internal_date: m.internal_date, |
| 534 | size: m.size, |
| 535 | raw: m.body, |
| 536 | whole: w, |
| 537 | arrival: m.uid > floor && !held.contains(m.uid), |
| 538 | }); |
| 539 | } |
| 540 | // What was asked for and not returned was expunged meanwhile; either way it leaves the backlog. |
| 541 | backlog = backlog.difference(&asked); |
| 542 | |
| 543 | d.more = !backlog.is_empty(); |
| 544 | d.cursor = Cursor { |
| 545 | uid_validity: st.uid_validity, |
| 546 | highest_uid: highest, |
| 547 | highest_modseq: st.highest_modseq, |
| 548 | polls, |
| 549 | backlog, |
| 550 | floor, |
| 551 | }; |
| 552 | Ok(d) |
| 553 | } |
| 554 | |
| 555 | /// How this copy is linked to its others: Gmail's message ID only where the server advertises |
| 556 | /// `X-GM-EXT-1`, otherwise the `Message-ID` field. `None` for a message with neither. |
| 557 | pub fn link_key(m: &FetchedMessage, gmail: bool) -> Option<LinkKey> { |
| 558 | if gmail { |
| 559 | if let Some(id) = m.gm_msgid { |
| 560 | return Some(LinkKey::GmMsgId(id)); |
| 561 | } |
| 562 | } |
| 563 | header_message_id(&m.body).map(LinkKey::MessageId) |
| 564 | } |
| 565 | |
| 566 | /// The `Message-ID` a header block (or whole message) names, without its angle brackets. |
| 567 | pub fn header_message_id(raw: &[u8]) -> Option<String> { |
| 568 | message::headers(raw).into_iter() |
| 569 | .find(|(n, _)| n == "message-id") |
| 570 | .map(|(_, v)| bare_message_id(&v).to_string()) |
| 571 | .filter(|v| !v.is_empty()) |
| 572 | } |
| 573 | |
| 574 | /// A Message-ID without its angle brackets or surrounding space. |
| 575 | pub fn bare_message_id(mid: &str) -> &str { |
| 576 | let t = mid.trim(); |
| 577 | t.strip_prefix('<').and_then(|t| t.strip_suffix('>')).unwrap_or(t).trim() |
| 578 | } |