Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_mail/src/sync/mod.rs

24.0 KiB, 22 runs

created by r1870400018:75207, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The folder sync engine: what changed in one IMAP folder since the device last looked, as a
2//! bounded [`Delta`] the device applies to its cache.
3//!
4//! The engine is written once, against [`MailSession`], so the gateway's `ImapClient` and a
5//! browser's tunnel drive the same algorithm. It owns no socket, no clock and no runtime, and
6//! builds for `wasm32-unknown-unknown` with the crate's default features off.
7//!
8//! A sync works by capability tier ([`Tier`]):
9//! - **QRESYNC** (RFC 7162): `SELECT (QRESYNC ...)` returns the changed flags and the `VANISHED`
10//! UIDs directly;
11//! - **CONDSTORE**: `UID FETCH held (FLAGS) (CHANGEDSINCE m)`, then `UID SEARCH UID held` for
12//! what vanished;
13//! - **full reconcile** (Steel, Office 365): `UID FETCH (FLAGS)` over the window on every poll and
14//! over the whole held set every `sweep_every`th, so a message read elsewhere long ago does not
15//! keep a stale state here; `UID SEARCH ALL` against the held set for what vanished.
16//!
17//! Each new message says whether it is an arrival ([`NewMessage::arrival`]): only what came after
18//! the folder became known to the device is, never what a capped first read or a rebuild left in
19//! the backlog, so a trigger that fires on arrivals does not fire on old mail.
20//!
21//! The device advances its [`Cursor`] only once the delta's writes have landed ([`apply`]), so a
22//! crash replays a delta and never skips one, and a replayed apply changes nothing. A UIDVALIDITY
23//! change makes the delta a [`Reset`]: the device sets its one old generation aside, rebuilds,
24//! and drops the old one when the rebuild completes.
25//!
26//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
27//! Anthropic Claude
28
29pub mod act;
30pub mod apply;
31pub mod query;
32pub mod imap;
33#[cfg(feature = "jdat")]
34pub mod wire;
35
36pub use act::{
37 act,
38 date_year,
39 file_sent,
40 find_message_id,
41 save_draft,
42 ActOutcome,
43 Action,
44 ArchiveRule,
45 Entry,
46 Filed,
47 Provider,
48 Saved,
49 SentFiling,
50 Target,
51};
52pub use apply::{
53 apply,
54 maildir_name,
55 parse_maildir_name,
56 rebase_flags,
57 Applied,
58 MsgKey,
59 SyncStore,
60};
61
62use crate::message;
63
64use oxedyne_fe2o3_core::prelude::*;
65pub use oxedyne_fe2o3_net::imap::{
66 types::{
67 AppendUid,
68 BodyPart,
69 CopyUid,
70 FetchChanged,
71 FetchWhat,
72 FetchedMessage,
73 MailboxInfo,
74 MailboxStatus,
75 SelectWith,
76 SpecialUse,
77 },
78 search::Segment,
79 uid_set::UidSet,
80};
81
82use std::collections::BTreeMap;
83
84
85// ┌───────────────────────────────────────────────────────────────────────────┐
86// │ THE SESSION │
87// └───────────────────────────────────────────────────────────────────────────┘
88
89/// One logged-in IMAP conversation, as the engine needs it: the wire verbs and nothing of policy,
90/// so that every rule (tiers, the Message-ID confirmation, the outcomes of an action) lives in
91/// the engine, once, whichever transport carries it.
92///
93/// `async fn` without a `Send` bound on purpose: a browser's tunnel is single-threaded, and the
94/// engine is only ever driven by one task. A UID the server does not hold is passed over in a
95/// fetch, a store or an expunge, never an error.
96#[allow(async_fn_in_trait)]
97pub trait MailSession {
98 async fn capabilities(&mut self) -> Outcome<Vec<String>>;
99 async fn list(&mut self) -> Outcome<Vec<MailboxInfo>>;
100 async fn create(&mut self, mailbox: &str) -> Outcome<()>;
101 /// `SELECT`; a QRESYNC select must have QRESYNC enabled first, which the session does.
102 async fn select(&mut self, mailbox: &str, with: &SelectWith) -> Outcome<MailboxStatus>;
103 /// `EXAMINE`: the mailbox opened read-only, so the session can write no flag and `\Recent`
104 /// is left for the next `SELECT`.
105 async fn examine(&mut self, mailbox: &str, with: &SelectWith) -> Outcome<MailboxStatus>;
106 async fn uid_fetch(&mut self, uids: &UidSet, what: FetchWhat) -> Outcome<Vec<FetchedMessage>>;
107 async fn uid_fetch_changed(&mut self, uids: &UidSet, what: FetchWhat, modseq: u64)
108 -> Outcome<FetchChanged>;
109 async fn uid_search(&mut self, criteria: &str) -> Outcome<Vec<u32>>;
110 /// `UID SEARCH` with the arguments `SearchKey::wire` writes. A session that cannot send a
111 /// literal takes 7-bit text only and refuses the rest.
112 async fn uid_search_key(&mut self, args: &[Segment]) -> Outcome<Vec<u32>> {
113 let mut t = String::new();
114 for a in args {
115 match a {
116 Segment::Text(x) if x.is_ascii() => t.push_str(x),
117 _ => return Err(err!(
118 "This session cannot send the search text {:?}, which needs a literal.", a; Unimplemented, Input)),
119 }
120 }
121 self.uid_search(&t).await
122 }
123 /// `BODY.PEEK[section]<0.max>`: `""` the whole message, `1.2` a part; `None` for a UID the
124 /// server does not hold.
125 async fn uid_fetch_section(&mut self, uid: u32, section: &str, max: u32)
126 -> Outcome<Option<FetchedMessage>>;
127 /// The bytes read from the server so far, which a search's byte bound charges.
128 fn read_bytes(&self) -> u64;
129 /// `STATUS (MESSAGES)`, without selecting the mailbox.
130 async fn count(&mut self, mailbox: &str) -> Outcome<u32>;
131 /// `+FLAGS` and `-FLAGS` only; there is no absolute set.
132 async fn uid_store_flags(&mut self, uids: &UidSet, add: &[&str], remove: &[&str]) -> Outcome<()>;
133 async fn uid_copy(&mut self, uids: &UidSet, to: &str) -> Outcome<Option<CopyUid>>;
134 async fn uid_move(&mut self, uids: &UidSet, to: &str) -> Outcome<Option<CopyUid>>;
135 async fn uid_expunge(&mut self, uids: &UidSet) -> Outcome<()>;
136 async fn append(&mut self, mailbox: &str, flags: &[&str], raw: &[u8]) -> Outcome<Option<AppendUid>>;
137}
138
139
140// ┌───────────────────────────────────────────────────────────────────────────┐
141// │ CAPABILITIES AND ROLES │
142// └───────────────────────────────────────────────────────────────────────────┘
143
144/// What the engine needs to know of a server's `CAPABILITY` list.
145#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
146pub struct Caps {
147 pub qresync: bool,
148 pub condstore: bool, // true wherever `qresync` is
149 pub mov: bool, // RFC 6851 `MOVE`
150 pub uidplus: bool, // RFC 4315, for `UID EXPUNGE`
151 pub gmail: bool, // `X-GM-EXT-1`, so `X-GM-MSGID` is fetched
152}
153
154impl Caps {
155 pub fn from_list(caps: &[String]) -> Self {
156 let has = |n: &str| caps.iter().any(|c| c.eq_ignore_ascii_case(n));
157 let qresync = has("QRESYNC");
158 Self {
159 qresync,
160 condstore: qresync || has("CONDSTORE"),
161 mov: has("MOVE"),
162 uidplus: has("UIDPLUS"),
163 gmail: has("X-GM-EXT-1"),
164 }
165 }
166
167 pub fn tier(&self) -> Tier {
168 match (self.qresync, self.condstore) {
169 (true, _) => Tier::QResync,
170 (false, true) => Tier::CondStore,
171 (false, false) => Tier::Full,
172 }
173 }
174}
175
176#[derive(Clone, Copy, Debug, Eq, PartialEq)]
177pub enum Tier {
178 QResync,
179 CondStore,
180 Full,
181}
182
183/// The folders of one account by RFC 6154 role. A role is found by its attribute only, never by
184/// a folder's name, which is localised ("[Gmail]/Gesendet") and which a user may reuse.
185#[derive(Clone, Debug, Default)]
186pub struct Roles {
187 pub inbox: String, // as the server spells it
188 pub roles: BTreeMap<RoleKey, String>, // role to mailbox name
189 pub delimiter: Option<char>, // the hierarchy delimiter of the first entry
190 pub mailboxes: Vec<MailboxInfo>, // every mailbox, as listed
191 pub archive: Option<String>, // where no folder has the \Archive role: the caller's, from config
192}
193
194/// `SpecialUse` as a map key.
195#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
196pub struct RoleKey(u8);
197
198impl From<SpecialUse> for RoleKey {
199 fn from(r: SpecialUse) -> Self {
200 RoleKey(match r {
201 SpecialUse::Sent => 0,
202 SpecialUse::Drafts => 1,
203 SpecialUse::Trash => 2,
204 SpecialUse::Junk => 3,
205 SpecialUse::Archive => 4,
206 SpecialUse::All => 5,
207 SpecialUse::Flagged => 6,
208 })
209 }
210}
211
212impl Roles {
213 /// The first mailbox to declare a role keeps it.
214 pub fn from_list(list: &[MailboxInfo]) -> Self {
215 let mut r = Self {
216 inbox: "INBOX".to_string(),
217 delimiter: list.iter().find_map(|m| m.delimiter),
218 mailboxes: list.to_vec(),
219 ..Default::default()
220 };
221 for m in list {
222 if m.name.eq_ignore_ascii_case("INBOX") {
223 r.inbox = m.name.clone();
224 }
225 if let Some(role) = m.special_use() {
226 r.roles.entry(role.into()).or_insert_with(|| m.name.clone());
227 }
228 }
229 r
230 }
231
232 pub fn role(&self, role: SpecialUse) -> Option<&str> {
233 self.roles.get(&role.into()).map(|s| s.as_str())
234 }
235
236 pub fn exists(&self, mailbox: &str) -> bool {
237 self.mailboxes.iter().any(|m| m.name == mailbox)
238 }
239
240 /// Every mailbox that can be selected, the order the server listed them in.
241 pub fn selectable(&self) -> impl Iterator<Item = &str> {
242 self.mailboxes.iter().filter(|m| m.selectable()).map(|m| m.name.as_str())
243 }
244
245 /// Every folder a sync follows: the inbox, each role, and each subscribed-or-listed
246 /// selectable folder, each named once.
247 pub fn synced(&self) -> Vec<String> {
248 let mut out = vec![self.inbox.clone()];
249 for name in self.roles.values().map(|s| s.as_str()).chain(self.selectable()) {
250 if !out.iter().any(|o| o == name) {
251 out.push(name.to_string());
252 }
253 }
254 out
255 }
256}
257
258
259// ┌───────────────────────────────────────────────────────────────────────────┐
260// │ THE REQUEST AND THE DELTA │
261// └───────────────────────────────────────────────────────────────────────────┘
262
263/// Where a device's copy of one folder stands. It is the engine's to compute and the device's to
264/// keep, and it is stored only once the delta that carried it has landed.
265#[derive(Clone, Debug, Default, Eq, PartialEq)]
266pub struct Cursor {
267 pub uid_validity: u32, // 0 for a folder never synced
268 pub highest_uid: u32, // every UID up to here is held, or in `backlog`
269 pub highest_modseq: Option<u64>, // CONDSTORE and QRESYNC only
270 pub polls: u32, // counts full-reconcile polls, for the sweep
271 pub backlog: UidSet, // seen arriving, not yet fetched; newest are fetched first
272 pub floor: u32, // the folder's last UID when it became known: none up to here arrived
273}
274
275/// What a device holds of one folder, sent with each sync.
276#[derive(Clone, Debug, Default, Eq, PartialEq)]
277pub struct Snapshot {
278 pub mailbox: String,
279 pub cursor: Cursor,
280 pub held: UidSet, // every UID held under `cursor.uid_validity`
281 pub window: UidSet, // the held UIDs a full-reconcile poll re-reads every time
282}
283
284/// The bound on one request, so a mailbox of 10^5 messages costs many small requests.
285#[derive(Clone, Copy, Debug, Eq, PartialEq)]
286pub struct Budget {
287 pub max_count: usize, // new messages per delta
288 pub max_bytes: u64, // of bodies and headers per delta; a body past it comes as its header
289 pub bodies: bool, // fetch whole new messages, where they fit
290 pub sweep_every: u32, // a full-reconcile poll re-reads every held flag this often
291}
292
293impl Default for Budget {
294 fn default() -> Self {
295 Self {
296 max_count: 200,
297 max_bytes: 4 * 1024 * 1024,
298 bodies: false,
299 sweep_every: 10,
300 }
301 }
302}
303
304/// The UIDVALIDITY changed: everything held under `old` is void.
305#[derive(Clone, Copy, Debug, Eq, PartialEq)]
306pub struct Reset {
307 pub old: u32,
308 pub new: u32,
309}
310
311/// How copies of one message are linked across folders, Gmail's All Mail above all.
312#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
313pub enum LinkKey {
314 GmMsgId(u64), // only where the server advertises `X-GM-EXT-1`
315 MessageId(String), // the general rule, without angle brackets
316}
317
318/// A message that arrived since the cursor.
319#[derive(Clone, Debug, Default, Eq, PartialEq)]
320pub struct NewMessage {
321 pub uid: u32,
322 pub flags: Vec<String>,
323 pub internal_date: String,
324 pub size: u32,
325 pub raw: Vec<u8>, // the whole message where `whole`, else its header block
326 pub whole: bool,
327 pub link: Option<LinkKey>,
328 pub arrival: bool, // came after the folder became known, so a trigger may fire on it
329}
330
331/// A held message's flags as the server now has them. [`apply`] turns them into `+`/`-` changes
332/// against the device's copy, rebased under its pending journal.
333#[derive(Clone, Debug, Default, Eq, PartialEq)]
334pub struct FlagState {
335 pub uid: u32,
336 pub flags: Vec<String>,
337}
338
339#[derive(Clone, Debug, Default, Eq, PartialEq)]
340pub struct Delta {
341 pub mailbox: String,
342 pub tier: Option<Tier>, // the tier this delta was read by
343 pub reset: Option<Reset>,
344 pub new: Vec<NewMessage>, // newest first
345 pub flags: Vec<FlagState>,
346 pub vanished: UidSet, // held UIDs the server no longer has
347 pub cursor: Cursor, // to be stored only once this delta has landed
348 pub more: bool, // call again with `cursor` for the rest
349}
350
351
352// ┌───────────────────────────────────────────────────────────────────────────┐
353// │ THE ENGINE │
354// └───────────────────────────────────────────────────────────────────────────┘
355
356/// One bounded sync of one folder.
357pub async fn sync_folder<S: MailSession>(
358 s: &mut S,
359 caps: &Caps,
360 snap: &Snapshot,
361 budget: &Budget,
362)
363 -> Outcome<Delta>
364{
365 let cur = &snap.cursor;
366 let fresh = cur.uid_validity == 0;
367 let tier = caps.tier();
368 let with = match (tier, cur.highest_modseq) {
369 (Tier::QResync, Some(m)) if !fresh => SelectWith::QResync {
370 uid_validity: cur.uid_validity,
371 modseq: m,
372 known: if snap.held.is_empty() { None } else { Some(snap.held.clone()) },
373 },
374 (Tier::QResync, _) | (Tier::CondStore, _) => SelectWith::CondStore,
375 (Tier::Full, _) => SelectWith::Plain,
376 };
377 let st = res!(s.select(&snap.mailbox, &with).await);
378 if st.uid_validity == 0 {
379 return Err(err!("The server gave {} no UIDVALIDITY, so nothing held from it could be \
380 trusted.", snap.mailbox; Invalid, Input, Network));
381 }
382
383 let mut d = Delta {
384 mailbox: snap.mailbox.clone(),
385 ..Default::default()
386 };
387 // A changed UIDVALIDITY voids every UID held: start again from nothing under the new one.
388 let (held, base) = if fresh || st.uid_validity != cur.uid_validity {
389 if !fresh {
390 d.reset = Some(Reset { old: cur.uid_validity, new: st.uid_validity });
391 }
392 (UidSet::new(), Cursor { uid_validity: st.uid_validity, ..Default::default() })
393 } else {
394 (snap.held.clone(), cur.clone())
395 };
396
397 // Changed flags and vanished UIDs, by tier.
398 let mut polls = base.polls;
399 if !held.is_empty() {
400 let (used, flags, gone) = match (&with, cur.highest_modseq, st.highest_modseq) {
401 (SelectWith::QResync { .. }, _, Some(_)) => {
402 (Tier::QResync, st.changed.clone(), st.vanished.intersection(&held))
403 }
404 (_, Some(was), Some(now)) if tier != Tier::Full => {
405 if was == now {
406 (Tier::CondStore, Vec::new(), UidSet::new())
407 } else {
408 let fc = res!(s.uid_fetch_changed(&held, FetchWhat::Flags, was).await);
409 let present = res!(s.uid_search(&fmt!("UID {}", held)).await);
410 let gone = held.difference(&UidSet::from(&present[..]));
411 (Tier::CondStore, fc.messages, gone)
412 }
413 }
414 // No mod-sequences to go by, from the server or yet from this device.
415 _ => {
416 let sweep = budget.sweep_every <= 1 || polls % budget.sweep_every == 0;
417 let set = if sweep { held.clone() } else { snap.window.intersection(&held) };
418 polls = polls.wrapping_add(1);
419 let flags = if set.is_empty() {
420 Vec::new()
421 } else {
422 res!(s.uid_fetch(&set, FetchWhat::Flags).await)
423 };
424 let present = res!(s.uid_search("ALL").await);
425 let gone = held.difference(&UidSet::from(&present[..]));
426 (Tier::Full, flags, gone)
427 }
428 };
429 d.tier = Some(used);
430 d.flags = flags.into_iter()
431 .filter(|m| held.contains(m.uid) && !gone.contains(m.uid))
432 .map(|m| FlagState { uid: m.uid, flags: m.flags })
433 .collect();
434 d.vanished = gone;
435 } else {
436 d.tier = Some(match &with {
437 SelectWith::QResync { .. } => Tier::QResync,
438 SelectWith::CondStore => Tier::CondStore,
439 SelectWith::Plain => Tier::Full,
440 });
441 // A first read counts as a sweep: every flag it holds is fresh.
442 if with == SelectWith::Plain {
443 polls = polls.wrapping_add(1);
444 }
445 }
446
447 // New messages: what arrived above the cursor, and what an earlier delta left in the backlog.
448 let backlog = base.backlog.clone();
449 let mut ask = backlog.clone();
450 if st.uid_next == 0 || st.uid_next > base.highest_uid.saturating_add(1) {
451 ask = ask.union(&UidSet::from_open(base.highest_uid.saturating_add(1)));
452 }
453 let mut sizes: Vec<(u32, u32)> = Vec::new();
454 if !ask.is_empty() {
455 // `n:*` past the last UID answers with the last message (RFC 3501 6.4.8): drop it.
456 for m in res!(s.uid_fetch(&ask, FetchWhat::Meta).await) {
457 if m.uid > base.highest_uid || backlog.contains(m.uid) {
458 sizes.push((m.uid, m.size));
459 }
460 }
461 }
462 sizes.sort();
463 sizes.dedup_by_key(|p| p.0);
464 let highest = sizes.last().map(|p| p.0.max(base.highest_uid)).unwrap_or(base.highest_uid);
465 // Arrivals are told from the floor: the folder's last UID when this device first read it,
466 // or rebuilt it under a new UIDVALIDITY. What was already there is fetched, a capped first
467 // read's remainder later from the backlog, without being an arrival; what came after is one,
468 // however long it waits in the backlog. UIDNEXT is read at the select, so a message delivered
469 // between the select and the fetch counts as arriving.
470 let floor = if fresh || d.reset.is_some() {
471 if st.uid_next > 0 { st.uid_next - 1 } else { highest }
472 } else {
473 base.floor
474 };
475 // A backlog UID the server no longer returns was expunged before it was fetched.
476 let mut backlog = UidSet::from(&sizes.iter().map(|p| p.0).collect::<Vec<_>>()[..]);
477
478 // Newest first, within the count; whole bodies within the byte budget, else headers.
479 let mut whole = UidSet::new();
480 let mut heads: Vec<(u32, u32)> = Vec::new();
481 let mut bytes = 0u64;
482 for (uid, size) in sizes.iter().rev().take(budget.max_count) {
483 if budget.bodies && bytes + (*size as u64) <= budget.max_bytes {
484 bytes += *size as u64;
485 whole.insert(*uid);
486 } else {
487 heads.push((*uid, *size));
488 }
489 }
490 let mut got = Vec::new();
491 if !whole.is_empty() {
492 got.extend(res!(s.uid_fetch(&whole, FetchWhat::Full).await).into_iter()
493 .filter(|m| whole.contains(m.uid))
494 .map(|m| (m, true)));
495 }
496 // Headers count against the same budget. A header's length is unknown until it arrives but
497 // never exceeds its message's size, so each batch is sized by that bound against what is
498 // left, and what is left is then charged with what actually came. The first message of a
499 // request moves however big its header, so no request is empty while any is due.
500 let mut asked = whole.clone();
501 let mut i = 0;
502 while i < heads.len() {
503 let left = budget.max_bytes.saturating_sub(bytes);
504 let mut batch = UidSet::new();
505 let mut bound = 0u64;
506 while i < heads.len() {
507 let size = heads[i].1 as u64;
508 if bound + size > left && !(batch.is_empty() && asked.is_empty()) {
509 break;
510 }
511 bound += size;
512 batch.insert(heads[i].0);
513 i += 1;
514 }
515 if batch.is_empty() {
516 break;
517 }
518 for m in res!(s.uid_fetch(&batch, FetchWhat::Headers).await) {
519 if batch.contains(m.uid) {
520 bytes += m.body.len() as u64;
521 got.push((m, false));
522 }
523 }
524 asked = asked.union(&batch);
525 }
526 got.sort_by(|a, b| b.0.uid.cmp(&a.0.uid));
527 got.dedup_by(|a, b| a.0.uid == b.0.uid);
528 for (m, w) in got {
529 d.new.push(NewMessage {
530 uid: m.uid,
531 link: link_key(&m, caps.gmail),
532 flags: m.flags,
533 internal_date: m.internal_date,
534 size: m.size,
535 raw: m.body,
536 whole: w,
537 arrival: m.uid > floor && !held.contains(m.uid),
538 });
539 }
540 // What was asked for and not returned was expunged meanwhile; either way it leaves the backlog.
541 backlog = backlog.difference(&asked);
542
543 d.more = !backlog.is_empty();
544 d.cursor = Cursor {
545 uid_validity: st.uid_validity,
546 highest_uid: highest,
547 highest_modseq: st.highest_modseq,
548 polls,
549 backlog,
550 floor,
551 };
552 Ok(d)
553}
554
555/// How this copy is linked to its others: Gmail's message ID only where the server advertises
556/// `X-GM-EXT-1`, otherwise the `Message-ID` field. `None` for a message with neither.
557pub fn link_key(m: &FetchedMessage, gmail: bool) -> Option<LinkKey> {
558 if gmail {
559 if let Some(id) = m.gm_msgid {
560 return Some(LinkKey::GmMsgId(id));
561 }
562 }
563 header_message_id(&m.body).map(LinkKey::MessageId)
564}
565
566/// The `Message-ID` a header block (or whole message) names, without its angle brackets.
567pub fn header_message_id(raw: &[u8]) -> Option<String> {
568 message::headers(raw).into_iter()
569 .find(|(n, _)| n == "message-id")
570 .map(|(_, v)| bare_message_id(&v).to_string())
571 .filter(|v| !v.is_empty())
572}
573
574/// A Message-ID without its angle brackets or surrounding space.
575pub fn bare_message_id(mid: &str) -> &str {
576 let t = mid.trim();
577 t.strip_prefix('<').and_then(|t| t.strip_suffix('>')).unwrap_or(t).trim()
578}