oxedyne/daimond — src/protocol.rs

118 KiB, 1 run

created by r2519314175:957, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1	//! WS protocol types for Daimond — JDAT serialisation.
2	//!
3	//! All messages between the browser and Steel use the existing syntax
4	//! protocol with JDAT values. This module defines the command and
5	//! response types and their JDAT conversion functions.
6
7	use crate::tools::CallOutcome;
8
9	use oxedyne_fe2o3_core::prelude::*;
10	use oxedyne_fe2o3_jdat::prelude::*;
11	use oxedyne_fe2o3_sbj::share;
12
13
14	// ┌───────────────────────────────────────────────────────────────┐
15	// │ Wall-clock helpers │
16	// └───────────────────────────────────────────────────────────────┘
17
18	/// Current Unix time in whole seconds.
19	///
20	/// The native path reads `std::time::SystemTime`; on `wasm32` that
21	/// panics ("time not implemented on this platform"), so the browser
22	/// path reads the core `Date.now()` clock shim instead.
23	#[cfg(not(target_arch = "wasm32"))]
24	fn now_secs() -> u64 {
25	std::time::SystemTime::now()
26	.duration_since(std::time::UNIX_EPOCH)
27	.map(\|d\| d.as_secs())
28	.unwrap_or(0)
29	}
30
31	/// Current Unix time in whole seconds (browser clock shim).
32	#[cfg(target_arch = "wasm32")]
33	fn now_secs() -> u64 {
34	(oxedyne_fe2o3_core::wasm::now_ms() / 1000.0) as u64
35	}
36
37	/// Current Unix time in whole milliseconds.
38	#[cfg(not(target_arch = "wasm32"))]
39	fn now_millis() -> u128 {
40	std::time::SystemTime::now()
41	.duration_since(std::time::UNIX_EPOCH)
42	.map(\|d\| d.as_millis())
43	.unwrap_or(0)
44	}
45
46	/// Current Unix time in whole milliseconds (browser clock shim).
47	#[cfg(target_arch = "wasm32")]
48	fn now_millis() -> u128 {
49	oxedyne_fe2o3_core::wasm::now_ms() as u128
50	}
51
52
53	// ┌───────────────────────────────────────────────────────────────┐
54	// │ Chat messages │
55	// └───────────────────────────────────────────────────────────────┘
56
57	/// A single tool call requested by the assistant.
58	#[derive(Clone, Debug, Eq, PartialEq)]
59	pub struct ToolCall {
60	pub id: String,
61	pub name: String,
62	/// Raw JSON arguments object as produced by the model.
63	pub arguments: String,
64	}
65
66	impl ToolCall {
67
68	/// Serialise to a JDAT map for the session store.
69	///
70	/// Flat, rather than the provider's `{"type":"function","function":{..}}` nesting:
71	/// this is Daimond's own record of what was asked for, and the wire form is built
72	/// separately by `llm::message_to_json` in whichever dialect the endpoint speaks.
73	pub fn to_datmap(&self) -> DaticleMap {
74	let mut m = DaticleMap::new();
75	m.insert(dat!("id"), dat!(self.id.clone()));
76	m.insert(dat!("name"), dat!(self.name.clone()));
77	m.insert(dat!("arguments"), dat!(self.arguments.clone()));
78	m
79	}
80
81	/// Deserialise from a JDAT map.
82	///
83	/// The id is required, because it is what pairs the call with its reply and a call
84	/// that cannot be paired is worse than one that was never read. The other two are
85	/// tolerated absent, since an empty argument object is a legal call.
86	pub fn from_datmap(m: &DaticleMap) -> Outcome<Self> {
87	let id = match m.get(&dat!("id")) {
88	Some(Dat::Str(s)) => s.clone(),
89	_ => return Err(err!("ToolCall: missing 'id'."; Invalid, Input)),
90	};
91	let name = match m.get(&dat!("name")) {
92	Some(Dat::Str(s)) => s.clone(),
93	_ => String::new(),
94	};
95	let arguments = match m.get(&dat!("arguments")) {
96	Some(Dat::Str(s)) => s.clone(),
97	_ => String::new(),
98	};
99	Ok(Self { id, name, arguments })
100	}
101	}
102
103	// ┌───────────────────────────────────────────────────────────────┐
104	// │ Message content │
105	// └───────────────────────────────────────────────────────────────┘
106
107	/// An image format both wire dialects accept.
108	///
109	/// An enum rather than a free `String` media type, because the set is closed: Anthropic's
110	/// Messages API takes exactly these four and rejects anything else, and OpenAI's `image_url`
111	/// carries the same four in a `data:` URL. A media type the model cannot decode is a provider
112	/// 400, which is the failure this type exists to make unreachable.
113	#[derive(Clone, Copy, Debug, Eq, PartialEq)]
114	pub enum ImageMedia {
115	Png,
116	Jpeg,
117	Gif,
118	WebP,
119	}
120
121	impl ImageMedia {
122
123	/// The IANA media type, as both dialects spell it.
124	pub fn mime(&self) -> &'static str {
125	match self {
126	Self::Png => "image/png",
127	Self::Jpeg => "image/jpeg",
128	Self::Gif => "image/gif",
129	Self::WebP => "image/webp",
130	}
131	}
132
133	/// The media type read from the bytes themselves, or `None` when they are not an image.
134	///
135	/// Sniffed rather than taken from the file extension: the extension is whatever the user
136	/// typed, and a `.png` holding JPEG bytes is a provider 400 with a message about the media
137	/// type that names the wrong one. The magic numbers are each format's own header --
138	/// PNG's signature, JPEG's start-of-image marker, GIF's version string, and RIFF/WEBP.
139	///
140	/// # Arguments
141	/// * `bytes` - The start of the file; four bytes are enough for three of the four.
142	pub fn sniff(bytes: &[u8]) -> Option<Self> {
143	const PNG: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];
144	if bytes.len() >= 8 && bytes[..8] == PNG {
145	return Some(Self::Png);
146	}
147	if bytes.len() >= 3 && bytes[..3] == [0xFF, 0xD8, 0xFF] {
148	return Some(Self::Jpeg);
149	}
150	if bytes.len() >= 6 && (&bytes[..6] == b"GIF87a" \|\| &bytes[..6] == b"GIF89a") {
151	return Some(Self::Gif);
152	}
153	// RIFF containers hold more than WebP, so the form marker at offset 8 decides.
154	if bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WEBP" {
155	return Some(Self::WebP);
156	}
157	None
158	}
159
160	/// The media type spelled the way [`mime`](Self::mime) spells it, for reading a stored part
161	/// back. An unknown string is refused rather than guessed: a part whose media type cannot be
162	/// named cannot be sent.
163	pub fn from_mime(s: &str) -> Option<Self> {
164	match s {
165	"image/png" => Some(Self::Png),
166	"image/jpeg" => Some(Self::Jpeg),
167	"image/gif" => Some(Self::Gif),
168	"image/webp" => Some(Self::WebP),
169	_ => None,
170	}
171	}
172	}
173
174	/// An image inside a message, held as the bytes that came off disk.
175	///
176	/// Bytes rather than base64: base64 is a wire encoding, and holding it would mean storing a third
177	/// more than the image weighs and re-decoding it to read the header. It is encoded once per
178	/// request, in whichever dialect is being spoken, by the serialiser that needs it.
179	#[derive(Clone, Debug, Eq, PartialEq)]
180	pub struct ImagePart {
181	/// What the bytes are, sniffed from the bytes.
182	pub media: ImageMedia,
183	/// The file's own bytes, undecoded.
184	pub data: Vec<u8>,
185	/// Where it came from, as the model asked for it.
186	///
187	/// The whole reason an elided image is not a loss: the line left behind names the file, and
188	/// the model can read it again. See `crate::compact::elide_bulk`.
189	pub source: String,
190	}
191
192	impl ImagePart {
193
194	/// A new part from bytes whose media type has already been settled.
195	///
196	/// # Arguments
197	/// * `media` - What the bytes are.
198	/// * `data` - The file's bytes.
199	/// * `source` - The path the model asked for.
200	pub fn new(media: ImageMedia, data: Vec<u8>, source: String) -> Self {
201	Self { media, data, source }
202	}
203
204	/// The image's pixel dimensions, or `None` when this build cannot read that format's header.
205	///
206	/// Header-only for both formats it knows -- neither reads a pixel -- because the only caller
207	/// is the token estimate, which runs on every request of a long turn. GIF and WebP return
208	/// `None` and are estimated from their bytes instead; see `crate::compact::image_tokens`.
209	pub fn dims(&self) -> Option<(usize, usize)> {
210	match self.media {
211	ImageMedia::Png => oxedyne_fe2o3_graphics::png::dimensions(&self.data).ok(),
212	ImageMedia::Jpeg => oxedyne_fe2o3_graphics::jpeg::dimensions(&self.data).ok(),
213	ImageMedia::Gif \| ImageMedia::WebP => None,
214	}
215	}
216
217	/// The bytes as base64, which is the form both dialects put on the wire.
218	pub fn base64(&self) -> String {
219	oxedyne_fe2o3_text::base64::encode(&self.data)
220	}
221
222	/// The one line an elided image leaves behind, naming the file so it can be read again.
223	pub fn elision(&self, why: Dropped) -> String {
224	match why {
225	Dropped::ToFit => fmt!(
226	"[image {} ({}, {} bytes) was dropped to fit the context window; read it again if \
227	you need to look at it]", self.source, self.media.mime(), self.data.len()),
228	// Deliberately NOT "read it again": on this endpoint that is a loop, and the whole
229	// point is that this model will never see the picture however often it is fetched.
230	// It is told not to describe it, because the failure a missing image invites is a
231	// confident description of something nobody showed it.
232	Dropped::Unseeable => fmt!(
233	"[image {} ({}, {} bytes) was left out because this model cannot be shown \
234	pictures. Do not describe what it looks like -- you have not seen it. Say so, \
235	and read it with \"as\":\"base64\" if you need its bytes to embed it]",
236	self.source, self.media.mime(), self.data.len()),
237	}
238	}
239	}
240
241	/// Why a picture came out of a message.
242	///
243	/// The two reasons want different words and the difference is load-bearing, so they are a type
244	/// rather than a boolean: one says "ask again", the other says "asking again will not help".
245	#[derive(Clone, Copy, Debug, Eq, PartialEq)]
246	pub enum Dropped {
247	/// Elided to fit the context window. The picture is still there to be fetched.
248	ToFit,
249	/// The endpoint will not take pictures at all, so fetching it again changes nothing.
250	Unseeable,
251	}
252
253	/// One piece of a message's content.
254	#[derive(Clone, Debug, Eq, PartialEq)]
255	pub enum ContentPart {
256	Text(String),
257	Image(ImagePart),
258	}
259
260	/// What a message carries.
261	///
262	/// Two shapes rather than always a list, because the shapes are not equally common: nearly every
263	/// message in a session is plain text, and a list-of-one-text-part would make every caller,
264	/// every serialiser and every byte count walk a vector to find the string it already had.
265	/// [`Text`](Self::Text) is that case named, and [`Parts`](Self::Parts) is the case that needs a
266	/// list -- which today means an image, and tomorrow whatever else a model can be shown.
267	///
268	/// The invariant that keeps the two from drifting: [`Parts`](Self::Parts) is only ever built by
269	/// [`parts`](Self::parts), which collapses an all-text list back to [`Text`](Self::Text). So two
270	/// contents that say the same thing compare equal, and no serialiser has to handle a `Parts`
271	/// carrying nothing but a string.
272	#[derive(Clone, Debug, Eq, PartialEq)]
273	pub enum MessageContent {
274	/// Plain text -- the overwhelmingly common case.
275	Text(String),
276	/// An ordered list of parts, at least one of which is not text.
277	Parts(Vec<ContentPart>),
278	}
279
280	impl Default for MessageContent {
281	fn default() -> Self {
282	Self::Text(String::new())
283	}
284	}
285
286	/// Content displays as its text, with each image named -- see [`MessageContent::as_text`].
287	///
288	/// Worth having rather than making every caller reach for `as_text`: the places that format a
289	/// message are error messages, log lines and test failures, and each of them wants exactly the
290	/// rendering `as_text` produces.
291	impl std::fmt::Display for MessageContent {
292	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
293	write!(f, "{}", self.as_text())
294	}
295	}
296
297	impl From<String> for MessageContent {
298	fn from(s: String) -> Self { Self::Text(s) }
299	}
300
301	impl From<&str> for MessageContent {
302	fn from(s: &str) -> Self { Self::Text(s.to_string()) }
303	}
304
305	impl MessageContent {
306
307	/// Plain text content -- one call, which is what the common case deserves.
308	pub fn text<S: Into<String>>(s: S) -> Self {
309	Self::Text(s.into())
310	}
311
312	/// Content from a list of parts, collapsed to [`Text`](Self::Text) when nothing in it needs
313	/// the list form. See the type's own note on why that collapse is the invariant.
314	pub fn parts(parts: Vec<ContentPart>) -> Self {
315	if parts.iter().all(\|p\| matches!(p, ContentPart::Text(_))) {
316	let mut s = String::new();
317	for p in &parts {
318	if let ContentPart::Text(t) = p {
319	if !s.is_empty() && !t.is_empty() {
320	s.push('\n');
321	}
322	s.push_str(t);
323	}
324	}
325	return Self::Text(s);
326	}
327	Self::Parts(parts)
328	}
329
330	/// The content as text, with each image standing in for itself by name.
331	///
332	/// Borrowed in the common case and built only when there are parts, so the panels, the ledger
333	/// and the fold rendering -- all of which want a string and none of which can look at an
334	/// image -- pay nothing on an ordinary message.
335	pub fn as_text(&self) -> std::borrow::Cow<'_, str> {
336	match self {
337	Self::Text(s) => std::borrow::Cow::Borrowed(s.as_str()),
338	Self::Parts(parts) => {
339	let mut out = String::new();
340	for p in parts {
341	if !out.is_empty() {
342	out.push('\n');
343	}
344	match p {
345	ContentPart::Text(t) => out.push_str(t),
346	ContentPart::Image(i) => out.push_str(&fmt!(
347	"[image {} ({}, {} bytes)]", i.source, i.media.mime(), i.data.len())),
348	}
349	}
350	std::borrow::Cow::Owned(out)
351	},
352	}
353	}
354
355	/// Bytes of TEXT this content carries; an image's payload is not counted here.
356	///
357	/// The separation is deliberate and is the whole of the token-accounting fix: an image costs
358	/// what its pixels cost, not what its bytes cost, so it is measured by
359	/// `crate::compact::image_bytes` instead and never by the text ratio.
360	pub fn text_len(&self) -> usize {
361	match self {
362	Self::Text(s) => s.len(),
363	Self::Parts(parts) => parts.iter().map(\|p\| match p {
364	ContentPart::Text(t) => t.len(),
365	// The `[image …]` stand-in a renderer would put here, near enough.
366	ContentPart::Image(i) => i.source.len() + 32,
367	}).sum(),
368	}
369	}
370
371	/// Whether there is nothing to send. An image alone is not empty.
372	pub fn is_empty(&self) -> bool {
373	match self {
374	Self::Text(s) => s.is_empty(),
375	Self::Parts(parts) => parts.is_empty(),
376	}
377	}
378
379	/// The images this content carries, in order.
380	pub fn images(&self) -> impl Iterator<Item = &ImagePart> {
381	let slice: &[ContentPart] = match self {
382	Self::Text(_) => &[],
383	Self::Parts(parts) => parts.as_slice(),
384	};
385	slice.iter().filter_map(\|p\| match p {
386	ContentPart::Image(i) => Some(i),
387	ContentPart::Text(_) => None,
388	})
389	}
390
391	/// Whether there is an image in here.
392	pub fn has_image(&self) -> bool {
393	self.images().next().is_some()
394	}
395
396	/// The same content with every image replaced by the line that names it.
397	///
398	/// What elision does to an image, and why elision is not a loss: the file is still named, and
399	/// `file_read` will fetch it again. See `crate::compact::elide_bulk`.
400	pub fn without_images(&self, why: Dropped) -> Self {
401	match self {
402	Self::Text(_) => self.clone(),
403	Self::Parts(parts) => Self::parts(parts.iter().map(\|p\| match p {
404	ContentPart::Text(t) => ContentPart::Text(t.clone()),
405	ContentPart::Image(i) => ContentPart::Text(i.elision(why)),
406	}).collect()),
407	}
408	}
409
410	/// Serialise to JDAT: a bare string for text, a list of typed maps for parts.
411	///
412	/// The text case keeps the shape the store has always written, which is what lets a session
413	/// snapshot round-trip without the reader knowing anything new.
414	pub fn to_dat(&self) -> Dat {
415	match self {
416	Self::Text(s) => dat!(s.clone()),
417	Self::Parts(parts) => {
418	let items: Vec<Dat> = parts.iter().map(\|p\| {
419	let mut m = DaticleMap::new();
420	match p {
421	ContentPart::Text(t) => {
422	m.insert(dat!("type"), dat!("text"));
423	m.insert(dat!("text"), dat!(t.clone()));
424	},
425	ContentPart::Image(i) => {
426	m.insert(dat!("type"), dat!("image"));
427	m.insert(dat!("media_type"), dat!(i.media.mime()));
428	m.insert(dat!("source"), dat!(i.source.clone()));
429	// Bytes, not base64: the store holds what the file held, and BU64 is
430	// the byte vector JDAT reads back without a decode step.
431	m.insert(dat!("data"), Dat::BU64(i.data.clone()));
432	},
433	}
434	Dat::Map(m)
435	}).collect();
436	Dat::List(items)
437	},
438	}
439	}
440
441	/// Read back what [`to_dat`](Self::to_dat) wrote.
442	///
443	/// A part that cannot be read -- an unknown media type, missing bytes -- is dropped rather
444	/// than refusing the message: half a conversation read back is worth more to the user than an
445	/// error, and the same reasoning already governs `ChatMessage::from_datmap`.
446	pub fn from_dat(d: &Dat) -> Outcome<Self> {
447	match d {
448	Dat::Str(s) => Ok(Self::Text(s.clone())),
449	Dat::List(items) => {
450	let mut parts = Vec::with_capacity(items.len());
451	for item in items {
452	let m = match item {
453	Dat::Map(m) => m,
454	_ => continue,
455	};
456	let kind = match m.get(&dat!("type")) {
457	Some(Dat::Str(s)) => s.as_str(),
458	_ => continue,
459	};
460	match kind {
461	"text" => {
462	if let Some(Dat::Str(t)) = m.get(&dat!("text")) {
463	parts.push(ContentPart::Text(t.clone()));
464	}
465	},
466	"image" => {
467	let media = match m.get(&dat!("media_type")) {
468	Some(Dat::Str(s)) => match ImageMedia::from_mime(s) {
469	Some(mt) => mt,
470	None => continue,
471	},
472	_ => continue,
473	};
474	let data = match m.get(&dat!("data")) {
475	Some(Dat::BU64(b)) => b.clone(),
476	_ => continue,
477	};
478	let source = match m.get(&dat!("source")) {
479	Some(Dat::Str(s)) => s.clone(),
480	_ => String::new(),
481	};
482	parts.push(ContentPart::Image(ImagePart::new(media, data, source)));
483	},
484	_ => continue,
485	}
486	}
487	Ok(Self::parts(parts))
488	},
489	_ => Err(err!("MessageContent: expected a string or a list of parts."; Invalid, Input)),
490	}
491	}
492	}
493
494
495	/// A single message in a conversation, mirroring the OpenAI API format.
496	#[derive(Clone, Debug, Eq, PartialEq)]
497	pub enum ChatMessage {
498	System { content: MessageContent },
499	User { content: MessageContent },
500	/// Assistant turn, with whatever tool calls it asked for.
501	///
502	/// The calls are part of the message and are persisted with it. They used to be
503	/// dropped on the way to storage, which made a reloaded session illegal rather than
504	/// merely lossy: the assistant turn came back bare and the `tool` replies that
505	/// followed it answered nothing, and an OpenAI-compatible provider rejects that
506	/// outright on every subsequent turn.
507	Assistant { content: MessageContent, tool_calls: Vec<ToolCall> },
508	/// Tool call result returned to the LLM.
509	Tool { tool_call_id: String, content: MessageContent },
510	}
511
512	impl ChatMessage {
513
514	/// A system message. One call, because the common case is a bare string.
515	pub fn system<C: Into<MessageContent>>(content: C) -> Self {
516	Self::System { content: content.into() }
517	}
518
519	/// A user message.
520	pub fn user<C: Into<MessageContent>>(content: C) -> Self {
521	Self::User { content: content.into() }
522	}
523
524	/// An assistant message that asked for nothing.
525	pub fn assistant<C: Into<MessageContent>>(content: C) -> Self {
526	Self::Assistant { content: content.into(), tool_calls: Vec::new() }
527	}
528
529	/// An assistant message and the tool calls it made.
530	pub fn assistant_calling<C: Into<MessageContent>>(content: C, tool_calls: Vec<ToolCall>)
531	-> Self
532	{
533	Self::Assistant { content: content.into(), tool_calls }
534	}
535
536	/// A tool reply, paired to the call it answers.
537	pub fn tool<C: Into<MessageContent>>(tool_call_id: String, content: C) -> Self {
538	Self::Tool { tool_call_id, content: content.into() }
539	}
540
541	/// Serialise to a JDAT map, for the session store and the session export.
542	///
543	/// Not the LLM request body: that is built by `llm::message_to_json`, in the
544	/// provider's own JSON and in whichever dialect the endpoint speaks. What this has
545	/// to survive is a round trip through storage, which means an assistant turn's tool
546	/// calls travel with it -- an assistant message that loses them leaves the `tool`
547	/// replies after it answering nothing, and a provider refuses such a conversation
548	/// outright.
549	///
550	/// Produces maps like:
551	/// { "role": "system", "content": "..." }
552	/// { "role": "user", "content": "..." }
553	/// { "role": "assistant", "content": "..." }
554	/// { "role": "assistant", "content": "...",
555	/// "tool_calls": [ { "id": "...", "name": "...", "arguments": "..." } ] }
556	/// { "role": "tool", "tool_call_id": "...", "content": "..." }
557	pub fn to_datmap(&self) -> DaticleMap {
558	let mut m = DaticleMap::new();
559	match self {
560	Self::System { content } => {
561	m.insert(dat!("role"), dat!("system"));
562	m.insert(dat!("content"), content.to_dat());
563	}
564	Self::User { content } => {
565	m.insert(dat!("role"), dat!("user"));
566	m.insert(dat!("content"), content.to_dat());
567	}
568	Self::Assistant { content, tool_calls } => {
569	m.insert(dat!("role"), dat!("assistant"));
570	m.insert(dat!("content"), content.to_dat());
571	// Written only when there are any, so an ordinary answer's map is the
572	// shape it always was and a reader of an older snapshot sees no change.
573	if !tool_calls.is_empty() {
574	let calls: Vec<Dat> = tool_calls.iter()
575	.map(\|tc\| Dat::Map(tc.to_datmap()))
576	.collect();
577	m.insert(dat!("tool_calls"), Dat::List(calls));
578	}
579	}
580	Self::Tool { tool_call_id, content } => {
581	m.insert(dat!("role"), dat!("tool"));
582	m.insert(dat!("tool_call_id"), dat!(tool_call_id.clone()));
583	m.insert(dat!("content"), content.to_dat());
584	}
585	}
586	m
587	}
588
589	/// Deserialise from a JDAT map.
590	pub fn from_datmap(m: &DaticleMap) -> Outcome<Self> {
591	let role = match m.get(&dat!("role")) {
592	Some(Dat::Str(s)) => s.clone(),
593	_ => return Err(err!("ChatMessage: missing 'role'."; Invalid, Input)),
594	};
595	let content = match m.get(&dat!("content")) {
596	Some(d) => res!(MessageContent::from_dat(d)),
597	None => return Err(err!("ChatMessage: missing 'content'."; Invalid, Input)),
598	};
599	match role.as_str() {
600	"system" => Ok(Self::System { content }),
601	"user" => Ok(Self::User { content }),
602	"assistant" => {
603	// Absent is an assistant turn that asked for nothing, and every snapshot
604	// written before the calls were persisted at all. A malformed entry is
605	// skipped rather than refusing the message: half a conversation read back
606	// is worth more to the user than an error, and `compact::orphan_count`
607	// already treats what is left as the broken pairing it is.
608	let mut tool_calls = Vec::new();
609	if let Some(Dat::List(list)) = m.get(&dat!("tool_calls")) {
610	for item in list {
611	if let Dat::Map(tc_m) = item {
612	if let Ok(tc) = ToolCall::from_datmap(tc_m) {
613	tool_calls.push(tc);
614	}
615	}
616	}
617	}
618	Ok(Self::Assistant { content, tool_calls })
619	}
620	"tool" => {
621	let tool_call_id = match m.get(&dat!("tool_call_id")) {
622	Some(Dat::Str(s)) => s.clone(),
623	_ => return Err(err!("ChatMessage: tool missing 'tool_call_id'."; Invalid, Input)),
624	};
625	Ok(Self::Tool { tool_call_id, content })
626	}
627	_ => Err(err!("ChatMessage: unknown role '{}'.", role; Invalid, Input)),
628	}
629	}
630
631	pub fn role(&self) -> &'static str {
632	match self {
633	Self::System { .. } => "system",
634	Self::User { .. } => "user",
635	Self::Assistant { .. } => "assistant",
636	Self::Tool { .. } => "tool",
637	}
638	}
639
640	/// What this message carries, whole.
641	pub fn content(&self) -> &MessageContent {
642	match self {
643	Self::System { content }
644	\| Self::User { content }
645	\| Self::Assistant { content, .. }
646	\| Self::Tool { content, .. } => content,
647	}
648	}
649
650	/// What this message says, as text, with any image standing in for itself by name.
651	///
652	/// Borrowed on an ordinary message; see [`MessageContent::as_text`].
653	pub fn text(&self) -> std::borrow::Cow<'_, str> {
654	self.content().as_text()
655	}
656
657	/// The same message with its content replaced. Role, tool calls and pairing id are kept,
658	/// which is what makes an in-place elision safe: nothing a provider pairs on is touched.
659	///
660	/// # Arguments
661	/// * `content` - What to put in place of the old content.
662	pub fn with_content(&self, content: MessageContent) -> Self {
663	match self {
664	Self::System { .. } => Self::System { content },
665	Self::User { .. } => Self::User { content },
666	Self::Assistant { tool_calls, .. } =>
667	Self::Assistant { content, tool_calls: tool_calls.clone() },
668	Self::Tool { tool_call_id, .. } =>
669	Self::Tool { tool_call_id: tool_call_id.clone(), content },
670	}
671	}
672	}
673
674
675	// ┌───────────────────────────────────────────────────────────────┐
676	// │ Session │
677	// └───────────────────────────────────────────────────────────────┘
678
679	/// A chat session belonging to a user.
680	#[derive(Clone, Debug)]
681	pub struct Session {
682	pub id: String,
683	pub name: String,
684	pub created_at: u64,
685	pub model: String,
686	pub messages: Vec<ChatMessage>,
687	/// Cumulative prompt tokens across all turns (for billing).
688	pub prompt_tokens: u64,
689	/// Cumulative completion tokens across all turns (for billing).
690	pub completion_tokens: u64,
691	/// Prompt tokens of the most recent request — the current context
692	/// window usage (not cumulative), for the live meter.
693	pub last_prompt_tokens: u64,
694	/// Cumulative prompt tokens the provider served from its cache, across all
695	/// turns. A subset of `prompt_tokens`, never added to it.
696	pub cached_tokens: u64,
697	/// Cumulative USD the provider says these turns actually cost.
698	///
699	/// Zero means no provider reported a figure, not that the session was
700	/// free; the caller prices those turns from its table instead.
701	pub cost_usd: f64,
702	}
703
704	impl Session {
705
706	pub fn new(id: String, name: String, model: String) -> Self {
707	Self {
708	id,
709	name,
710	created_at: now_secs(),
711	model,
712	messages: Vec::new(),
713	prompt_tokens: 0,
714	completion_tokens: 0,
715	last_prompt_tokens: 0,
716	cached_tokens: 0,
717	cost_usd: 0.0,
718	}
719	}
720
721	/// Serialise metadata (without messages) to a JDAT map.
722	pub fn to_meta_datmap(&self) -> DaticleMap {
723	let mut m = DaticleMap::new();
724	m.insert(dat!("id"), dat!(self.id.clone()));
725	m.insert(dat!("name"), dat!(self.name.clone()));
726	m.insert(dat!("created_at"), Dat::U64(self.created_at));
727	m.insert(dat!("model"), dat!(self.model.clone()));
728	m.insert(dat!("prompt_tokens"), Dat::U64(self.prompt_tokens));
729	m.insert(dat!("completion_tokens"), Dat::U64(self.completion_tokens));
730	m.insert(dat!("last_prompt_tokens"), Dat::U64(self.last_prompt_tokens));
731	m.insert(dat!("cached_tokens"), Dat::U64(self.cached_tokens));
732	m.insert(dat!("cost_usd"), dat!(self.cost_usd));
733	m
734	}
735
736	/// Serialise full session (with messages) to a JDAT map.
737	pub fn to_datmap(&self) -> DaticleMap {
738	let mut m = self.to_meta_datmap();
739	let msgs: Vec<Dat> = self.messages.iter()
740	.map(\|msg\| Dat::Map(msg.to_datmap()))
741	.collect();
742	m.insert(dat!("messages"), Dat::List(msgs));
743	m
744	}
745
746	/// Deserialise from a JDAT map.
747	pub fn from_datmap(m: &DaticleMap) -> Outcome<Self> {
748	let id = match m.get(&dat!("id")) {
749	Some(Dat::Str(s)) => s.clone(),
750	_ => return Err(err!("Session: missing 'id'."; Invalid, Input)),
751	};
752	let name = match m.get(&dat!("name")) {
753	Some(Dat::Str(s)) => s.clone(),
754	_ => return Err(err!("Session: missing 'name'."; Invalid, Input)),
755	};
756	let created_at = match m.get(&dat!("created_at")) {
757	Some(Dat::U64(n)) => *n,
758	_ => 0,
759	};
760	let model = match m.get(&dat!("model")) {
761	Some(Dat::Str(s)) => s.clone(),
762	_ => String::new(),
763	};
764	let messages = match m.get(&dat!("messages")) {
765	Some(Dat::List(list)) => {
766	let mut msgs = Vec::new();
767	for item in list {
768	if let Dat::Map(msg_m) = item {
769	msgs.push(res!(ChatMessage::from_datmap(msg_m)));
770	}
771	}
772	msgs
773	}
774	_ => Vec::new(),
775	};
776	let prompt_tokens = match m.get(&dat!("prompt_tokens")) {
777	Some(Dat::U64(n)) => *n,
778	_ => 0,
779	};
780	let completion_tokens = match m.get(&dat!("completion_tokens")) {
781	Some(Dat::U64(n)) => *n,
782	_ => 0,
783	};
784	let last_prompt_tokens = match m.get(&dat!("last_prompt_tokens")) {
785	Some(Dat::U64(n)) => *n,
786	_ => 0,
787	};
788	// OPTIONAL, and it has to be: a snapshot written before these two fields
789	// existed must still load. Every field here is decoded with a `_ => 0`
790	// arm for that reason -- a required field would refuse the user's own
791	// history the moment the shape grew, which is how a session store gets
792	// bricked by an upgrade.
793	let cached_tokens = match m.get(&dat!("cached_tokens")) {
794	Some(Dat::U64(n)) => *n,
795	_ => 0,
796	};
797	let cost_usd = match m.get(&dat!("cost_usd")) {
798	Some(Dat::F64(f)) => **f,
799	_ => 0.0,
800	};
801	Ok(Self {
802	id, name, created_at, model, messages,
803	prompt_tokens, completion_tokens, last_prompt_tokens,
804	cached_tokens, cost_usd,
805	})
806	}
807	}
808
809
810	// ┌───────────────────────────────────────────────────────────────┐
811	// │ User configuration │
812	// └───────────────────────────────────────────────────────────────┘
813
814	/// Per-user configuration stored in O3db.
815	///
816	/// Supports multi-user with individual model selection — the foundation
817	/// for a future commercial offering with billing.
818	#[derive(Clone, Debug)]
819	pub struct UserConfig {
820	pub username: String,
821	pub default_model: String,
822	pub created_at: u64,
823	}
824
825	impl UserConfig {
826
827	pub fn new(username: String, default_model: String) -> Self {
828	Self {
829	username,
830	default_model,
831	created_at: now_secs(),
832	}
833	}
834
835	pub fn to_datmap(&self) -> DaticleMap {
836	let mut m = DaticleMap::new();
837	m.insert(dat!("username"), dat!(self.username.clone()));
838	m.insert(dat!("default_model"), dat!(self.default_model.clone()));
839	m.insert(dat!("created_at"), Dat::U64(self.created_at));
840	m
841	}
842
843	pub fn from_datmap(m: &DaticleMap) -> Outcome<Self> {
844	let username = match m.get(&dat!("username")) {
845	Some(Dat::Str(s)) => s.clone(),
846	_ => return Err(err!("UserConfig: missing 'username'."; Invalid, Input)),
847	};
848	let default_model = match m.get(&dat!("default_model")) {
849	Some(Dat::Str(s)) => s.clone(),
850	_ => String::new(),
851	};
852	let created_at = match m.get(&dat!("created_at")) {
853	Some(Dat::U64(n)) => *n,
854	_ => 0,
855	};
856	Ok(Self { username, default_model, created_at })
857	}
858	}
859
860
861	/// Make a restored conversation legal: every tool call answered, every tool reply
862	/// answering something.
863	///
864	/// The provider's rule is not a preference. An assistant turn bearing `tool_calls`
865	/// must be followed by one `tool` message per call, and a `tool` message must follow
866	/// the assistant turn that asked for it; a conversation that breaks either is
867	/// rejected WHOLE, so one lost reply from one turn last Tuesday takes every turn
868	/// after it with it. A store merged across tabs, synced between devices and
869	/// restored from backups will eventually hand over such a list, so it is repaired
870	/// here rather than trusted.
871	///
872	/// Two edits, and only these two: a call with no reply in the run of `tool` messages
873	/// directly after it is dropped from the assistant turn (its prose stays), and a
874	/// reply that answers no call in the assistant turn directly before it is dropped
875	/// entirely. Nothing is reordered and nothing is invented — a call that lost its
876	/// result is a call the model must be allowed to make again, not one to answer with
877	/// a guess.
878	///
879	///
880	/// TWO CALLERS SINCE 2026-08-28, and that is why it lives here rather than beside the restore.
881	/// [`crate::wasm::app::DaimondApp::restore_session`] repairs a conversation coming back from the
882	/// store, and [`crate::agent::Agent`] repairs the round it is abandoning when a tool call dies on
883	/// the road: same rule, same edits, and the two must not be allowed to differ. They did differ
884	/// for as long as only the first existed -- a live session held whatever the turn left in it, and
885	/// only a reload put it right.
886	///
887	/// # Arguments
888	/// * `msgs` - The conversation to repair, oldest first.
889	pub fn pair_up(msgs: Vec<ChatMessage>) -> Vec<ChatMessage> {
890	let mut out: Vec<ChatMessage> = Vec::with_capacity(msgs.len());
891	let mut i = 0usize;
892	while i < msgs.len() {
893	match &msgs[i] {
894	ChatMessage::Assistant { content, tool_calls } if !tool_calls.is_empty() => {
895	// The run of tool replies that directly follows, which is the only
896	// place a reply to this turn may legally sit.
897	let mut answered: Vec<String> = Vec::new();
898	let mut j = i + 1;
899	while j < msgs.len() {
900	match &msgs[j] {
901	ChatMessage::Tool { tool_call_id, .. } => {
902	answered.push(tool_call_id.clone());
903	j += 1;
904	}
905	_ => break,
906	}
907	}
908	let kept: Vec<ToolCall> = tool_calls.iter()
909	.filter(\|tc\| answered.iter().any(\|id\| id == &tc.id))
910	.cloned()
911	.collect();
912	out.push(ChatMessage::Assistant {
913	content: content.clone(),
914	tool_calls: kept.clone(),
915	});
916	// Then the replies, keeping only those that answer a call we kept.
917	for k in (i + 1)..j {
918	if let ChatMessage::Tool { tool_call_id, content } = &msgs[k] {
919	if kept.iter().any(\|tc\| &tc.id == tool_call_id) {
920	out.push(ChatMessage::Tool {
921	tool_call_id: tool_call_id.clone(),
922	content: content.clone(),
923	});
924	}
925	}
926	}
927	i = j;
928	}
929	// A tool reply reached here without an asking turn in front of it.
930	ChatMessage::Tool { .. } => { i += 1; }
931	other => { out.push(other.clone()); i += 1; }
932	}
933	}
934	out
935	}
936
937	// ┌───────────────────────────────────────────────────────────────┐
938	// │ Agent events │
939	// └───────────────────────────────────────────────────────────────┘
940
941	/// Events emitted by the agent loop, sent to the client over WS.
942	#[derive(Clone, Debug)]
943	pub enum AgentEvent {
944	/// Streamed LLM response text (a token or chunk).
945	Text(String),
946	/// The agent is invoking a tool (name + raw JSON args).
947	ToolCall { id: String, name: String, args: String },
948	/// A tool returned a result: its name, the reply text, and what the call came to.
949	///
950	/// The outcome travels because the tool layer is the only place that knows it. It used to be
951	/// flattened into the reply's opening word and read back out of the prose by four separate
952	/// consumers, each with its own reading -- so a refusal, whose sentence opens "Refused" rather
953	/// than "Error", was drawn as a completed step, journalled as a success and reported to the
954	/// Optimiser as a tool that had worked. A daimon has nothing to check that against, and
955	/// reports the turn done.
956	ToolResult { name: String, result: String, outcome: CallOutcome },
957	/// The user said something while the turn was running, and it has now been put
958	/// into the conversation at the seam between two rounds.
959	///
960	/// Emitted so the thread can show WHERE the user cut in. A correction that
961	/// appears at the end, after the work it was meant to redirect, reads as though
962	/// it was ignored -- and the one thing an interjection must never look like is
963	/// something that arrived too late.
964	Interjected(String),
965	/// The conversation was folded to fit the model's context window.
966	///
967	/// Its own variant rather than a line of assistant text: a fold is something the APP
968	/// did, it is lossy, and the user is entitled to see it as an act rather than as prose
969	/// the model produced.
970	Compacted { folded: usize, kept: usize, note: String },
971	/// A picture could not be put in front of this model.
972	///
973	/// Its own variant and not an error: the turn goes on without the picture. The app did
974	/// something, it is lossy, and the user is entitled to see it as an act -- the same reasoning
975	/// as [`Compacted`](Self::Compacted).
976	///
977	/// It says WHICH model, because that is the fact anybody acting on this has to have: the
978	/// worker that is re-routed by it needs to name the model it left, and a conversation that
979	/// cannot be re-routed at all needs to name the model that would not look.
980	Unseeable { images: usize, model: String },
981	/// A tool call is being made again because the ROAD failed under it, not the far end.
982	///
983	/// Its own variant and not [`Text`](Self::Text), for the reason [`Compacted`](Self::Compacted)
984	/// is: this is something the APP is doing. The model did not say it and -- the whole point
985	/// of the mechanism -- will never read it. A road failure that reached the model as a tool
986	/// result is what produced "I can't get through to the web right now to look this up" on a
987	/// real iPhone, so nothing about one may enter the conversation.
988	///
989	/// It exists because the alternative is silence. The ladder can climb for up to two minutes,
990	/// and a turn retrying quietly is indistinguishable from a turn that has hung -- which is the
991	/// one thing a spinner cannot say. The provider ladder already prints its own banner
992	/// (`[daimond: …; retrying in …]`, src/llm.rs); this is the same courtesy one layer down, in
993	/// a form the page can draw as furniture rather than as prose.
994	Roading { name: String, attempt: u32, of: u32, wait_ms: u64 },
995	/// The provider stopped generating because the reply reached the output limit.
996	///
997	/// Said outright rather than inferred. A tool call cut at the limit arrives as
998	/// malformed JSON, so the browser had to guess truncation from arguments that would
999	/// not parse -- a guess that is right in practice and cannot see a plain text reply
1000	/// cut short, which is the case with no other symptom at all.
1001	///
1002	/// It is not an error. The request succeeded; a setting was reached.
1003	Truncated,
1004	/// A chunk of the model's own reasoning, as it arrives.
1005	///
1006	/// Its own variant and not [`Text`](Self::Text), because the two are different KINDS of
1007	/// content on the wire -- the provider sends `thinking` blocks and `text` blocks, and a
1008	/// page that ran them together could never fold one and stream the other. The user pays
1009	/// for these tokens and they decide the answer, so drawing none of them was the app
1010	/// holding something back that was already bought.
1011	///
1012	/// Delivered as it ARRIVES, one delta per event, from the moment the model starts
1013	/// thinking. It used to be sent whole at the end of the round, on the reasoning that a
1014	/// shut tile has nothing for a stream to fill -- which was true of the tile and wrong
1015	/// about the wait. A GLM round measured on 2026-08-28 spent 230 of its 300 output
1016	/// tokens reasoning, and a DeepSeek round spent 84 seconds and 1.8 MB on it; all of
1017	/// that was time the page had nothing to show, because the one thing happening was the
1018	/// thing being held back. So the tile opens as the first delta lands and fills while
1019	/// the round runs. A reader who wants none of it collapses it, and it stays collapsed.
1020	///
1021	/// Consecutive events belong to ONE tile. The page appends them to the tile it is
1022	/// filling and closes it off when anything else is drawn, so a round's working is one
1023	/// disclosure and not four hundred.
1024	///
1025	/// Empty for every endpoint that does not return reasoning, which is most of them. The
1026	/// tokens are billed either way, so a provider that thinks and says nothing costs the
1027	/// same and shows nothing -- that is the provider's doing and the tile simply does not
1028	/// appear.
1029	Thinking(String),
1030	/// How the turn ended, and what its tool log came to.
1031	///
1032	/// Always emitted, on every path out of a turn, and drawn as FURNITURE -- one quiet line
1033	/// in the register of a timestamp. That is not decoration: an app that appends a warning
1034	/// to every turn teaches its reader to skip the one that mattered, which is the failure
1035	/// this exists to prevent. `dev/CONTRACT_CLAIMS.md` §3 binds the drawing to it, and
1036	/// `TurnEnding::unaccounted` is the only thing that promotes the line to a notice.
1037	///
1038	/// It exists because a model announced work and then ended its turn having done none, the
1039	/// spinner stopped -- correctly, the turn HAD ended -- and the owner was left with no way
1040	/// to tell that from a turn that finished. Three endings were explained before this and
1041	/// every other one was silence.
1042	///
1043	/// `missing` is paths a completed call SAID it left on the store and which are not there,
1044	/// read off the call's arguments and never off the model's words.
1045	Ended {
1046	how: String, // answered \| stopped \| capped \| silent \| failed
1047	offered: usize, // tools this turn was allowed to call
1048	rounds: usize,
1049	calls: usize,
1050	refused: usize,
1051	failed: usize,
1052	missing: Vec<String>,
1053	},
1054	/// Agent turn complete.
1055	Done,
1056	/// Error occurred.
1057	Error(String),
1058	}
1059
1060	impl AgentEvent {
1061
1062	/// Convert to a JDAT map suitable for a WS `data` response.
1063	pub fn to_datmap(&self) -> DaticleMap {
1064	let mut m = DaticleMap::new();
1065	match self {
1066	Self::Text(text) => {
1067	m.insert(dat!("type"), dat!("text"));
1068	m.insert(dat!("content"), dat!(text.clone()));
1069	}
1070	Self::Thinking(text) => {
1071	m.insert(dat!("type"), dat!("thinking"));
1072	m.insert(dat!("content"), dat!(text.clone()));
1073	}
1074	Self::Ended { how, offered, rounds, calls, refused, failed, missing } => {
1075	m.insert(dat!("type"), dat!("ended"));
1076	m.insert(dat!("how"), dat!(how.clone()));
1077	m.insert(dat!("offered"), Dat::U64(*offered as u64));
1078	m.insert(dat!("rounds"), Dat::U64(*rounds as u64));
1079	m.insert(dat!("calls"), Dat::U64(*calls as u64));
1080	m.insert(dat!("refused"), Dat::U64(*refused as u64));
1081	m.insert(dat!("failed"), Dat::U64(*failed as u64));
1082	m.insert(dat!("missing"),
1083	Dat::List(missing.iter().map(\|p\| dat!(p.clone())).collect()));
1084	}
1085	Self::ToolCall { name, args, .. } => {
1086	m.insert(dat!("type"), dat!("tool_call"));
1087	m.insert(dat!("name"), dat!(name.clone()));
1088	m.insert(dat!("args"), dat!(args.clone()));
1089	}
1090	Self::ToolResult { name, result, outcome } => {
1091	m.insert(dat!("type"), dat!("tool_result"));
1092	m.insert(dat!("name"), dat!(name.clone()));
1093	m.insert(dat!("content"), dat!(result.clone()));
1094	m.insert(dat!("outcome"), dat!(outcome.wire()));
1095	}
1096	Self::Interjected(text) => {
1097	m.insert(dat!("type"), dat!("interjected"));
1098	m.insert(dat!("content"), dat!(text.clone()));
1099	}
1100	Self::Compacted { folded, kept, note } => {
1101	m.insert(dat!("type"), dat!("compacted"));
1102	m.insert(dat!("folded"), Dat::U64(*folded as u64));
1103	m.insert(dat!("kept"), Dat::U64(*kept as u64));
1104	m.insert(dat!("content"), dat!(note.clone()));
1105	}
1106	Self::Unseeable { images, model } => {
1107	m.insert(dat!("type"), dat!("unseeable"));
1108	m.insert(dat!("images"), Dat::U64(*images as u64));
1109	m.insert(dat!("model"), dat!(model.clone()));
1110	}
1111	Self::Roading { name, attempt, of, wait_ms } => {
1112	m.insert(dat!("type"), dat!("roading"));
1113	m.insert(dat!("name"), dat!(name.clone()));
1114	m.insert(dat!("attempt"), Dat::U64(*attempt as u64));
1115	m.insert(dat!("of"), Dat::U64(*of as u64));
1116	m.insert(dat!("wait_ms"), Dat::U64(*wait_ms));
1117	}
1118	Self::Truncated => {
1119	m.insert(dat!("type"), dat!("truncated"));
1120	}
1121	Self::Done => {
1122	m.insert(dat!("type"), dat!("done"));
1123	}
1124	Self::Error(msg) => {
1125	m.insert(dat!("type"), dat!("error"));
1126	m.insert(dat!("content"), dat!(msg.clone()));
1127	}
1128	}
1129	m
1130	}
1131	}
1132
1133
1134	// ┌───────────────────────────────────────────────────────────────┐
1135	// │ O3db key helpers │
1136	// └───────────────────────────────────────────────────────────────┘
1137
1138	/// Build the O3db key for a user's session list.
1139	pub fn sessions_key(username: &str) -> Dat {
1140	Dat::Str(fmt!("daimond:{}:sessions", username))
1141	}
1142
1143	/// Build the O3db key for a specific session.
1144	pub fn session_key(session_id: &str) -> Dat {
1145	Dat::Str(fmt!("daimond:session:{}", session_id))
1146	}
1147
1148	/// Build the O3db key for a user's config.
1149	pub fn user_config_key(username: &str) -> Dat {
1150	Dat::Str(fmt!("daimond:user:{}", username))
1151	}
1152
1153	/// Generate a unique session ID (8 hex chars from timestamp + counter).
1154	pub fn generate_session_id() -> String {
1155	use std::sync::atomic::{AtomicU64, Ordering};
1156	static COUNTER: AtomicU64 = AtomicU64::new(0);
1157	let ts = now_millis();
1158	let n = COUNTER.fetch_add(1, Ordering::Relaxed);
1159	fmt!("{:x}{:x}", ts, n)
1160	}
1161
1162
1163	// ┌───────────────────────────────────────────────────────────────┐
1164	// │ The Diamond pack │
1165	// └───────────────────────────────────────────────────────────────┘
1166
1167	/// What a pack is, which decides what an importer may do with it.
1168	///
1169	/// The whole difference between the two doors: a `Diamond` is written OVER the id it names, and
1170	/// whatever stood there is deleted first ([`crate::wasm::diamond::import_diamond`]); a `Template`
1171	/// is opened as a NEW Diamond whatever id it names. A reader that could not tell them apart would
1172	/// eventually take one for the other, and one of those two mistakes destroys the Diamond the pack
1173	/// happened to be named after.
1174	#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1175	pub enum PackKind {
1176	Diamond, // a whole Diamond, id and all
1177	Template, // a shape to open as a new one
1178	}
1179
1180	impl PackKind {
1181
1182	/// The word this kind travels as.
1183	pub fn wire(&self) -> &'static str {
1184	match self {
1185	Self::Diamond => "diamond",
1186	Self::Template => "template",
1187	}
1188	}
1189
1190	/// What a pack says it is.
1191	///
1192	/// Read from the head of the pack -- everything before the `files` map -- and not from the
1193	/// whole of it, because a file called `kind` inside `files` would be a key of exactly the shape
1194	/// [`crate::llm::extract_json_string`] is looking for. A pack that says nothing is a whole
1195	/// Diamond: nothing that predates this field could pack anything else.
1196	pub fn of(json: &str) -> Self {
1197	let head = match json.find("\"files\":") {
1198	Some(i) => &json[..i],
1199	None => json,
1200	};
1201	match crate::llm::extract_json_string(head, PACK_KIND).as_deref() {
1202	Some(w) if w == Self::Template.wire() => Self::Template,
1203	_ => Self::Diamond,
1204	}
1205	}
1206	}
1207
1208	// The key a pack says its kind under. Absent means [`PackKind::Diamond`].
1209	pub const PACK_KIND: &str = "kind";
1210
1211	// The key a template carries its display name under.
1212	pub const PACK_NAME: &str = "name";
1213
1214	/// The JSON a whole Diamond travels in, between two devices and inside a share.
1215	///
1216	/// # A file that is not text used to be destroyed on the way out
1217	///
1218	/// Every file went through `String::from_utf8_lossy`, which is not a lenient decoding but a
1219	/// REPLACEMENT: each byte sequence that is not valid UTF-8 becomes U+FFFD and the original bytes are
1220	/// gone. A PNG came out the far end as a PNG-shaped ruin of about the right size, so a share could
1221	/// not carry a picture in either direction and neither end reported a fault. That is why the
1222	/// encoding lives here, target-agnostic, where a test can put real bytes in and compare what comes
1223	/// out -- `wasm::diamond` is compiled only for the browser and nothing native could reach it.
1224	///
1225	/// # Two maps, because of what an older build does with a pack it does not understand
1226	///
1227	/// `import_diamond` reads `files` and ignores what it does not know, so a device that has not been
1228	/// updated lays every text file down exactly as before and simply does not receive the pictures --
1229	/// which is what it does today, minus the corruption. A scheme that tagged entries inside `files`
1230	/// would have had the old importer write the base64 TEXT into the file.
1231	///
1232	/// # Arguments
1233	/// * `id` - The Diamond's id, which becomes a directory name on arrival.
1234	/// * `touched` - When anything under it last changed, repeated at the top level so a carrier can
1235	/// tell two copies apart without opening a file inside the pack.
1236	/// * `files` - Every file under the Diamond's directory, by its path relative to it.
1237	pub fn pack_diamond(id: &str, touched: u64, files: &[(String, Vec<u8>)]) -> String {
1238	pack(id, touched, PackKind::Diamond, "", files)
1239	}
1240
1241	/// The same pack, said to be a template and carrying the display name.
1242	///
1243	/// The name travels at the top level because `.daimond/meta.json` does not travel at all (see
1244	/// [`template_carries`]), and a Diamond that arrived with no name would be a tile with nothing on
1245	/// it but a cog. It is the same place `fe2o3_sbj`'s share format keeps it, and for the same
1246	/// reason.
1247	///
1248	/// # Arguments
1249	/// * `id` - Where the template came from. Provenance and nothing else: [`PackKind::Template`]
1250	/// says an importer must mint its own.
1251	/// * `name` - What to call the Diamond the template opens as.
1252	pub fn pack_template(
1253	id: &str,
1254	touched: u64,
1255	name: &str,
1256	files: &[(String, Vec<u8>)],
1257	)
1258	-> String
1259	{
1260	pack(id, touched, PackKind::Template, name, files)
1261	}
1262
1263	fn pack(
1264	id: &str,
1265	touched: u64,
1266	kind: PackKind,
1267	name: &str,
1268	files: &[(String, Vec<u8>)],
1269	)
1270	-> String
1271	{
1272	// Sorted, so two packs of an unchanged Diamond are the same bytes and a caller comparing states
1273	// sees no change where there is none.
1274	let mut sorted: Vec<&(String, Vec<u8>)> = files.iter().collect();
1275	sorted.sort_by(\|a, b\| a.0.cmp(&b.0));
1276	let mut text = Vec::new();
1277	let mut binary = Vec::new();
1278	for (path, bytes) in sorted {
1279	match std::str::from_utf8(bytes) {
1280	// Valid UTF-8 round-trips through JSON exactly, so it goes as itself and a pack stays
1281	// readable by a person.
1282	Ok(s) => text.push(fmt!("\"{}\":\"{}\"",
1283	crate::llm::json_escape(path), crate::llm::json_escape(s))),
1284	Err(_) => binary.push(fmt!("\"{}\":\"{}\"",
1285	crate::llm::json_escape(path), oxedyne_fe2o3_text::base64::encode(bytes))),
1286	}
1287	}
1288	// A whole Diamond says nothing, so a pack from a build that predates this field is byte for
1289	// byte what it always was -- which matters, because the sync compares a Diamond's pack with the
1290	// one it last sent and a new constant field would make every Diamond on every device look
1291	// changed once. Absence is unambiguous in the other direction too: nothing before this could
1292	// pack anything but a whole Diamond.
1293	let head = match kind {
1294	PackKind::Diamond => String::new(),
1295	PackKind::Template => fmt!(",\"{}\":\"{}\",\"{}\":\"{}\"",
1296	PACK_KIND, kind.wire(), PACK_NAME, crate::llm::json_escape(name)),
1297	};
1298	fmt!(
1299	"{{\"id\":\"{}\",\"touched\":{}{},\"files\":{{{}}},\"{}\":{{{}}}}}",
1300	crate::llm::json_escape(id), touched, head, text.join(","), PACK_BINARY, binary.join(","),
1301	)
1302	}
1303
1304	/// The key the files that are not text travel under.
1305	pub const PACK_BINARY: &str = "binary";
1306
1307	// ┌───────────────────────────────────────────────────────────────┐
1308	// │ What a pack will weigh, without building it │
1309	// └───────────────────────────────────────────────────────────────┘
1310
1311	// The JSON an entry costs beyond its path and its content: two pairs of quotes,
1312	// a colon and a comma.
1313	pub const PACK_ENTRY_OVERHEAD: u64 = 6;
1314
1315	/// How one file's bytes are weighed when a pack is estimated ahead of being
1316	/// built.
1317	///
1318	/// Not a guess about content -- a guess about PUNCTUATION, which is the only
1319	/// thing [`json_escape`](crate::llm::json_escape) charges for. A quote, a
1320	/// backslash, a newline, a tab or a return costs two bytes instead of one, and a
1321	/// control character six. Everything else, multibyte included, travels as
1322	/// itself.
1323	#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1324	enum Weigh {
1325	Base64, // not text, so `ceil(n/3) * 4` exactly -- an arithmetic fact, not an estimate
1326	Quoted, // three for two: JSON, where every quote doubles
1327	Markup, // five for four: attribute-dense but newline-broken
1328	Prose, // nine for eight: punctuation is rare in it
1329	}
1330
1331	/// How a file with this path will be carried, judged from its extension alone.
1332	///
1333	/// Unknown means [`Weigh::Quoted`], the heaviest of the four, so a file kind
1334	/// nobody has classified is never weighed lighter than it might travel.
1335	fn weigh(path: &str) -> Weigh {
1336	let ext = match path.rfind('.') {
1337	// A dot in a directory name is not this file's extension.
1338	Some(at) if !path[at..].contains('/') => &path[at + 1..],
1339	_ => return Weigh::Quoted,
1340	};
1341	match ext {
1342	"html" \| "htm" => Weigh::Markup,
1343	"md" \| "txt" => Weigh::Prose,
1344	// A Diamond's own patches, and the shapes a worker leaves beside them.
1345	// Anything not named here that is nonetheless binary is weighed as
1346	// quoted text instead, which is heavier than base64 and so still safe.
1347	"jpatch" \| "hpatch" => Weigh::Base64,
1348	"png" \| "jpg" \| "jpeg" \| "gif" \| "webp" => Weigh::Base64,
1349	"pdf" \| "zip" \| "docx" \| "xlsx" \| "pptx" => Weigh::Base64,
1350	"wasm" \| "woff" \| "woff2" \| "ttf" \| "otf" => Weigh::Base64,
1351	"mp3" \| "mp4" \| "m4a" \| "webm" \| "mov" => Weigh::Base64,
1352	// `.json`, `.jsonl`, the extensionless log, and everything unrecognised.
1353	_ => Weigh::Quoted,
1354	}
1355	}
1356
1357	/// What one file will weigh inside a pack, from its path and its size on disk.
1358	///
1359	/// **This is the estimate `export_size` spends a sync budget against, and every
1360	/// file used to be weighed at four bytes for three as though it travelled
1361	/// base64.** That figure was wrong in both directions at once, which is why it
1362	/// is now four figures and not one.
1363	///
1364	/// TOO HEAVY FOR MARKUP, WHICH IS THE BULK OF A DIAMOND. Valid UTF-8 travels as
1365	/// ITSELF through JSON, so what escaping adds to a page is a few per cent, not
1366	/// thirty-three. Over the 78 HTML files in this repository the cost runs from
1367	/// 1.025x to 1.126x, median 1.056x; over its markdown, at most 1.026x. The cost
1368	/// of the old figure was not theoretical: thirteen Diamonds at the page ceiling
1369	/// with five edits each, weighed that way, admitted five where the pack itself
1370	/// admitted seven -- two Diamonds refused a sync they would have fitted, and
1371	/// told the user they had been left behind.
1372	///
1373	/// FIVE-FOR-FOUR AND NOT NINE-FOR-EIGHT, because nine-for-eight was tried and
1374	/// fails: `www/console/index.html` escapes at 1.1262x and comes out twenty-nine
1375	/// bytes short of what the pack charges for it, which is the one direction this
1376	/// may not be wrong in. The margin over the worst real file is the price of the
1377	/// answer being safe, and it is why the estimate still reads high for a page
1378	/// that escapes as lightly as a capp's.
1379	///
1380	/// TOO LIGHT FOR JSON, WHICH NOBODY HAD NOTICED. A JSON file is dense in quotes
1381	/// and every one of them doubles. `lanes/diet.json` in the shipped capp measures
1382	/// 1.302x and `{"a":"b"}` measures 1.47x, so four-for-three was never a margin
1383	/// there -- it was an under-estimate, and the test that found it is
1384	/// `test_json_is_weighed_heavier_than_base64_because_every_quote_doubles`.
1385	/// Three-for-two covers both.
1386	///
1387	/// AND NOT ACTUALLY BASE64 EITHER, because base64 rounds up to a multiple of
1388	/// four and pads: eight bytes cost twelve, where four-for-three said ten. The
1389	/// binary figure is now the arithmetic rather than a ratio.
1390	///
1391	/// AN ESTIMATE, NOT A PROOF. What it has to be right about is the direction a
1392	/// REFUSAL points: a Diamond this says will not fit is never built, so that
1393	/// answer has to hold. One it lets through is measured again exactly, against
1394	/// the pack that actually travels, so an over-optimistic answer costs one
1395	/// Diamond materialised and dropped -- which is what every Diamond cost before
1396	/// this existed -- and never a parcel over the door.
1397	pub fn packed_weight(path: &str, size: u64) -> u64 {
1398	let bytes = match weigh(path) {
1399	// `ceil(n / 3) * 4`, which is what base64 of n bytes is.
1400	Weigh::Base64 => size.saturating_add(2) / 3 * 4,
1401	Weigh::Quoted => size.saturating_mul(3) / 2,
1402	Weigh::Markup => size.saturating_mul(5) / 4,
1403	Weigh::Prose => size.saturating_mul(9) / 8,
1404	};
1405	bytes.saturating_add(path.len() as u64).saturating_add(PACK_ENTRY_OVERHEAD)
1406	}
1407
1408
1409	/// The bytes of one file from a pack's `binary` map.
1410	///
1411	/// One function, so the browser's importer and the test that proves a picture survives are decoding
1412	/// by the same rule. A value that is not base64 is refused rather than written as whatever decoded:
1413	/// half a picture over a good one is the corruption this exists to remove, and it would arrive
1414	/// looking like a successful sync.
1415	pub fn unpack_binary(path: &str, body: &str) -> Outcome<Vec<u8>> {
1416	Ok(res!(oxedyne_fe2o3_text::base64::decode(body).map_err(\|e\| err!(e,
1417	"The file '{}' in a Diamond export is not valid base64, so its bytes cannot be recovered.",
1418	path; Invalid, Input, Decode))))
1419	}
1420
1421	// ┌───────────────────────────────────────────────────────────────┐
1422	// │ A template: the shape without the contents │
1423	// └───────────────────────────────────────────────────────────────┘
1424
1425	// The path prefixes a template drops, over and above the ones the share format already refuses.
1426	//
1427	// `log/` is a capp's ENTRIES. It is refused by name rather than by inspection because that is
1428	// where the shipped Log Life page puts what its user records, and `cappNeverDelivered` in
1429	// `www/js/daimond.js` already refuses a served template the same path from the other direction.
1430	// One rule, in the two places that have to agree about it.
1431	const TEMPLATE_DROP_PREFIXES: [&str; 1] = ["log/"];
1432
1433	// The exact paths a template drops.
1434	//
1435	// `crystal.json` is the Diamond's MEMORY -- what its folds accumulated -- and `crystal.md` is the
1436	// same memory before the migration. `transcript.md` is a conversation kept whole. A bare `log`
1437	// is the file form of the prefix above.
1438	// `triggers.json` is dropped for a reason of its own, and it is the only entry here that is
1439	// not about the sender's privacy: a trigger is automation that fires WITHOUT anybody pressing
1440	// anything, so a template carrying one would start work on a stranger's machine because they
1441	// opened a file. Disarming rather than dropping was considered and refused: `on: false` does
1442	// NOT disarm a trigger -- the pause tree is the authority and a trigger left in the file stays
1443	// armed under it -- so a template that carried one and claimed it was off would be worse than
1444	// one that carries none. The receiver sets up their own, which is a sentence of documentation
1445	// and not a data problem.
1446	const TEMPLATE_DROP_EXACT: [&str; 5] =
1447	["crystal.json", "crystal.md", "transcript.md", "log", "triggers.json"];
1448
1449	/// Does a template carry this file?
1450	///
1451	/// The rule is DROP A KNOWN LIST AND KEEP THE REST, which is the way round it has to be: a capp
1452	/// keeps its data beside itself, in folders this build has never heard of, and a template built
1453	/// from a list of what to keep would silently lose whichever folder the refinement was in. That is
1454	/// the failure `cappManifest` exists to avoid on the delivery side.
1455	///
1456	/// Three of the drops are `fe2o3_sbj`'s and are not invented here -- `.daimond/`, `versions/` and
1457	/// the `capp.json` delivery record -- so an unsealed template and a sealed share refuse the same
1458	/// paths for the same stated reasons and the two cannot drift. What this adds is the memory, the
1459	/// kept conversation and a capp's entries, because a share is a copy of a Diamond for one named
1460	/// person and a template is its shape for anybody at all.
1461	///
1462	/// # Arguments
1463	/// * `rel` - The path relative to the Diamond's own directory.
1464	pub fn template_carries(rel: &str) -> bool {
1465	for prefix in share::REFUSED_PREFIXES {
1466	if rel.starts_with(prefix) {
1467	return false;
1468	}
1469	}
1470	if rel == share::REFUSED_EXACT {
1471	return false;
1472	}
1473	for prefix in TEMPLATE_DROP_PREFIXES.iter() {
1474	if rel.starts_with(prefix) {
1475	return false;
1476	}
1477	}
1478	!TEMPLATE_DROP_EXACT.contains(&rel)
1479	}
1480
1481	/// What of a Diamond's directory goes into a template.
1482	///
1483	/// `with_conversation` is the door back to a complete copy: everything travels, exactly as
1484	/// [`pack_diamond`] would have carried it, and the pack is still a [`PackKind::Template`], so it
1485	/// still opens as a new Diamond rather than over the one it came from.
1486	pub fn template_files(
1487	files: &[(String, Vec<u8>)],
1488	with_conversation: bool,
1489	)
1490	-> Vec<(String, Vec<u8>)>
1491	{
1492	files.iter()
1493	.filter(\|(rel, _)\| with_conversation \|\| template_carries(rel))
1494	.cloned()
1495	.collect()
1496	}
1497
1498	// The files inside a Diamond that name it by id, and so have to be readdressed when a copy opens
1499	// under a new one.
1500	//
1501	// Both are the app's own records rather than anybody's content, and neither is in a shape-only
1502	// template at all -- they arrive only with `with_conversation`. The log's `delta_ref` holds a
1503	// PATH written when the fold was applied, which is what `rewrite_delta_refs` exists for on the
1504	// move between Diamond roots; a link's ends are `diamond:<id>` and `file:diamonds/<id>/...`. Left
1505	// alone, every stored delta and every link in a complete copy would point at the directory it was
1506	// copied from.
1507	const TEMPLATE_RETARGET: [&str; 2] = [".daimond/log", ".daimond/links.jsonl"];
1508
1509	/// One file of a template, readdressed from the id it was packed under to the one it is landing at.
1510	///
1511	/// Only the two files that carry an id are touched, and only where they are text. A blanket
1512	/// search and replace over everything would reach a page's prose and a picture's bytes, where the
1513	/// id is not a reference to anything.
1514	pub fn retarget(
1515	rel: &str,
1516	body: Vec<u8>,
1517	old_id: &str,
1518	new_id: &str,
1519	)
1520	-> Vec<u8>
1521	{
1522	if old_id == new_id \|\| old_id.is_empty() \|\| !TEMPLATE_RETARGET.contains(&rel) {
1523	return body;
1524	}
1525	let text = match String::from_utf8(body) {
1526	Ok(t) => t,
1527	Err(e) => return e.into_bytes(), // not text: no reference here to readdress
1528	};
1529	text
1530	.replace(&fmt!("diamonds/{}/", old_id), &fmt!("diamonds/{}/", new_id))
1531	.replace(&fmt!("diamond:{}", old_id), &fmt!("diamond:{}", new_id))
1532	.into_bytes()
1533	}
1534
1535	// How many ids are tried before an import gives up.
1536	//
1537	// A collision needs two ids minted in the same millisecond by the same counter, so one retry would
1538	// almost certainly do. The figure is what turns "almost certainly" into a bounded loop that fails
1539	// loudly, rather than a guess that fails silently over somebody's Diamond.
1540	const FRESH_ID_TRIES: usize = 32;
1541
1542	/// An id no Diamond in `taken` is using.
1543	///
1544	/// This is what stops an imported template writing over a Diamond that is already there. The
1545	/// pack's id is NOT consulted: a template says where it came from, and a Log Life template imported
1546	/// by the person who made it would otherwise land on the Log Life it was made from and delete it.
1547	/// That is the shape this project has already lost a user's tags to.
1548	///
1549	/// # Arguments
1550	/// * `taken` - Every id the store already holds.
1551	pub fn fresh_id(taken: &[String]) -> Outcome<String> {
1552	fresh_id_from(taken, generate_session_id)
1553	}
1554
1555	/// [`fresh_id`], with the mint named, so a test can hand it one that collides on purpose.
1556	pub fn fresh_id_from(taken: &[String], mint: fn() -> String) -> Outcome<String> {
1557	for _ in 0..FRESH_ID_TRIES {
1558	let id = mint();
1559	if !taken.iter().any(\|t\| t == &id) {
1560	return Ok(id);
1561	}
1562	}
1563	Err(err!(
1564	"{} ids were minted for an imported template and every one of them was already a Diamond \
1565	on this device, so the import stopped rather than write over one.", FRESH_ID_TRIES;
1566	Conflict, Data))
1567	}
1568
1569
1570	// ┌───────────────────────────────────────────────────────────────┐
1571	// │ Tests │
1572	// └───────────────────────────────────────────────────────────────┘
1573
1574	#[cfg(test)]
1575	mod content_tests {
1576	use super::*;
1577
1578	/// The one-pixel PNG Anthropic prints in its vision documentation, decoded.
1579	///
1580	/// Source: `platform.claude.com/docs/en/build-with-claude/vision`. Real bytes from a real
1581	/// provider document rather than a header this test invented, so the sniffer is being checked
1582	/// against a file the world agrees is a PNG.
1583	fn doc_png() -> Vec<u8> {
1584	oxedyne_fe2o3_text::base64::decode(
1585	"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLv\
1586	AAAAAElFTkSuQmCC").expect("the documented base64 must decode")
1587	}
1588
1589	/// A picture put into a Diamond pack comes out of it BYTE FOR BYTE.
1590	///
1591	/// This is the test the corruption escaped for want of. `export_diamond` ran every file through
1592	/// `String::from_utf8_lossy`, which replaces each byte sequence that is not valid UTF-8 with
1593	/// U+FFFD -- so a PNG arrived at the far end at roughly the right size, shaped like a PNG, and
1594	/// would not open. Neither end reported a fault, because as far as either could see the file had
1595	/// travelled. Run against that code this fails on the very first assertion: the pack's `files`
1596	/// map holds the picture's path and its bytes are not the picture's.
1597	#[test]
1598	fn test_the_models_working_travels_as_its_own_kind_and_not_as_text() {
1599	// The wire name is the whole contract with the page: `type: "thinking"` is what
1600	// decides whether a chunk is drawn in the answer or in a shut tile beside it. It is
1601	// asserted here rather than left to the browser, because a reasoning block that
1602	// arrived typed as `text` would be appended to the reply -- the model's working
1603	// mistaken for what it said, and then stored and sent back as if it had said it.
1604	let m = AgentEvent::Thinking(fmt!("1071 = 2 x 462 + 147")).to_datmap();
1605	assert_eq!(m.get(&dat!("type")), Some(&dat!("thinking")),
1606	"reasoning did not travel as its own kind");
1607	assert_eq!(m.get(&dat!("content")), Some(&dat!("1071 = 2 x 462 + 147")),
1608	"the working was not carried whole");
1609	// And it is NOT the same shape as an answer, which is the distinction the page
1610	// depends on. A single map with both spellings would be indistinguishable.
1611	let t = AgentEvent::Text(fmt!("1071 = 2 x 462 + 147")).to_datmap();
1612	assert_ne!(t.get(&dat!("type")), m.get(&dat!("type")),
1613	"an answer and the working behind it are the same kind on the wire");
1614	}
1615
1616	#[test]
1617	fn test_a_picture_survives_a_diamond_pack() {
1618	let png = doc_png();
1619	assert!(std::str::from_utf8(&png).is_err(), "the fixture has to be bytes that are not text");
1620	let files = vec![
1621	// No braces in it: the files map is found below by brace matching, and a `}` inside a
1622	// string value would cut the search short and make the check that follows vacuous.
1623	("notes.txt".to_string(), b"a \"quoted\" word\nand a \\ backslash\n".to_vec()),
1624	("img/one.png".to_string(), png.clone()),
1625	];
1626	let pack = pack_diamond("d1", 42, &files);
1627
1628	// The picture is NOT in `files`, because `files` is the text map and an older importer
1629	// would write whatever is there straight to disk. Located by brace matching rather than by
1630	// `extract_json_string`, which reads a STRING value and would answer nothing for an object
1631	// -- and a check that can only pass is not a check.
1632	let at = pack.find("\"files\":{").expect("the pack has no files map");
1633	let end = pack[at..].find('}').expect("the files map does not close") + at;
1634	let text_map = &pack[at..end];
1635	assert!(text_map.contains("notes.txt"),
1636	"the text file is not in the text map, so this test is looking in the wrong place: {}",
1637	text_map);
1638	assert!(!text_map.contains("img/one.png"),
1639	"a binary file was put in the text map, where an older build would write it as text");
1640	assert!(pack.contains(&fmt!("\"{}\":{{\"img/one.png\"", PACK_BINARY)),
1641	"the picture is not in the binary map: {}", pack);
1642
1643	// And it IS recoverable, exactly.
1644	let body = crate::llm::extract_json_string(&pack, "img/one.png")
1645	.expect("the pack does not carry the picture at all");
1646	let back = unpack_binary("img/one.png", &body).expect("the picture's base64 must decode");
1647	assert_eq!(back, png, "the picture did not survive the round trip");
1648
1649	// The text file still travels as text, and its escaping survives.
1650	let note = crate::llm::extract_json_string(&pack, "notes.txt")
1651	.expect("the pack does not carry the text file");
1652	assert_eq!(note.as_bytes(), files[0].1.as_slice(),
1653	"a text file's own quotes did not survive the pack");
1654	}
1655
1656	/// The pack is the same bytes twice over, so a carrier comparing states sees no change where
1657	/// there is none.
1658	#[test]
1659	fn test_a_pack_of_an_unchanged_diamond_is_the_same_bytes() {
1660	let files = vec![
1661	("b.txt".to_string(), b"second".to_vec()),
1662	("a.png".to_string(), vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00]),
1663	];
1664	let one = pack_diamond("d1", 7, &files);
1665	let mut other = files.clone();
1666	other.reverse();
1667	assert_eq!(one, pack_diamond("d1", 7, &other),
1668	"the order the directory walk happened to return changed the pack");
1669	}
1670
1671	/// Each format is recognised from its own header, and a RIFF container that is not a WebP is
1672	/// not mistaken for one.
1673	#[test]
1674	fn test_the_media_type_is_read_from_the_bytes() {
1675	assert_eq!(Some(ImageMedia::Png), ImageMedia::sniff(&doc_png()));
1676	assert_eq!(Some(ImageMedia::Jpeg), ImageMedia::sniff(&[0xFF, 0xD8, 0xFF, 0xE0, 0, 0]));
1677	assert_eq!(Some(ImageMedia::Gif), ImageMedia::sniff(b"GIF89a\x01\x00"));
1678	assert_eq!(Some(ImageMedia::WebP), ImageMedia::sniff(b"RIFF\x24\x00\x00\x00WEBPVP8 "));
1679	// A WAV is a RIFF too, and is not an image.
1680	assert_eq!(None, ImageMedia::sniff(b"RIFF\x24\x00\x00\x00WAVEfmt "));
1681	assert_eq!(None, ImageMedia::sniff(b"fn main() {}"));
1682	assert_eq!(None, ImageMedia::sniff(b""));
1683	}
1684
1685	/// The media type survives a round trip through its own spelling, and an unknown spelling is
1686	/// refused rather than guessed.
1687	#[test]
1688	fn test_a_media_type_round_trips_through_its_mime() {
1689	for m in [ImageMedia::Png, ImageMedia::Jpeg, ImageMedia::Gif, ImageMedia::WebP] {
1690	assert_eq!(Some(m), ImageMedia::from_mime(m.mime()));
1691	}
1692	assert_eq!(None, ImageMedia::from_mime("image/tiff"));
1693	}
1694
1695	/// A list of nothing but text collapses back to text, so two contents that say the same thing
1696	/// are the same content.
1697	#[test]
1698	fn test_an_all_text_parts_list_collapses() {
1699	let c = MessageContent::parts(vec![
1700	ContentPart::Text("one".to_string()),
1701	ContentPart::Text("two".to_string()),
1702	]);
1703	assert_eq!(MessageContent::text("one\ntwo"), c);
1704	assert!(!c.has_image());
1705
1706	let with_image = MessageContent::parts(vec![
1707	ContentPart::Text("look".to_string()),
1708	ContentPart::Image(ImagePart::new(ImageMedia::Png, doc_png(), "a.png".to_string())),
1709	]);
1710	assert!(matches!(with_image, MessageContent::Parts(_)));
1711	assert!(with_image.has_image());
1712	assert_eq!(1, with_image.images().count());
1713	}
1714
1715	/// An image's bytes are not counted as text, and its stand-in names it.
1716	#[test]
1717	fn test_an_image_is_named_in_the_text_and_not_weighed_as_text() {
1718	let c = MessageContent::parts(vec![
1719	ContentPart::Text("look".to_string()),
1720	ContentPart::Image(ImagePart::new(
1721	ImageMedia::Png, vec![0u8; 100_000], "shots/a.png".to_string())),
1722	]);
1723	assert!(c.text_len() < 200, "the image's bytes were counted as text: {}", c.text_len());
1724	let t = c.as_text();
1725	assert!(t.contains("shots/a.png"), "{}", t);
1726	assert!(t.contains("image/png"), "{}", t);
1727	}
1728
1729	/// Dropping the images leaves a line naming each, and nothing else changes.
1730	#[test]
1731	fn test_dropping_the_images_leaves_their_names() {
1732	let c = MessageContent::parts(vec![
1733	ContentPart::Text("here it is".to_string()),
1734	ContentPart::Image(ImagePart::new(
1735	ImageMedia::Png, doc_png(), "shots/a.png".to_string())),
1736	]);
1737	let out = c.without_images(Dropped::ToFit);
1738	assert!(!out.has_image());
1739	let t = out.as_text();
1740	assert!(t.contains("here it is"), "the prose was lost: {}", t);
1741	assert!(t.contains("shots/a.png"), "the file was not named: {}", t);
1742	assert!(t.contains("read it again"), "the model was not told what to do: {}", t);
1743
1744	// The other reason, which must NOT say that -- on an endpoint that will not take
1745	// pictures, "read it again" is a loop, and the two reasons are a type precisely so
1746	// that one cannot be told to do the thing that cannot help.
1747	let blind = c.without_images(Dropped::Unseeable);
1748	let b = blind.as_text();
1749	assert!(b.contains("here it is"), "the prose was lost: {}", b);
1750	assert!(b.contains("shots/a.png"), "the file was not named: {}", b);
1751	assert!(!b.contains("read it again"), "it was told to retry a read that cannot help: {}", b);
1752	assert!(b.contains("cannot be shown"), "it was not told why: {}", b);
1753	assert!(b.contains("Do not describe"),
1754	"nothing stopped it describing a picture it never saw: {}", b);
1755	}
1756
1757	/// A message carrying an image survives storage byte for byte.
1758	///
1759	/// The store is where a session lives between turns; a part that cannot be written and read
1760	/// back is a part the model loses on the first reload, silently.
1761	#[test]
1762	fn test_a_message_with_an_image_round_trips_through_the_store() {
1763	let img = ImagePart::new(ImageMedia::Png, doc_png(), "shots/a.png".to_string());
1764	let msg = ChatMessage::tool("call_1".to_string(), MessageContent::parts(vec![
1765	ContentPart::Text("Read the image shots/a.png.".to_string()),
1766	ContentPart::Image(img.clone()),
1767	]));
1768	let back = ChatMessage::from_datmap(&msg.to_datmap()).expect("read back");
1769	assert_eq!(msg, back);
1770	let got = back.content().images().next().expect("the image was lost").clone();
1771	assert_eq!(img.data, got.data, "the bytes changed in storage");
1772	assert_eq!(ImageMedia::Png, got.media);
1773	assert_eq!("shots/a.png", got.source);
1774	}
1775
1776	/// Plain text still writes the bare string the store has always held, so a snapshot from
1777	/// before this change reads back unchanged.
1778	#[test]
1779	fn test_plain_text_keeps_the_shape_the_store_always_had() {
1780	let msg = ChatMessage::user("hello".to_string());
1781	let m = msg.to_datmap();
1782	assert_eq!(Some(&dat!("hello")), m.get(&dat!("content")),
1783	"text content is no longer a bare string in the store");
1784	assert_eq!(msg, ChatMessage::from_datmap(&m).expect("read back"));
1785	}
1786	}
1787
1788	#[cfg(test)]
1789	mod tests {
1790	use super::*;
1791
1792	// ── What a pack will weigh, without building it ──
1793
1794	/// The shipped Log Life capp page, and the heaviest-escaping HTML in this
1795	/// repository -- 1.126x, against a median of 1.056x over its 78 HTML files.
1796	/// A factor that holds for the worst one holds for the rest.
1797	const CAPP_PAGE: &str = include_str!("../www/capps/lifelog/crystal.html");
1798	const DENSE_HTML: &str = include_str!("../www/console/index.html");
1799
1800	/// What a pack really charges for one file: the pack with it, less the pack
1801	/// without it.
1802	///
1803	/// Measured through [`pack_diamond`] rather than by a second copy of its
1804	/// rules, because a test that re-implements the thing it is checking agrees
1805	/// with the bug as readily as with the code.
1806	fn charged(path: &str, body: &[u8]) -> u64 {
1807	let with = pack_diamond("d", 1, &[(path.to_string(), body.to_vec())]);
1808	let without = pack_diamond("d", 1, &[]);
1809	(with.len() - without.len()) as u64
1810	}
1811
1812	/// The property the sync budget rests on. A file the estimate weighs is
1813	/// never weighed lighter than the pack will charge for it -- so a Diamond
1814	/// the estimate refuses really would not have fitted, and that Diamond is
1815	/// the one that is never built.
1816	#[test]
1817	fn test_no_file_is_weighed_lighter_than_the_pack_charges_for_it() {
1818	let log = b"{\"id\":\"a\",\"ts\":1,\"kind\":\"fold\",\"note\":\"a \\\"quoted\\\" thing\"}\n";
1819	let cases: Vec<(&str, &[u8])> = vec![
1820	("crystal.html", CAPP_PAGE.as_bytes()),
1821	("versions/0021.html", DENSE_HTML.as_bytes()),
1822	("crystal.json", br#"{"title":"A crystal","notes":["one","two"]}"#),
1823	("crystal.md", b"# A crystal\n\nWords, a \"quote\", and a tab\there.\n"),
1824	(".daimond/log", log),
1825	(".daimond/links.jsonl", br#"{"from":"a","to":"b","rel":"cites"}"#),
1826	("versions/0022.hpatch", &[0u8, 0xff, 0x41, 0x0a, 0x22, 0x5c, 0x7f, 0x80]),
1827	("notes.txt", b"plain words, and a newline\n"),
1828	("worker/shot.png", &[0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
1829	("worker/no_extension", b"could be anything at all"),
1830	("worker/asset.png", &[0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]),
1831	("odd.dir.name/inside.html", b"<p>a dot in the directory is not the file's</p>"),
1832	];
1833	for (path, body) in cases {
1834	let want = charged(path, body);
1835	let got = packed_weight(path, body.len() as u64);
1836	assert!(
1837	got >= want,
1838	"'{}' is weighed at {} and the pack charges {}, so a Diamond holding it could be \
1839	refused a parcel it would have fitted", path, got, want);
1840	}
1841	}
1842
1843	/// The consequence, stated as the thing the user loses. A capp Diamond at
1844	/// the page ceiling, weighed the old way, was refused parcels it would have
1845	/// fitted. The test is not how close the estimate is; it is how many
1846	/// Diamonds a 4 MiB budget admits.
1847	#[test]
1848	fn test_more_diamonds_fit_the_parcel_than_the_old_weighing_admitted() {
1849	// A Diamond at the page ceiling with a handful of edits: the live page,
1850	// two keyframes, the patches between them, the memory and its snapshots.
1851	let page = CAPP_PAGE.len() as u64 * 5; // roughly the 512 KiB ceiling
1852	let mut files: Vec<(String, u64)> = vec![
1853	("crystal.html".to_string(), page),
1854	("versions/0001.html".to_string(), page),
1855	("crystal.json".to_string(), 15_000),
1856	("versions/0001.json".to_string(), 15_000),
1857	(".daimond/log".to_string(), 4_000),
1858	];
1859	for v in 2..=6u64 {
1860	files.push((fmt!("versions/{:04}.hpatch", v), 900));
1861	files.push((fmt!("versions/{:04}.jpatch", v), 300));
1862	}
1863	let one_new: u64 = files.iter().map(\|(p, n)\| packed_weight(p, *n)).sum();
1864	let one_old: u64 = files.iter()
1865	.map(\|(p, n)\| n.saturating_mul(4) / 3 + p.len() as u64).sum();
1866	// What the pack really charges for the same set, measured through it.
1867	let bodies: Vec<(String, Vec<u8>)> = files.iter().map(\|(p, n)\| {
1868	let body = if p.ends_with(".html") {
1869	CAPP_PAGE.as_bytes().iter().cycle().take(*n as usize).cloned().collect()
1870	} else if p.ends_with("patch") {
1871	(0..*n).map(\|i\| (i % 251) as u8).collect()
1872	} else {
1873	let mut b = Vec::new();
1874	while (b.len() as u64) < *n { b.extend_from_slice(br#"{"k":"v","j":"w"},"#); }
1875	b.truncate(*n as usize);
1876	b
1877	};
1878	(p.clone(), body)
1879	}).collect();
1880	let one_real = (pack_diamond("d", 1, &bodies).len()
1881	- pack_diamond("d", 1, &[]).len()) as u64;
1882	let budget = 4u64 * 1024 * 1024;
1883	let fits = \|each: u64\| if each == 0 { 0 } else { budget / each };
1884	assert!(
1885	one_new >= one_real,
1886	"the new weighing is optimistic: {} against a real {}", one_new, one_real);
1887	assert!(
1888	fits(one_new) > fits(one_old),
1889	"the same store admits {} Diamonds under the new weighing and {} under the old, so \
1890	nothing was recovered (real {})", fits(one_new), fits(one_old), fits(one_real));
1891	assert_eq!(
1892	fits(one_new), fits(one_real),
1893	"the new weighing admits {} where the pack itself admits {}",
1894	fits(one_new), fits(one_real));
1895	}
1896
1897	/// The half nobody had noticed. Four-for-three was not a margin for
1898	/// JSON, it was an under-estimate: every quote doubles, so a quote-dense
1899	/// object costs more than base64 of the same bytes would.
1900	#[test]
1901	fn test_json_is_weighed_heavier_than_base64_because_every_quote_doubles() {
1902	let dense = br#"{"a":"b","c":"d","e":"f","g":"h","i":"j","k":"l"}"#;
1903	let want = charged("crystal.json", dense);
1904	let base64ish = dense.len() as u64 * 4 / 3 + "crystal.json".len() as u64;
1905	assert!(
1906	want > base64ish,
1907	"quote-dense JSON was expected to cost more than four for three; it charged {} where \
1908	four for three is {}", want, base64ish);
1909	assert!(
1910	packed_weight("crystal.json", dense.len() as u64) >= want,
1911	"weighed {} against a charge of {}",
1912	packed_weight("crystal.json", dense.len() as u64), want);
1913	}
1914
1915	/// Base64 rounds up to a multiple of four and pads, so four-for-three is
1916	/// short for every length that is not a multiple of three.
1917	#[test]
1918	fn test_a_patch_is_weighed_by_the_arithmetic_of_base64_and_not_by_a_ratio() {
1919	for n in 1..=64usize {
1920	let body: Vec<u8> = (0..n).map(\|i\| if i == 0 { 0xff } else { (i % 251) as u8 }).collect();
1921	let want = charged("versions/0002.hpatch", &body);
1922	let got = packed_weight("versions/0002.hpatch", n as u64);
1923	assert!(got >= want, "{} bytes weighed {} against a charge of {}", n, got, want);
1924	}
1925	}
1926
1927	/// A file kind nobody has classified is weighed heavily, so a new one
1928	/// arriving in a Diamond cannot quietly be under-weighed.
1929	///
1930	/// The two roads to that answer are both walked: an extension the match
1931	/// does not know, and a name with no extension for it to read at all. The
1932	/// first list here was all of the second kind, which left the arm that
1933	/// actually decides an unknown EXTENSION covered by nothing -- found by
1934	/// `dev/breakproof_deltalog.sh`, which changed that arm and watched this
1935	/// test stay green.
1936	#[test]
1937	fn test_an_unknown_kind_is_weighed_at_the_heavy_figure() {
1938	let n = 30_000u64;
1939	let heavy = n * 4 / 3;
1940	let paths = [
1941	// An extension the match does not know.
1942	"worker/data.sqlite", "worker/thing.bin", "crystal.json",
1943	".daimond/links.jsonl", "worker/notes.rtf",
1944	// And no extension to read.
1945	"worker/thing", ".daimond/log", "a.b/c",
1946	// A known-binary kind, which must still not come out under base64.
1947	"worker/thing.wasm",
1948	];
1949	for path in paths {
1950	let got = packed_weight(path, n);
1951	assert!(
1952	got >= heavy,
1953	"'{}' is weighed at {} and an unclassified file must be weighed at least {}",
1954	path, got, heavy);
1955	}
1956	}
1957
1958	#[test]
1959	fn test_chat_message_roundtrip() {
1960	let msg = ChatMessage::user("Hello".to_string());
1961	let dm = msg.to_datmap();
1962	let msg2 = ChatMessage::from_datmap(&dm).unwrap();
1963	assert_eq!(msg, msg2);
1964	}
1965
1966	#[test]
1967	fn test_chat_message_tool_roundtrip() {
1968	let msg = ChatMessage::Tool {
1969	tool_call_id: "call_123".to_string(),
1970	content: MessageContent::text("42"),
1971	};
1972	let dm = msg.to_datmap();
1973	let msg2 = ChatMessage::from_datmap(&dm).unwrap();
1974	assert_eq!(msg, msg2);
1975	}
1976
1977	#[test]
1978	fn test_session_roundtrip() {
1979	let mut s = Session::new("s1".to_string(), "Test".to_string(), "glm-5p2".to_string());
1980	s.messages.push(ChatMessage::user("Hi".to_string()));
1981	s.messages.push(ChatMessage::assistant("Hello!".to_string()));
1982	let dm = s.to_datmap();
1983	let s2 = Session::from_datmap(&dm).unwrap();
1984	assert_eq!(s.id, s2.id);
1985	assert_eq!(s.name, s2.name);
1986	assert_eq!(s.model, s2.model);
1987	assert_eq!(s.messages.len(), s2.messages.len());
1988	assert_eq!(s.messages[0], s2.messages[0]);
1989	assert_eq!(s.messages[1], s2.messages[1]);
1990	}
1991
1992	#[test]
1993	fn test_user_config_roundtrip() {
1994	let uc = UserConfig::new("jason".to_string(), "glm-5p2".to_string());
1995	let dm = uc.to_datmap();
1996	let uc2 = UserConfig::from_datmap(&dm).unwrap();
1997	assert_eq!(uc.username, uc2.username);
1998	assert_eq!(uc.default_model, uc2.default_model);
1999	}
2000
2001	#[test]
2002	fn test_agent_event_text() {
2003	let ev = AgentEvent::Text("hello".to_string());
2004	let dm = ev.to_datmap();
2005	assert_eq!(dm.get(&dat!("type")), Some(&dat!("text")));
2006	assert_eq!(dm.get(&dat!("content")), Some(&dat!("hello")));
2007	}
2008
2009	#[test]
2010	fn test_agent_event_done() {
2011	let ev = AgentEvent::Done;
2012	let dm = ev.to_datmap();
2013	assert_eq!(dm.get(&dat!("type")), Some(&dat!("done")));
2014	}
2015
2016	#[test]
2017	fn test_session_id_unique() {
2018	let id1 = generate_session_id();
2019	let id2 = generate_session_id();
2020	assert_ne!(id1, id2);
2021	}
2022
2023	#[test]
2024	fn test_session_datmap_round_trip() {
2025	let mut s = Session::new("s1".to_string(), "Test".to_string(), "glm-5.2".to_string());
2026	s.prompt_tokens = 10240;
2027	s.completion_tokens = 512;
2028	s.last_prompt_tokens = 8192;
2029	s.cached_tokens = 9216;
2030	s.cost_usd = 0.0021;
2031	let back = match Session::from_datmap(&s.to_datmap()) {
2032	Ok(v) => v,
2033	Err(e) => panic!("round trip failed: {}", e),
2034	};
2035	assert_eq!(back.cached_tokens, 9216);
2036	assert_eq!(back.cost_usd, 0.0021);
2037	assert_eq!(back.prompt_tokens, 10240);
2038	}
2039
2040	#[test]
2041	fn test_session_datmap_without_new_fields() {
2042	// A snapshot written before `cached_tokens` and `cost_usd` existed. It
2043	// must still load, at zero -- this is what makes the two fields optional
2044	// in practice, and a required field here would refuse a real history.
2045	let mut m = DaticleMap::new();
2046	m.insert(dat!("id"), dat!("s1"));
2047	m.insert(dat!("name"), dat!("Old"));
2048	m.insert(dat!("created_at"), Dat::U64(1));
2049	m.insert(dat!("model"), dat!("glm-5.2"));
2050	m.insert(dat!("prompt_tokens"), Dat::U64(7));
2051	m.insert(dat!("completion_tokens"), Dat::U64(3));
2052	m.insert(dat!("last_prompt_tokens"), Dat::U64(7));
2053	let s = match Session::from_datmap(&m) {
2054	Ok(v) => v,
2055	Err(e) => panic!("an older snapshot must still load: {}", e),
2056	};
2057	assert_eq!(s.prompt_tokens, 7);
2058	assert_eq!(s.cached_tokens, 0);
2059	assert_eq!(s.cost_usd, 0.0);
2060	}
2061
2062	// ── What a session store must give back ─────────────────────────────
2063	//
2064	// The rule every OpenAI-compatible provider enforces, written here and nowhere
2065	// else in the file: an assistant message bearing `tool_calls` must be followed
2066	// by one `tool` reply per call, in order, and a `tool` reply must answer a
2067	// call. Deliberately not `compact::orphan_count` -- a round trip checked with
2068	// the app's own notion of pairing proves only that the app agrees with itself.
2069
2070	/// Tool calls with no reply, plus tool replies with no call.
2071	fn unanswered(msgs: &[ChatMessage]) -> usize {
2072	let mut n = 0;
2073	let mut i = 0;
2074	while i < msgs.len() {
2075	match &msgs[i] {
2076	ChatMessage::Assistant { tool_calls, .. } if !tool_calls.is_empty() => {
2077	let mut k = 0;
2078	while k < tool_calls.len() {
2079	match msgs.get(i + 1 + k) {
2080	Some(ChatMessage::Tool { tool_call_id, .. })
2081	if *tool_call_id == tool_calls[k].id => k += 1,
2082	_ => break,
2083	}
2084	}
2085	n += tool_calls.len() - k;
2086	i += 1 + k;
2087	}
2088	ChatMessage::Tool { .. } => { n += 1; i += 1; }
2089	_ => i += 1,
2090	}
2091	}
2092	n
2093	}
2094
2095	/// A session in the shape a tool loop leaves behind: a question, the assistant
2096	/// turn that asked for a tool, the reply, and the answer.
2097	fn session_with_a_tool_call() -> Session {
2098	let mut s = Session::new(fmt!("s1"), fmt!("Work"), fmt!("glm-5.2"));
2099	s.messages.push(ChatMessage::user(fmt!("read the parser")));
2100	s.messages.push(ChatMessage::Assistant {
2101	content: MessageContent::text(fmt!("Looking.")),
2102	tool_calls: vec![ToolCall {
2103	id: fmt!("call_abc"),
2104	name: fmt!("file_read"),
2105	arguments: fmt!("{{\"path\":\"src/parse.rs\"}}"),
2106	}],
2107	});
2108	s.messages.push(ChatMessage::Tool {
2109	tool_call_id: fmt!("call_abc"),
2110	content: MessageContent::text(fmt!("fn parse() {{}}")),
2111	});
2112	s.messages.push(ChatMessage::Assistant {
2113	content: MessageContent::text(fmt!("It is one function.")), tool_calls: Vec::new(),
2114	});
2115	s
2116	}
2117
2118	/// Store and fetch a session the way `SessionStore` does: a `Dat::Map` written
2119	/// as bytes and read back.
2120	fn round_trip(s: &Session) -> Session {
2121	let stored = Dat::Map(s.to_datmap());
2122	let bytes = match stored.to_bytes(Vec::new()) {
2123	Ok(b) => b,
2124	Err(e) => panic!("a session must be storable: {}", e),
2125	};
2126	let (back, _) = match Dat::from_bytes(&bytes) {
2127	Ok(v) => v,
2128	Err(e) => panic!("a stored session must be readable: {}", e),
2129	};
2130	match back {
2131	Dat::Map(m) => match Session::from_datmap(&m) {
2132	Ok(v) => v,
2133	Err(e) => panic!("a stored session must decode: {}", e),
2134	},
2135	other => panic!("a session stored as a map came back as {:?}", other.kind()),
2136	}
2137	}
2138
2139	#[test]
2140	fn test_a_reloaded_session_is_one_a_provider_would_accept_00() {
2141	// The bug this replaces: `to_datmap` dropped `tool_calls`, so a session read
2142	// back had a bare assistant turn followed by a `tool` message answering
2143	// nothing. Every OpenAI-compatible provider rejects that outright -- so the
2144	// native and Steel paths broke on any reload after a tool call, and went on
2145	// breaking, because the same conversation was sent every turn.
2146	let s = session_with_a_tool_call();
2147	assert_eq!(unanswered(&s.messages), 0, "the fixture must start whole");
2148	let back = round_trip(&s);
2149	assert_eq!(unanswered(&back.messages), 0,
2150	"a reloaded session has {} unpaired tool calls; a provider refuses it",
2151	unanswered(&back.messages));
2152	}
2153
2154	#[test]
2155	fn test_a_reloaded_session_still_knows_what_it_asked_for_00() {
2156	// Pairing alone is not enough: the call has to come back whole, or the model
2157	// reads a conversation in which it asked for nothing and was answered anyway.
2158	let back = round_trip(&session_with_a_tool_call());
2159	assert_eq!(back.messages, session_with_a_tool_call().messages);
2160	match &back.messages[1] {
2161	ChatMessage::Assistant { tool_calls, .. } => {
2162	assert_eq!(tool_calls.len(), 1);
2163	assert_eq!(tool_calls[0].id, "call_abc");
2164	assert_eq!(tool_calls[0].name, "file_read");
2165	assert!(tool_calls[0].arguments.contains("src/parse.rs"),
2166	"the arguments were lost: {}", tool_calls[0].arguments);
2167	}
2168	other => panic!("message 1 came back as {}", other.role()),
2169	}
2170	}
2171
2172	#[test]
2173	fn test_an_assistant_turn_that_asked_for_nothing_is_stored_as_it_always_was_00() {
2174	// The key is written only when there are calls, so nothing changes for the
2175	// ordinary answer -- and a reader of an older snapshot sees the same map.
2176	let m = ChatMessage::assistant(fmt!("Hello."));
2177	assert!(m.to_datmap().get(&dat!("tool_calls")).is_none());
2178	assert_eq!(ChatMessage::from_datmap(&m.to_datmap()).ok(), Some(m));
2179	}
2180
2181	#[test]
2182	fn test_a_snapshot_written_before_tool_calls_were_kept_still_loads_00() {
2183	// What is already in every user's store. It loads, with no calls, exactly as
2184	// it did -- a required field here would refuse their own history.
2185	let mut m = DaticleMap::new();
2186	m.insert(dat!("role"), dat!("assistant"));
2187	m.insert(dat!("content"), dat!("older"));
2188	match ChatMessage::from_datmap(&m) {
2189	Ok(ChatMessage::Assistant { content, tool_calls }) => {
2190	assert_eq!(content.as_text(), "older");
2191	assert!(tool_calls.is_empty());
2192	}
2193	Ok(other) => panic!("it came back as {}", other.role()),
2194	Err(e) => panic!("an older snapshot must still load: {}", e),
2195	}
2196	}
2197
2198	#[test]
2199	fn test_a_call_with_no_id_is_dropped_rather_than_losing_the_message_00() {
2200	// An id is what pairs a call with its reply, so a call without one can never
2201	// be answered. Half a conversation is worth more to the user than an error,
2202	// and what is left reads as the broken pairing it is.
2203	let mut bad = DaticleMap::new();
2204	bad.insert(dat!("name"), dat!("file_read"));
2205	let mut m = DaticleMap::new();
2206	m.insert(dat!("role"), dat!("assistant"));
2207	m.insert(dat!("content"), dat!("hm"));
2208	m.insert(dat!("tool_calls"), Dat::List(vec![Dat::Map(bad)]));
2209	match ChatMessage::from_datmap(&m) {
2210	Ok(ChatMessage::Assistant { tool_calls, .. }) => assert!(tool_calls.is_empty()),
2211	Ok(other) => panic!("it came back as {}", other.role()),
2212	Err(e) => panic!("a malformed call must not lose the message: {}", e),
2213	}
2214	}
2215
2216	// ── A fold is its own event ─────────────────────────────────────────
2217
2218	#[test]
2219	fn test_a_fold_announces_itself_as_a_fold_00() {
2220	// It used to borrow the tool-call surface, which drew it as an action row the
2221	// model had taken. A fold is something the APP did to the user's
2222	// conversation, and it is lossy; it says so in its own variant, with the two
2223	// counts beside the sentence so a client need not parse prose to draw it.
2224	let ev = AgentEvent::Compacted {
2225	folded: 41, kept: 7, note: fmt!("Folded 41 earlier messages."),
2226	};
2227	let dm = ev.to_datmap();
2228	assert_eq!(dm.get(&dat!("type")), Some(&dat!("compacted")));
2229	assert_eq!(dm.get(&dat!("folded")), Some(&Dat::U64(41)));
2230	assert_eq!(dm.get(&dat!("kept")), Some(&Dat::U64(7)));
2231	assert_eq!(dm.get(&dat!("content")), Some(&dat!("Folded 41 earlier messages.")));
2232	// And it is not a tool row: a client keying off `type` must not confuse the
2233	// two, because one is collapsible machine output and the other is a notice.
2234	assert_ne!(dm.get(&dat!("type")), Some(&dat!("tool_call")));
2235	assert_ne!(dm.get(&dat!("type")), Some(&dat!("tool_result")));
2236	}
2237
2238	#[test]
2239	fn test_a_cut_reply_says_so_and_is_not_an_error_00() {
2240	// The browser used to infer this from tool arguments that would not parse, which
2241	// says nothing about a plain text reply cut short. It is not an `error` either:
2242	// the request succeeded and a setting was reached, and a client that drew it in
2243	// red would be reporting a failure that did not happen.
2244	let dm = AgentEvent::Truncated.to_datmap();
2245	assert_eq!(dm.get(&dat!("type")), Some(&dat!("truncated")));
2246	assert_ne!(dm.get(&dat!("type")), Some(&dat!("error")));
2247	assert_ne!(dm.get(&dat!("type")), Some(&dat!("done")));
2248	}
2249
2250	#[test]
2251	fn test_o3db_keys() {
2252	assert_eq!(sessions_key("jason"), Dat::Str("daimond:jason:sessions".to_string()));
2253	assert_eq!(session_key("s1"), Dat::Str("daimond:session:s1".to_string()));
2254	assert_eq!(user_config_key("jason"), Dat::Str("daimond:user:jason".to_string()));
2255	}
2256	}
2257
2258
2259	// ┌───────────────────────────────────────────────────────────────┐
2260	// │ Template tests │
2261	// └───────────────────────────────────────────────────────────────┘
2262
2263	#[cfg(test)]
2264	mod template_tests {
2265	use super::*;
2266
2267	/// A Log Life Diamond as one is on disk after a year of use.
2268	///
2269	/// Every path here is one something in this codebase really writes: the page and the seeded
2270	/// tables come from `CAPP_TEMPLATES.lifelog`, `log/` is where the shipped page puts what its
2271	/// user records, `transcript.md` is what "Keep as a Diamond" writes, and the three under
2272	/// `.daimond/` are the store's own. The bodies stand in for content, but each is distinctive
2273	/// enough that a test can say whether it travelled.
2274	fn log_life() -> Vec<(String, Vec<u8>)> {
2275	vec![
2276	(fmt!("crystal.html"), b"<p>Log Life</p>".to_vec()),
2277	(fmt!("crystal.json"), b"{\"summary\":\"I weigh 84kg\"}".to_vec()),
2278	(fmt!("index.json"), b"{\"v\":2}".to_vec()),
2279	(fmt!("lanes/gym.json"), b"{\"lane\":\"gym\"}".to_vec()),
2280	(fmt!("cat/gym.json"), b"{\"squat\":1}".to_vec()),
2281	(fmt!("triggers.json"), b"{\"actions\":[]}".to_vec()),
2282	(fmt!("capp.json"), b"{\"capp\":\"lifelog\"}".to_vec()),
2283	(fmt!("log/2026-08-21.json"), b"{\"ate\":\"two eggs\"}".to_vec()),
2284	(fmt!("transcript.md"), b"# what I told it about my heart".to_vec()),
2285	(fmt!("versions/0007.json"), b"{\"summary\":\"I weighed 91kg\"}".to_vec()),
2286	(fmt!("versions/0007.html"), b"<p>older page</p>".to_vec()),
2287	(fmt!(".daimond/meta.json"), b"{\"name\":\"Log Life\"}".to_vec()),
2288	(fmt!(".daimond/log"), b"{\"task\":\"my blood results\"}".to_vec()),
2289	(fmt!(".daimond/links.jsonl"), b"{\"from\":\"diamond:d1\"}".to_vec()),
2290	(fmt!(".daimond/deltas/0007.md"), b"user: I have been drinking".to_vec()),
2291	]
2292	}
2293
2294	// The paths a template must carry, and the paths it must not.
2295	//
2296	// Named here rather than inline so the two directions are read together: a rule that keeps
2297	// everything passes the first list, and a rule that keeps nothing passes the second.
2298	const KEPT: [&str; 4] = [
2299	"crystal.html", "index.json", "lanes/gym.json", "cat/gym.json",
2300	];
2301	const DROPPED: [&str; 11] = [
2302	// Automation, and the only entry here dropped for the RECEIVER's sake rather than
2303	// the sender's: a trigger fires without anybody pressing anything.
2304	"triggers.json",
2305	"crystal.json", "capp.json", "log/2026-08-21.json", "transcript.md",
2306	"versions/0007.json", "versions/0007.html",
2307	".daimond/meta.json", ".daimond/log", ".daimond/links.jsonl", ".daimond/deltas/0007.md",
2308	];
2309
2310	/// A TEMPLATE CARRIES NO MESSAGES, and no memory, and no entries.
2311	///
2312	/// Asserted against the PACK's bytes rather than against the filtered list, because the pack is
2313	/// what leaves the device: a filter that dropped a file and a packer that put it back would
2314	/// pass a test on the list and still send the user's life log. Each dropped path is named in
2315	/// the failure, so the assertion says which one escaped.
2316	#[test]
2317	fn test_a_template_carries_no_conversation_no_memory_and_no_entries() {
2318	let files = log_life();
2319	let kept = template_files(&files, false);
2320	let pack = pack_template("d1", 42, "Log Life", &kept);
2321
2322	for path in DROPPED.iter() {
2323	assert!(!pack.contains(path),
2324	"a template carried '{}', which is the owner's own record and not the shape of \
2325	anything: {}", path, pack);
2326	}
2327	// And the bodies, not only the paths: a pack that named no file and carried the bytes
2328	// anyway would pass the loop above.
2329	for body in ["I weigh 84kg", "two eggs", "my heart", "I weighed 91kg", "blood results",
2330	"been drinking"].iter() {
2331	assert!(!pack.contains(body),
2332	"a template carried the words '{}' out of the owner's Diamond: {}", body, pack);
2333	}
2334	// The other direction, so this cannot be passed by a template that carries nothing at all.
2335	for path in KEPT.iter() {
2336	assert!(pack.contains(path),
2337	"a template did not carry '{}', without which it is not the Diamond it is a \
2338	template of: {}", path, pack);
2339	}
2340	}
2341
2342	/// The classification, path by path, as the table in the brief has it.
2343	#[test]
2344	fn test_each_path_falls_on_the_side_it_was_classified_on() {
2345	for path in KEPT.iter() {
2346	assert!(template_carries(path), "'{}' is shape and was dropped", path);
2347	}
2348	for path in DROPPED.iter() {
2349	assert!(!template_carries(path), "'{}' is contents and was carried", path);
2350	}
2351	// `capp.json` is refused AT THE ROOT and nowhere else, which is the distinction
2352	// `fe2o3_sbj` draws: one inside a folder of the user's own making is ordinary data.
2353	assert!(template_carries("recipes/capp.json"),
2354	"a file that merely shares a name with the delivery record was dropped");
2355	// A bare `log` file is the file form of the `log/` prefix, and `logbook.md` is not.
2356	assert!(!template_carries("log"), "the entries file was carried");
2357	assert!(template_carries("logbook.md"), "a file whose name begins with 'log' was dropped");
2358	}
2359
2360	/// The door back to a complete copy carries everything -- and is still a template.
2361	#[test]
2362	fn test_the_conversation_door_carries_everything_and_is_still_a_template() {
2363	let files = log_life();
2364	let all = template_files(&files, true);
2365	assert_eq!(all.len(), files.len(), "a complete copy left something behind");
2366	let pack = pack_template("d1", 42, "Log Life", &all);
2367	for (path, _) in files.iter() {
2368	assert!(pack.contains(path.as_str()), "a complete copy did not carry '{}'", path);
2369	}
2370	assert_eq!(PackKind::Template, PackKind::of(&pack),
2371	"a complete copy did not say it was a template, so it would be imported OVER the \
2372	Diamond it was copied from");
2373	}
2374
2375	/// A template and a whole Diamond are distinguishable, and a whole Diamond's pack is exactly
2376	/// the bytes it always was.
2377	#[test]
2378	fn test_a_pack_says_which_kind_it_is() {
2379	let files = vec![(fmt!("crystal.html"), b"<p>x</p>".to_vec())];
2380	let whole = pack_diamond("d1", 42, &files);
2381	assert_eq!(PackKind::Diamond, PackKind::of(&whole));
2382	assert!(!whole.contains(PACK_KIND),
2383	"a whole Diamond's pack grew a field, so every Diamond on every device would look \
2384	changed to the sync once: {}", whole);
2385
2386	let tmpl = pack_template("d1", 42, "Log Life", &files);
2387	assert_eq!(PackKind::Template, PackKind::of(&tmpl));
2388	assert!(tmpl.contains("\"name\":\"Log Life\""),
2389	"a template did not carry what to call the Diamond it opens as: {}", tmpl);
2390
2391	// The kind is read from the HEAD of the pack. A Diamond holding a file called `kind` must
2392	// not be able to say what kind of pack it is in.
2393	let sneaky = pack_diamond("d1", 42, &vec![(fmt!("kind"), b"template".to_vec())]);
2394	assert_eq!(PackKind::Diamond, PackKind::of(&sneaky),
2395	"a file inside the Diamond decided what kind of pack it was travelling in: {}", sneaky);
2396	}
2397
2398	/// What a colliding mint answers, so the retry below has something that collides.
2399	fn always_taken() -> String { fmt!("d1") }
2400
2401	/// The first two ids are already Diamonds and the third is not.
2402	fn third_time_lucky() -> String {
2403	use std::sync::atomic::{AtomicU64, Ordering};
2404	static N: AtomicU64 = AtomicU64::new(0);
2405	fmt!("d{}", N.fetch_add(1, Ordering::Relaxed) + 1)
2406	}
2407
2408	/// AN EXISTING DIAMOND IS NEVER WRITTEN OVER.
2409	///
2410	/// The pack's own id is in `taken`, which is the case that matters: the person most likely to
2411	/// open a Log Life template is the one whose Log Life it was made from.
2412	#[test]
2413	fn test_an_existing_diamond_is_never_written_over() {
2414	let taken: Vec<String> = vec![fmt!("d1"), fmt!("d2"), fmt!("19fb11892cdc")];
2415	let id = fresh_id(&taken).expect("an id had to be minted");
2416	assert!(!taken.contains(&id),
2417	"an imported template was to be written at '{}', where there is already a Diamond", id);
2418	assert_ne!(fmt!("d1"), id, "an imported template took the id the pack named");
2419
2420	// The retry itself, with a mint that is made to collide. Two ids are already Diamonds, so
2421	// a `fresh_id` that minted once and hoped would answer 'd1' and destroy it.
2422	let id = fresh_id_from(&taken, third_time_lucky).expect("the third id was free");
2423	assert_eq!(fmt!("d3"), id, "the mint was not tried again after it collided");
2424
2425	// And a mint that never gets clear of the store FAILS, rather than answering an id that is
2426	// taken. Silence here would be a deletion.
2427	assert!(fresh_id_from(&taken, always_taken).is_err(),
2428	"an id that is already a Diamond was answered as a fresh one");
2429	}
2430
2431	/// Two imports in a row do not land on each other.
2432	#[test]
2433	fn test_two_templates_opened_in_a_row_get_different_ids() {
2434	let one = fresh_id(&[]).expect("an id had to be minted");
2435	let two = fresh_id(&[one.clone()]).expect("a second id had to be minted");
2436	assert_ne!(one, two, "two templates opened in a row took the same id");
2437	}
2438
2439	/// A complete copy's records point at the Diamond it landed in, not the one it came from.
2440	#[test]
2441	fn test_a_complete_copy_is_readdressed_to_where_it_landed() {
2442	let log = b"{\"delta_ref\":\"diamonds/d1/.daimond/deltas/0007.md\"}".to_vec();
2443	let out = retarget(".daimond/log", log, "d1", "d9");
2444	assert_eq!("{\"delta_ref\":\"diamonds/d9/.daimond/deltas/0007.md\"}",
2445	String::from_utf8_lossy(&out),
2446	"a stored delta still points at the Diamond the copy was made from");
2447
2448	let links = b"{\"from\":\"diamond:d1\",\"to\":\"file:diamonds/d1/lanes/gym.json\"}".to_vec();
2449	let out = retarget(".daimond/links.jsonl", links, "d1", "d9");
2450	assert!(!String::from_utf8_lossy(&out).contains("d1"),
2451	"a link still names the Diamond the copy was made from: {}",
2452	String::from_utf8_lossy(&out));
2453
2454	// And NOTHING ELSE is rewritten. A page that mentions the id is prose, not a reference,
2455	// and a picture is not text at all.
2456	let page = b"<p>made in diamonds/d1/</p>".to_vec();
2457	assert_eq!(page.clone(), retarget("crystal.html", page, "d1", "d9"),
2458	"a search and replace reached a file that holds no reference");
2459	}
2460
2461	/// EXPORT A TEMPLATE, OPEN IT, AND THE ORIGINAL IS UNTOUCHED.
2462	///
2463	/// The store is a list of workspace-relative paths and their bytes, which is what OPFS holds;
2464	/// `wasm::diamond` is compiled only for the browser, so the walk and the writes are simulated
2465	/// here over the same pure functions the browser calls -- `template_files` to choose,
2466	/// `pack_template` to pack, `fresh_id` to mint, `retarget` to readdress.
2467	#[test]
2468	fn test_a_template_round_trip_leaves_the_original_alone() {
2469	let mut store: Vec<(String, Vec<u8>)> = log_life().into_iter()
2470	.map(\|(rel, body)\| (fmt!("diamonds/d1/{}", rel), body))
2471	.collect();
2472	let before = store.clone();
2473
2474	// Export.
2475	let files = log_life();
2476	let kept = template_files(&files, false);
2477	let pack = pack_template("d1", 42, "Log Life", &kept);
2478	assert_eq!(PackKind::Template, PackKind::of(&pack));
2479
2480	// Open. The id is minted against what the store already holds, and never taken from the
2481	// pack.
2482	let taken: Vec<String> = vec![fmt!("d1")];
2483	let new_id = fresh_id(&taken).expect("an id had to be minted");
2484	assert_ne!(fmt!("d1"), new_id, "the template landed on the Diamond it was made from");
2485	for (rel, body) in kept.iter() {
2486	let bytes = retarget(rel, body.clone(), "d1", &new_id);
2487	// An UPSERT, because that is what a write to a path does: an import that reused the
2488	// pack's id would land on the original's files and replace them, and a store that
2489	// merely appended would hide exactly the loss this test is here to catch.
2490	let at = fmt!("diamonds/{}/{}", new_id, rel);
2491	match store.iter().position(\|(p, _)\| *p == at) {
2492	Some(i) => store[i] = (at, bytes),
2493	None => store.push((at, bytes)),
2494	}
2495	}
2496
2497	// The original, byte for byte, still there.
2498	for (path, body) in before.iter() {
2499	let found = store.iter().find(\|(p, _)\| p == path);
2500	match found {
2501	Some((_, now)) => assert_eq!(body, now,
2502	"the original Diamond's '{}' was rewritten by an import", path),
2503	None => panic!("the original Diamond lost '{}' to an import", path),
2504	}
2505	}
2506	// Nothing new on disk means every write landed on a path that was already there, which is
2507	// what an import that took the pack's id does: it replaces the original file by file.
2508	assert!(store.len() > before.len(),
2509	"the import added no file to the store, so it wrote over the Diamond it came from");
2510
2511	// The new Diamond has the shape.
2512	let home = fmt!("diamonds/{}/", new_id);
2513	let mine: Vec<&String> = store.iter().map(\|(p, _)\| p).filter(\|p\| p.starts_with(&home))
2514	.collect();
2515	for path in KEPT.iter() {
2516	assert!(mine.iter().any(\|p\| p.ends_with(path)),
2517	"the new Diamond has no '{}', so it is not the Diamond it is a template of", path);
2518	}
2519	for path in DROPPED.iter() {
2520	assert!(!mine.iter().any(\|p\| p.ends_with(path)),
2521	"the new Diamond arrived holding '{}', out of somebody else's Diamond", path);
2522	}
2523	}
2524	}