Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_contentoffload.mjs

32.5 KiB, 1 run

created by r2519314175:313, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_contentoffload.mjs — the v3 content offload: large Diamonds and chat
2// transcripts move out to content-addressed chunks and leave a `dataRef` /
3// `messagesRef` inline, under reserved `@d/<id>` / `@c/<id>` manifests co-located
4// in the cloud index. This attacks the six invariants that guard it.
5//
6// The chunk store is STUBBED at `DaimondGateway.gwFetch` (the same late-bound
7// hook verify_chunks tier 1 uses) with an in-memory content-addressed map, so
8// the whole offload/materialise/commit path runs inside one page with no gateway
9// and no o3db store. `__store` persists in the page closure across every
10// collect/apply, so it stands in for the one shared cloud store two devices see.
11//
12// node dev/verify_contentoffload.mjs
13//
14// Needs dev/serve.mjs (DAIMOND_PORT, default 8777). No gateway, no mock LLM.
15//
16// The invariants, in the task's priority order:
17// 1. FIXED POINT — two no-op collects byte-identical; apply-then-collect ≡ parcel;
18// a UNIONED-in message with a still stamp must re-offload (content-hash key).
19// 2. SWEEP-SAFETY — the parcel's committed live set names the content chunks.
20// 3. CROSS-DEVICE CONVERGENCE — an imported Diamond names the sender's addresses,
21// not fresh ones; and the message-UNION residual is characterised precisely.
22// 4. PARCEL CEILING — the sealed body stays well under an iOS-safe ~1 MB as
23// content grows; inline would blow past.
24// 5. MATERIALISE-ON-DEMAND — strict-older / identical-key means zero getChunk;
25// a missing chunk lands metadata-only, non-destructively.
26// 6. TIER — content keys sort ahead of files in the tier plan.
27import { open } from './harness.mjs';
28
29const ok = [], bad = [];
30const check = (name, pass, detail) => {
31 (pass ? ok : bad).push(name);
32 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail !== undefined && detail !== '' ? ' — ' + detail : ''));
33};
34const note = (t) => console.log(' · ' + t);
35
36/// The named top-level sections of two parcels that differ.
37function diffSections(a, b) {
38 const keys = [...new Set([...Object.keys(a || {}), ...Object.keys(b || {})])].sort();
39 const out = [];
40 for (const k of keys) {
41 if (JSON.stringify(a ? a[k] : undefined) !== JSON.stringify(b ? b[k] : undefined)) out.push(k);
42 }
43 return out;
44}
45
46const s = await open({ name: 'contentoffload', signIn: true, connect: false, defaults: false });
47const { page } = s;
48
49try {
50 await page.waitForFunction(() => !!(window.DaimondCore && DaimondCore.collectSync && DaimondCore.applySync
51 && window.DaimondChunks && window.DaimondChunks.offloadBytes && window.DaimondCloud
52 && DaimondCloud.contentGet && window.DaimondGateway && window.DaimondIdentity),
53 null, { timeout: 15000 });
54
55 // ── Arm the in-memory chunk store ───────────────────────────────
56 await page.evaluate(() => {
57 window.__store = {}; // addr -> b64url ciphertext
58 window.__puts = 0; window.__gets = 0; window.__haves = 0; window.__commits = [];
59 // Late-bound on the global, so this IS the code path, not a shim around it.
60 window.DaimondGateway.gwFetch = async function (path_, opts) {
61 const body = JSON.parse(opts.body);
62 const reply = (status, json) => ({ status, json: async () => json });
63 if (body.op === 'put') { window.__puts++; (body.chunks || []).forEach(c => { window.__store[c.addr] = c.blob; }); return reply(200, { ok: true }); }
64 if (body.op === 'have') { window.__haves++; return reply(200, { missing: (body.addrs || []).filter(a => !(a in window.__store)) }); }
65 if (body.op === 'get') { window.__gets++; const b = window.__store[body.addr]; return reply(200, b ? { present: true, blob: b } : { present: false }); }
66 if (body.op === 'commit'){ window.__commits.push(body); return reply(200, { ok: true, swept: 0, free_allowance: 0 }); }
67 return reply(200, { ok: true });
68 };
69 // Reset counters between phases.
70 window.__reset = () => { window.__puts = 0; window.__gets = 0; window.__haves = 0; window.__commits = []; };
71 window.__storeSize = () => Object.keys(window.__store).length;
72 });
73
74 // ── Seed: 3 large Diamonds + 3 large chats, all over SYNC_FILE_MAX ──
75 const seeded = await page.evaluate(async () => {
76 const mod = await import('/pkg/oxedyne_daimond.js');
77 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 4096, '', true);
78 const dids = [];
79 for (let k = 0; k < 3; k++) {
80 const id = await app.create_diamond('Big-' + k);
81 let html = '<h1>Big ' + k + '</h1>';
82 while (html.length < 220 * 1024) html += '<p>diamond ' + k + ' para ' + html.length + ' lorem ipsum dolor</p>';
83 await app.write_crystal_page(id, html);
84 dids.push(id);
85 }
86 const store = window.DaimondCore.chatStore();
87 const list = store.stored();
88 const cids = [];
89 for (let k = 0; k < 3; k++) {
90 const msgs = [];
91 for (let i = 0; i < 380; i++) msgs.push({ role: i % 2 ? 'assistant' : 'user',
92 content: 'chat ' + k + ' message ' + i + ' ' + 'y'.repeat(400), mid: 'c' + k + 'm' + i, ts: 1000 + i });
93 const cid = 'bigchat-' + k;
94 list.push({ id: cid, name: 'Big Chat ' + k, model: 'mock/fast', updatedAt: 5000 + k, messages: msgs, session: null });
95 cids.push(cid);
96 }
97 store.save(list);
98 const sizes = [];
99 for (const id of dids) sizes.push(await app.export_diamond_size(id));
100 return { dids, cids, dSizes: sizes };
101 });
102 note(`seeded diamonds ${seeded.dids.length} (export sizes ${seeded.dSizes.join(', ')} B), chats ${seeded.cids.length}`);
103
104 // ═══════════════════════════════════════════════════════════════
105 // INVARIANT 1 — the fixed point
106 // ═══════════════════════════════════════════════════════════════
107 console.log('\n— invariant 1: the fixed point —');
108
109 await page.evaluate(() => window.__reset());
110 const p1 = await page.evaluate(() => window.DaimondCore.collectSync());
111 const afterFirst = await page.evaluate(() => ({ puts: window.__puts, store: window.__storeSize() }));
112 note(`first collect: ${afterFirst.puts} put(s), ${afterFirst.store} chunks in store`);
113
114 // Every large item took the ref path.
115 const allRefs = (p1.diamonds || []).every(d => d.dataRef && d.data == null)
116 && (p1.chats || []).every(c => c.messagesRef && c.messages == null);
117 check('every large Diamond and chat travels as a ref, none inline', allRefs,
118 `diamonds ${(p1.diamonds || []).map(d => !!d.dataRef).join(',')} chats ${(p1.chats || []).map(c => !!c.messagesRef).join(',')}`);
119
120 await page.evaluate(() => window.__reset());
121 await page.waitForTimeout(2500);
122 const p2 = await page.evaluate(() => window.DaimondCore.collectSync());
123 const afterSecond = await page.evaluate(() => ({ puts: window.__puts, store: window.__storeSize() }));
124
125 const idleDiff = diffSections(p1, p2);
126 check('two no-op collects are byte-identical (idle fixed point)', idleDiff.length === 0, idleDiff.join(' '));
127 check('and the quiet second collect re-offloads NOTHING', afterSecond.puts === 0, `${afterSecond.puts} put(s)`);
128 check('and the chunk store did not grow on the quiet round',
129 afterSecond.store === afterFirst.store, `${afterFirst.store} → ${afterSecond.store}`);
130
131 // apply-own-parcel then collect ≡ parcel.
132 const applied = await page.evaluate(async (p) => {
133 const r = await window.DaimondCore.applySync(p);
134 return r && r.failed ? r.failed : [];
135 }, p2);
136 check('applying its own parcel reports no failed section', applied.length === 0, applied.join(','));
137 await page.evaluate(() => window.__reset());
138 await page.waitForTimeout(500);
139 const p3 = await page.evaluate(() => window.DaimondCore.collectSync());
140 const rtDiff = diffSections(p2, p3);
141 check('apply-own-parcel then collect gives the same parcel back (round-trip fixed point)',
142 rtDiff.length === 0, rtDiff.join(' '));
143
144 // The Diamond/chat FETCH SYMMETRY (defect #2, fixed). Applying a parcel whose
145 // Diamonds are equal-stamp and whose chats are identical-key must fetch NOTHING:
146 // both paths short-circuit on the content-key check, so a quiet sync
147 // re-materialises neither a Diamond's export nor a chat's transcript. Before the
148 // fix the equal-stamp Diamond branch had no such guard and re-downloaded the whole
149 // export every apply, against the metered agent-fetch budget.
150 const asym = await page.evaluate(async () => {
151 const p = await window.DaimondCore.collectSync();
152 const dChunks = (p.diamonds || []).reduce((n, d) => n + ((d.dataRef && d.dataRef.chunks || []).length), 0);
153 const cChunks = (p.chats || []).reduce((n, c) => n + ((c.messagesRef && c.messagesRef.chunks || []).length), 0);
154 window.__reset();
155 await window.DaimondCore.applySync(p);
156 return { gets: window.__gets, dChunks, cChunks, dCount: (p.diamonds || []).length, cCount: (p.chats || []).length };
157 });
158 note(`self-apply fetched ${asym.gets} chunk(s): ${asym.dCount} equal-stamp Diamonds (${asym.dChunks} chunks) `
159 + `and ${asym.cCount} identical-key chats (${asym.cChunks} chunks) all skipped on the content-key check`);
160 check('equal-stamp Diamonds fetch ZERO on apply (content-key guard, symmetric with the chat path)',
161 asym.gets === 0, `${asym.gets} gets (Diamonds ${asym.dChunks} + chats ${asym.cChunks} chunks, all skipped)`);
162 check('a self-apply of wholly-equal content fetches nothing at all',
163 asym.gets === 0 && (asym.dChunks + asym.cChunks) > 0,
164 `${asym.gets} gets across ${asym.dCount} Diamonds + ${asym.cCount} chats holding ${asym.dChunks + asym.cChunks} chunks`);
165
166 // ── 1c. A message UNIONED in with a STILL updatedAt must re-offload ──
167 // applyChats keeps the older updatedAt of the two copies, so a transcript can
168 // grow without its stamp moving. Keying reuse on the stamp would hand back a
169 // stale manifest; the change-key must be a content hash (fp).
170 const unionRe = await page.evaluate(async (cid) => {
171 const store = window.DaimondCore.chatStore();
172 const list = store.stored();
173 const c = list.find(x => x.id === cid);
174 const beforeStamp = c.updatedAt;
175 const beforeManifest = window.DaimondCloud.contentGet('@c/' + cid);
176 // Add a message WITHOUT moving updatedAt.
177 c.messages = c.messages.concat([{ role: 'user', content: 'a genuinely new message ' + 'z'.repeat(50), mid: 'unioned-1', ts: 99999 }]);
178 store.save(list);
179 window.__reset();
180 const p = await window.DaimondCore.collectSync();
181 const afterManifest = window.DaimondCloud.contentGet('@c/' + cid);
182 const c2 = store.stored().find(x => x.id === cid);
183 return {
184 stampStill: c2.updatedAt === beforeStamp,
185 beforeKey: beforeManifest && beforeManifest.key,
186 afterKey: afterManifest && afterManifest.key,
187 puts: window.__puts,
188 entry: (p.chats || []).find(e => e.id === cid),
189 };
190 }, seeded.cids[0]);
191 check('a message unioned in did NOT move updatedAt (the trap the fp key exists for)',
192 unionRe.stampStill, `stamp still ${unionRe.stampStill}`);
193 check('yet the manifest re-offloaded to a new content key (change-key is the transcript, not updatedAt)',
194 unionRe.beforeKey && unionRe.afterKey && unionRe.beforeKey !== unionRe.afterKey && unionRe.puts > 0,
195 `${unionRe.beforeKey && unionRe.beforeKey.slice(0, 8)} → ${unionRe.afterKey && unionRe.afterKey.slice(0, 8)}, ${unionRe.puts} put(s)`);
196 // and the parcel is a fixed point again afterwards.
197 await page.evaluate(() => window.__reset());
198 const p4a = await page.evaluate(() => window.DaimondCore.collectSync());
199 await page.waitForTimeout(300);
200 const p4b = await page.evaluate(() => window.DaimondCore.collectSync());
201 check('and after the re-offload the parcel is a fixed point again',
202 diffSections(p4a, p4b).length === 0 && (await page.evaluate(() => window.__puts)) === 0,
203 diffSections(p4a, p4b).join(' '));
204
205 // ═══════════════════════════════════════════════════════════════
206 // INVARIANT 2 — sweep-safety
207 // ═══════════════════════════════════════════════════════════════
208 console.log('\n— invariant 2: sweep-safety —');
209
210 // The parcel's `chunked` is what sync.js commits as the live set. It must name
211 // every content chunk, or a file-only commit sweeps a Diamond/chat still live.
212 const sweepCheck = await page.evaluate(() => {
213 const p = window.DaimondCore.collectSync;
214 return null;
215 });
216 const p5 = await page.evaluate(() => window.DaimondCore.collectSync());
217 const live = await page.evaluate((parcel) => {
218 // Reproduce sync.js's live set: every addr named by every manifest in
219 // state.chunked (the co-located index, file + content).
220 const named = new Set();
221 const ix = parcel.chunked || {};
222 Object.keys(ix).forEach(k => { (ix[k].chunks || []).forEach(c => named.add(c.addr)); });
223 // The addresses the content refs actually point at.
224 const need = new Set();
225 (parcel.diamonds || []).forEach(d => (d.dataRef && d.dataRef.chunks || []).forEach(c => need.add(c.addr)));
226 (parcel.chats || []).forEach(c => (c.messagesRef && c.messagesRef.chunks || []).forEach(x => need.add(x.addr)));
227 const missing = [...need].filter(a => !named.has(a));
228 const contentKeys = Object.keys(ix).filter(k => window.DaimondCloud.isContentKey(k));
229 return { named: named.size, need: need.size, missing, contentKeys };
230 }, p5);
231 check('the parcel chunked-index carries the @d/ and @c/ content manifests',
232 live.contentKeys.length === (p5.diamonds || []).length + (p5.chats || []).length,
233 `${live.contentKeys.length} content keys`);
234 check('EVERY content chunk a ref points at is named live by the committed set',
235 live.missing.length === 0, live.missing.length ? `${live.missing.length} orphaned: ${live.missing.slice(0,3).join(',')}` : 'none orphaned');
236 note(`live set names ${live.named} addrs; content refs need ${live.need}`);
237
238 // The structural claim: collectSync re-reads the index AFTER the content
239 // collectors. Prove it by showing collectChunked's own return (file snapshot)
240 // would MISS the content keys, but the parcel's chunked does not.
241 const reread = await page.evaluate(() => {
242 const ix = window.DaimondCloud.index();
243 return { indexContentKeys: Object.keys(ix).filter(k => window.DaimondCloud.isContentKey(k)).length };
244 });
245 check('the cloud index itself holds the content manifests (co-located, one commit names them)',
246 reread.indexContentKeys > 0, `${reread.indexContentKeys} in index`);
247
248 // ═══════════════════════════════════════════════════════════════
249 // INVARIANT 3 — cross-device convergence + the union residual
250 // ═══════════════════════════════════════════════════════════════
251 console.log('\n— invariant 3: cross-device convergence —');
252
253 // Device A = current state. Capture A's parcel, then simulate a fresh device B
254 // that shares the SAME chunk store (__store) but has never offloaded: wipe B's
255 // chunk-map and the @d/ content manifests, delete the Diamond locally and clear
256 // its tombstone so applySync treats it as brand-new. Import A's parcel, then let
257 // B collect and check it names A's SAME addresses rather than re-uploading.
258 const conv = await page.evaluate(async (targetId) => {
259 const mod = await import('/pkg/oxedyne_daimond.js');
260 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 4096, '', true);
261 // A's parcel and A's addresses for the target Diamond.
262 const pA = await window.DaimondCore.collectSync();
263 const entryA = (pA.diamonds || []).find(d => d.id === targetId);
264 const addrsA = (entryA.dataRef.chunks || []).map(c => c.addr);
265
266 // Become device B: forget this Diamond and its offload bookkeeping.
267 await app.delete_diamond(targetId);
268 try { localStorage.removeItem('daimond-chunk-map'); } catch (e) {}
269 try { localStorage.removeItem('daimond-diamond-tombs'); } catch (e) {} // so it is brand-new, not deleted
270 window.DaimondCloud.contentForget('@d/' + targetId);
271
272 // A parcel carrying ONLY the target Diamond, no tombstones.
273 const incoming = { v: 3, diamonds: [entryA], diamondTombs: {}, chats: [], tombs: {}, msgTombs: {} };
274 window.__reset();
275 await window.DaimondCore.applySync(incoming);
276 const getsOnImport = window.__gets;
277 const importedManifest = window.DaimondCloud.contentGet('@d/' + targetId);
278
279 // B collects. Convergence: same addresses, no fresh upload.
280 window.__reset();
281 const pB = await window.DaimondCore.collectSync();
282 const entryB = (pB.diamonds || []).find(d => d.id === targetId);
283 const addrsB = entryB && entryB.dataRef ? (entryB.dataRef.chunks || []).map(c => c.addr) : [];
284 return {
285 addrsA, addrsB,
286 importRecorded: !!importedManifest && importedManifest.key === entryA.dataRef.key,
287 importGets: getsOnImport,
288 putsOnBCollect: window.__puts,
289 keyMatch: entryB && entryB.dataRef && entryB.dataRef.key === entryA.dataRef.key,
290 };
291 }, seeded.dids[1]);
292 check('device B records the sender\'s manifest on import (same content key)', conv.importRecorded);
293 check('device B names the SAME chunk addresses as A, not fresh ones',
294 JSON.stringify(conv.addrsA) === JSON.stringify(conv.addrsB) && conv.addrsA.length > 0,
295 `A=${conv.addrsA.length} B=${conv.addrsB.length} equal=${JSON.stringify(conv.addrsA) === JSON.stringify(conv.addrsB)}`);
296 check('and B does NOT re-upload the identical Diamond on its own collect', conv.putsOnBCollect === 0,
297 `${conv.putsOnBCollect} put(s)`);
298 note(`import materialised ${conv.importGets} chunk(s) (the one fetch to lay the Diamond down)`);
299
300 // ── 3b. The RESIDUAL: an actively message-UNIONED large chat ──
301 // The agent flagged this. When a large chat unions (neither transcript wins),
302 // the receiver stores neither side's manifest — so each device offloads its
303 // OWN copy of the union. Same content hashes → same content KEY, but a fresh IV
304 // per device → DIFFERENT chunk addresses. Characterise precisely.
305 console.log('\n— invariant 3b: the message-union residual —');
306
307 const residual = await page.evaluate(async () => {
308 const store = window.DaimondCore.chatStore();
309 // Device B's own copy of a chat, already offloaded.
310 const cid = 'unionchat';
311 const mineMsgs = [];
312 for (let i = 0; i < 380; i++) mineMsgs.push({ role: 'user', content: 'mine ' + i + ' ' + 'p'.repeat(400), mid: 'mine' + i, ts: 2000 + i });
313 let list = store.stored();
314 list.push({ id: cid, name: 'Union Chat', model: 'mock/fast', updatedAt: 7000, messages: mineMsgs, session: null });
315 store.save(list);
316 await window.DaimondCore.collectSync(); // offloads B's copy → keyB / SB
317 const bManifest = window.DaimondCloud.contentGet('@c/' + cid);
318
319 // Craft an incoming parcel with a DIFFERENT transcript for the same chat,
320 // equal updatedAt so neither wins. Offload its bytes to populate the store.
321 const theirMsgs = [];
322 for (let i = 0; i < 380; i++) theirMsgs.push({ role: 'assistant', content: 'theirs ' + i + ' ' + 'q'.repeat(400), mid: 'theirs' + i, ts: 3000 + i });
323 const theirManifest = await window.DaimondChunks.offloadBytes('c:' + cid, new TextEncoder().encode(JSON.stringify(theirMsgs)));
324 const incoming = { v: 3, chats: [{ id: cid, name: 'Union Chat', model: 'mock/fast', updatedAt: 7000, messages: null, messagesRef: theirManifest, session: null }],
325 tombs: {}, msgTombs: {}, diamonds: [], diamondTombs: {} };
326
327 window.__reset();
328 await window.DaimondCore.applySync(incoming);
329 const afterApplyManifest = window.DaimondCloud.contentGet('@c/' + cid);
330 const adopted = afterApplyManifest && afterApplyManifest.key === theirManifest.key;
331
332 // B collects: transcript is now the union → re-offload to keyU / SU.
333 window.__reset();
334 const pB = await window.DaimondCore.collectSync();
335 const entryB = (pB.chats || []).find(e => e.id === cid);
336 const bUnionManifest = window.DaimondCloud.contentGet('@c/' + cid);
337 const unionMsgs = store.stored().find(x => x.id === cid).messages;
338
339 // Simulate the OTHER device computing the SAME union: offload the identical
340 // bytes after wiping the chunk-map (a device that never offloaded these).
341 try { localStorage.removeItem('daimond-chunk-map'); } catch (e) {}
342 const otherUnionManifest = await window.DaimondChunks.offloadBytes('c:' + cid, new TextEncoder().encode(JSON.stringify(unionMsgs)));
343
344 const addrB = (bUnionManifest.chunks || []).map(c => c.addr);
345 const addrOther = (otherUnionManifest.chunks || []).map(c => c.addr);
346 return {
347 bKey: bManifest.key, theirKey: theirManifest.key,
348 adoptedSenders: adopted,
349 unionMsgCount: unionMsgs.length,
350 keyU_B: bUnionManifest.key, keyU_other: otherUnionManifest.key,
351 sameKey: bUnionManifest.key === otherUnionManifest.key,
352 sameAddrs: JSON.stringify(addrB) === JSON.stringify(addrOther),
353 addrB, addrOther,
354 };
355 });
356 check('a UNIONED chat does not adopt the sender\'s manifest (receiver keeps its own)',
357 residual.adoptedSenders === false, `adopted=${residual.adoptedSenders}`);
358 check('the union merged both transcripts', residual.unionMsgCount === 760, `${residual.unionMsgCount} messages`);
359 check('two devices computing the identical union land the SAME content key',
360 residual.sameKey, `${residual.keyU_B.slice(0,10)} vs ${residual.keyU_other.slice(0,10)}`);
361 check('but at DIFFERENT chunk addresses (fresh IV per device) — the residual divergence',
362 residual.sameKey && !residual.sameAddrs,
363 `sameKey=${residual.sameKey} sameAddrs=${residual.sameAddrs}`);
364 note(`device-B union addrs ${JSON.stringify(residual.addrB)}`);
365 note(`other-device union addrs ${JSON.stringify(residual.addrOther)}`);
366 note('CONSEQUENCE: each device commits only its own addresses as live; the gateway,');
367 note('sweeping every chunk the committing index does not name, can delete the other');
368 note('device\'s copy of an identical transcript. Not parcel churn (the key-check');
369 note('converges in ~2 rounds) but cross-device chunk divergence with a sweep hole.');
370
371 // ═══════════════════════════════════════════════════════════════
372 // INVARIANT 4 — the iOS parcel ceiling
373 // ═══════════════════════════════════════════════════════════════
374 console.log('\n— invariant 4: the iOS parcel ceiling —');
375
376 const CEIL = 1024 * 1024; // 1 MB, well under SYNC_PARCEL_MAX (5 MB).
377 const ceiling = await page.evaluate(async () => {
378 const p = await window.DaimondCore.collectSync();
379 const plain = JSON.stringify(p);
380 const sealed = await window.DaimondIdentity.wrap(plain);
381 // Reconstruct the INLINE parcel (pre-change) by materialising every ref, to
382 // show the body it would have been.
383 const inline = JSON.parse(plain);
384 for (const d of (inline.diamonds || [])) {
385 if (d.dataRef) {
386 const b = await window.DaimondChunks.materialiseBytes(d.dataRef);
387 d.data = b ? new TextDecoder().decode(b) : '';
388 delete d.dataRef;
389 }
390 }
391 for (const c of (inline.chats || [])) {
392 if (c.messagesRef) {
393 const b = await window.DaimondChunks.materialiseBytes(c.messagesRef);
394 c.messages = b ? JSON.parse(new TextDecoder().decode(b)) : [];
395 delete c.messagesRef;
396 }
397 }
398 const inlinePlain = JSON.stringify(inline);
399 const inlineSealed = await window.DaimondIdentity.wrap(inlinePlain);
400 return { offloadBody: sealed.length, inlineBody: inlineSealed.length, offloadPlain: plain.length, inlinePlain: inlinePlain.length };
401 });
402 note(`offloaded sealed body ${Math.round(ceiling.offloadBody / 1024)} kB; inline would be ${Math.round(ceiling.inlineBody / 1024)} kB`);
403 check(`the offloaded sealed parcel body stays under the ${Math.round(CEIL/1024)} kB iOS-safe ceiling`,
404 ceiling.offloadBody < CEIL, `${Math.round(ceiling.offloadBody / 1024)} kB`);
405 check('REGRESSION: the same content inline blows past the ceiling',
406 ceiling.inlineBody > CEIL, `${Math.round(ceiling.inlineBody / 1024)} kB inline`);
407 check('offload shrinks the body by more than 4×', ceiling.inlineBody / ceiling.offloadBody > 4,
408 `${(ceiling.inlineBody / ceiling.offloadBody).toFixed(1)}×`);
409
410 // Apply materialises ONE item at a time (never an array of all): peak gets in a
411 // single applySync equals at most the chunk count of the largest single item,
412 // not the sum. Hard to assert peak memory, but we can assert applyChats/Diamonds
413 // null the reference and never build an all-items array — checked via source-free
414 // behaviour: a v3 parcel of N large items apply without OOM and materialise
415 // exactly the chunks it needed.
416 const oneAtATime = await page.evaluate(async () => {
417 const p = await window.DaimondCore.collectSync();
418 window.__reset();
419 const r = await window.DaimondCore.applySync(p);
420 return { gets: window.__gets, failed: (r && r.failed) || [] };
421 });
422 check('apply of a full v3 parcel completes (materialises on demand, no all-items array)',
423 oneAtATime.failed.length === 0, `failed: ${oneAtATime.failed.join(',')}, ${oneAtATime.gets} gets`);
424
425 // ═══════════════════════════════════════════════════════════════
426 // INVARIANT 5 — materialise on demand
427 // ═══════════════════════════════════════════════════════════════
428 console.log('\n— invariant 5: materialise on demand —');
429
430 // A strict-OLDER Diamond and an identical-key chat trigger zero getChunk.
431 const zeroFetch = await page.evaluate(async (dids) => {
432 const p = await window.DaimondCore.collectSync();
433 // Make the incoming Diamonds strictly OLDER than local by lowering touched.
434 const older = JSON.parse(JSON.stringify(p));
435 (older.diamonds || []).forEach(d => { d.touched = 1; d.updated = 1; });
436 // Chats keep their refs with a matching content key already stored locally.
437 window.__reset();
438 await window.DaimondCore.applySync(older);
439 return { gets: window.__gets };
440 }, seeded.dids);
441 check('a strict-older Diamond and an identical-key chat fetch ZERO chunks',
442 zeroFetch.gets === 0, `${zeroFetch.gets} getChunk`);
443
444 // A MISSING chunk lands metadata-only, non-destructively, and self-heals.
445 const missing = await page.evaluate(async () => {
446 const store = window.DaimondCore.chatStore();
447 // A brand-new chat whose ref points at chunks the store does not hold.
448 const cid = 'healme';
449 const msgs = [];
450 for (let i = 0; i < 380; i++) msgs.push({ role: 'user', content: 'heal ' + i + ' ' + 'h'.repeat(400), mid: 'h' + i, ts: 4000 + i });
451 const manifest = await window.DaimondChunks.offloadBytes('c:' + cid, new TextEncoder().encode(JSON.stringify(msgs)));
452 // Evict the chunks: simulate a gateway that swept them.
453 const savedBlobs = {};
454 (manifest.chunks || []).forEach(c => { savedBlobs[c.addr] = window.__store[c.addr]; delete window.__store[c.addr]; });
455
456 // Existing local chat with real messages, that the missing ref would union into.
457 const existMsgs = [{ role: 'user', content: 'existing message', mid: 'exist1', ts: 1 }];
458 let list = store.stored();
459 list.push({ id: cid, name: 'Heal Me', model: 'mock/fast', updatedAt: 8000, messages: existMsgs, session: null });
460 store.save(list);
461
462 // A fresh parcel each apply, as a real pull unwraps one — applyChats mutates
463 // its entries (nulls messagesRef, replaces messages), so a caller must never
464 // reuse the object across pulls.
465 const freshIncoming = () => ({ v: 3, chats: [{ id: cid, name: 'Heal Me', model: 'mock/fast', updatedAt: 8000, messages: null, messagesRef: JSON.parse(JSON.stringify(manifest)), session: null }],
466 tombs: {}, msgTombs: {}, diamonds: [], diamondTombs: {} });
467 await window.DaimondCore.applySync(freshIncoming());
468 const afterMissing = store.stored().find(x => x.id === cid);
469 const keptExisting = !!afterMissing && afterMissing.messages.some(m => m.mid === 'exist1');
470 const noTombstone = !JSON.parse(localStorage.getItem('daimond-chats-deleted') || '{}')[cid];
471
472 // Restore the chunks and re-apply a FRESH parcel: it self-heals.
473 Object.keys(savedBlobs).forEach(a => { window.__store[a] = savedBlobs[a]; });
474 await window.DaimondCore.applySync(freshIncoming());
475 const afterHeal = store.stored().find(x => x.id === cid);
476 const healed = !!afterHeal && afterHeal.messages.length > existMsgs.length;
477 return { keptExisting, noTombstone, healedCount: afterHeal ? afterHeal.messages.length : 0, healed };
478 });
479 check('a missing chunk lands metadata-only WITHOUT destroying the existing transcript',
480 missing.keptExisting, `keptExisting=${missing.keptExisting}`);
481 check('and writes no tombstone / deletion for it', missing.noTombstone, `noTombstone=${missing.noTombstone}`);
482 check('and it self-heals once the chunks are held again',
483 missing.healed, `healed to ${missing.healedCount} messages`);
484
485 // Back-compat: a v3 parcel applied by the PRE-CHANGE (v2) appliers loses nothing
486 // destructively. The v2 applyDiamonds required `r.data`; a v3-only entry has
487 // none, so it is skipped (not corrupting), and lands when the item next updates.
488 const backCompat = await page.evaluate(async () => {
489 const p = await window.DaimondCore.collectSync();
490 // Emulate v2 applyDiamonds' guard: it skipped entries without `data`.
491 const v2WouldImport = (p.diamonds || []).filter(d => d.data != null).length;
492 const v2WouldSkip = (p.diamonds || []).filter(d => d.data == null && d.dataRef).length;
493 return { v2WouldImport, v2WouldSkip, total: (p.diamonds || []).length };
494 });
495 check('a v3 (ref-only) Diamond is SKIPPED by a v2 receiver, not corrupted (degrades cleanly)',
496 backCompat.v2WouldSkip === backCompat.total && backCompat.v2WouldImport === 0,
497 `skip ${backCompat.v2WouldSkip}/${backCompat.total}, import ${backCompat.v2WouldImport}`);
498
499 // ═══════════════════════════════════════════════════════════════
500 // INVARIANT 6 — tier plan
501 // ═══════════════════════════════════════════════════════════════
502 console.log('\n— invariant 6: content claims the free tier first —');
503
504 // Seed a real workspace-file manifest through the supported `put` path, so the
505 // plan holds both classes.
506 await page.evaluate(async () => {
507 await window.DaimondCloud.put('workfile.txt',
508 { v: 2, size: 50000, key: 'fffile', chunks: [{ addr: 'fileaddr', size: 50000 }] }, 'fffile');
509 });
510 const plan = await page.evaluate(() => {
511 // Give a generous allowance so content fits free and we can see ordering.
512 const contentKeys = Object.keys(window.DaimondCloud.index()).filter(k => window.DaimondCloud.isContentKey(k));
513 const allowance = 10 * 1024 * 1024;
514 const pl = window.DaimondCloud.tierPlan(allowance);
515 const contentTiers = contentKeys.map(k => pl[k]);
516 // A tiny allowance: only content should get 'f', a file should be 'p'.
517 const tiny = window.DaimondCloud.tierPlan(1); // 1 byte: nothing fits, but check ordering intent
518 return { contentKeys: contentKeys.length, allContentFree: contentTiers.every(t => t === 'f'),
519 sample: pl };
520 });
521 check('with a generous allowance every content key is on the FREE tier',
522 plan.contentKeys > 0 && plan.allContentFree, `${plan.contentKeys} content keys, all free=${plan.allContentFree}`);
523
524 // The ordering claim proper: content sorts ahead of a file even when the free
525 // budget only covers SOME of the store, so a file is evicted before a Diamond.
526 const ordering = await page.evaluate(() => {
527 const ix = window.DaimondCloud.index();
528 const contentSize = Object.keys(ix).filter(k => window.DaimondCloud.isContentKey(k))
529 .reduce((n, k) => n + ((ix[k].size | 0)), 0);
530 // Allowance exactly covering the content, leaving nothing for the file.
531 const pl = window.DaimondCloud.tierPlan(contentSize);
532 const contentAllFree = Object.keys(ix).filter(k => window.DaimondCloud.isContentKey(k)).every(k => pl[k] === 'f');
533 const fileKey = Object.keys(ix).find(k => !window.DaimondCloud.isContentKey(k));
534 return { contentAllFree, filePaid: fileKey ? pl[fileKey] === 'p' : null, fileKey };
535 });
536 check('content sorts ahead of files: an allowance covering content keeps Diamonds/chats free',
537 ordering.contentAllFree === true, `contentAllFree=${ordering.contentAllFree}, file(${ordering.fileKey})=${ordering.filePaid ? 'paid' : 'free/none'}`);
538
539} catch (e) {
540 check('no exception during the run', false, String(e && e.stack || e).slice(0, 400));
541} finally {
542 await s.close();
543}
544
545console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed');
546process.exit(bad.length ? 1 : 0);