Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_cloud.mjs

21.2 KiB, 1 run

created by r2519314175:299, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_cloud.mjs — cloud storage is where the workspace lives, and the device
2// holds as much of it as it can.
3//
4// The load-bearing test here is the FIRST one. The gateway sweeps every chunk
5// the committed index does not name, and the client used to rebuild that index
6// by walking its own sandbox — so a device that was not holding a file omitted
7// it, and the sweep deleted it for everyone. Under a residency model, where a
8// file is MEANT to live in cloud storage alone, that is straightforward data
9// loss. The index is merged state now, and a device that lacks a file must
10// still carry it forward.
11//
12// Drives the REAL client (cloud.js + chunks.js + daimond.js) against the REAL
13// gateway (/api/chunk + /api/sync). Starts its own gateway, or uses one already
14// on :9002.
15import { spawn } from 'node:child_process';
16import path from 'node:path';
17import { fileURLToPath } from 'node:url';
18import { requireFreshGateway, procLog, GWCWD } from './gwbin.mjs';
19import { open } from './harness.mjs';
20import { makePagePro } from './pro.mjs';
21import { GW_PORT, GW_URL } from './ports.mjs';
22
23const __dirname = path.dirname(fileURLToPath(import.meta.url));
24const GWDIR = path.resolve(__dirname, '..', 'gateway');
25/// What the gateway says while this runs -- a sweep it declined, a commit it
26/// called stale -- so a residency failure can be read from both ends. Silent
27/// when this run reuses a gateway it did not start.
28const GW_LOG = procLog('verify_cloud');
29/// A fault to inject on purpose, so a check can be shown going red:
30///
31/// --break=reseal every chunk of the file changes, not only the last
32/// --break=noedit the file is written back unchanged
33///
34/// Both bear on the partial-upload pair at the end, which is the one place here
35/// that had to be corrected rather than left to fail.
36const BREAK = (process.argv.find(a => a.startsWith('--break=')) || '').slice(8);
37
38const ok = [], bad = [];
39const check = (name, pass, detail) => {
40 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
41 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
42};
43
44let gw = null;
45async function waitFor(fn, ms = 20000, gap = 300) {
46 const t0 = Date.now();
47 while (Date.now() - t0 < ms) {
48 try { if (await fn()) return true; } catch (e) { /* keep waiting */ }
49 await new Promise(r => setTimeout(r, gap));
50 }
51 return false;
52}
53
54requireFreshGateway();
55
56const alreadyUp = await waitFor(async () => (await fetch(`${GW_URL}/api/health`)).ok, 800, 200);
57if (alreadyUp) {
58 console.log(` ok using the gateway already on :${GW_PORT}`);
59} else {
60 gw = spawn(path.join(GWDIR, 'target/release/daimond_gateway'), [], {
61 cwd: GWCWD, env: { ...process.env, APP_MODE: 'sandbox' }, stdio: GW_LOG.stdio,
62 });
63 check('gateway starts', await waitFor(async () => (await fetch(`${GW_URL}/api/health`)).ok));
64}
65
66const s = await open({ name: 'cloud', signIn: true, connect: false });
67const { page } = s;
68
69await page.waitForFunction(
70 () => !!window.DaimondSync && !!window.DaimondChunks && !!window.DaimondCloud
71 && !!window.DaimondCore && !!window.DaimondGateway && DaimondGateway.state().authed,
72 null, { timeout: 12000 },
73).catch(() => {});
74
75try {
76 check('the cloud module and an authed session are live',
77 await page.evaluate(() => !!window.DaimondCloud && DaimondGateway.state().authed));
78
79 // Cloud storage is a Pro capability: without the licence every push is a
80 // 402, nothing is ever offloaded, and the residency assertions below would
81 // be measuring the gate instead of the feature.
82 const lic = await makePagePro(page, GWDIR, GW_URL);
83 check('the account holds Pro, so cloud storage is allowed to run',
84 lic.pro === true, `webhook ${lic.status}, pro=${lic.pro}`);
85
86 // Two large files, so one can be evicted while the other is pinned.
87 const MARK = 'CLOUDMARK-7788';
88 await page.evaluate(async (mark) => {
89 const mod = await import('../pkg/oxedyne_daimond.js');
90 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
91 const body = (n) => { let b = ''; while (b.length < n) b += mark + ' the quick brown fox jumps. '; return b; };
92 await app.run_tool('file_write', JSON.stringify({ path: 'papers/alpha.txt', content: body(200 * 1024) }));
93 await app.run_tool('file_write', JSON.stringify({ path: 'papers/beta.txt', content: body(160 * 1024) }));
94 await window.DaimondSync.push();
95 }, MARK);
96
97 const seeded = await page.evaluate(() => ({
98 alpha: !!window.DaimondCloud.manifest('papers/alpha.txt'),
99 beta: !!window.DaimondCloud.manifest('papers/beta.txt'),
100 }));
101 check('both large files are recorded in the cloud index', seeded.alpha && seeded.beta);
102
103 // ── The regression: a device that lacks a file must not delete it ──
104 // Simulate the smallest device. Drop alpha locally, keep the merged index,
105 // push. The gateway must still hold alpha's chunks afterwards.
106 const survived = await page.evaluate(async () => {
107 const mod = await import('../pkg/oxedyne_daimond.js');
108 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
109 const mani = window.DaimondCloud.manifest('papers/alpha.txt');
110 const addr = mani.chunks[0].addr;
111 // The body is gone from this device, but the file is NOT deleted — this is
112 // eviction, which must never be mistaken for a delete.
113 const root = await navigator.storage.getDirectory();
114 const dir = await root.getDirectoryHandle('papers');
115 await dir.removeEntry('alpha.txt');
116 await window.DaimondCloud.refreshPaths();
117 await window.DaimondSync.push();
118 const g = await fetch('/api/chunk', {
119 method: 'POST', credentials: 'same-origin',
120 headers: { 'content-type': 'application/json', 'x-daimond-api': '1' },
121 body: JSON.stringify({ op: 'get', addr }),
122 });
123 const gj = await g.json();
124 return { held: !!gj.present, stillIndexed: !!window.DaimondCloud.manifest('papers/alpha.txt') };
125 });
126 check('a push from a device NOT holding the file keeps its chunks in cloud storage', survived.held);
127 check('and keeps it in the index, because absence is not deletion', survived.stillIndexed);
128
129 // ── Fetching it back ──
130 const round = await page.evaluate(async (mark) => {
131 const res = await window.DaimondCloud.fetch('papers/alpha.txt');
132 let back = null;
133 try {
134 const root = await navigator.storage.getDirectory();
135 const dir = await root.getDirectoryHandle('papers');
136 back = await (await (await dir.getFileHandle('alpha.txt')).getFile()).text();
137 } catch (e) { back = null; }
138 return { res, size: back ? back.length : 0, hasMark: !!back && back.includes(mark) };
139 }, MARK);
140 check('fetching brings it back byte-for-byte', round.hasMark && round.size > 128 * 1024,
141 'size=' + round.size);
142
143 // ── Freeing space keeps the file, and a pin refuses ──
144 const freed = await page.evaluate(async () => {
145 window.DaimondCloud.pin('papers/beta.txt', true);
146 const evictBeta = await window.DaimondCloud.evict('papers/beta.txt');
147 const evictAlpha = await window.DaimondCloud.evict('papers/alpha.txt');
148 const away = window.DaimondCloud.awayPaths();
149 return {
150 pinRefused: evictBeta.indexOf('OK') !== 0,
151 alphaFreed: evictAlpha.indexOf('OK') === 0,
152 alphaAway: Object.prototype.hasOwnProperty.call(away, 'papers/alpha.txt'),
153 alphaKnown: !!window.DaimondCloud.manifest('papers/alpha.txt'),
154 betaStillHere: !Object.prototype.hasOwnProperty.call(away, 'papers/beta.txt'),
155 };
156 });
157 check('a pinned file refuses to be freed', freed.pinRefused);
158 check('freeing space drops the local copy', freed.alphaFreed && freed.alphaAway);
159 check('but the file remains in cloud storage', freed.alphaKnown);
160 check('the pinned file is still on this device', freed.betaStillHere);
161
162 // ── Non-ASCII must be freeable too ──
163 // Every recorded length must be BYTES on disk. Mixing in a character count
164 // once made eviction compare the two, and since they agree only for pure
165 // ASCII, a single accented character marked the file permanently "edited" and
166 // it could never be freed. The pipeline is byte-shaped throughout now, so the
167 // test's job is to hold it that way.
168 const accented = await page.evaluate(async () => {
169 const mod = await import('../pkg/oxedyne_daimond.js');
170 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
171 let body = '';
172 while (body.length < 200 * 1024) body += 'naïve café façade — Ünicode, mesure de qualité. ';
173 await app.run_tool('file_write', JSON.stringify({ path: 'papers/accented.txt', content: body }));
174 await window.DaimondSync.push();
175 const m = window.DaimondCloud.manifest('papers/accented.txt');
176 const onDisk = (await window.DaimondCloud.fileAt('papers/accented.txt')).size;
177 const res = await window.DaimondCloud.evict('papers/accented.txt');
178 return { res, chars: body.length, size: m && m.size, bytes: m && m.bytes, onDisk };
179 });
180 check('a non-ASCII file records its length in bytes, not characters',
181 accented.size === accented.onDisk && accented.bytes === accented.onDisk
182 && accented.onDisk > accented.chars,
183 'chars=' + accented.chars + ' bytes=' + accented.onDisk);
184 check('and can still be freed', accented.res.indexOf('OK') === 0, accented.res.slice(0, 80));
185
186 // ── The agent sees it, and is told plainly ──
187 const agentView = await page.evaluate(async () => {
188 const mod = await import('../pkg/oxedyne_daimond.js');
189 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
190 const listing = String(await app.run_tool('file_list', JSON.stringify({ path: 'papers' })));
191 const read = String(await app.run_tool('file_read', JSON.stringify({ path: 'papers/alpha.txt' })));
192 return { listing, read };
193 });
194 check('file_list shows the away file, marked as in cloud storage',
195 /alpha\.txt\s+\(\d+ bytes, in cloud storage\)/.test(agentView.listing),
196 agentView.listing.replace(/\n/g, ' | ').slice(0, 100));
197 check('file_read refuses honestly and names the remedy',
198 /in cloud storage/i.test(agentView.read) && /file_fetch/.test(agentView.read),
199 agentView.read.slice(0, 90));
200
201 // ── The gateway refuses to sweep for a stale device ──
202 const stale = await page.evaluate(async () => {
203 const r = await fetch('/api/chunk', {
204 method: 'POST', credentials: 'same-origin',
205 headers: { 'content-type': 'application/json', 'x-daimond-api': '1' },
206 body: JSON.stringify({ op: 'commit', chunks: [], blob_version: 0 }),
207 });
208 return { status: r.status, json: await r.json().catch(() => null) };
209 });
210 check('the gateway refuses a commit derived from a stale view', stale.status === 409,
211 'status=' + stale.status);
212 const stillThere = await page.evaluate(async () => {
213 const mani = window.DaimondCloud.manifest('papers/alpha.txt');
214 const g = await fetch('/api/chunk', {
215 method: 'POST', credentials: 'same-origin',
216 headers: { 'content-type': 'application/json', 'x-daimond-api': '1' },
217 body: JSON.stringify({ op: 'get', addr: mani.chunks[0].addr }),
218 });
219 return (await g.json()).present;
220 });
221 check('and the empty index it sent swept nothing', !!stillThere);
222
223 // ── Save a copy must write the WHOLE file ──
224 // The export once read through file_read, which truncates at 60 KB, so every
225 // larger file was silently shortened on the way to the user's disk. The
226 // folder picker is native and cannot be driven, but an OPFS directory handle
227 // implements the same interface — so stub the picker with one and read back
228 // exactly what a real folder would have received. The destination is dotted so
229 // the walk skips it: a real folder is on disk, outside OPFS, and can never be
230 // inside the tree being copied.
231 await page.evaluate(() => { try { DaimondPanels.open('work'); } catch (e) { /* already */ } });
232 await page.waitForTimeout(800);
233 const exported = await page.evaluate(async () => {
234 const mod = await import('../pkg/oxedyne_daimond.js');
235 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
236 let big = '';
237 while (big.length < 150 * 1024) big += 'export fidelity matters, every byte of it. ';
238 await app.run_tool('file_write', JSON.stringify({ path: 'out/big.txt', content: big }));
239
240 const root = await navigator.storage.getDirectory();
241 const dest = await root.getDirectoryHandle('.export_target', { create: true });
242 const realPicker = window.showDirectoryPicker;
243 window.showDirectoryPicker = async () => dest;
244 try {
245 const btns = Array.from(document.querySelectorAll('#panel-work .files-mode-btn'));
246 const save = btns.find(b => /Save a copy/i.test(b.textContent));
247 if (!save) return { ran: false };
248 save.click();
249 // Give the walk time to finish; it writes file by file.
250 for (let i = 0; i < 60; i++) {
251 await new Promise(r => setTimeout(r, 250));
252 try {
253 const d = await dest.getDirectoryHandle('out');
254 const f = await (await d.getFileHandle('big.txt')).getFile();
255 if (f.size >= big.length) return { ran: true, wrote: f.size, want: big.length };
256 } catch (e) { /* not there yet */ }
257 }
258 let got = 0;
259 try {
260 const d = await dest.getDirectoryHandle('out');
261 got = (await (await d.getFileHandle('big.txt')).getFile()).size;
262 } catch (e) { got = 0; }
263 return { ran: true, wrote: got, want: big.length };
264 } finally { window.showDirectoryPicker = realPicker; }
265 });
266 check('save a copy writes the file whole, not truncated at the tool cap',
267 exported.ran && exported.wrote === exported.want,
268 'wrote=' + exported.wrote + ' want=' + exported.want);
269
270 // ── The agent cannot download without limit ──
271 const budget = await page.evaluate(async () => {
272 const before = window.DaimondCloud.agentAllowance();
273 // Pretend the agent has already pulled its fill this window.
274 localStorage.setItem('daimond-cloud-agent-fetches',
275 JSON.stringify([{ at: Date.now(), n: 200 * 1024 * 1024 }]));
276 const after = window.DaimondCloud.agentAllowance();
277 const refusal = await window.__daimondCloudFetch('papers/alpha.txt')
278 .catch(e => String(e));
279 localStorage.removeItem('daimond-cloud-agent-fetches');
280 // The user is not subject to the agent's budget.
281 return { before, after, refusal };
282 });
283 check('the agent has a download allowance', budget.before > 0, 'bytes=' + budget.before);
284 check('and is refused once it is spent, told to come back through the user',
285 budget.after === 0 && /past what may be downloaded automatically/.test(budget.refusal),
286 String(budget.refusal).slice(0, 80));
287
288 // ── Binary files travel, byte for byte ──
289 // The workspace was text-only until now: anything binary was silently
290 // skipped, so a picture stayed on the one device that made it. The pipeline
291 // carries bytes end to end, so the test uses bytes that are NOT valid text —
292 // every value 0-255, NULs included — and demands them back identically.
293 const binary = await page.evaluate(async () => {
294 const N = 700 * 1024;
295 const src = new Uint8Array(N);
296 for (let i = 0; i < N; i++) src[i] = (i * 7 + (i >> 8)) & 0xff;
297 await window.DaimondCloud.writeBlob('media/pattern.bin', new Blob([src]));
298
299 // The server version may have moved under this page earlier in the run,
300 // so the first push can come back 409: sync pulls, merges and leaves the
301 // send for the next round rather than clobbering. That is the design, so
302 // the test pushes the way the app does -- again -- instead of asserting
303 // that one attempt must always win a race.
304 for (let i = 0; i < 3; i++) {
305 await window.DaimondSync.push();
306 if (window.DaimondCloud.manifest('media/pattern.bin')) break;
307 await new Promise(r => setTimeout(r, 400));
308 }
309 const m = window.DaimondCloud.manifest('media/pattern.bin');
310 // A bare `false` here says nothing about WHY, and this has failed for two
311 // different reasons already. Carry back what the modules know.
312 if (!m) {
313 let summary = null;
314 try { summary = await window.DaimondCloud.summary(); } catch (e) { /* none */ }
315 return { offloaded: false, why: JSON.stringify({
316 indexed: Object.keys(window.DaimondCloud.index() || {}),
317 allowance: window.DaimondCloud.allowance ? window.DaimondCloud.allowance() : null,
318 summary,
319 }) };
320 }
321
322 // Drop it locally and bring it back down from cloud storage.
323 await window.DaimondCloud.evict('media/pattern.bin');
324 const away = !!window.DaimondCloud.awayPaths()['media/pattern.bin'];
325 const res = await window.DaimondCloud.fetch('media/pattern.bin');
326
327 const back = new Uint8Array(await (await window.DaimondCloud.fileAt('media/pattern.bin')).arrayBuffer());
328 let same = back.length === src.length;
329 if (same) for (let i = 0; i < src.length; i++) { if (back[i] !== src[i]) { same = false; break; } }
330 return { offloaded: true, v: m.v, chunks: m.chunks.length, away, res, size: back.length, same };
331 });
332 check('a binary file is offloaded to cloud storage rather than skipped', binary.offloaded,
333 binary.why || '');
334 check('it is sealed chunk by chunk, not as one whole-file blob', binary.v === 2,
335 'v=' + binary.v + ' chunks=' + binary.chunks);
336 check('it can be freed like any other file', binary.away);
337 check('and comes back byte-for-byte identical, NULs and all',
338 binary.same && binary.size === 700 * 1024, 'size=' + binary.size + ' identical=' + binary.same);
339
340 // ── Only changed chunks are re-uploaded ──
341 // A seal draws a fresh IV each time, so without the plaintext-chunk map an
342 // edit to a large file would re-encrypt and re-send all of it.
343 //
344 // ONE `push()` IS NOT ONE PUSH. `sync.js` returns early when a push is
345 // already in flight (`if (inFlight) { schedule(); return; }`), and the fetch
346 // two blocks above sets one going, so the first call here can come back
347 // having done nothing at all: measured on 2026-08-13, after push #0 the
348 // manifest still carried the pre-edit mtime (…290 against a file stamped
349 // …758) and the pre-edit key, and only push #1 offloaded. The block above
350 // already loops for the same reason and says so; this one did not, and read
351 // "unchanged=3/3" — the file untouched — as a failure to reuse chunks. So it
352 // pushes until the identity moves, which is the app's own behaviour and not
353 // a weaker assertion: what is asserted about the chunks is unchanged.
354 const partial = await page.evaluate(async (mode) => {
355 const before = window.DaimondCloud.manifest('media/pattern.bin');
356 const src = new Uint8Array(await (await window.DaimondCloud.fileAt('media/pattern.bin')).arrayBuffer());
357 if (mode === 'reseal') { for (let i = 0; i < src.length; i++) src[i] = (src[i] + 1) & 0xff; }
358 else if (mode !== 'noedit') { src[src.length - 5] ^= 0xff; } // touch the LAST chunk only.
359 await window.DaimondCloud.writeBlob('media/pattern.bin', new Blob([src]));
360 let pushes = 0;
361 for (let i = 0; i < 4; i++) {
362 pushes++;
363 await window.DaimondSync.push();
364 if (window.DaimondCloud.manifest('media/pattern.bin').key !== before.key) break;
365 await new Promise(r => setTimeout(r, 400));
366 }
367 const after = window.DaimondCloud.manifest('media/pattern.bin');
368 let held = 0;
369 for (let i = 0; i < Math.min(before.chunks.length, after.chunks.length); i++) {
370 if (before.chunks[i].addr === after.chunks[i].addr) held++;
371 }
372 return { n: after.chunks.length, held, pushes, keyChanged: before.key !== after.key };
373 }, BREAK);
374 check('editing one chunk of a file leaves the others at their old addresses',
375 partial.held === partial.n - 1 && partial.n > 1,
376 'unchanged=' + partial.held + '/' + partial.n + ' after ' + partial.pushes + ' push(es)');
377 check('and the file identity changes with the edit', partial.keyChanged);
378
379 // ── The free tier must survive a lapse ──
380 // At the end of grace the gateway evicts the paid tier and keeps the free
381 // one. The client used to tag every chunk paid, which would have taken a
382 // lapsed account's whole store instead of its overflow.
383 const tiers = await page.evaluate(async () => {
384 // Pretend the gateway granted a small allowance, enough for one file.
385 window.DaimondCloud.setAllowance(250 * 1024);
386 const ix = window.DaimondCloud.index();
387 // Make one file plainly the most recently used.
388 window.DaimondCloud.touch('papers/beta.txt');
389 const plan = window.DaimondCloud.tierPlan(window.DaimondCloud.allowance());
390 const free = Object.keys(plan).filter(k => plan[k] === 'f');
391 const paid = Object.keys(plan).filter(k => plan[k] === 'p');
392 return { free, paid, n: Object.keys(ix).length };
393 });
394 check('the most recently used file is tagged free, inside the allowance',
395 tiers.free.includes('papers/beta.txt'), 'free=' + JSON.stringify(tiers.free));
396 check('and the rest is paid overflow, so a lapse takes only the overflow',
397 tiers.paid.length > 0 && tiers.free.length < tiers.n,
398 'free=' + tiers.free.length + ' paid=' + tiers.paid.length);
399
400 // ── An explicit delete DOES remove it ──
401 const deleted = await page.evaluate(async () => {
402 const mod = await import('../pkg/oxedyne_daimond.js');
403 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
404 const out = String(await app.run_tool('file_delete', JSON.stringify({ path: 'papers/alpha.txt' })));
405 return { out, known: !!window.DaimondCloud.manifest('papers/alpha.txt') };
406 });
407 check('deleting an away file succeeds and forgets it', !deleted.known,
408 deleted.out.slice(0, 80));
409} catch (e) {
410 check('no exception during the run', false, String(e && e.message || e));
411} finally {
412 try { await s.browser.close(); } catch (e) { /* ignore */ }
413 if (gw) { try { gw.kill('SIGTERM'); } catch (e) { /* ignore */ } }
414}
415
416if (bad.length) GW_LOG.report();
417console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed');
418process.exit(bad.length ? 1 : 0);