Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_chunks.mjs

33.5 KiB, 1 run

created by r2519314175:293, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_chunks.mjs — the content-addressed chunk store, in two tiers.
2//
3// TIER 1 (no gateway). The two things a user can act on, which had no way to be
4// acted on at all:
5//
6// a. A HELD-BACK DELETION CAN BE CONFIRMED. When the gateway declines a large
7// sweep, `chunks.js` parks the whole commit body plus the token and paints
8// an amber chip. Until 2026-08-14 that chip was a `role="status"` div and
9// `confirmHeldSweep` had NO PRODUCTION CALLER anywhere in the tree, so the
10// notice was the entire feature: a permanent pill saying a deletion was
11// standing, with nothing on any surface that could carry it out. The chip
12// is a button now, it asks before it deletes, and the parked commit is
13// written to localStorage — a token can only be minted by the gateway, so a
14// reload used to abandon a deletion the account goes on paying for.
15// b. A REFUSED UPLOAD ARRIVES AS ITS OWN SENTENCE. `putChunks` threw
16// `chunk put failed: 507` and discarded `res.json.error`. The gateway
17// composes four sentences on this route and each names the remedy; the user
18// got a number. The sentences the stub answers with are READ OUT OF
19// `gateway/src/handlers/chunk.rs`, so the fixture cannot drift from what
20// the server actually says.
21//
22// The gateway is stubbed at `DaimondGateway.gwFetch`, which `chunks.js` looks up
23// on the global at every call for exactly this reason. That makes tier 1
24// runnable inside a world, with no :9002 and no gateway binary.
25//
26// TIER 2 (needs a gateway on :9002). The original round trip: a workspace file
27// too large for the sync blob travels to a second device through the chunk store
28// and comes back byte-for-byte, without the gateway ever seeing its plaintext.
29//
30// 1. Sign in. Write a 200 KiB file — well over the 128 KiB inline ceiling, so
31// it is offloaded to chunks rather than carried in the blob.
32// 2. Push. The sync blob must NOT contain the file's plaintext (it holds only
33// chunk references), and a fetched chunk must be ciphertext (marker absent).
34// 3. Second device: delete the file, wipe the offload cache and cursors, pull.
35// The file is reconstructed from its chunks, identical to the original.
36//
37// ── Running it ──────────────────────────────────────────────────────
38//
39// bash dev/world.sh 14 --up ; eval "$(bash dev/world.sh 14 --env)"
40// node dev/verify_chunks.mjs --no-gateway # tier 1 only
41// node dev/verify_chunks.mjs # both; needs :9002
42//
43// `--no-gateway` skips tier 2 and says so. It does NOT soften tier 2: without
44// the flag a missing or stale gateway binary still fails the run, because a
45// release gate that quietly stops testing the round trip is worse than one that
46// stops.
47//
48// ── Proved red ──────────────────────────────────────────────────────
49//
50// `--break <name>` serves a deliberately damaged `js/chunks.js` to the real page
51// and the run is EXPECTED TO FAIL. An anchor that does not appear exactly once
52// aborts rather than passing quietly.
53//
54// node dev/verify_chunks.mjs --no-gateway --break nocaller # the chip is a notice again
55// node dev/verify_chunks.mjs --no-gateway --break status # the status code is back
56// node dev/verify_chunks.mjs --no-gateway --break memory # the deletion dies on reload
57// node dev/verify_chunks.mjs --no-gateway --break anyone # a stranger inherits it
58import fs from 'node:fs';
59import path from 'node:path';
60import { spawn } from 'node:child_process';
61import { fileURLToPath } from 'node:url';
62import { requireFreshGateway, procLog, GWCWD } from './gwbin.mjs';
63import { open, signInAs } from './harness.mjs';
64import { makePagePro } from './pro.mjs';
65import { GW_PORT, GW_URL } from './ports.mjs';
66
67const __dirname = path.dirname(fileURLToPath(import.meta.url));
68const ROOT = path.resolve(__dirname, '..');
69const WWW = path.join(ROOT, 'www');
70const GWDIR = path.join(ROOT, 'gateway');
71const SRC = 'js/chunks.js';
72
73const ok = [], bad = [];
74const check = (name, pass, detail) => {
75 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
76 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
77};
78
79const NO_GATEWAY = process.argv.includes('--no-gateway');
80const BREAK = (() => {
81 const i = process.argv.indexOf('--break');
82 return i > 0 ? String(process.argv[i + 1] || '') : '';
83})();
84
85// ── The breaks ──────────────────────────────────────────────────────────
86//
87// Each is the defect exactly as it stood, so a green run under one would mean
88// the check below it had stopped measuring anything.
89
90/// The line that makes the chip a control rather than a notice.
91const LISTENER = "\t\tc.addEventListener('click', function () { onChipClick(); });\n";
92
93/// The refusal path as it shipped: the gateway's sentence on the floor.
94const SENTENCE = "\t\t\t\tvar msg = (res.json && (res.json.error || res.json.message))\n"
95 + "\t\t\t\t\t|| ('HTTP ' + res.status);\n"
96 + "\t\t\t\tstandRefused(msg, res.status);\n"
97 + "\t\t\t\tvar e = new Error(msg);\n"
98 + "\t\t\t\te.status = res.status;\t\t// for a caller that wants to branch on it.\n"
99 + "\t\t\t\tthrow e;\n";
100const STATUS_ONLY = "\t\t\t\tthrow new Error('chunk put failed: ' + res.status);\n";
101
102/// The write that lets a standing deletion outlive the page.
103const PERSIST = "\t\ttry { localStorage.setItem(HELD_KEY, s); persisted = true; }\n"
104 + "\t\tcatch (e) { /* quota or private mode: it stands for this sitting only */ }\n";
105
106/// The guard that stops one identity inheriting another's parked deletion.
107const OWNER = "\t\tvar fp = whoseFp();\n"
108 + "\t\tif (!fp || h.fp !== fp) {\n"
109 + "\t\t\ttry { localStorage.removeItem(HELD_KEY); } catch (e) { /* private mode */ }\n"
110 + "\t\t\treturn null;\n"
111 + "\t\t}\n";
112
113const BREAKS = {
114 // The chip goes back to being a pill nobody can press. `confirmHeldSweep`
115 // still exists and still works — which is the whole point of the finding, and
116 // why a check that called it directly would have gone green throughout.
117 nocaller: [{ file: SRC, find: LISTENER, with: '' }],
118 // `chunk put failed: 507`, as the first user to fill their allowance saw it.
119 status: [{ file: SRC, find: SENTENCE, with: STATUS_ONLY }],
120 // In memory only, so a reload abandons the deletion.
121 memory: [{ file: SRC, find: PERSIST, with: '' }],
122 // Any identity picks up any parked deletion, which is what an un-namespaced
123 // key left behind by a forget would hand the next person in this browser.
124 anyone: [{ file: SRC, find: OWNER, with: '' }],
125};
126
127if (BREAK && !BREAKS[BREAK]) {
128 console.error(`unknown break '${BREAK}'; one of: ${Object.keys(BREAKS).join(', ')}`);
129 process.exit(2);
130}
131
132/// The damaged source, or a hard stop. Nothing is served that was not verified
133/// to differ from the file on disk.
134function damaged(src, spec) {
135 const n = src.split(spec.find).length - 1;
136 if (n !== 1) {
137 console.error(`break '${BREAK}': the anchor appears ${n} times in ${spec.file}, `
138 + 'so nothing was broken and the run below would prove nothing.');
139 process.exit(2);
140 }
141 return src.replace(spec.find, spec.with);
142}
143
144/// Serve the damaged file to the page, before anything navigates.
145async function breakInto(page) {
146 const bodies = {};
147 for (const spec of BREAKS[BREAK]) {
148 const disk = fs.readFileSync(path.join(WWW, spec.file), 'utf8');
149 damaged(disk, spec); // checked against DISK, so two edits cannot mask each other.
150 bodies[spec.file] = (bodies[spec.file] || disk).replace(spec.find, spec.with);
151 }
152 for (const file of Object.keys(bodies)) {
153 await page.route('**/' + file, (r) => r.fulfill({
154 status: 200, contentType: 'application/javascript', body: bodies[file],
155 }));
156 }
157}
158
159// ── What the gateway actually says ──────────────────────────────────────
160//
161// Read out of the handler rather than typed here. A fixture that quotes the
162// server from memory is a fixture that goes on passing after the server's words
163// change, which is the difference between checking the seam and checking a copy
164// of one side of it.
165function gatewaySentences() {
166 const rs = fs.readFileSync(path.join(GWDIR, 'src/handlers/chunk.rs'), 'utf8');
167 const re = /err_response\(\s*HttpStatus::([A-Za-z]+)\s*,\s*"((?:[^"\\]|\\[\s\S])*)"/g;
168 const out = [];
169 let m;
170 while ((m = re.exec(rs))) {
171 // A Rust `\` at end of line eats the newline and the indent after it.
172 out.push({ status: m[1], text: m[2].replace(/\\\s*\n\s*/g, '').trim() });
173 }
174 return out;
175}
176
177const SENTENCES = gatewaySentences();
178const AT_CEILING = (SENTENCES.find((s) => /reached its cloud storage limit/.test(s.text)) || {}).text;
179
180// ┌───────────────────────────────────────────────────────────────────┐
181// │ TIER 1 — the two controls, with the gateway stubbed │
182// └───────────────────────────────────────────────────────────────────┘
183
184console.log('\n— tier 1: the standing deletion, and the refused upload —');
185if (BREAK) console.log(`\n*** RUNNING UNDER --break ${BREAK}: failures below are the point ***\n`);
186
187check('the four sentences the gateway composes were read out of chunk.rs',
188 SENTENCES.length >= 4 && !!AT_CEILING, `${SENTENCES.length} found`);
189check('one of them is the 507 that names the remedy',
190 !!AT_CEILING && /Delete something/.test(AT_CEILING), AT_CEILING);
191
192const t1 = await open({
193 name: 'chunkctl',
194 signIn: true,
195 connect: false,
196 route: BREAK ? breakInto : null,
197});
198const p = t1.page;
199
200/// Put the stub gateway and the stub dialog on the page. Re-run after a reload,
201/// which throws both away along with everything else.
202async function arm(page) {
203 await page.evaluate(() => {
204 window.__gw = {
205 calls: [],
206 token: 'sweep-token-abcdef',
207 held: 4, // chunks the account holds, all of them doomed.
208 putStatus: 0, // 0 = accept the upload.
209 putError: '',
210 };
211 // `chunks.js` reads `DaimondGateway.gwFetch` off the global at every call
212 // — late-bound, never captured — so replacing it here is the real code
213 // path and not a shim around it.
214 window.DaimondGateway.gwFetch = async function (path_, opts) {
215 const body = JSON.parse(opts.body);
216 window.__gw.calls.push(body);
217 const reply = (status, json) => ({ status, json: async () => json });
218 if (body.op === 'put') {
219 if (window.__gw.putStatus) {
220 return reply(window.__gw.putStatus, { ok: false, error: window.__gw.putError });
221 }
222 return reply(200, { ok: true });
223 }
224 if (body.op === 'have') return reply(200, { missing: (body.addrs || []).slice() });
225 if (body.op === 'commit') {
226 // The interlock: the identical body, quoting the token, carries the
227 // deletion out. Anything else is held back again.
228 if (body.sweep_token === window.__gw.token) {
229 return reply(200, { ok: true, swept: window.__gw.held, free_allowance: 0 });
230 }
231 return reply(200, {
232 ok: true, swept: 0,
233 sweep_held_back: window.__gw.held,
234 sweep_held: window.__gw.held,
235 sweep_token: window.__gw.token,
236 free_allowance: 0,
237 });
238 }
239 return reply(200, { ok: true });
240 };
241 // The app's own confirm box, recorded rather than drawn. What matters is
242 // that the control ASKS and honours the answer; the dialog itself is
243 // daimond.js's and has its own checks.
244 window.__asked = [];
245 window.__answer = true;
246 window.DaimondCore.confirm = function (message, okLabel, opts) {
247 window.__asked.push({ message: message, okLabel: okLabel, title: (opts || {}).title });
248 return Promise.resolve(window.__answer);
249 };
250 });
251}
252
253/// What the chip is, as the DOM has it. `exists` is reported separately from
254/// `shown` on purpose: an element that is not there answers "hidden" to every
255/// visibility question, so an absence check with no presence check beside it
256/// passes for a chip that was never built.
257const chipState = () => p.evaluate(() => {
258 const c = document.getElementById('chunk-chip');
259 if (!c) return { exists: false, shown: false, tag: '', text: '', title: '', label: '' };
260 return {
261 exists: true,
262 shown: getComputedStyle(c).display !== 'none',
263 tag: c.tagName,
264 text: (c.textContent || '').trim(),
265 title: c.title || '',
266 label: c.getAttribute('aria-label') || '',
267 };
268});
269
270try {
271 await p.waitForFunction(
272 () => !!window.DaimondChunks && !!window.DaimondGateway && !!window.DaimondCore,
273 null, { timeout: 15000 });
274 await arm(p);
275
276 // ── 1a. Nothing standing, and the chip proved absent for the right reason ──
277 const idle = await chipState();
278 check('with nothing standing the chip is not on screen', !idle.shown,
279 `exists=${idle.exists} shown=${idle.shown}`);
280 check('and localStorage holds no standing deletion',
281 (await p.evaluate(() => localStorage.getItem('daimond-chunk-held'))) === null);
282
283 // ── 1b. A held-back sweep stands, and the notice is a CONTROL ─────────────
284 //
285 // An index naming nothing: the one case `refusalToConfirm` deliberately never
286 // clears by itself, and the case the escape hatch exists for.
287 const stood = await p.evaluate(async () => {
288 await window.DaimondChunks.commit({}, 1, null);
289 return window.DaimondChunks.state();
290 });
291 check('an index naming nothing leaves the deletion standing',
292 stood.standing === true && stood.why === 'names_nothing' && stood.heldBack === 4,
293 JSON.stringify(stood));
294
295 const chip = await chipState();
296 // The pair that the two invisible features needed and did not have.
297 check('the chip EXISTS in the document', chip.exists, chip.tag || '(absent)');
298 check('and it is on screen', chip.shown);
299 check('and it is a BUTTON, not a status region nobody can press',
300 chip.tag === 'BUTTON', chip.tag);
301 check('it says something, in words rather than an i18n key',
302 chip.text.length > 0 && !/^chunks\./.test(chip.text) && !/^chunks\./.test(chip.title),
303 chip.text);
304 check('and it carries an accessible name that includes the reason',
305 chip.label.length > chip.text.length, chip.label.slice(0, 80));
306
307 // ── 1c. Saying NO deletes nothing ─────────────────────────────────────────
308 //
309 // Before the yes, because a control that deleted on any click would pass the
310 // next check and be a far worse defect than the one being fixed.
311 await p.evaluate(() => { window.__answer = false; window.__gw.calls.length = 0; });
312 await p.click('#chunk-chip');
313 await p.waitForTimeout(300);
314 // A COMMIT QUOTING THE TOKEN, not any commit. The sync engine is running in
315 // this page and commits its own live set on its own schedule; counting every
316 // commit would make these checks depend on whether a background round
317 // happened to land inside the window, and one already did during a `--break`
318 // run — turning a check that should have gone red green.
319 const said = await p.evaluate(() => ({
320 asked: window.__asked.length,
321 tokened: window.__gw.calls.filter((c) => c.op === 'commit' && c.sweep_token).length,
322 st: window.DaimondChunks.state(),
323 }));
324 check('pressing the chip ASKS before it deletes', said.asked === 1, `${said.asked} question(s)`);
325 check('and answering no authorises no deletion', said.tokened === 0,
326 `${said.tokened} commit(s) quoting a token`);
327 check('so the deletion is still standing', said.st.standing === true, JSON.stringify(said.st));
328
329 // ── 1d. Saying YES carries the deletion out ───────────────────────────────
330 await p.evaluate(() => { window.__answer = true; window.__gw.calls.length = 0; });
331 await p.click('#chunk-chip');
332 await p.waitForFunction(() => window.DaimondChunks.state().standing === false,
333 null, { timeout: 8000 }).catch(() => {});
334 const done = await p.evaluate(() => ({
335 asked: window.__asked.length,
336 tokened: window.__gw.calls.filter((c) => c.op === 'commit' && c.sweep_token),
337 st: window.DaimondChunks.state(),
338 held: localStorage.getItem('daimond-chunk-held'),
339 }));
340 check('answering yes authorises the deletion exactly once, never in a loop',
341 done.tokened.length === 1, `${done.tokened.length} commit(s) quoting a token`);
342 check('and it quotes the token the gateway minted, so the gateway can check it',
343 done.tokened.length === 1 && done.tokened[0].sweep_token === 'sweep-token-abcdef',
344 JSON.stringify(done.tokened[0] && done.tokened[0].sweep_token));
345 check('the chunks actually go: the client records the sweep as confirmed',
346 done.st.confirmed === 1 && done.st.standing === false, JSON.stringify(done.st));
347 const cleared = await chipState();
348 check('the chip goes with them, and the element is still there to be hidden',
349 cleared.exists === true && cleared.shown === false,
350 `exists=${cleared.exists} shown=${cleared.shown}`);
351 check('and nothing is left in storage to raise it from the dead next boot',
352 done.held === null, String(done.held));
353
354 // ── 1e. A standing deletion survives a reload ─────────────────────────────
355 //
356 // The half that bites. Only the gateway can mint a token; this client cannot
357 // re-derive one. A reload used to drop the body and the token together, and
358 // on a device where sync is not running the commit that would raise it again
359 // never comes — so the chunks sit there, referenced by nothing, swept by
360 // nothing, and billed.
361 await p.evaluate(async () => {
362 window.__gw.calls.length = 0;
363 await window.DaimondChunks.commit({}, 1, null);
364 });
365 const beforeReload = await p.evaluate(() => ({
366 st: window.DaimondChunks.state(),
367 saved: localStorage.getItem('daimond-chunk-held'),
368 }));
369 check('the standing deletion is written to storage', !!beforeReload.saved,
370 beforeReload.saved ? (beforeReload.saved.length + ' bytes') : 'nothing written');
371 check('and the client says so, rather than leaving it to be guessed',
372 beforeReload.st.persisted === true, JSON.stringify(beforeReload.st));
373
374 await p.reload({ waitUntil: 'domcontentloaded' });
375 await signInAs(t1, t1.name);
376 await p.waitForFunction(() => !!window.DaimondChunks && !!window.DaimondCore,
377 null, { timeout: 15000 });
378
379 const survived = await p.evaluate(() => window.DaimondChunks.state());
380 check('after a reload the deletion is STILL standing',
381 survived.standing === true && survived.heldBack === 4, JSON.stringify(survived));
382 const afterChip = await chipState();
383 check('and the chip is drawn again from storage, unprompted',
384 afterChip.exists === true && afterChip.shown === true && afterChip.tag === 'BUTTON',
385 JSON.stringify({ exists: afterChip.exists, shown: afterChip.shown, tag: afterChip.tag }));
386
387 // And it is the SAME deletion: the restored token is the one the gateway
388 // minted, which is the only thing that makes the restored body worth keeping.
389 await arm(p);
390 await p.evaluate(() => { window.__answer = true; window.__gw.calls.length = 0; });
391 await p.click('#chunk-chip');
392 await p.waitForFunction(() => window.DaimondChunks.state().standing === false,
393 null, { timeout: 8000 }).catch(() => {});
394 const resumed = await p.evaluate(() => ({
395 tokened: window.__gw.calls.filter((c) => c.op === 'commit' && c.sweep_token),
396 st: window.DaimondChunks.state(),
397 }));
398 check('a deletion recovered from storage can be carried out',
399 resumed.st.standing === false && resumed.st.confirmed === 1, JSON.stringify(resumed.st));
400 check('and it quotes the token from the sitting before, not a new one',
401 resumed.tokened.length === 1 && resumed.tokened[0].sweep_token === 'sweep-token-abcdef',
402 JSON.stringify(resumed.tokened[0] && resumed.tokened[0].sweep_token));
403
404 // ── 1e². And it belongs to whoever it was stored for ──────────────────────
405 //
406 // `forgetIdentity` sweeps a NAMED list of keys, and the primary account's keys
407 // are un-namespaced -- so anything not on that list is inherited whole by the
408 // next identity made in this browser. A commit body for an account that no
409 // longer exists would paint a chip for a stranger and send a token that can
410 // only be refused. The record carries the identity fingerprint for that.
411 const stranger = await p.evaluate(async () => {
412 await window.DaimondChunks.commit({}, 1, null);
413 // Tolerant of a build that wrote nothing, so a `--break` run reports every
414 // check below rather than stopping at a null.
415 let raw = null;
416 try { raw = JSON.parse(localStorage.getItem('daimond-chunk-held') || 'null'); }
417 catch (e) { raw = null; }
418 const wasFp = raw ? String(raw.fp || '') : '';
419 if (raw) {
420 raw.fp = 'ffff ffff ffff ffff'; // as though another identity had left it.
421 localStorage.setItem('daimond-chunk-held', JSON.stringify(raw));
422 }
423 return { wasFp: wasFp, standing: window.DaimondChunks.state().standing };
424 });
425 check('the record names whose deletion it is', !!stranger.wasFp && stranger.standing === true,
426 JSON.stringify(stranger));
427
428 await p.reload({ waitUntil: 'domcontentloaded' });
429 await signInAs(t1, t1.name);
430 await p.waitForFunction(() => !!window.DaimondChunks, null, { timeout: 15000 });
431 const notMine = await p.evaluate(() => ({
432 st: window.DaimondChunks.state(),
433 saved: localStorage.getItem('daimond-chunk-held'),
434 }));
435 check('another identity\'s standing deletion is NOT adopted',
436 notMine.st.standing === false, JSON.stringify(notMine.st));
437 check('and it is dropped rather than left to be found again',
438 notMine.saved === null, String(notMine.saved));
439 const strayChip = await chipState();
440 check('so no chip is raised for it — and the element is there to have been raised',
441 strayChip.exists === true && strayChip.shown === false,
442 `exists=${strayChip.exists} shown=${strayChip.shown}`);
443
444 await arm(p);
445
446 // ── 1f. A refused upload arrives as its own sentence ──────────────────────
447 await p.evaluate((sentence) => {
448 window.__gw.putStatus = 507;
449 window.__gw.putError = sentence;
450 window.__asked.length = 0;
451 }, AT_CEILING);
452
453 const refused = await p.evaluate(async () => {
454 const bytes = new Uint8Array(4096).map((_, i) => (i * 7) % 251);
455 const file = new File([bytes], 'big.bin');
456 let message = '', status = 0;
457 try { await window.DaimondChunks.offloadFile('big.bin', file); }
458 catch (e) { message = String(e && e.message || e); status = (e && e.status) | 0; }
459 return { message: message, status: status, st: window.DaimondChunks.state() };
460 });
461 check('the upload really was refused, so the checks below are not vacuous',
462 refused.message.length > 0, refused.message || '(nothing thrown)');
463 check('THE GATEWAY\'S OWN SENTENCE REACHES THE CALLER',
464 refused.message === AT_CEILING, refused.message);
465 check('and it is not the status code standing in for it',
466 !/^chunk put failed/.test(refused.message) && refused.message !== 'HTTP 507'
467 && refused.message.indexOf('507') === -1,
468 refused.message);
469 check('the status is kept beside it for a caller that wants to branch',
470 refused.status === 507, String(refused.status));
471 check('and the module holds the sentence, not a number',
472 refused.st.refused === AT_CEILING && refused.st.refusedStatus === 507,
473 JSON.stringify({ refused: refused.st.refused, status: refused.st.refusedStatus }));
474
475 // Said, not merely thrown. `collectChunked` in daimond.js catches every
476 // offload failure and discards it, so a perfect sentence on an exception
477 // still reaches nobody.
478 const refChip = await chipState();
479 check('the refusal is ON SCREEN, on a chip that exists',
480 refChip.exists === true && refChip.shown === true, JSON.stringify(refChip));
481 check('and the whole sentence is reachable from it, not only the short label',
482 refChip.label.indexOf(AT_CEILING) !== -1 || refChip.title === AT_CEILING,
483 refChip.title.slice(0, 80));
484
485 await p.click('#chunk-chip');
486 await p.waitForTimeout(300);
487 const told = await p.evaluate(() => window.__asked.slice());
488 check('and pressing it puts the sentence in front of the user',
489 told.length === 1 && String(told[0].message).indexOf(AT_CEILING) !== -1,
490 told.length ? String(told[0].message).slice(0, 90) : '(nothing said)');
491
492 // The other half: a batch that lands lifts the refusal, or the chip would be
493 // a permanent amber pill for a ceiling that was raised an hour ago.
494 const lifted = await p.evaluate(async () => {
495 window.__gw.putStatus = 0;
496 const bytes = new Uint8Array(4096).map((_, i) => (i * 3) % 251);
497 await window.DaimondChunks.offloadFile('big2.bin', new File([bytes], 'big2.bin'));
498 return window.DaimondChunks.state();
499 });
500 check('an upload that lands clears the refusal', lifted.refused === '',
501 JSON.stringify(lifted.refused));
502} catch (e) {
503 check('no exception during tier 1', false, String(e && e.message || e));
504} finally {
505 try { await t1.browser.close(); } catch (e) { /* ignore */ }
506}
507
508// ┌───────────────────────────────────────────────────────────────────┐
509// │ TIER 2 — the round trip, against a real gateway on :9002 │
510// └───────────────────────────────────────────────────────────────────┘
511
512if (NO_GATEWAY) {
513 console.log('\n— tier 2 SKIPPED (--no-gateway): the offload round trip was not run —');
514} else {
515 console.log('\n— tier 2: the offload round trip, against a real gateway —');
516
517 /// What the gateway says while this runs. A chunk request answering 500 says
518 /// only that something went wrong; the reason is logged beside it, here.
519 /// Silent when this run reuses a gateway it did not start.
520 const GW_LOG = procLog('verify_chunks');
521
522 let gw = null;
523 const waitFor = async (fn, ms = 20000, gap = 300) => {
524 const t0 = Date.now();
525 while (Date.now() - t0 < ms) {
526 try { if (await fn()) return true; } catch (e) { /* keep waiting */ }
527 await new Promise((r) => setTimeout(r, gap));
528 }
529 return false;
530 };
531 const startGateway = async () => {
532 gw = spawn(path.join(GWDIR, 'target/release/daimond_gateway'), [], {
533 cwd: GWCWD,
534 env: { ...process.env, APP_MODE: 'sandbox' },
535 stdio: GW_LOG.stdio,
536 });
537 return await waitFor(async () => (await fetch(`${GW_URL}/api/health`)).ok);
538 };
539
540 requireFreshGateway();
541
542 // Use a gateway already up (started outside for environments where spawning a
543 // child here is unreliable), otherwise start our own.
544 const alreadyUp = await waitFor(async () => (await fetch(`${GW_URL}/api/health`)).ok, 800, 200);
545 if (alreadyUp) {
546 console.log(` ok using the gateway already on :${GW_PORT}`);
547 gw = null; // not ours to kill.
548 } else {
549 check('gateway starts', await startGateway());
550 }
551
552 const s = await open({ name: 'chunks', signIn: true, connect: false });
553 const { page } = s;
554
555 await page.waitForFunction(
556 () => !!window.DaimondSync && !!window.DaimondChunks && !!window.DaimondCore
557 && !!window.DaimondGateway && DaimondGateway.state().authed,
558 null, { timeout: 12000 },
559 ).catch(() => {});
560
561 try {
562 check('the chunk module and an authed session are live',
563 await page.evaluate(() => !!window.DaimondChunks && DaimondGateway.state().authed));
564
565 // Sync and the chunk store are Pro capabilities, so a free account is
566 // refused at the door (402) and nothing below could ever happen. Buy the
567 // licence the way a user does -- a signed checkout event -- rather than
568 // testing the gate instead of the feature.
569 const lic = await makePagePro(page, GWDIR, GW_URL);
570 check('the account holds Pro, so sync is allowed to run',
571 lic.pro === true, `webhook ${lic.status}, pro=${lic.pro}`);
572
573 // A 200 KiB file: over the 128 KiB inline ceiling, so it must be offloaded.
574 const MARK = 'CHUNKMARK-' + '4242';
575 const built = await page.evaluate(async (mark) => {
576 const mod = await import('../pkg/oxedyne_daimond.js');
577 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
578 // 200 KiB of text with the marker sprinkled through it.
579 let body = '';
580 while (body.length < 200 * 1024) body += mark + ' lorem ipsum dolor sit amet, consectetur. ';
581 await app.run_tool('file_write', JSON.stringify({ path: 'big-note.txt', content: body }));
582 return { size: body.length };
583 }, MARK);
584 check('a 200 KiB workspace file exists (over the inline ceiling)', built.size > 128 * 1024,
585 'size=' + built.size);
586
587 // Push: offload to chunks, then the referencing blob.
588 const pushed = await page.evaluate(async () => {
589 await window.DaimondSync.push();
590 const r = await fetch('/api/sync', { credentials: 'same-origin', headers: { 'x-daimond-api': '1' } });
591 const j = await r.json();
592 return { version: j.version, present: j.present, blob: j.blob || '' };
593 });
594 check('after a push the mailbox holds a version >= 1', pushed.present && pushed.version >= 1,
595 'version=' + pushed.version);
596
597 // The blob is small references, not the body: the plaintext marker is absent.
598 check('the large file is NOT inline in the sync blob (offloaded to chunks)',
599 !pushed.blob.includes(MARK));
600
601 // The blob names the file under `chunked`, and a fetched chunk is ciphertext.
602 const chunkCheck = await page.evaluate(async (mark) => {
603 const plain = await window.DaimondIdentity.unwrap(document ? (await (async () => {
604 const r = await fetch('/api/sync', { credentials: 'same-origin', headers: { 'x-daimond-api': '1' } });
605 return (await r.json()).blob;
606 })()) : '');
607 const state = JSON.parse(plain);
608 const ref = state.chunked && state.chunked['big-note.txt'];
609 if (!ref || !ref.chunks || !ref.chunks.length) return { referenced: false };
610 const addr = ref.chunks[0].addr;
611 const g = await fetch('/api/chunk', {
612 method: 'POST', credentials: 'same-origin',
613 headers: { 'content-type': 'application/json', 'x-daimond-api': '1' },
614 body: JSON.stringify({ op: 'get', addr }),
615 });
616 const gj = await g.json();
617 // The stored chunk is base64url ciphertext: decode and confirm the marker
618 // is not in it.
619 let cipherHasMark = false;
620 try {
621 const t = atob(String(gj.blob || '').replace(/-/g, '+').replace(/_/g, '/'));
622 cipherHasMark = t.includes(mark);
623 } catch (e) { /* undecodable is fine */ }
624 return { referenced: true, chunkCount: ref.chunks.length, present: !!gj.present, cipherHasMark };
625 }, MARK);
626 check('the blob references the file in its chunk manifest', chunkCheck.referenced,
627 chunkCheck.referenced ? ('chunks=' + chunkCheck.chunkCount) : 'no chunked entry');
628 check('the gateway holds the referenced chunk', chunkCheck.present);
629 check('a stored chunk is ciphertext (plaintext marker absent)', !chunkCheck.cipherHasMark);
630
631 // Second device: drop the local copy, wipe the offload cache and cursors,
632 // pull. The file must NOT be downloaded — it stays in cloud storage until
633 // asked for, which is what lets a workspace be larger than the device — and
634 // must then come back byte-for-byte when it is fetched.
635 const restored = await page.evaluate(async (mark) => {
636 const mod = await import('../pkg/oxedyne_daimond.js');
637 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
638 await app.run_tool('file_delete', JSON.stringify({ path: 'big-note.txt' }));
639 localStorage.removeItem('daimond-chunk-map'); // a fresh device has never offloaded.
640 localStorage.removeItem('daimond-sync-version');
641 localStorage.removeItem('daimond-sync-filebase');
642 await window.DaimondSync.pull();
643
644 const onDisk = async () => {
645 try {
646 const root = await navigator.storage.getDirectory();
647 return await (await (await root.getFileHandle('big-note.txt')).getFile()).text();
648 } catch (e) { return null; }
649 };
650 const afterPull = await onDisk();
651 const away = window.DaimondCloud.awayPaths();
652 const known = !!window.DaimondCloud.manifest('big-note.txt');
653 // The agent is told where it is rather than that it is missing.
654 const readErr = String(await app.run_tool('file_read', JSON.stringify({ path: 'big-note.txt' })));
655 // And fetching it is a deliberate, separate act.
656 const fetched = String(await app.run_tool('file_fetch', JSON.stringify({ path: 'big-note.txt' })));
657 const back = await onDisk();
658 return {
659 lazy: afterPull === null,
660 known: known,
661 away: Object.prototype.hasOwnProperty.call(away, 'big-note.txt'),
662 readErr: readErr,
663 fetchedOk: /^\s*OK/.test(fetched) || /fetched/i.test(fetched),
664 size: back ? back.length : 0,
665 hasMark: !!back && back.includes(mark),
666 };
667 }, MARK);
668 check('a pull does NOT download the large file (it stays in cloud storage)', restored.lazy);
669 check('the device still knows the file exists, as a cloud manifest', restored.known);
670 check('the file is listed as away from this device', restored.away);
671 check('file_read tells the agent it is in cloud storage, not that it is missing',
672 /in cloud storage/i.test(restored.readErr), restored.readErr.slice(0, 90));
673 check('file_fetch brings it down on request', restored.fetchedOk, restored.fetchedOk ? '' : 'fetch refused');
674 check('the fetched file is byte-for-byte the original',
675 restored.hasMark && restored.size > 128 * 1024, 'size=' + restored.size);
676 } catch (e) {
677 check('no exception during the run', false, String(e && e.message || e));
678 } finally {
679 try { await s.browser.close(); } catch (e) { /* ignore */ }
680 if (gw) { try { gw.kill('SIGTERM'); } catch (e) { /* ignore */ } }
681 }
682
683 if (bad.length) GW_LOG.report();
684}
685
686console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed'
687 + (NO_GATEWAY ? ' (tier 2 skipped)' : ''));
688process.exit(bad.length ? 1 : 0);