Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/www/js/cloud.js

35.2 KiB, 1 run

created by r2519314175:1351, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1/* ============================================================
2 Daimond — cloud residency (cloud.js)
3 ------------------------------------------------------------
4 Where the workspace lives, and how much of it this device holds.
5
6 The workspace is ONE set of files with one set of paths. The
7 browser's OPFS sandbox is a CACHE of it, bounded by whatever
8 storage the browser grants — small on iOS, and evictable by
9 Safari without asking. Cloud storage is what lets the workspace
10 be larger than the device.
11
12 So a file has a RESIDENCY, not a location:
13
14 held — on this device and in cloud storage.
15 cloud — in cloud storage, not on this device right now.
16 pinned — held, and never evicted to make room.
17
18 A cloud-only file is still the user's file and still appears in
19 the tree. It is safe; it is simply not here at the moment.
20
21 THE INDEX IS SHARED STATE. `daimond-cloud-index` maps a path to
22 the manifest that reconstructs it. It travels in the sync blob
23 and is MERGED across devices — never rebuilt from one device's
24 local view. That distinction is load-bearing: the gateway sweeps
25 every chunk the committed index does not name, so a device that
26 rebuilt the index from its own sandbox would delete the files it
27 happened not to be holding. Absence from disk means "not here";
28 only an explicit delete means "gone".
29 ============================================================ */
30(function () {
31 'use strict';
32
33 var IX_KEY = 'daimond-cloud-index'; // path -> {size, hash, chunks:[{addr,size}]}
34 var PIN_KEY = 'daimond-cloud-pins'; // path -> 1, device-local
35 var ATIME_KEY = 'daimond-cloud-atime'; // path -> ms, for least-recently-used reclaim
36 var PATHS_KEY = 'daimond-cloud-paths'; // path -> size, DERIVED: in cloud, not on this device
37
38 // Reclaim thresholds, as a fraction of the storage the browser grants us.
39 // Reclaiming to a little under the trigger stops it running on every write.
40 var PRESSURE_HIGH = 0.85;
41 var PRESSURE_TARGET = 0.70;
42
43 // Reclaim runs after a push, and a push can be frequent. Without these two
44 // the device could free a file, have the user open it again, fetch it back,
45 // and free it once more -- each cycle costing a download the user pays for.
46 // So: a floor on how often reclaim may run at all, and a cool-down during
47 // which a file that was just used is not a candidate however cold it looks.
48 var RECLAIM_MIN_GAP_MS = 5 * 60 * 1000;
49 var RECENT_USE_MS = 30 * 60 * 1000;
50 var RECLAIM_AT_KEY = 'daimond-cloud-reclaimed';
51
52 // A ceiling on what the AGENT may pull down unprompted, over a rolling
53 // window. A fetch spends the user's credits, and an agent in a loop can ask
54 // for a great deal very quickly; the user clicking a file is a different
55 // thing entirely and is not counted here.
56 var AGENT_FETCH_WINDOW_MS = 10 * 60 * 1000;
57 var AGENT_FETCH_BUDGET = 128 * 1024 * 1024;
58 var AGENT_FETCH_KEY = 'daimond-cloud-agent-fetches';
59 var ALLOWANCE_KEY = 'daimond-cloud-allowance'; // free bytes, as the gateway last reported them.
60
61 function log(/* ...args */) {
62 try { if (window.console && console.debug) console.debug.apply(console, ['[cloud]'].concat([].slice.call(arguments))); }
63 catch (e) { /* ignore */ }
64 }
65
66 // ── Stored state ───────────────────────────────────────────
67 // localStorage is namespaced per account by accounts.js, so these keys need
68 // no prefixing of their own.
69
70 function readJson(key, fallback) {
71 try { return JSON.parse(localStorage.getItem(key) || 'null') || fallback; }
72 catch (e) { return fallback; }
73 }
74 function writeJson(key, val) {
75 try { localStorage.setItem(key, JSON.stringify(val)); return true; }
76 catch (e) { return false; } // quota: recomputed next round rather than corrupted.
77 }
78
79 function index() { return readJson(IX_KEY, {}); }
80 function setIndex(ix) { return writeJson(IX_KEY, ix || {}); }
81 function pins() { return readJson(PIN_KEY, {}); }
82 function atimes() { return readJson(ATIME_KEY, {}); }
83
84 // ── Content manifests, co-located under a reserved prefix ──────
85 // Diamonds and chats too large for the inline parcel are offloaded like a
86 // file, but they are not workspace files: they have no path, no `.synced`
87 // merge, and no residency the file tools should report. So their manifests
88 // live in THIS index under a reserved-prefix namespace — `@d/<id>` for a
89 // Diamond, `@c/<id>` for a chat — rather than in a store of their own.
90 //
91 // Co-location is the whole point, and it is load-bearing. The one commit in
92 // sync.js declares the live set from this index; the gateway sweeps every
93 // chunk that set does not name. A Diamond or chat chunk kept in a separate
94 // store would be swept by that file-only commit the moment it ran. Sharing
95 // the index means the commit names them WITHOUT a line changing at its call
96 // site — the sweep-safety is structural rather than remembered.
97 //
98 // What keeps them apart from the file paths beside them is `isContentKey`:
99 // the 3-way merge, the residency list and the cloud view each skip a content
100 // key, because the collector that owns the Diamond or the chat is the only
101 // writer of its manifest, and none of those file mechanisms mean anything for
102 // it.
103
104 /// Is this index key a content manifest rather than a workspace path?
105 function isContentKey(p) { return /^@[dc]\//.test(String(p)); }
106
107 /// The content manifest stored under `key`, or null. Shape-checked the same
108 /// way `manifest` checks a file's, so a half-written entry never becomes a
109 /// reference the collector reuses.
110 function contentGet(key) {
111 var m = index()[key];
112 return (m && Array.isArray(m.chunks)) ? m : null;
113 }
114
115 /// Record the content manifest for `key`. The record carries the collector's
116 /// own change-key beside `{v,size,key,chunks}` — `touched` for a Diamond, `fp`
117 /// for a chat — so the next collect can tell an unchanged item from a moved
118 /// one WITHOUT re-offloading it, which is what keeps the parcel a fixed point.
119 function contentSet(key, rec) {
120 var ix = index();
121 ix[key] = rec;
122 setIndex(ix);
123 return ix;
124 }
125
126 /// Drop the content manifest at `key`, so its chunks stop being named live
127 /// and the next commit sweeps them. Used when a Diamond or chat drops below
128 /// the inline threshold and no longer needs a reference at all.
129 function contentForget(key) {
130 var ix = index();
131 if (!Object.prototype.hasOwnProperty.call(ix, key)) return false;
132 delete ix[key];
133 setIndex(ix);
134 return true;
135 }
136
137 /// Drop every content manifest under `prefix` whose id is not in `live`.
138 /// Called by the Diamond and chat collectors once they have enumerated what
139 /// still exists, so a deleted item's manifest does not linger and go on
140 /// naming chunks nothing refers to. Writes only when something actually goes,
141 /// so a collect where nothing was deleted leaves the index byte-identical.
142 function contentReap(prefix, live) {
143 var ix = index(), changed = false;
144 Object.keys(ix).forEach(function (k) {
145 if (k.slice(0, prefix.length) !== prefix) return;
146 var id = k.slice(prefix.length);
147 if (!live || !live[id]) { delete ix[k]; changed = true; }
148 });
149 if (changed) setIndex(ix);
150 return changed;
151 }
152
153 /// The manifest for a path, or null if cloud storage does not hold it.
154 function manifest(path) {
155 var m = index()[path];
156 return (m && Array.isArray(m.chunks)) ? m : null;
157 }
158
159 /// A cheap content fingerprint, deliberately identical to daimond.js's
160 /// `fileHash` — the two modules must agree on whether a file changed, and
161 /// each has to work if the other failed to load.
162 ///
163 /// Good enough to decide a MERGE, where being wrong means an unnecessary
164 /// sidecar. Not good enough to decide a DELETION, which is why eviction
165 /// verifies with `sha256` below instead.
166 function hash(s) {
167 var h = 5381;
168 for (var i = 0; i < s.length; i++) { h = ((h << 5) + h + s.charCodeAt(i)) | 0; }
169 return (h >>> 0).toString(36) + ':' + s.length;
170 }
171
172 /// SHA-256 of a string, hex. Used where being wrong costs the user a file:
173 /// dropping the only local copy on the strength of a 32-bit fingerprint is
174 /// not a risk worth carrying when the real hash is one call away.
175 async function sha256(s) {
176 var d = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(s));
177 var b = new Uint8Array(d), out = '';
178 for (var i = 0; i < b.length; i++) {
179 out += (b[i] >>> 4).toString(16);
180 out += (b[i] & 15).toString(16);
181 }
182 return out;
183 }
184
185 // ── OPFS, honouring the account namespace ──────────────────
186 // A non-primary account lives in an OPFS subdirectory, exactly as the wasm
187 // file tools resolve it. Reading the raw root instead would look in the
188 // primary's workspace.
189
190 /// The directory this account's OPFS files live in: the origin root for the
191 /// primary account, its own `d~<id>` subdirectory for every other. Published,
192 /// because the backup path in daimond.js needs the same answer and a second
193 /// implementation of it is a second chance to walk the wrong root.
194 async function opfsRoot() {
195 var root = await navigator.storage.getDirectory();
196 var ns = '';
197 try { ns = (window.DaimondAccounts && DaimondAccounts.opfsNs()) || ''; } catch (e) { ns = ''; }
198 if (!ns) return root;
199 return await root.getDirectoryHandle(ns, { create: true });
200 }
201
202 function parts(path) {
203 return String(path).split('/').filter(function (x) { return x && x !== '.' && x !== '..'; });
204 }
205
206 // ── One name, two spellings ────────────────────────────────
207 // A workspace name is not always a filesystem name. A Maildir message is called
208 // `<uid>.<uidvalidity>.daimond:2,<flags>`, and a colon is refused by every File System Access
209 // root except a modern browser's own sandbox — including the real folder the user may have
210 // open. `src/fsname.rs` is the codec; this is the same codec in JavaScript, because these
211 // walkers reach the handles directly rather than through the wasm, and a name the wasm writes
212 // one way and this reads another is a file that has gone missing.
213 //
214 // The two implementations are held to each other by dev/verify_mailnames.mjs, which drives
215 // both over one corpus and compares character for character. Change one, change the other.
216
217 // `/` is deliberately absent: a path component cannot hold one, so escaping it would be
218 // dead code — while recognising `%2F` on the way back would turn a saved-URL name like
219 // `https%3A%2F%2Fexample.com.html` into three path components. See src/fsname.rs.
220 var FS_RESERVED = /["*:<>?\\|\x00-\x1F\x7F]/;
221
222 /// The byte an escape at `i` stands for, or -1 when it is not one this codec emits.
223 function fsEscaped(s, i) {
224 if (i + 2 >= s.length || s.charAt(i) !== '%') return -1;
225 var h = s.substr(i + 1, 2);
226 if (!/^[0-9A-F]{2}$/.test(h)) return -1; // upper case only: the encoder emits no other
227 var v = parseInt(h, 16);
228 if (v === 0x25 || FS_RESERVED.test(String.fromCharCode(v))) return v;
229 return -1;
230 }
231
232 /// Spell one path component so a filesystem will take it. The identity on every ordinary name.
233 function diskName(name) {
234 var s = String(name), out = '';
235 for (var i = 0; i < s.length; i++) {
236 var c = s.charAt(i);
237 var esc = FS_RESERVED.test(c) ? c.charCodeAt(0)
238 : (c === '%' && fsEscaped(s, i) >= 0) ? 0x25
239 : -1;
240 if (esc < 0) { out += c; continue; }
241 out += '%' + (esc < 16 ? '0' : '') + esc.toString(16).toUpperCase();
242 }
243 return out;
244 }
245
246 /// Read a stored name back as the workspace spells it. The exact inverse of `diskName`.
247 function logicalName(name) {
248 var s = String(name), out = '';
249 for (var i = 0; i < s.length; ) {
250 var v = fsEscaped(s, i);
251 if (v < 0) { out += s.charAt(i); i += 1; continue; }
252 out += String.fromCharCode(v);
253 i += 3;
254 }
255 return out;
256 }
257
258 /// The name a component is actually stored under inside `dir`.
259 ///
260 /// Mirrors `disk_name` in src/wasm/opfs.rs, legacy tolerance included: a name the codec does
261 /// not touch is used as it is, and one it does is looked for UNESCAPED first, because a store
262 /// written before the codec existed holds it that way.
263 async function diskNameIn(dir, name) {
264 var enc = diskName(name);
265 if (enc === name) return enc;
266 try { await dir.getFileHandle(name); return name; } catch (e) { /* not there */ }
267 try { await dir.getDirectoryHandle(name); return name; } catch (e) { /* nor there */ }
268 return enc;
269 }
270
271 /// The directory handle holding `path`, or null when a component is absent.
272 async function dirFor(path, create) {
273 var p = parts(path);
274 if (!p.length) return null;
275 var dir = await opfsRoot();
276 for (var i = 0; i < p.length - 1; i++) {
277 try { dir = await dir.getDirectoryHandle(await diskNameIn(dir, p[i]), { create: !!create }); }
278 catch (e) { return null; }
279 }
280 return dir;
281 }
282
283 /// Whether this device currently holds the file.
284 async function isHeld(path) {
285 var p = parts(path);
286 if (!p.length) return false;
287 var dir = await dirFor(path, false);
288 if (!dir) return false;
289 try { await dir.getFileHandle(await diskNameIn(dir, p[p.length - 1])); return true; }
290 catch (e) { return false; }
291 }
292
293 async function readText(path) {
294 var p = parts(path);
295 var dir = await dirFor(path, false);
296 if (!dir) throw new Error('No such directory: ' + path);
297 var fh = await dir.getFileHandle(await diskNameIn(dir, p[p.length - 1]));
298 return await (await fh.getFile()).text();
299 }
300
301 async function writeText(path, content) {
302 var p = parts(path);
303 var dir = await dirFor(path, true);
304 if (!dir) throw new Error('Cannot create directory for: ' + path);
305 var fh = await dir.getFileHandle(await diskNameIn(dir, p[p.length - 1]), { create: true });
306 var w = await fh.createWritable();
307 await w.write(new TextEncoder().encode(content));
308 await w.close();
309 }
310
311 /// The File at a path, or null. The handle is how a file is read in slices
312 /// rather than whole.
313 async function fileAt(path) {
314 var p = parts(path);
315 var dir = await dirFor(path, false);
316 if (!dir) return null;
317 try { return await (await dir.getFileHandle(await diskNameIn(dir, p[p.length - 1]))).getFile(); }
318 catch (e) { return null; }
319 }
320
321 /// Open a writable stream at a path, so a large file lands on disk piece by
322 /// piece instead of being assembled in memory first.
323 async function openWrite(path) {
324 var p = parts(path);
325 var dir = await dirFor(path, true);
326 if (!dir) throw new Error('Cannot create directory for: ' + path);
327 var fh = await dir.getFileHandle(await diskNameIn(dir, p[p.length - 1]), { create: true });
328 return await fh.createWritable();
329 }
330
331 /// Write a Blob or File straight to a path. Streams, and carries bytes rather
332 /// than text, so a picture arrives as a picture.
333 async function writeBlob(path, blob) {
334 var w = await openWrite(path);
335 await w.write(blob);
336 await w.close();
337 }
338
339 /// Re-derive a file's identity by streaming it: the hash of its chunk
340 /// hashes, exactly as the offload computed it. Never holds the file, so this
341 /// is affordable at any size — which matters, because it is what stands
342 /// between a user and the deletion of their only local copy.
343 async function fileKey(file, chunkSize) {
344 var CH = chunkSize || 256 * 1024;
345 var n = Math.max(1, Math.ceil(file.size / CH));
346 var hashes = '';
347 for (var i = 0; i < n; i++) {
348 var off = i * CH;
349 var len = Math.min(CH, file.size - off);
350 var slice = new Uint8Array(await file.slice(off, off + Math.max(0, len)).arrayBuffer());
351 hashes += await sha256Bytes(slice);
352 }
353 return await sha256Bytes(new TextEncoder().encode(hashes));
354 }
355
356 /// SHA-256 of a byte array, hex.
357 async function sha256Bytes(bytes) {
358 var d = await crypto.subtle.digest('SHA-256', bytes);
359 var b = new Uint8Array(d), out = '';
360 for (var i = 0; i < b.length; i++) {
361 out += (b[i] >>> 4).toString(16);
362 out += (b[i] & 15).toString(16);
363 }
364 return out;
365 }
366
367 async function removeLocal(path) {
368 var p = parts(path);
369 var dir = await dirFor(path, false);
370 if (!dir) return false;
371 try { await dir.removeEntry(p[p.length - 1]); return true; }
372 catch (e) { return false; }
373 }
374
375 // ── The derived path list the Rust file tools read ─────────
376 // `file_read` and `file_list` in wasm consult `daimond-cloud-paths` to tell
377 // the agent that a file exists in cloud storage rather than reporting it
378 // missing. It holds only what is NOT held here, so it is recomputed
379 // whenever residency changes.
380
381 async function refreshPaths() {
382 var ix = index(), out = {};
383 var keys = Object.keys(ix);
384 for (var i = 0; i < keys.length; i++) {
385 var p = keys[i];
386 // A content manifest is not a workspace file: the agent's file tools
387 // must not be told a Diamond or a chat is a path in cloud storage.
388 if (isContentKey(p)) continue;
389 if (!(await isHeld(p))) out[p] = (ix[p] && ix[p].size) | 0;
390 }
391 writeJson(PATHS_KEY, out);
392 return out;
393 }
394
395 /// The paths in cloud storage that this device is not holding.
396 function awayPaths() { return readJson(PATHS_KEY, {}); }
397
398 // ── Merging the index across devices ───────────────────────
399
400 /// Merge a pulled index into the stored one by the same 3-way compare the
401 /// inline files use, against the per-path baseline hashes of the last agreed
402 /// sync. Nothing is fetched here — a manifest is a reference, and adopting
403 /// one costs no bytes.
404 ///
405 /// A path changed on BOTH sides differently keeps the local manifest and
406 /// records the remote one at `<path>.synced`, mirroring the sidecar rule for
407 /// inline files. No download is needed to preserve it, because the sidecar
408 /// is only a second reference to chunks the gateway already holds.
409 function merge(remoteIx, baseline) {
410 var local = index(), base = baseline || {}, out = {}, seen = {};
411 remoteIx = (remoteIx && typeof remoteIx === 'object') ? remoteIx : {};
412
413 Object.keys(local).forEach(function (p) { seen[p] = 1; });
414 Object.keys(remoteIx).forEach(function (p) { seen[p] = 1; });
415
416 Object.keys(seen).forEach(function (p) {
417 // A content manifest is owned by the Diamond or chat collector on THIS
418 // device, never reconciled across devices: the reference that travels
419 // rides inline with its Diamond or chat, so a remote copy here is nothing
420 // to adopt and nothing to sidecar. Keep whatever this device holds and
421 // drop the rest.
422 if (isContentKey(p)) {
423 if (Object.prototype.hasOwnProperty.call(local, p)) out[p] = local[p];
424 return;
425 }
426 var l = local[p], r = remoteIx[p];
427 if (!r) { out[p] = l; return; } // only here: keep, it will push.
428 if (!l) { out[p] = r; return; } // only there: adopt the reference.
429 if (l.hash === r.hash) { out[p] = l; return; } // same file.
430 var b = base[p] || null;
431 var localChanged = (l.hash !== b);
432 var remoteChanged = (r.hash !== b);
433 if (remoteChanged && !localChanged) { out[p] = r; return; }
434 if (localChanged && !remoteChanged) { out[p] = l; return; }
435 out[p] = l; // both diverged: keep ours,
436 // and preserve theirs beside it -- but never chain sidecars onto
437 // sidecars, or a path that keeps diverging grows a tail of
438 // `.synced.synced.synced` that nobody will ever read.
439 if (!/\.synced$/.test(p)) out[p + '.synced'] = r;
440 });
441 // Drop a sidecar whose original is gone: it was only ever meaningful as
442 // "the other version of that file", and on its own it is landfill the
443 // user is paying to store.
444 Object.keys(out).forEach(function (p) {
445 var m = /^(.*)\.synced$/.exec(p);
446 if (m && !out[m[1]]) delete out[p];
447 });
448 setIndex(out);
449 return out;
450 }
451
452 /// Record (or replace) the manifest for a path this device just offloaded.
453 ///
454 /// `at` is when the upload happened and `bytes` is the file's true length on
455 /// disk; together they let a later eviction satisfy itself that the local
456 /// copy is the uploaded one without reading it back.
457 ///
458 /// `bytes` is measured, not taken from the manifest: a manifest's `size` is
459 /// the string's length in UTF-16 code units, which equals the byte length
460 /// only for pure ASCII. Comparing that against a file's real size would
461 /// declare every accented character an unsaved edit.
462 async function put(path, mani, h) {
463 var ix = index();
464 var f = await fileAt(path);
465 ix[path] = {
466 v: mani.v || 1,
467 size: mani.size, // plaintext bytes on disk.
468 bytes: f ? f.size : mani.size,
469 mtime: f ? f.lastModified : 0, // with size, the cheap "did it change" test.
470 hash: h, // the merge fingerprint.
471 key: mani.key || null, // what eviction verifies against.
472 chunks: mani.chunks,
473 at: Date.now(),
474 };
475 setIndex(ix);
476 return ix;
477 }
478
479 /// Drop a path from the index — the file is GONE, not merely absent. Its
480 /// chunks are swept on the next commit. Only an explicit delete does this.
481 function forget(path) {
482 var ix = index();
483 if (!Object.prototype.hasOwnProperty.call(ix, path)) return false;
484 delete ix[path];
485 setIndex(ix);
486 var a = atimes(); delete a[path]; writeJson(ATIME_KEY, a);
487 var p = pins(); delete p[path]; writeJson(PIN_KEY, p);
488 refreshPaths();
489 return true;
490 }
491
492 // ── Residency actions ──────────────────────────────────────
493
494 /// Note that a path was just used, so reclaim evicts the coldest first.
495 function touch(path) {
496 var a = atimes();
497 a[path] = Date.now();
498 writeJson(ATIME_KEY, a);
499 }
500
501 /// Bring a cloud-only file down onto this device. Returns a message string
502 /// beginning `OK` or `Error`, which is also what the agent's `file_fetch`
503 /// tool reports.
504 /// Decide which paths ride in the free allowance and which are paid overflow.
505 ///
506 /// This matters at exactly one moment, and it is the worst one: at the end of
507 /// grace the gateway evicts the paid tier and keeps the free one. The client
508 /// used to tag everything paid, so a lapsed account would have lost its whole
509 /// store rather than its overflow -- the opposite of what the policy promises.
510 ///
511 /// Most recently used first, because the free tier is meant to be the working
512 /// set: the files someone is actually using are the ones that should survive
513 /// a lapse.
514 function tierPlan(allowance) {
515 var ix = index(), a = atimes(), plan = {};
516 // Diamonds and chats are CORE content, not overflow, so they claim the
517 // free allowance BEFORE the workspace files do. The tier decides one thing
518 // and it is the worst one: at the end of grace the gateway keeps the free
519 // tier and evicts the paid. A Diamond left paid would be lost at a lapse
520 // the way overflow is, where today it rides the free sealed parcel and
521 // survives — so a lapsed account must keep its Diamonds and its chats, and
522 // spend the paid tier on the workspace files instead. Within each class,
523 // most recently used first, because the free tier is the working set.
524 var paths = Object.keys(ix).sort(function (x, y) {
525 var cx = isContentKey(x) ? 0 : 1, cy = isContentKey(y) ? 0 : 1;
526 if (cx !== cy) return cx - cy; // content first, then files.
527 return (a[y] || 0) - (a[x] || 0); // most recently used first.
528 });
529 var free = 0, budget = allowance | 0;
530 paths.forEach(function (p) {
531 var size = (ix[p].bytes | 0) || (ix[p].size | 0);
532 if (free + size <= budget) { plan[p] = 'f'; free += size; }
533 else { plan[p] = 'p'; }
534 });
535 return plan;
536 }
537
538 /// The free allowance the gateway last reported, in bytes.
539 function allowance() {
540 var n = parseInt(localStorage.getItem(ALLOWANCE_KEY) || '0', 10);
541 return isNaN(n) ? 0 : n;
542 }
543 function setAllowance(n) {
544 try { localStorage.setItem(ALLOWANCE_KEY, String(n | 0)); } catch (e) { /* best effort */ }
545 }
546
547 /// What the agent has pulled down in the last window, pruned as it is read.
548 function agentFetches() {
549 var all = readJson(AGENT_FETCH_KEY, []), now = Date.now();
550 return all.filter(function (r) { return r && (now - r.at) < AGENT_FETCH_WINDOW_MS; });
551 }
552
553 /// Bytes the agent may still pull down unprompted.
554 function agentFetchAllowance() {
555 var used = agentFetches().reduce(function (n, r) { return n + (r.n | 0); }, 0);
556 return Math.max(0, AGENT_FETCH_BUDGET - used);
557 }
558
559 function noteAgentFetch(n) {
560 var all = agentFetches();
561 all.push({ at: Date.now(), n: n | 0 });
562 writeJson(AGENT_FETCH_KEY, all);
563 }
564
565 async function fetchDown(path, viaAgent) {
566 var m = manifest(path);
567 if (!m) return 'Error: ' + path + ' is not in cloud storage.';
568 if (await isHeld(path)) { touch(path); return 'OK: ' + path + ' is already on this device.'; }
569 if (!window.DaimondChunks) return 'Error: the chunk transport is not loaded.';
570 // An agent asking is not the same as a person asking. A person clicking a
571 // file has seen its size and been warned if it is large; an agent can ask
572 // for a hundred files in a loop, and every one of them is billed. So the
573 // agent gets a budget, and past it must come back through the user.
574 if (viaAgent) {
575 var left = agentFetchAllowance();
576 var want = (m.bytes | 0) || (m.size | 0);
577 if (want > left) {
578 return 'Error: fetching ' + path + ' (' + want + ' bytes) would go past what may be ' +
579 'downloaded automatically; ' + left + ' bytes are left in this window. Ask the user ' +
580 'to fetch it from the workspace panel, or wait.';
581 }
582 }
583 var written = 0;
584 if ((m.v | 0) >= 2) {
585 // Straight to disk, one piece at a time: a file too large to hold is
586 // exactly the file this exists for.
587 var w;
588 try { w = await openWrite(path); }
589 catch (e) { return 'Error: could not write ' + path + ' to this device: ' + (e && e.message ? e.message : e); }
590 var okAll = false;
591 try {
592 okAll = await DaimondChunks.materialiseStream(m, async function (bytes) {
593 await w.write(bytes);
594 written += bytes.length;
595 });
596 await w.close();
597 } catch (e) {
598 try { await w.close(); } catch (e2) { /* already gone */ }
599 okAll = false;
600 }
601 if (!okAll) {
602 // Never leave a truncated file standing in for a whole one.
603 await removeLocal(path);
604 return 'Error: ' + path + ' could not be fetched; cloud storage no longer holds all of its parts.';
605 }
606 } else {
607 // An older whole-file manifest, from before the streaming pipeline.
608 var content;
609 try { content = await DaimondChunks.materialiseV1(m); }
610 catch (e) { return 'Error: could not fetch ' + path + ': ' + (e && e.message ? e.message : e); }
611 if (content == null) return 'Error: ' + path + ' could not be fetched; cloud storage no longer holds all of its parts.';
612 try { await writeText(path, content); }
613 catch (e) { return 'Error: could not write ' + path + ' to this device: ' + (e && e.message ? e.message : e); }
614 written = content.length;
615 }
616 touch(path);
617 if (viaAgent) noteAgentFetch(written);
618 await refreshPaths();
619 log('fetched', path, written);
620 return 'OK: fetched ' + path + ' (' + written + ' bytes) onto this device.';
621 }
622
623 /// Drop this device's copy, keeping the file in cloud storage. Refused
624 /// unless cloud storage holds THIS content — never evict what is not backed,
625 /// and never evict a pinned file.
626 /// The largest file we will read whole just to fingerprint it before freeing.
627 /// Above this, size and modification time have to carry the decision, because
628 /// pulling a 60 MB file into a string to check a hash — on the very device
629 /// that is short of memory — would be perverse.
630 var VERIFY_READ_MAX = 4 * 1024 * 1024;
631
632 async function evict(path) {
633 var m = manifest(path);
634 if (!m) return 'Error: ' + path + ' is not in cloud storage, so it cannot be freed.';
635 if (isPinned(path)) return 'Error: ' + path + ' is pinned to this device.';
636
637 var file = await fileAt(path);
638 if (!file) return 'OK: ' + path + ' was already not on this device.';
639
640 // Cheap rejection first: a different length on disk means an edit that has
641 // not been pushed yet.
642 if (typeof m.bytes === 'number' && file.size !== m.bytes) {
643 return 'Error: ' + path + ' has changed since it was last uploaded; it will be freed after the next sync.';
644 }
645 // Then prove it, by re-deriving the same identity the offload computed.
646 // Streaming makes this affordable at any size, so there is no size above
647 // which a deletion rests on a guess.
648 if (m.key) {
649 var live;
650 try { live = await fileKey(file, window.DaimondChunks ? DaimondChunks.chunkSizeFor(file.size) : 0); }
651 catch (e) { return 'Error: could not verify ' + path + ' before freeing it.'; }
652 if (live !== m.key) {
653 return 'Error: ' + path + ' has changed since it was last uploaded; it will be freed after the next sync.';
654 }
655 } else if (m.at && file.lastModified && file.lastModified > m.at) {
656 // An older entry, stored before file keys were recorded.
657 return 'Error: ' + path + ' has been edited since it was last uploaded; it will be freed after the next sync.';
658 }
659
660 if (!(await removeLocal(path))) return 'Error: could not free ' + path + '.';
661 await refreshPaths();
662 log('evicted', path, file.size);
663 return 'OK: freed ' + file.size + ' bytes; ' + path + ' remains in cloud storage.';
664 }
665
666 function isPinned(path) { return !!pins()[path]; }
667
668 /// Pin a file to this device, or release it. A pinned file is never
669 /// reclaimed automatically, which is what makes automatic reclaim safe to
670 /// leave switched on.
671 function pin(path, on) {
672 var p = pins();
673 if (on) p[path] = 1; else delete p[path];
674 writeJson(PIN_KEY, p);
675 return !!p[path];
676 }
677
678 // ── Reclaiming space ───────────────────────────────────────
679
680 /// What the browser has granted and how much of it is used.
681 async function pressure() {
682 var est = { usage: 0, quota: 0 };
683 try {
684 if (navigator.storage && navigator.storage.estimate) est = await navigator.storage.estimate();
685 } catch (e) { /* unsupported: report no pressure rather than guess */ }
686 var usage = est.usage || 0, quota = est.quota || 0;
687 return { usage: usage, quota: quota, ratio: quota ? (usage / quota) : 0 };
688 }
689
690 /// Free the coldest unpinned, cloud-backed files until the sandbox is
691 /// comfortably under its quota again. Does nothing when there is no
692 /// pressure, and never touches a file cloud storage does not hold.
693 async function reclaim(force) {
694 var pr = await pressure();
695 if (!pr.quota) return { freed: 0, evicted: [], ratio: pr.ratio };
696 if (!force && pr.ratio < PRESSURE_HIGH) return { freed: 0, evicted: [], ratio: pr.ratio };
697 // Reclaim rides on the push, and a push can come every few seconds. Left
698 // ungoverned it would free a file the user is still working with, who
699 // opens it again, pays to fetch it, and has it freed once more.
700 var now = Date.now();
701 if (!force) {
702 var last = parseInt(localStorage.getItem(RECLAIM_AT_KEY) || '0', 10) || 0;
703 if (now - last < RECLAIM_MIN_GAP_MS) return { freed: 0, evicted: [], ratio: pr.ratio };
704 }
705 try { localStorage.setItem(RECLAIM_AT_KEY, String(now)); } catch (e) { /* best effort */ }
706
707 var ix = index(), a = atimes(), pinned = pins();
708 var candidates = [];
709 var keys = Object.keys(ix);
710 for (var i = 0; i < keys.length; i++) {
711 var p = keys[i];
712 if (pinned[p]) continue;
713 // Just used is not cold, however long ago it was used before that.
714 if (a[p] && (now - a[p]) < RECENT_USE_MS) continue;
715 if (!(await isHeld(p))) continue;
716 candidates.push({ path: p, size: (ix[p].bytes | 0) || (ix[p].size | 0), at: a[p] || 0 });
717 }
718 candidates.sort(function (x, y) { return x.at - y.at; }); // coldest first.
719
720 var target = pr.quota * PRESSURE_TARGET;
721 var usage = pr.usage, freed = 0, evicted = [];
722 for (var j = 0; j < candidates.length && usage > target; j++) {
723 var r = await evict(candidates[j].path);
724 if (r.indexOf('OK') !== 0) continue;
725 usage -= candidates[j].size;
726 freed += candidates[j].size;
727 evicted.push(candidates[j].path);
728 }
729 if (evicted.length) log('reclaimed', freed, 'bytes from', evicted.length, 'files');
730 return { freed: freed, evicted: evicted, ratio: pr.quota ? (usage / pr.quota) : 0 };
731 }
732
733 // ── Totals, for the workspace chips and the cloud view ─────
734
735 /// What cloud storage holds for this account, and how much of it is here.
736 async function summary() {
737 var ix = index(), away = awayPaths();
738 var total = 0, awayBytes = 0, files = 0, awayFiles = 0;
739 Object.keys(ix).forEach(function (p) {
740 if (isContentKey(p)) return; // Diamonds and chats are not workspace files.
741 var s = (ix[p] && ix[p].size) | 0;
742 total += s; files++;
743 if (Object.prototype.hasOwnProperty.call(away, p)) { awayBytes += s; awayFiles++; }
744 });
745 var pr = await pressure();
746 return {
747 bytes: total, // everything cloud storage holds for this account.
748 files: files,
749 awayBytes: awayBytes, // the part not on this device.
750 awayFiles: awayFiles,
751 usage: pr.usage, // what the browser sandbox is using.
752 quota: pr.quota,
753 ratio: pr.ratio,
754 };
755 }
756
757 /// Whether cloud storage is usable at all: the transport is loaded and the
758 /// identity that seals a chunk is unlocked.
759 function available() {
760 if (!window.DaimondChunks) return false;
761 try { return !!(window.DaimondIdentity && DaimondIdentity.isUnlocked && DaimondIdentity.isUnlocked()); }
762 catch (e) { return false; }
763 }
764
765 // ── The bridge the wasm file tools call ────────────────────
766 // The agent's own tool calls are dispatched inside Rust, not through JS, so
767 // these are globals rather than module exports: `file_fetch` and
768 // `file_delete` reach them from there.
769
770 // The agent's own fetches come through here, and are budgeted as such.
771 window.__daimondCloudFetch = function (path) { return fetchDown(String(path), true); };
772 window.__daimondCloudForget = function (path) {
773 return Promise.resolve(forget(String(path))
774 ? 'OK: ' + path + ' removed from cloud storage.'
775 : 'OK: ' + path + ' was not in cloud storage.');
776 };
777
778 // ── Public surface ─────────────────────────────────────────
779 window.DaimondCloud = {
780 index: index,
781 manifest: manifest,
782 merge: merge,
783 put: put,
784 forget: forget,
785 // Content manifests (Diamonds, chats) co-located under a reserved prefix.
786 // Owned by the sync collectors, skipped by every file mechanism.
787 isContentKey: isContentKey,
788 contentGet: contentGet,
789 contentSet: contentSet,
790 contentForget: contentForget,
791 contentReap: contentReap,
792 fetch: fetchDown,
793 evict: evict,
794 pin: pin,
795 isPinned: isPinned,
796 isHeld: isHeld,
797 touch: touch,
798 awayPaths: awayPaths,
799 refreshPaths: refreshPaths,
800 reclaim: reclaim,
801 pressure: pressure,
802 summary: summary,
803 available: available,
804 hash: hash,
805 sha256: sha256,
806 fileAt: fileAt,
807 fileKey: fileKey,
808 writeBlob: writeBlob,
809 // What the agent may still pull down unprompted, in bytes.
810 agentAllowance: agentFetchAllowance,
811 // The free/paid split, and the allowance it is drawn against.
812 tierPlan: tierPlan,
813 allowance: allowance,
814 setAllowance: setAllowance,
815 // OPFS access that honours the account namespace; the sync path uses
816 // these for large files, which must not go through the truncating
817 // `file_read` tool.
818 readText: readText,
819 writeText: writeText,
820 // And the root those resolve against, for the backup path, which walks
821 // the whole tree rather than one named file.
822 opfsRoot: opfsRoot,
823 // The filesystem-name codec, published for the same reason `opfsRoot` is: the workspace
824 // walkers in daimond.js reach the same handles, and a second implementation of the
825 // spelling rule is a second chance to look in the wrong place.
826 diskName: diskName,
827 logicalName: logicalName,
828 diskNameIn: diskNameIn,
829 };
830})();