Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/probe_provider.mjs

4.6 KiB, 1 run

created by r2519314175:65, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Replay a real agentic loop's requests at a REAL provider, and read the clock.
2//
3// The mock can say what Daimond sends and what Daimond's own overhead costs. It
4// cannot say what the provider does with a 57 KB request twelve times in a row --
5// whether the prefix is served from a cache, and what the first token costs when
6// it is not. So the bodies captured by dev/latmock.mjs (LAT_KEEP_BODY=1) are
7// replayed verbatim, in order, with only the model swapped.
8//
9// node dev/probe_provider.mjs <captured.log> <model> [--rounds 13] [--pass 2]
10//
11// The key is read from the environment and from nowhere else:
12//
13// OPENROUTER_API_KEY=$(cat ~/.config/oxedyne/daimond/openrouter.key) \
14// node dev/probe_provider.mjs dev/latmock-x.log z-ai/glm-5.2
15//
16// Every round reports time to first token, total time, the prompt tokens the
17// provider counted, how many of them it served from its cache, and what it says
18// the call cost. Nothing is written to the log but numbers.
19
20import fs from 'node:fs';
21
22const LOG = process.argv[2];
23const MODEL = process.argv[3];
24const arg = (n, d) => { const i = process.argv.indexOf('--' + n); return i === -1 ? d : process.argv[i + 1]; };
25const ROUNDS = Number(arg('rounds', 99));
26const PASSES = Number(arg('pass', 1));
27const MAXTOK = Number(arg('maxtok', 200));
28const URL_ = arg('url', 'https://openrouter.ai/api/v1/chat/completions');
29
30if (!LOG || !MODEL) {
31 console.error('usage: probe_provider.mjs <captured.log> <model> [--rounds n] [--pass n]');
32 process.exit(2);
33}
34const KEY = process.env.OPENROUTER_API_KEY;
35if (!KEY) {
36 console.error('Set OPENROUTER_API_KEY before running this probe.');
37 process.exit(2);
38}
39
40const bodies = fs.readFileSync(LOG, 'utf8').trim().split('\n')
41 .filter(Boolean).map(l => JSON.parse(l)).filter(l => l.raw).map(l => JSON.parse(l.raw))
42 .slice(0, ROUNDS);
43if (!bodies.length) {
44 console.error(`${LOG} holds no captured bodies -- run the probe with LAT_KEEP_BODY=1.`);
45 process.exit(2);
46}
47
48const now = () => Number(process.hrtime.bigint() / 1000n) / 1000;
49
50/// One replayed round: the same messages and the same tool schemas, at a real model.
51async function round(body) {
52 const send = {
53 ...body,
54 model: MODEL,
55 max_tokens: MAXTOK,
56 stream: true,
57 stream_options: { include_usage: true },
58 // OpenRouter reports what the call cost in the final usage chunk when asked.
59 usage: { include: true },
60 };
61 const raw = JSON.stringify(send);
62 const t0 = now();
63 const r = await fetch(URL_, {
64 method: 'POST',
65 headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${KEY}` },
66 body: raw,
67 });
68 if (!r.ok) {
69 const t = await r.text();
70 return { err: `${r.status} ${t.slice(0, 200)}` };
71 }
72 let first = 0, usage = null, out = 0;
73 const reader = r.body.getReader();
74 const dec = new TextDecoder();
75 let buf = '';
76 for (;;) {
77 const { done, value } = await reader.read();
78 if (done) break;
79 if (!first) first = now();
80 buf += dec.decode(value, { stream: true });
81 let nl;
82 while ((nl = buf.indexOf('\n')) !== -1) {
83 const line = buf.slice(0, nl).trim();
84 buf = buf.slice(nl + 1);
85 if (!line.startsWith('data: ')) continue;
86 const d = line.slice(6);
87 if (d === '[DONE]') continue;
88 let j; try { j = JSON.parse(d); } catch { continue; }
89 if (j.usage) usage = j.usage;
90 const ch = (j.choices || [])[0];
91 if (ch && ch.delta) out += ((ch.delta.content || '') + (ch.delta.reasoning || '')).length;
92 }
93 }
94 const end = now();
95 const u = usage || {};
96 const det = u.prompt_tokens_details || {};
97 return {
98 bytes: Buffer.byteLength(raw, 'utf8'),
99 ttft: +(first - t0).toFixed(0),
100 total: +(end - t0).toFixed(0),
101 ptok: u.prompt_tokens || 0,
102 cached: det.cached_tokens || 0,
103 ctok: u.completion_tokens || 0,
104 cost: u.cost || 0,
105 chars: out,
106 };
107}
108
109console.log(`model ${MODEL} rounds ${bodies.length} pass(es) ${PASSES}`);
110let spend = 0;
111for (let p = 1; p <= PASSES; p++) {
112 console.log(`\npass ${p}`);
113 console.log('round reqKB promptTok cached %cached ttft total outTok cost');
114 for (let i = 0; i < bodies.length; i++) {
115 const r = await round(bodies[i]);
116 if (r.err) { console.log(`${String(i + 1).padStart(5)} ERROR ${r.err}`); continue; }
117 spend += r.cost;
118 const pc = r.ptok ? (100 * r.cached / r.ptok) : 0;
119 console.log([
120 String(i + 1).padStart(5),
121 (r.bytes / 1024).toFixed(1).padStart(7),
122 String(r.ptok).padStart(11),
123 String(r.cached).padStart(8),
124 pc.toFixed(0).padStart(8),
125 String(r.ttft).padStart(6),
126 String(r.total).padStart(7),
127 String(r.ctok).padStart(8),
128 r.cost.toFixed(5).padStart(8),
129 ].join(''));
130 }
131}
132console.log(`\nspent on ${MODEL}: $${spend.toFixed(4)}`);