Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_search_gateway.mjs

15.2 KiB, 1 run

created by r2519314175:665, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_search_gateway.mjs — /api/web/search, and the refusals that are most
2// of its behaviour.
3//
4// A search endpoint is mostly a set of answers to requests it will NOT serve,
5// and each of those answers is a decision somebody could quietly change: the
6// engine is the user's setting rather than the model's choice, and one engine
7// is off the credits tier on purpose. Unit tests hold the decisions; this holds
8// the wire, because a decision that never reaches an HTTP status is a decision
9// the browser cannot act on.
10//
11// WHAT IS MEASURED, and why each one would otherwise go quiet:
12//
13// 1. A request with no query is refused, naming `query`. A search tool whose
14// empty call 500s teaches a model to stop calling it.
15// 2. An engine nobody has heard of is refused, LISTING the ones that work.
16// "Unknown engine" leaves a model guessing; the list ends the guessing.
17// 3. An engine named with no key is refused, naming the KEY. This is the
18// commonest real failure — a user picks Exa in Settings and does not
19// paste a key — and "search failed" would send them to the wrong place.
20// 4. A key sent with `credits` is refused rather than ignored. Ignoring it
21// would spend the account's credits on a search the user believed their
22// own key was paying for.
23// 5. **Serper on the credits tier is refused.** §3 of the search contract:
24// serper resells another engine's results, so Oxedyne does not bill for
25// it. The only way to reach that decision over HTTP is to configure the
26// gateway wrongly on purpose, which is what the second fixture below
27// does — a request cannot ask for it, and that is itself the point.
28// 6. An operator who has chosen an engine and set no key gets a `503` that
29// says the OPERATOR has not configured search. They read logs; the user
30// cannot fix it.
31// 7. GET is refused. The query is in the body and a query string is a URL,
32// and a URL is where searches end up in somebody's access log.
33// 8. No response body ever echoes a key that was sent.
34// 9. `limit` opens no path of its own. It is now a number with a price on it
35// — a credits search is metered by what the vendor charges, and on Exa the
36// results past the tenth are money — so a request that asks for the
37// largest allowed must still be refused for exactly the reason a small one
38// is, rather than finding a way past the routing.
39//
40// WHAT IS NOT MEASURED HERE. What a search COSTS. Every case below is refused
41// before a socket opens, so nothing is metered and nothing is charged; the
42// arithmetic is held by the unit tests in `gateway/src/handlers/web.rs`, which
43// can price a search without spending anybody's vendor quota to do it.
44//
45// HOW IT RUNS. Two throwaway gateways, each on its own port with its own store
46// and a CWD holding a patched `app.jdat` — one configured to search with
47// `serper` (the misconfiguration), one with `brave` and no key (the unset
48// operator). `DAIMOND_GW_DEV=1`, so no account, no session and no credits are
49// needed to reach the refusals. Nothing touches the real store, and no request
50// leaves this machine: every case here is refused before a socket is opened.
51//
52// node dev/verify_search_gateway.mjs
53//
54// It needs a built gateway binary. It does NOT need dev/serve.mjs, a browser,
55// or the gateway on :9002.
56
57import fs from 'node:fs';
58import os from 'node:os';
59import path from 'node:path';
60import { spawn } from 'node:child_process';
61import { fileURLToPath } from 'node:url';
62import { requireFreshGateway } from './gwbin.mjs';
63
64const HERE = path.dirname(fileURLToPath(import.meta.url));
65const ROOT = path.join(HERE, '..');
66const GWDIR = path.join(ROOT, 'gateway');
67
68const ok = [], bad = [];
69const check = (name, pass, detail) => {
70 (pass ? ok : bad).push(name);
71 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
72};
73const sleep = ms => new Promise(r => setTimeout(r, ms));
74
75async function waitFor(fn, ms = 15000, gap = 200) {
76 const t0 = Date.now();
77 for (;;) {
78 try { if (await fn()) return true; } catch (e) {}
79 if (Date.now() - t0 > ms) return false;
80 await sleep(gap);
81 }
82}
83
84/// The binary to drive. The isolated slot target is preferred when it is
85/// newer, exactly as the other gateway verifiers do it — a stale binary would
86/// answer 404 here and read as "the route was never wired".
87function binary() {
88 const cands = [
89 path.join(os.homedir(), '.cache/cargo-targets/gateway_target/release/daimond_gateway'),
90 path.join(GWDIR, 'target/release/daimond_gateway'),
91 ].filter(p => fs.existsSync(p));
92 cands.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs);
93 return cands[0] || null;
94}
95
96/// A working directory holding a patched `app.jdat`: its own port, its own
97/// store, and a `/api/web/search` route configured as the case under test
98/// needs it.
99///
100/// The route is INSERTED rather than edited, because `gateway/app.jdat` is the
101/// deployed configuration and the record of intent. A verifier that edited it
102/// would be one interrupted run away from committing a test fixture as the
103/// shipped config.
104function fixture(name, port, engine) {
105 const out = path.join(HERE, 'searchgw', name);
106 // Cleared rather than reused: the store inside it belongs to one run, and a
107 // gateway that found a store from an older build would answer about that
108 // one. `dev/searchgw/` wants a line in `.gitignore` beside `dev/devgw/`.
109 fs.rmSync(out, { recursive: true, force: true });
110 fs.mkdirSync(out, { recursive: true });
111 // The keys are the real sandbox keys: `app.jdat` reads several of them with
112 // `{file:…}` and the config will not load without them. Symlinked, never
113 // copied. The STORE is this fixture's own, and not because the real one is
114 // locked — o3db excludes nobody, and a second process opening it is refused
115 // nothing. It is worse than that: it would get an index of its own built at
116 // open, so neither process would see the other's writes, and a garbage
117 // collector of its own over the same files. A fixture that shared the store
118 // would measure a picture nothing else holds.
119 const keys = path.join(out, 'keys');
120 if (!fs.existsSync(keys)) fs.symlinkSync(path.join(GWDIR, 'keys'), keys);
121
122 let cfg = fs.readFileSync(path.join(GWDIR, 'app.jdat'), 'utf8');
123 cfg = cfg.replace(/"listen_port":\s*\(u16\|\d+\)/, `"listen_port": (u16|${port})`);
124
125 // The vendor prices are written out even though no case here reaches a
126 // vendor: they are what a credits search is metered by, and a gateway that
127 // would not START with them configured is a failure this file should be the
128 // one to find. `search_min_charge_minor` is deliberately absent — the floor
129 // under a per-request charge went when the charge stopped being per request.
130 const route = ` { "path": "/api/web/search", "handler": "web_search", "config": {
131 "search_engine": "${engine}",
132 "search_byok_minor": "1",
133 "search_cost_brave_per_1k_minor": "500",
134 "search_cost_exa_per_1k_minor": "700",
135 "search_cost_tavily_per_1k_minor": "800",
136 "search_cost_serper_per_1k_minor": "100",
137 "search_extra_result_per_1k_minor": "100",
138 "search_fee_bps": "550",
139 "max_redirects": "3"
140 }},\n`;
141 // The real route is REMOVED first, not merely preceded. It did not exist
142 // when this fixture was written; once it was wired into app.jdat, inserting
143 // ahead of it left two routes on one path and the router took the shipped
144 // one -- so a fixture configured for serper quietly measured a gateway
145 // configured for brave, and the refusal it asserted could never appear.
146 // A duplicate is worse than an override precisely because it looks like it
147 // worked.
148 cfg = cfg.replace(
149 /^[ \t]*\{ "path": "\/api\/web\/search",[\s\S]*?^[ \t]*\}\},[ \t]*\n/m, '');
150
151 // Anchored on the fetch route's own line, so a config change elsewhere
152 // cannot quietly move where this lands.
153 const anchor = ' { "path": "/api/web/fetch", "handler": "web_fetch", "config": {';
154 if (cfg.indexOf(anchor) === -1) {
155 console.log(' FAIL dev/verify_search_gateway cannot find the fetch route in app.jdat');
156 process.exit(1);
157 }
158 cfg = cfg.replace(anchor, route + anchor);
159 fs.writeFileSync(path.join(out, 'app.jdat'), cfg);
160 return out;
161}
162
163const procs = [];
164function cleanup() {
165 for (const p of procs) { try { p.kill('SIGKILL'); } catch (e) {} }
166}
167
168/// Start a gateway in `cwd` and wait for it to answer.
169async function start(bin, cwd, port) {
170 const p = spawn(bin, [], {
171 cwd,
172 env: { ...process.env, APP_MODE: 'sandbox', DAIMOND_GW_DEV: '1' },
173 stdio: 'ignore',
174 });
175 procs.push(p);
176 const up = await waitFor(async () => (await fetch(`http://127.0.0.1:${port}/api/health`)).ok);
177 return up;
178}
179
180const H = { 'content-type': 'application/json', 'x-daimond-api': '1' };
181const post = (port, body) => fetch(`http://127.0.0.1:${port}/api/web/search`,
182 { method: 'POST', headers: H, body: JSON.stringify(body) });
183
184/// The status and the text of a refusal, so an assertion can name both.
185async function refusal(port, body) {
186 const r = await post(port, body);
187 let text = '';
188 try { text = JSON.stringify(await r.json()); } catch (e) { text = '<not json>'; }
189 return { status: r.status, text };
190}
191
192(async () => {
193 requireFreshGateway();
194 const bin = binary();
195 if (!bin) {
196 console.log('SKIP verify_search_gateway — no daimond_gateway binary built');
197 process.exit(0);
198 }
199
200 // ── Fixture A: the operator has set the credits tier to serper ──
201 const A_PORT = 9013;
202 const upA = await start(bin, fixture('serper', A_PORT, 'serper'), A_PORT);
203 check('a gateway starts with the credits tier set to serper', upA, bin);
204 if (!upA) { cleanup(); process.exit(1); }
205
206 // If the route is not wired the endpoint 404s and every check below would
207 // pass or fail for the wrong reason, so it is settled first and loudly.
208 const probe = await post(A_PORT, { query: 'ada lovelace' });
209 if (probe.status === 404) {
210 console.log('SKIP verify_search_gateway — /api/web/search answers 404, so the ' +
211 'handler is not registered. Lane `gateway` owns handlers/web.rs and not the ' +
212 'wiring: it still needs `pub use web::WebSearch` in handlers/mod.rs, an ' +
213 '`api_reg.insert_boxed("web_search", …)` in app_main.rs, the route in ' +
214 'gateway/app.jdat, and the proxy line in gateway/config.jdat.');
215 cleanup();
216 process.exit(0);
217 }
218 check('the search endpoint is wired', probe.status !== 404, 'HTTP ' + probe.status);
219
220 // 5. Serper on the credits tier. Nothing in the REQUEST can ask for this —
221 // it is reachable only by configuring the gateway wrongly, which is what
222 // this fixture does.
223 const serper = await refusal(A_PORT, { query: 'ada lovelace' });
224 check('the credits tier refuses serper', serper.status === 503,
225 'HTTP ' + serper.status);
226 check('and the refusal says serper is reachable with your own key',
227 /serper/i.test(serper.text) && /own key/i.test(serper.text), serper.text);
228 check('and it names the operator, who is the one who can fix it',
229 /operator/i.test(serper.text), serper.text);
230
231 // Naming credits explicitly is the same request by another spelling, and
232 // must not find a way through.
233 const serperNamed = await refusal(A_PORT, { query: 'q', engine: 'credits' });
234 check('naming the credits tier explicitly is refused the same way',
235 serperNamed.status === 503, 'HTTP ' + serperNamed.status);
236
237 // Serper with the caller's OWN key is the one way it is reachable, and it
238 // must not be refused by the routing. It is not sent here: asserting that
239 // the refusal is not the routing's is enough, and a real search would spend
240 // somebody's quota.
241 const serperByok = await refusal(A_PORT, { query: 'q', engine: 'serper', key: 'x'.repeat(40) });
242 check('serper with your own key is not refused by the routing',
243 serperByok.status !== 400 || !/only|credits cannot/i.test(serperByok.text),
244 'HTTP ' + serperByok.status + ' ' + serperByok.text);
245 check('and no reply echoes the key that was sent',
246 serperByok.text.indexOf('x'.repeat(40)) === -1, serperByok.text);
247
248 // ── The request-shape refusals, on the same gateway ──
249 const noQuery = await refusal(A_PORT, {});
250 check('a request with no query is refused, naming query',
251 noQuery.status === 400 && /query/i.test(noQuery.text),
252 'HTTP ' + noQuery.status + ' ' + noQuery.text);
253
254 const unknown = await refusal(A_PORT, { query: 'q', engine: 'bing', key: 'k'.repeat(20) });
255 check('an unknown engine is refused', unknown.status === 400,
256 'HTTP ' + unknown.status);
257 check('and the refusal lists the engines that do work',
258 ['brave', 'exa', 'tavily', 'serper'].every(e => unknown.text.indexOf(e) !== -1),
259 unknown.text);
260
261 const noKey = await refusal(A_PORT, { query: 'q', engine: 'exa' });
262 check('an engine with no key is refused, naming the key',
263 noKey.status === 400 && /key/i.test(noKey.text) && /exa/i.test(noKey.text),
264 'HTTP ' + noKey.status + ' ' + noKey.text);
265
266 const keyWithCredits = await refusal(A_PORT, { query: 'q', engine: 'credits', key: 'z'.repeat(30) });
267 check('a key sent with the credits tier is refused rather than ignored',
268 keyWithCredits.status === 400,
269 'HTTP ' + keyWithCredits.status + ' ' + keyWithCredits.text);
270 check('and that refusal does not echo the key either',
271 keyWithCredits.text.indexOf('z'.repeat(30)) === -1, keyWithCredits.text);
272
273 const kindWrong = await refusal(A_PORT, { query: 'q', kind: 'images' });
274 check('a kind that is not one of the three is refused',
275 kindWrong.status === 400 && /web|news|academic/i.test(kindWrong.text),
276 'HTTP ' + kindWrong.status + ' ' + kindWrong.text);
277
278 // 9. A limit is a number with a price on it, and it must not also be a way
279 // through. The largest one this endpoint allows is refused exactly as the
280 // default is — same status, same sentence.
281 const plain = await refusal(A_PORT, { query: 'ada lovelace' });
282 const big = await refusal(A_PORT, { query: 'ada lovelace', limit: 20 });
283 check('the largest limit is refused exactly as the default one is',
284 big.status === plain.status && big.text === plain.text,
285 'HTTP ' + big.status + ' ' + big.text);
286
287 const got = await fetch(`http://127.0.0.1:${A_PORT}/api/web/search?query=q`,
288 { headers: { 'x-daimond-api': '1' } });
289 check('GET is refused: a query belongs in a body, not in somebody\'s access log',
290 got.status === 405, 'HTTP ' + got.status);
291
292 // ── Fixture B: a well-configured engine with no key set ──
293 const B_PORT = 9014;
294 const upB = await start(bin, fixture('nokey', B_PORT, 'brave'), B_PORT);
295 check('a second gateway starts with brave chosen and no key set', upB);
296 if (upB) {
297 const unset = await refusal(B_PORT, { query: 'ada lovelace' });
298 check('an operator who set no key gets 503, not a generic failure',
299 unset.status === 503, 'HTTP ' + unset.status);
300 check('and the message says the OPERATOR has not configured search',
301 /operator has not configured search/i.test(unset.text), unset.text);
302 check('and it names the engine whose key is missing',
303 /brave/i.test(unset.text), unset.text);
304 check('and nothing was charged',
305 /nothing has been charged/i.test(unset.text), unset.text);
306 }
307
308 cleanup();
309 console.log(`\n${ok.length} ok, ${bad.length} failed`);
310 process.exit(bad.length ? 1 : 0);
311})().catch(e => {
312 cleanup();
313 console.log(' FAIL verify_search_gateway threw — ' + (e && e.stack || e));
314 process.exit(1);
315});