Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/src/search.rs

41.0 KiB, 66 runs

created by r1870400018:21413, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A bring-your-own-key client for the web search APIs, in the half that is not a socket.
2//!
3//! # What it is, and is not
4//!
5//! One query, one list: a phrase goes up, titles and links come back. It is not an answer engine,
6//! holds no session, and asks for no page contents -- a caller wanting the text of a result fetches
7//! that result itself, and pays the tokens knowingly rather than by surprise.
8//!
9//! # Why it does not send anything
10//!
11//! [`Engine::request`] returns the *parts* of a call -- host, port, path, method, headers, body --
12//! and stops there. The caller dials. That is the whole point rather than an oversight: a caller
13//! that resolves a host, refuses a private address and repeats the refusal on every redirect hop
14//! has built a gate, and a module that opens its own socket walks around it. There is deliberately
15//! no convenience here that sends.
16//!
17//! # Four vendors, four dialects
18//!
19//! Unlike [`crate::llm`], where three providers speak one dialect and differ only in address, these
20//! four agree on nothing: one is a `GET` with the query in the path, three are a `POST` with it in a
21//! JSON body, and each names the key header differently. So the enum carries real per-arm code, and
22//! [`SearchResult`] is the narrow common shape all four are flattened into -- title, url, snippet,
23//! age, every field a string.
24//!
25//! # Forgiving by the row, strict by the document
26//!
27//! [`Engine::parse`] drops a result that has no title or no url and returns the rest, because one
28//! malformed row out of twenty is not a reason to answer a user with nothing. It returns an error
29//! only when the *document* is an error document, and then it names the engine and repeats what the
30//! engine said, since that text is where a vendor explains a rejected key or an unknown parameter.
31//!
32//! `age` is passed through exactly as the engine wrote it and is never parsed into a timestamp. The
33//! engines disagree about what it measures -- when a page was published, when it was last crawled,
34//! how long ago either was -- and a confidently wrong date is worse than an honest blank.
35//!
36//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
37//! Anthropic Claude
38
39use oxedyne_fe2o3_core::prelude::*;
40use oxedyne_fe2o3_jdat::{
41 prelude::*,
42 string::dec::DecoderConfig,
43 usr::{
44 UsrKind,
45 UsrKindCode,
46 UsrKindId,
47 },
48};
49
50use std::collections::BTreeMap;
51
52use crate::http::{
53 header::HttpMethod,
54 pct,
55};
56
57
58/// What a query is looking for, and therefore which of an engine's endpoints answers it.
59///
60/// Three kinds rather than a free string, because an engine either has a corner of its index for a
61/// kind or it does not, and [`Engine::supports`] can only answer a closed question.
62#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
63pub enum Kind {
64 #[default]
65 Web, // the open web, and the default when a caller says nothing
66 News, // recent journalism, from whichever corner of the index holds it
67 Academic, // papers, preprints, journal articles; not every engine has any
68}
69
70impl Kind {
71
72 /// The word a kind is named by on the wire, in a request and in a stored setting.
73 pub fn id(&self) -> &'static str {
74 match self {
75 Self::Web => "web",
76 Self::News => "news",
77 Self::Academic => "academic",
78 }
79 }
80
81 /// The kind a word names, or `None` if it names no kind.
82 ///
83 /// `None` rather than a default, so a caller can tell a kind it was not given from a kind it was
84 /// given wrongly, and answer the second with an error instead of silently searching the web.
85 pub fn from_id(s: &str) -> Option<Self> {
86 match s {
87 "web" => Some(Self::Web),
88 "news" => Some(Self::News),
89 "academic" => Some(Self::Academic),
90 _ => None,
91 }
92 }
93}
94
95/// A search engine, which is to say a request shape and a reply shape that go together.
96#[derive(Clone, Copy, Debug, Eq, PartialEq)]
97pub enum Engine {
98 Brave, // its own crawler, keyed by a subscription-token header
99 Exa, // neural retrieval over an embedding index
100 Tavily, // a retrieval API built for agents, bearer-authenticated
101 Serper, // another engine's results, resold as JSON
102}
103
104impl Engine {
105
106 // In a fixed order, so a caller listing them need not keep its own copy in step.
107 pub const ALL: [Self; 4] = [Self::Brave, Self::Exa, Self::Tavily, Self::Serper];
108
109 /// The word an engine is named by: in a request, in a stored setting, and in a ledger line.
110 ///
111 /// One string in every one of those places. They are not independent spellings that happen to
112 /// agree, and a typo in any of them is a mismatch nothing reports.
113 pub fn id(&self) -> &'static str {
114 match self {
115 Self::Brave => "brave",
116 Self::Exa => "exa",
117 Self::Tavily => "tavily",
118 Self::Serper => "serper",
119 }
120 }
121
122 /// The engine a word names, or `None` if it names no engine.
123 pub fn from_id(s: &str) -> Option<Self> {
124 match s {
125 "brave" => Some(Self::Brave),
126 "exa" => Some(Self::Exa),
127 "tavily" => Some(Self::Tavily),
128 "serper" => Some(Self::Serper),
129 _ => None,
130 }
131 }
132
133 /// The host to dial and to validate the TLS certificate against.
134 pub fn host(&self) -> &'static str {
135 match self {
136 Self::Brave => "api.search.brave.com",
137 Self::Exa => "api.exa.ai",
138 Self::Tavily => "api.tavily.com",
139 Self::Serper => "google.serper.dev",
140 }
141 }
142
143 /// Can this engine answer this kind at all?
144 ///
145 /// Two of the four have no scholarly index and say so here, so a caller can offer a kind the
146 /// configured engine can actually serve rather than discovering it in a rejected request.
147 pub fn supports(&self, kind: Kind) -> bool {
148 match (self, kind) {
149 // No scholarly corner of the index, and no parameter that would ask for one.
150 (Self::Brave, Kind::Academic) => false,
151 (Self::Tavily, Kind::Academic) => false,
152 _ => true,
153 }
154 }
155
156 /// Host, path, method, headers and body for one query.
157 ///
158 /// **Not a request that is sent.** The caller owns the transport, the address check and the TLS;
159 /// see the module documentation for why that division is deliberate.
160 pub fn request(&self, key: &str, q: &SearchQuery) -> Outcome<SearchCall> {
161 if q.query.trim().is_empty() {
162 return Err(err!(
163 "An empty query was passed to the {} engine.", self.id();
164 Invalid, Input, Missing));
165 }
166 // A missing key is refused here rather than sent, because the vendor answers a keyless
167 // request with a bare 401 and no caller can read the reason out of that.
168 if key.is_empty() {
169 return Err(err!(
170 "No API key was given for the {} engine.", self.id();
171 Invalid, Input, Missing));
172 }
173 if !self.supports(q.kind) {
174 return Err(err!(
175 "The {} engine cannot answer a '{}' search.", self.id(), q.kind.id();
176 Invalid, Input, Unknown));
177 }
178 let n = self.clamped(q); // Results asked for, within what the engine allows.
179
180 match self {
181 Self::Brave => {
182 // The query rides in the path, so it is escaped as a browser would escape it.
183 let seg = match q.kind {
184 Kind::News => "news",
185 _ => "web",
186 };
187 let path = fmt!("/res/v1/{}/search?q={}&count={}",
188 seg, pct::encode_component(q.query), n);
189 Ok(SearchCall {
190 host: self.host().to_string(),
191 port: 443,
192 path,
193 method: HttpMethod::GET,
194 headers: self.headers(key),
195 body: Vec::new(),
196 })
197 },
198 Self::Exa => {
199 let mut b = DaticleMap::new();
200 b.insert(dat!("query"), dat!(q.query.to_string()));
201 b.insert(dat!("numResults"), dat!(n as u64));
202 // The category names the slice of the index; the web is the whole of it, so it
203 // asks for no category at all.
204 match q.kind {
205 Kind::News => { b.insert(dat!("category"), dat!("news")); },
206 // Scholarly work is filed under publications, which is what this vendor
207 // calls papers, preprints and journal articles.
208 Kind::Academic => { b.insert(dat!("category"), dat!("publication")); },
209 Kind::Web => {},
210 }
211 self.post(key, "/search", res!(Dat::Map(b).json()))
212 },
213 Self::Tavily => {
214 let mut b = DaticleMap::new();
215 b.insert(dat!("query"), dat!(q.query.to_string()));
216 b.insert(dat!("max_results"), dat!(n as u64));
217 b.insert(dat!("topic"), match q.kind {
218 Kind::News => dat!("news"),
219 _ => dat!("general"),
220 });
221 self.post(key, "/search", res!(Dat::Map(b).json()))
222 },
223 Self::Serper => {
224 // Here the kind is the path rather than a parameter.
225 let path = match q.kind {
226 Kind::Web => "/search",
227 Kind::News => "/news",
228 Kind::Academic => "/scholar",
229 };
230 let mut b = DaticleMap::new();
231 b.insert(dat!("q"), dat!(q.query.to_string()));
232 b.insert(dat!("num"), dat!(n as u64));
233 self.post(key, path, res!(Dat::Map(b).json()))
234 },
235 }
236 }
237
238 /// The engine's answer, whatever its shape, as the common list.
239 ///
240 /// Forgiving by the row and strict by the document: see the module documentation.
241 pub fn parse(&self, body: &[u8]) -> Outcome<Vec<SearchResult>> {
242 let txt = match std::str::from_utf8(body) {
243 Ok(t) => t,
244 Err(e) => return Err(err!(e,
245 "The {} engine answered with bytes that are not text.", self.id();
246 Network, Data, Decode)),
247 };
248 let dat = res!(Dat::decode_string_with_config(txt.to_string(), &json_decoder()));
249 let map = match &dat {
250 Dat::Map(m) => m,
251 other => return Err(err!(
252 "The {} engine answered with a JSON {:?} rather than an object.",
253 self.id(), other.kind();
254 Network, Data, Mismatch)),
255 };
256
257 // An error document is surfaced with the engine's own words, since that is the only place a
258 // rejected key or an unknown parameter is ever explained.
259 if let Some(why) = error_message(map) {
260 return Err(err!(
261 "The {} engine returned an error: {}", self.id(), why;
262 Network, Data));
263 }
264
265 let rows = match self.rows(map) {
266 Some(l) => l,
267 None => {
268 // Some engines report a refusal as a bare message beside a status code, with no
269 // wrapper this reads as an error document, so it is named here instead.
270 let tail = match map.get(&dat!("message")) {
271 Some(Dat::Str(s)) => s.clone(),
272 _ => clip(txt),
273 };
274 return Err(err!(
275 "The {} engine answered with no results: {}", self.id(), tail;
276 Network, Data, Missing));
277 },
278 };
279
280 let mut out = Vec::with_capacity(rows.len());
281 for row in rows {
282 let m = match row {
283 Dat::Map(m) => m,
284 _ => continue, // Not an object, so not a result.
285 };
286 if let Some(r) = self.row(m) {
287 out.push(r);
288 }
289 }
290 Ok(out)
291 }
292
293 /// The greatest number of results this engine will return for this kind.
294 ///
295 /// Each is the vendor's own published ceiling; asking for more is a rejected request rather than
296 /// a longer list.
297 fn max_results(&self, kind: Kind) -> usize {
298 match (self, kind) {
299 (Self::Brave, Kind::News) => 50,
300 (Self::Brave, _) => 20,
301 (Self::Exa, _) => 100,
302 (Self::Tavily, _) => 20,
303 (Self::Serper, _) => 100,
304 }
305 }
306
307 /// The number of results to ask for: what the caller wanted, held between one and the ceiling.
308 fn clamped(&self, q: &SearchQuery) -> usize {
309 let max = self.max_results(q.kind);
310 if q.limit == 0 { 1 } else if q.limit > max { max } else { q.limit }
311 }
312
313 /// The headers for one call, including the key under whichever name this vendor reads it.
314 ///
315 /// `Accept-Encoding: identity` because the caller owns the socket and need not own a
316 /// decompressor as well; a body it cannot inflate is a body it cannot parse.
317 fn headers(&self, key: &str) -> Vec<(String, String)> {
318 let mut h = vec![
319 ("Host".to_string(), self.host().to_string()),
320 ("Accept".to_string(), "application/json".to_string()),
321 ("Accept-Encoding".to_string(), "identity".to_string()),
322 ];
323 // Four vendors, four spellings. None of them takes the key as a query parameter, so it never
324 // reaches a path, a log line or a referrer.
325 let (name, value) = match self {
326 Self::Brave => ("X-Subscription-Token", key.to_string()),
327 Self::Exa => ("x-api-key", key.to_string()),
328 Self::Tavily => ("Authorization", fmt!("Bearer {}", key)),
329 Self::Serper => ("X-API-KEY", key.to_string()),
330 };
331 h.push((name.to_string(), value));
332 h
333 }
334
335 /// A JSON `POST` to this engine, which is the shape of three of the four.
336 fn post(&self, key: &str, path: &str, body: String) -> Outcome<SearchCall> {
337 let mut headers = self.headers(key);
338 headers.push(("Content-Type".to_string(), "application/json".to_string()));
339 Ok(SearchCall {
340 host: self.host().to_string(),
341 port: 443,
342 path: path.to_string(),
343 method: HttpMethod::POST,
344 headers,
345 body: body.into_bytes(),
346 })
347 }
348
349 /// The list of rows in this engine's answer, wherever it keeps them.
350 ///
351 /// [`Engine::parse`] is not told the kind, and two engines file a news answer somewhere other
352 /// than a web one, so each candidate place is tried in turn.
353 fn rows<'a>(&self, map: &'a DaticleMap) -> Option<&'a Vec<Dat>> {
354 match self {
355 Self::Brave => {
356 // A web answer nests its list under `web`; a news answer puts it at the top.
357 if let Some(Dat::Map(w)) = map.get(&dat!("web")) {
358 if let Some(Dat::List(l)) = w.get(&dat!("results")) {
359 return Some(l);
360 }
361 }
362 list(map, "results")
363 },
364 Self::Exa => list(map, "results"),
365 Self::Tavily => list(map, "results"),
366 // Web and scholarly answers are both `organic`; a news answer is `news`.
367 Self::Serper => list(map, "organic").or_else(|| list(map, "news")),
368 }
369 }
370
371 /// One row of this engine's answer as a common result, or `None` if it is not usable.
372 ///
373 /// A row with no title or no url is dropped rather than passed on empty: a link with nothing to
374 /// click and a heading with nothing under it are both worse than one fewer result.
375 fn row(&self, m: &DaticleMap) -> Option<SearchResult> {
376 let (title, url, snippet, age) = match self {
377 Self::Brave => (
378 text(m, "title"),
379 text(m, "url"),
380 text(m, "description"),
381 // The relative phrase if the engine gave one, the crawl stamp otherwise.
382 first_of(m, &["age", "page_age"]),
383 ),
384 Self::Exa => (
385 text(m, "title"),
386 text(m, "url"),
387 // Page contents are not asked for, so this is normally empty. The full text, if
388 // an engine sends it unbidden, is deliberately not folded in: a snippet is a
389 // line and a page is a token bill.
390 match m.get(&dat!("highlights")) {
391 Some(Dat::List(l)) => match l.first() {
392 Some(Dat::Str(s)) => s.clone(),
393 _ => text(m, "summary"),
394 },
395 _ => text(m, "summary"),
396 },
397 text(m, "publishedDate"),
398 ),
399 Self::Tavily => (
400 text(m, "title"),
401 text(m, "url"),
402 text(m, "content"),
403 text(m, "published_date"),
404 ),
405 Self::Serper => (
406 text(m, "title"),
407 text(m, "link"),
408 // A scholarly row carries its authors and venue where a web row carries a
409 // snippet, and that line is the useful one to show.
410 first_of(m, &["snippet", "publicationInfo"]),
411 // A scholarly row dates itself with a bare year, which is rendered rather than
412 // interpreted.
413 first_of(m, &["date", "year"]),
414 ),
415 };
416 if title.is_empty() || url.is_empty() {
417 return None;
418 }
419 Some(SearchResult { title, url, snippet, age })
420 }
421}
422
423/// One result, flattened out of whatever the engine called its fields.
424///
425/// Every field is a string, `snippet` and `age` may be empty, and `title` and `url` may not -- a row
426/// missing either never becomes one of these.
427#[derive(Clone, Debug, Default, Eq, PartialEq)]
428pub struct SearchResult {
429 pub title: String,
430 pub url: String,
431 pub snippet: String, // the engine's excerpt, which may be empty
432 // Whatever freshness the engine reported, verbatim and unparsed. Never a timestamp this
433 // module worked out.
434 pub age: String,
435}
436
437/// What is being asked for, before any engine has been chosen.
438#[derive(Clone, Copy, Debug)]
439pub struct SearchQuery<'a> {
440 pub query: &'a str,
441 pub kind: Kind, // which corner of the index to search
442 pub limit: usize, // clamped by the engine's own maximum; nought is read as one
443}
444
445/// The parts of one call, for a caller that will make it.
446///
447/// Everything needed to dial and to write the request, and nothing that dials. See the module
448/// documentation for why this type exists instead of a function that sends.
449#[derive(Clone, Debug)]
450pub struct SearchCall {
451 pub host: String, // dialled, and the name the certificate is validated against
452 pub port: u16, // always 443 for these vendors, carried rather than assumed
453 pub path: String, // with any query string already escaped into it
454 pub method: HttpMethod,
455 pub headers: Vec<(String, String)>, // the key among them, named as this vendor names it
456 pub body: Vec<u8>, // empty for a `GET`
457}
458
459fn json_decoder() -> DecoderConfig<
460 BTreeMap<UsrKindCode, UsrKind>,
461 BTreeMap<String, UsrKindId>,
462>
463{
464 DecoderConfig::json(None)
465}
466
467/// A string field, or the empty string if it is absent or is neither a string nor a whole number.
468fn text(m: &DaticleMap, key: &str) -> String {
469 match m.get(&dat!(key)) {
470 Some(Dat::Str(s)) => s.clone(),
471 Some(other) => whole(other).unwrap_or_default(),
472 None => String::new(),
473 }
474}
475
476/// A whole number written out, or `None` if the value is not one.
477///
478/// One engine dates a paper with a bare integer, and the decoder narrows an unannotated number to
479/// the smallest kind that holds it -- so every width has to be answered here, not just the widest.
480/// Rendering is not parsing: the digits go through as they arrived.
481fn whole(d: &Dat) -> Option<String> {
482 match d {
483 Dat::U8(n) => Some(fmt!("{}", n)),
484 Dat::U16(n) => Some(fmt!("{}", n)),
485 Dat::U32(n) => Some(fmt!("{}", n)),
486 Dat::U64(n) => Some(fmt!("{}", n)),
487 Dat::U128(n) => Some(fmt!("{}", n)),
488 Dat::C64(n) => Some(fmt!("{}", n)),
489 Dat::I8(n) => Some(fmt!("{}", n)),
490 Dat::I16(n) => Some(fmt!("{}", n)),
491 Dat::I32(n) => Some(fmt!("{}", n)),
492 Dat::I64(n) => Some(fmt!("{}", n)),
493 Dat::I128(n) => Some(fmt!("{}", n)),
494 Dat::Aint(n) => Some(fmt!("{}", n)),
495 _ => None,
496 }
497}
498
499/// The first of several fields that is present and not empty.
500fn first_of(m: &DaticleMap, keys: &[&str]) -> String {
501 for k in keys {
502 let v = text(m, k);
503 if !v.is_empty() {
504 return v;
505 }
506 }
507 String::new()
508}
509
510/// A list field, or `None` if it is absent or is not a list.
511fn list<'a>(m: &'a DaticleMap, key: &str) -> Option<&'a Vec<Dat>> {
512 match m.get(&dat!(key)) {
513 Some(Dat::List(l)) => Some(l),
514 _ => None,
515 }
516}
517
518/// The message out of an error document, or `None` if the document is not one.
519///
520/// The four vendors wrap it four ways -- a bare string, an object with a `detail`, an object with a
521/// `message`, an object holding an object -- so each shape is unwrapped rather than any one being
522/// assumed.
523fn error_message(map: &DaticleMap) -> Option<String> {
524 for outer in ["error", "detail"] {
525 match map.get(&dat!(outer)) {
526 Some(Dat::Str(s)) => {
527 // A short word beside a longer explanation: report both, in that order.
528 let extra = text(map, "message");
529 return Some(if extra.is_empty() || &extra == s {
530 s.clone()
531 } else {
532 fmt!("{}: {}", s, extra)
533 });
534 },
535 Some(Dat::Map(inner)) => {
536 let v = first_of(inner, &["detail", "message", "error"]);
537 return Some(if v.is_empty() { fmt!("{:?}", inner) } else { v });
538 },
539 _ => {},
540 }
541 }
542 None
543}
544
545/// A body cut short enough to put in an error message, on a character boundary.
546fn clip(s: &str) -> String {
547 const MAX: usize = 200; // Enough to recognise the document, short enough to read.
548 if s.len() <= MAX {
549 return s.to_string();
550 }
551 let mut end = MAX;
552 while end > 0 && !s.is_char_boundary(end) {
553 end -= 1;
554 }
555 fmt!("{}...", &s[..end])
556}
557
558
559#[cfg(test)]
560mod tests {
561 use super::*;
562
563 // Sample bodies, written to the shapes the four vendors publish. Values are invented; the field
564 // names, their nesting and their spelling are the point.
565
566 // A web answer, nesting its list under `web`, with one row dated relatively, one only by its
567 // crawl stamp, and one not at all.
568 const BRAVE_WEB: &str = r#"{
569 "query": {"original": "iron oxide", "more_results_available": true},
570 "web": {"results": [
571 {"title": "Iron oxide", "url": "https://enc.example.org/iron-oxide",
572 "description": "Iron oxides are chemical compounds.",
573 "age": "3 days ago", "page_age": "2026-08-07T11:00:00"},
574 {"title": "How rust forms", "url": "https://chem.example.com/rust",
575 "description": "Corrosion in the presence of water.",
576 "page_age": "2026-07-01T09:30:00"},
577 {"title": "Red pigments", "url": "https://pigment.example.net/red",
578 "description": ""}
579 ]}
580 }"#;
581
582 // A news answer, which puts its list at the top level instead.
583 const BRAVE_NEWS: &str = r#"{
584 "type": "news",
585 "results": [
586 {"title": "Foundry reopens", "url": "https://news.example.org/foundry",
587 "description": "The plant restarts.", "age": "5 hours ago"},
588 {"title": "Ore prices ease", "url": "https://news.example.com/ore",
589 "description": "Down on the week.", "age": "1 day ago"}
590 ]
591 }"#;
592
593 // Two results, one carrying a highlight and a null author, one carrying neither.
594 const EXA: &str = r#"{
595 "requestId": "req-1",
596 "results": [
597 {"title": "A study of iron oxide", "url": "https://arxiv.example.org/abs/2401.00001",
598 "id": "https://arxiv.example.org/abs/2401.00001",
599 "publishedDate": "2024-01-02T00:00:00.000Z", "author": null},
600 {"title": "Corrosion review", "url": "https://journal.example.com/c/12",
601 "id": "c12", "publishedDate": "2023-05-05T00:00:00.000Z",
602 "author": "A. Smith",
603 "highlights": ["Corrosion of steel in seawater."]}
604 ],
605 "costDollars": {"total": 0.005}
606 }"#;
607
608 // Two results, the second dated in the way a news answer dates them.
609 const TAVILY: &str = r#"{
610 "query": "iron oxide",
611 "results": [
612 {"title": "Iron oxide", "url": "https://example.org/a",
613 "content": "Iron oxides are compounds.", "score": 0.98},
614 {"title": "Ochre", "url": "https://example.org/b",
615 "content": "A natural pigment.", "score": 0.81,
616 "published_date": "Mon, 04 Aug 2026 09:00:00 GMT"}
617 ],
618 "response_time": 1.2
619 }"#;
620
621 // A web answer, whose list is `organic` and whose address field is `link`.
622 const SERPER_WEB: &str = r#"{
623 "searchParameters": {"q": "iron oxide", "type": "search"},
624 "organic": [
625 {"title": "Iron oxide - Encyclopaedia", "link": "https://enc.example.org/iron-oxide",
626 "snippet": "Iron oxides are chemical compounds.", "position": 1},
627 {"title": "Rust never sleeps", "link": "https://blog.example.com/rust",
628 "snippet": "On corrosion.", "date": "12 Jul 2026", "position": 2}
629 ],
630 "credits": 1
631 }"#;
632
633 // A news answer, whose list is `news` instead.
634 const SERPER_NEWS: &str = r#"{
635 "news": [
636 {"title": "Foundry reopens", "link": "https://news.example.org/foundry",
637 "snippet": "The plant restarts.", "date": "2 hours ago", "source": "Example Times"}
638 ]
639 }"#;
640
641 // A scholarly answer: no snippet, a venue line instead, and a bare year for a date.
642 const SERPER_SCHOLAR: &str = r#"{
643 "organic": [
644 {"title": "Attention is all you need", "link": "https://p.example.org/7181",
645 "publicationInfo": "A Vaswani, N Shazeer - Advances in neural information processing",
646 "year": 2017, "citedBy": 119097}
647 ]
648 }"#;
649
650 // Error documents, one per vendor, each wrapping its message differently.
651
652 const BRAVE_ERR: &str = r#"{"type":"ErrorResponse","error":{"id":"e1","status":422,
653 "code":"VALIDATION","detail":"Unable to validate request parameter(s)","meta":{}},
654 "time":1754800000}"#;
655 const EXA_ERR: &str = r#"{"error":"Unauthorized","message":"Invalid API key","statusCode":401}"#;
656 const TAVILY_ERR: &str = r#"{"detail":{"error":"Unauthorized: missing or invalid API key."}}"#;
657 const SERPER_ERR: &str = r#"{"message":"Unauthorized.","statusCode":403}"#;
658
659 /// The urls of a parsed body, which is what a test compares: the whole list, in order, rather
660 /// than a count that would still pass with the wrong rows in it.
661 fn urls(rs: &[SearchResult]) -> Vec<&str> {
662 rs.iter().map(|r| r.url.as_str()).collect()
663 }
664
665 /// A query of a given kind, so the tests below read as what they are varying.
666 fn q<'a>(query: &'a str, kind: Kind, limit: usize) -> SearchQuery<'a> {
667 SearchQuery { query, kind, limit }
668 }
669
670 /// The header of a call, by name, case-insensitively as HTTP reads them.
671 fn hdr(c: &SearchCall, name: &str) -> Option<String> {
672 c.headers.iter()
673 .find(|(k, _)| k.eq_ignore_ascii_case(name))
674 .map(|(_, v)| v.clone())
675 }
676
677 /// Every engine's word round-trips, and a word that names no engine is refused rather than
678 /// guessed at. The same string is the wire format in four places, so one typo is a silent
679 /// mismatch nothing else would catch.
680 #[test]
681 fn test_an_engine_names_itself_00() -> Outcome<()> {
682 for e in Engine::ALL {
683 let id = e.id();
684 assert!(!id.is_empty(), "{:?} has no id", e);
685 assert_eq!(Engine::from_id(id), Some(e), "'{}' did not round-trip", id);
686 assert!(e.host().contains('.'), "'{}' has no host", id);
687 }
688 // Every id is distinct, which a match arm copied and half-edited would break.
689 for a in Engine::ALL {
690 for b in Engine::ALL {
691 if a != b {
692 assert_ne!(a.id(), b.id(), "{:?} and {:?} share an id", a, b);
693 }
694 }
695 }
696 assert_eq!(Engine::from_id("bing"), None, "an unknown engine was accepted");
697 assert_eq!(Engine::from_id("Brave"), None, "the id is not case-insensitive");
698 assert_eq!(Engine::from_id(""), None, "an empty id was accepted");
699 Ok(())
700 }
701
702 /// A kind's word round-trips too, since it crosses the same wire.
703 #[test]
704 fn test_a_kind_names_itself_01() -> Outcome<()> {
705 for k in [Kind::Web, Kind::News, Kind::Academic] {
706 assert_eq!(Kind::from_id(k.id()), Some(k), "'{}' did not round-trip", k.id());
707 }
708 assert_eq!(Kind::from_id("scholar"), None, "an unknown kind was accepted");
709 assert_eq!(Kind::default(), Kind::Web, "the default kind is the web");
710 Ok(())
711 }
712
713 /// Each vendor reads the key from its own header, and none of them reads it from the path -- so
714 /// a key never reaches a request line, and a header named wrongly is a 401 with no explanation.
715 #[test]
716 fn test_the_key_goes_where_the_vendor_wants_it_02() -> Outcome<()> {
717 let key = "test-key-value";
718 let expect: &[(Engine, &str, &str)] = &[
719 (Engine::Brave, "X-Subscription-Token", "test-key-value"),
720 (Engine::Exa, "x-api-key", "test-key-value"),
721 (Engine::Tavily, "Authorization", "Bearer test-key-value"),
722 (Engine::Serper, "X-API-KEY", "test-key-value"),
723 ];
724 for (e, name, value) in expect {
725 let c = res!(e.request(key, &q("iron oxide", Kind::Web, 5)));
726 assert_eq!(hdr(&c, name).as_deref(), Some(*value),
727 "{} did not carry its key in {}", e.id(), name);
728 assert_eq!(hdr(&c, "Host").as_deref(), Some(e.host()),
729 "{} did not name its host", e.id());
730 assert!(!c.path.contains(key),
731 "{} put the key in the path: {}", e.id(), c.path);
732 let body = String::from_utf8_lossy(&c.body).to_string();
733 assert!(!body.contains(key),
734 "{} put the key in the body: {}", e.id(), body);
735 // No other vendor's header name is present, which a copied arm would leave behind.
736 // Compared as HTTP compares them, since two of these vendors ask for the same name
737 // in different case and that is one header, not two.
738 for (_, other, _) in expect {
739 if !other.eq_ignore_ascii_case(name) {
740 assert!(hdr(&c, other).is_none(),
741 "{} also carried {}", e.id(), other);
742 }
743 }
744 }
745 // A call with no key at all is refused here rather than sent to be refused there.
746 for e in Engine::ALL {
747 assert!(e.request("", &q("iron oxide", Kind::Web, 5)).is_err(),
748 "{} accepted an empty key", e.id());
749 assert!(e.request("k", &q(" ", Kind::Web, 5)).is_err(),
750 "{} accepted an empty query", e.id());
751 }
752 Ok(())
753 }
754
755 /// The query reaches the wire intact: escaped into the path where it rides in the path, and
756 /// quoted into JSON where it rides in a body.
757 #[test]
758 fn test_a_query_reaches_the_wire_intact_03() -> Outcome<()> {
759 // Every character a naive concatenation would break out of, in both directions.
760 let awkward = "rust & \"iron\" +oxide/water";
761
762 let c = res!(Engine::Brave.request("k", &q(awkward, Kind::Web, 5)));
763 assert_eq!(c.method, HttpMethod::GET, "brave is a GET");
764 assert!(c.body.is_empty(), "a GET carries no body");
765 assert!(c.path.starts_with("/res/v1/web/search?q="), "wrong path: {}", c.path);
766 // The ampersand and the quote are escaped, so neither starts a parameter nor ends a word.
767 assert!(!c.path.contains(" ") && !c.path.contains("\""),
768 "the query was not escaped: {}", c.path);
769 assert!(c.path.contains("%26"), "the ampersand was not escaped: {}", c.path);
770 let raw = match c.path.split("q=").nth(1) {
771 Some(t) => match t.split('&').next() {
772 Some(v) => v,
773 None => return Err(err!("no q value: {}", c.path; Test)),
774 },
775 None => return Err(err!("no q parameter: {}", c.path; Test)),
776 };
777 assert_eq!(res!(pct::decode_str(raw)), awkward, "the query did not survive escaping");
778
779 for e in [Engine::Exa, Engine::Tavily, Engine::Serper] {
780 let c = res!(e.request("k", &q(awkward, Kind::Web, 5)));
781 assert_eq!(c.method, HttpMethod::POST, "{} is a POST", e.id());
782 assert_eq!(hdr(&c, "Content-Type").as_deref(), Some("application/json"),
783 "{} did not declare JSON", e.id());
784 let txt = String::from_utf8_lossy(&c.body).to_string();
785 // It parses back as JSON, which a hand-built body with a bare quote would not.
786 let dat = res!(Dat::decode_string_with_config(txt.clone(), &json_decoder()));
787 let m = match dat {
788 Dat::Map(m) => m,
789 _ => return Err(err!("{} sent no object", e.id(); Test)),
790 };
791 let field = match e {
792 Engine::Serper => "q",
793 _ => "query",
794 };
795 assert_eq!(text(&m, field), awkward,
796 "{} did not carry the query intact: {}", e.id(), txt);
797 }
798 Ok(())
799 }
800
801 /// A caller asking for more than the vendor allows is held at the ceiling, and one asking for
802 /// nothing still asks for something.
803 #[test]
804 fn test_the_limit_is_held_within_the_ceiling_04() -> Outcome<()> {
805 // The number of results a call asks for, wherever that engine writes it.
806 fn asked(e: Engine, c: &SearchCall) -> Outcome<usize> {
807 match e {
808 Engine::Brave => {
809 let s = match c.path.split("count=").nth(1) {
810 Some(s) => s,
811 None => return Err(err!(
812 "no count parameter: {}", c.path; Test)),
813 };
814 Ok(res!(s.parse::<usize>()))
815 },
816 _ => {
817 let txt = String::from_utf8_lossy(&c.body).to_string();
818 let dat = res!(Dat::decode_string_with_config(txt, &json_decoder()));
819 let m = match dat {
820 Dat::Map(m) => m,
821 _ => return Err(err!("no object"; Test)),
822 };
823 let field = match e {
824 Engine::Exa => "numResults",
825 Engine::Tavily => "max_results",
826 _ => "num",
827 };
828 Ok(res!(text(&m, field).parse::<usize>()))
829 },
830 }
831 }
832 for e in Engine::ALL {
833 for kind in [Kind::Web, Kind::News, Kind::Academic] {
834 if !e.supports(kind) {
835 continue;
836 }
837 let ceiling = e.max_results(kind);
838 let big = res!(e.request("k", &q("iron", kind, 5_000)));
839 let n = res!(asked(e, &big));
840 assert_eq!(n, ceiling,
841 "{} '{}' asked for {} against a ceiling of {}",
842 e.id(), kind.id(), n, ceiling);
843
844 let none = res!(e.request("k", &q("iron", kind, 0)));
845 assert!(res!(asked(e, &none)) >= 1,
846 "{} '{}' asked for no results at all", e.id(), kind.id());
847
848 let some = res!(e.request("k", &q("iron", kind, 3)));
849 assert_eq!(res!(asked(e, &some)), 3,
850 "{} '{}' did not pass a modest limit through", e.id(), kind.id());
851 }
852 }
853 Ok(())
854 }
855
856 /// An engine with no scholarly index says so, and refuses the request rather than sending one it
857 /// knows will come back empty or rejected.
858 #[test]
859 fn test_an_engine_is_honest_about_the_kinds_it_answers_05() -> Outcome<()> {
860 for e in Engine::ALL {
861 assert!(e.supports(Kind::Web), "{} cannot search the web", e.id());
862 assert!(e.supports(Kind::News), "{} cannot search news", e.id());
863 }
864 assert!(!Engine::Brave.supports(Kind::Academic), "brave claimed a scholarly index");
865 assert!(!Engine::Tavily.supports(Kind::Academic), "tavily claimed a scholarly index");
866 assert!(Engine::Exa.supports(Kind::Academic), "exa has a publications category");
867 assert!(Engine::Serper.supports(Kind::Academic), "serper has a scholar endpoint");
868
869 // What is not supported is refused, and the refusal names both the engine and the kind.
870 for e in Engine::ALL {
871 let r = e.request("k", &q("iron oxide", Kind::Academic, 5));
872 assert_eq!(r.is_err(), !e.supports(Kind::Academic),
873 "{} disagreed with its own supports()", e.id());
874 if let Err(err) = r {
875 let msg = fmt!("{}", err);
876 assert!(msg.contains(e.id()), "the refusal did not name {}: {}", e.id(), msg);
877 assert!(msg.contains("academic"), "the refusal did not name the kind: {}", msg);
878 }
879 }
880 // The kind that is supported reaches the wire as that vendor spells it.
881 let exa = res!(Engine::Exa.request("k", &q("iron", Kind::Academic, 5)));
882 let txt = String::from_utf8_lossy(&exa.body).to_string();
883 assert!(txt.contains("publication"), "exa did not ask for publications: {}", txt);
884 let ser = res!(Engine::Serper.request("k", &q("iron", Kind::Academic, 5)));
885 assert_eq!(ser.path, "/scholar", "serper did not use its scholar endpoint");
886 let ser_news = res!(Engine::Serper.request("k", &q("iron", Kind::News, 5)));
887 assert_eq!(ser_news.path, "/news", "serper did not use its news endpoint");
888 Ok(())
889 }
890
891 /// Each engine's answer flattens to the same list, in the engine's own order.
892 #[test]
893 fn test_a_sample_answer_becomes_the_common_list_06() -> Outcome<()> {
894 let cases: &[(Engine, &str, &[&str])] = &[
895 (Engine::Brave, BRAVE_WEB, &[
896 "https://enc.example.org/iron-oxide",
897 "https://chem.example.com/rust",
898 "https://pigment.example.net/red",
899 ]),
900 (Engine::Brave, BRAVE_NEWS, &[
901 "https://news.example.org/foundry",
902 "https://news.example.com/ore",
903 ]),
904 (Engine::Exa, EXA, &[
905 "https://arxiv.example.org/abs/2401.00001",
906 "https://journal.example.com/c/12",
907 ]),
908 (Engine::Tavily, TAVILY, &[
909 "https://example.org/a",
910 "https://example.org/b",
911 ]),
912 (Engine::Serper, SERPER_WEB, &[
913 "https://enc.example.org/iron-oxide",
914 "https://blog.example.com/rust",
915 ]),
916 (Engine::Serper, SERPER_NEWS, &["https://news.example.org/foundry"]),
917 (Engine::Serper, SERPER_SCHOLAR, &["https://p.example.org/7181"]),
918 ];
919 for (e, body, want) in cases {
920 let got = res!(e.parse(body.as_bytes()));
921 assert_eq!(urls(&got), want.to_vec(), "{} parsed the wrong rows", e.id());
922 for r in &got {
923 assert!(!r.title.is_empty(), "{} emitted a titleless row", e.id());
924 }
925 }
926
927 // The fields each vendor calls something else arrive under the common names.
928 let brave = res!(Engine::Brave.parse(BRAVE_WEB.as_bytes()));
929 assert_eq!(brave[0].snippet, "Iron oxides are chemical compounds.");
930 let serper = res!(Engine::Serper.parse(SERPER_WEB.as_bytes()));
931 assert_eq!(serper[1].title, "Rust never sleeps");
932 assert_eq!(serper[1].snippet, "On corrosion.");
933 let tavily = res!(Engine::Tavily.parse(TAVILY.as_bytes()));
934 assert_eq!(tavily[0].snippet, "Iron oxides are compounds.");
935 let exa = res!(Engine::Exa.parse(EXA.as_bytes()));
936 assert_eq!(exa[1].snippet, "Corrosion of steel in seawater.");
937 assert_eq!(exa[0].snippet, "", "contents were not asked for, so there is no snippet");
938 let scholar = res!(Engine::Serper.parse(SERPER_SCHOLAR.as_bytes()));
939 assert!(scholar[0].snippet.starts_with("A Vaswani"),
940 "a scholarly row lost its venue line: {}", scholar[0].snippet);
941 Ok(())
942 }
943
944 /// Whatever the engine said about freshness is what comes out, character for character.
945 #[test]
946 fn test_age_is_passed_through_unparsed_07() -> Outcome<()> {
947 let brave = res!(Engine::Brave.parse(BRAVE_WEB.as_bytes()));
948 assert_eq!(brave[0].age, "3 days ago", "a relative phrase was not left alone");
949 assert_eq!(brave[1].age, "2026-07-01T09:30:00", "the crawl stamp was not used as a fallback");
950 assert_eq!(brave[2].age, "", "an undated row was given a date");
951
952 let exa = res!(Engine::Exa.parse(EXA.as_bytes()));
953 assert_eq!(exa[0].age, "2024-01-02T00:00:00.000Z", "an ISO stamp was reshaped");
954
955 let tavily = res!(Engine::Tavily.parse(TAVILY.as_bytes()));
956 assert_eq!(tavily[0].age, "", "an undated row was given a date");
957 assert_eq!(tavily[1].age, "Mon, 04 Aug 2026 09:00:00 GMT", "an RFC date was reshaped");
958
959 let serper = res!(Engine::Serper.parse(SERPER_WEB.as_bytes()));
960 assert_eq!(serper[1].age, "12 Jul 2026", "a written date was reshaped");
961 let scholar = res!(Engine::Serper.parse(SERPER_SCHOLAR.as_bytes()));
962 assert_eq!(scholar[0].age, "2017", "a bare year was not rendered as it stands");
963 Ok(())
964 }
965
966 /// A row with nothing to click, or nothing to read, is dropped and the rest are kept: one bad
967 /// row out of a page is not a reason to answer with nothing.
968 #[test]
969 fn test_a_row_with_no_title_or_no_url_is_dropped_08() -> Outcome<()> {
970 // Per engine: a good row, then one missing a title, one missing a url, one with an empty
971 // url, one with an empty title, and one that is not an object at all.
972 let brave = r#"{"web":{"results":[
973 {"title":"Kept one","url":"https://example.org/1","description":"d"},
974 {"url":"https://example.org/no-title","description":"d"},
975 {"title":"No url","description":"d"},
976 {"title":"Empty url","url":"","description":"d"},
977 {"title":"","url":"https://example.org/empty-title","description":"d"},
978 "not an object",
979 {"title":"Kept two","url":"https://example.org/2","description":"d"}
980 ]}}"#;
981 let exa = r#"{"results":[
982 {"title":"Kept one","url":"https://example.org/1"},
983 {"url":"https://example.org/no-title"},
984 {"title":"No url"},
985 {"title":"Empty url","url":""},
986 {"title":"","url":"https://example.org/empty-title"},
987 42,
988 {"title":"Kept two","url":"https://example.org/2"}
989 ]}"#;
990 let tavily = r#"{"results":[
991 {"title":"Kept one","url":"https://example.org/1","content":"c"},
992 {"url":"https://example.org/no-title","content":"c"},
993 {"title":"No url","content":"c"},
994 {"title":"Empty url","url":"","content":"c"},
995 {"title":"","url":"https://example.org/empty-title","content":"c"},
996 {"title":"Kept two","url":"https://example.org/2","content":"c"}
997 ]}"#;
998 let serper = r#"{"organic":[
999 {"title":"Kept one","link":"https://example.org/1","snippet":"s"},
1000 {"link":"https://example.org/no-title","snippet":"s"},
1001 {"title":"No url","snippet":"s"},
1002 {"title":"Empty url","link":"","snippet":"s"},
1003 {"title":"","link":"https://example.org/empty-title","snippet":"s"},
1004 {"title":"Kept two","link":"https://example.org/2","snippet":"s"}
1005 ]}"#;
1006 let cases: &[(Engine, &str)] = &[
1007 (Engine::Brave, brave),
1008 (Engine::Exa, exa),
1009 (Engine::Tavily, tavily),
1010 (Engine::Serper, serper),
1011 ];
1012 let want = vec!["https://example.org/1", "https://example.org/2"];
1013 for (e, body) in cases {
1014 let got = res!(e.parse(body.as_bytes()));
1015 assert_eq!(urls(&got), want, "{} kept or dropped the wrong rows", e.id());
1016 for r in &got {
1017 assert!(!r.title.is_empty() && !r.url.is_empty(),
1018 "{} emitted an empty row", e.id());
1019 }
1020 }
1021 // A list with nothing usable in it is an empty answer, not an error: the engine replied.
1022 let none = res!(Engine::Tavily.parse(br#"{"results":[]}"#));
1023 assert!(none.is_empty(), "an empty list should parse to an empty list");
1024 Ok(())
1025 }
1026
1027 /// An error document is an error, and the error says which engine refused and what it said.
1028 ///
1029 /// Each case also names something that appears only in the raw document -- a wrapper key, a
1030 /// status code -- and insists it is absent. Without that, a parser that recognised no error at
1031 /// all would still pass by dumping the whole body, since the body contains the message.
1032 #[test]
1033 fn test_an_error_document_names_the_engine_and_its_message_09() -> Outcome<()> {
1034 let cases: &[(Engine, &str, &[&str], &[&str])] = &[
1035 (Engine::Brave, BRAVE_ERR,
1036 &["Unable to validate request parameter(s)"],
1037 &["VALIDATION", "ErrorResponse"]),
1038 (Engine::Exa, EXA_ERR,
1039 &["Unauthorized", "Invalid API key"],
1040 &["statusCode"]),
1041 (Engine::Tavily, TAVILY_ERR,
1042 &["Unauthorized: missing or invalid API key."],
1043 &["detail"]),
1044 (Engine::Serper, SERPER_ERR,
1045 &["Unauthorized."],
1046 &["statusCode", "403"]),
1047 ];
1048 for (e, body, said, unsaid) in cases {
1049 let r = e.parse(body.as_bytes());
1050 assert!(r.is_err(), "{} read an error document as results", e.id());
1051 if let Err(err) = r {
1052 let msg = fmt!("{}", err);
1053 assert!(msg.contains(e.id()), "the error did not name {}: {}", e.id(), msg);
1054 for want in *said {
1055 assert!(msg.contains(want),
1056 "the error did not repeat what {} said ('{}'): {}",
1057 e.id(), want, msg);
1058 }
1059 for raw in *unsaid {
1060 assert!(!msg.contains(raw),
1061 "the error dumped the document rather than reading it ('{}'): {}",
1062 raw, msg);
1063 }
1064 }
1065 }
1066 // A body that is not an object, and one that is not text, are errors that name the engine.
1067 for bad in [&b"[]"[..], &b"not json at all"[..], &[0xffu8, 0xfe][..]] {
1068 let r = Engine::Brave.parse(bad);
1069 assert!(r.is_err(), "a body of {:?} was read as results", bad);
1070 }
1071 Ok(())
1072 }
1073
1074 /// A call carries everything the caller needs to dial, and nothing that dials.
1075 #[test]
1076 fn test_a_call_is_only_its_parts_10() -> Outcome<()> {
1077 for e in Engine::ALL {
1078 let c = res!(e.request("k", &q("iron oxide", Kind::Web, 5)));
1079 assert_eq!(c.host, e.host(), "{} named the wrong host", e.id());
1080 assert_eq!(c.port, 443, "{} is not on 443", e.id());
1081 assert!(c.path.starts_with('/'), "{} has no leading slash: {}", e.id(), c.path);
1082 assert_eq!(hdr(&c, "Accept").as_deref(), Some("application/json"),
1083 "{} did not ask for JSON", e.id());
1084 // The caller owns the socket and not a decompressor, so nothing arrives compressed.
1085 assert_eq!(hdr(&c, "Accept-Encoding").as_deref(), Some("identity"),
1086 "{} may be answered with a body the caller cannot inflate", e.id());
1087 assert_eq!(c.method.body_required(), !c.body.is_empty(),
1088 "{} disagrees with its own method about a body", e.id());
1089 }
1090 Ok(())
1091 }
1092}