Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/src/llm.rs

17.7 KiB, 44 runs

created by r1870400018:17330, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A bring-your-own-key client for an OpenAI-compatible chat-completions API.
2//!
3//! # What it is, and is not
4//!
5//! One request, one reply: a system instruction and a piece of text go up, an assistant message comes
6//! back. It is not an agent, holds no conversation, and streams nothing -- a caller that wants a post
7//! tidied or a comment judged sends the whole thing and reads the whole answer. That is the shape both
8//! callers this was built for need, and a narrower client is a smaller thing to get wrong.
9//!
10//! # Why one client covers three providers
11//!
12//! OpenRouter, Fireworks and Mistral all speak the OpenAI chat-completions dialect: a `POST` of
13//! `{"model", "messages":[{"role","content"}...]}` to a `/chat/completions` path, bearer-authenticated,
14//! answered with `choices[0].message.content`. So a provider is a base address and nothing more, and
15//! adding a fourth that speaks the same dialect is a line in one enum. A provider that spoke a
16//! different dialect -- Anthropic's `messages` API, say -- would be a second [`complete`], not a second
17//! [`Provider`] arm; this one does not pretend to abstract over that.
18//!
19//! # Built pure, wrapped thin
20//!
21//! [`chat_body`] builds the request and [`chat_reply`] reads the answer, both pure functions over
22//! strings, tested without a socket -- because a test cannot reach a live model with a key it does not
23//! have, and what it cannot reach it cannot catch. What it *can* pin -- the JSON a provider is sent,
24//! the text pulled from what it returns, the error surfaced rather than swallowed -- it does. The
25//! network wrapper [`complete`] is as thin as the send seam it borrows from [`crate::http::client`].
26//!
27//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
28//! Anthropic Claude
29
30use oxedyne_fe2o3_core::prelude::*;
31use oxedyne_fe2o3_jdat::{
32 prelude::*,
33 string::dec::DecoderConfig,
34 usr::{
35 UsrKind,
36 UsrKindCode,
37 UsrKindId,
38 },
39};
40
41use std::{
42 collections::BTreeMap,
43 sync::Arc,
44};
45
46use tokio_rustls::rustls::ClientConfig;
47
48use crate::http::{
49 client::https_request,
50 header::{
51 HttpHeadline,
52 HttpMethod,
53 },
54};
55
56
57/// An OpenAI-compatible provider: the host that answers, and the path it answers on.
58///
59/// The three named here were asked for; [`Provider::Custom`] is the escape hatch for a fourth that
60/// speaks the same dialect, so a new endpoint needs no new code here.
61#[derive(Clone, Debug, Eq, PartialEq)]
62pub enum Provider {
63 OpenRouter, // a router in front of many models
64 Fireworks,
65 Mistral,
66 // Any other host and path speaking the same dialect.
67 Custom {
68 host: String, // dialled and certificate-checked, no scheme, e.g. `api.example.com`
69 path: String, // leading slash, e.g. `/v1/chat/completions`
70 },
71}
72
73impl Provider {
74
75 /// Not a lenient default: a config that meant `mistral` and typed `mistrral` should hear about it
76 /// rather than quietly reach a host that does not exist.
77 pub fn of(s: &str) -> Outcome<Self> {
78 match s {
79 "openrouter" => Ok(Self::OpenRouter),
80 "fireworks" => Ok(Self::Fireworks),
81 "mistral" => Ok(Self::Mistral),
82 _ => Err(err!(
83 "Unknown LLM provider '{}': expected openrouter, fireworks or mistral.", s;
84 Invalid, Input)),
85 }
86 }
87
88 /// [`Provider::Custom`] has no stored word, since it is named by its own host and path rather than
89 /// by a key in this enum.
90 pub fn as_str(&self) -> Option<&'static str> {
91 match self {
92 Self::OpenRouter => Some("openrouter"),
93 Self::Fireworks => Some("fireworks"),
94 Self::Mistral => Some("mistral"),
95 Self::Custom { .. } => None,
96 }
97 }
98
99 /// Dialled, and the name the TLS certificate is validated against.
100 pub fn host(&self) -> &str {
101 match self {
102 Self::OpenRouter => "openrouter.ai",
103 Self::Fireworks => "api.fireworks.ai",
104 Self::Mistral => "api.mistral.ai",
105 Self::Custom { host, .. } => host,
106 }
107 }
108
109 pub fn path(&self) -> &str {
110 match self {
111 Self::OpenRouter => "/api/v1/chat/completions",
112 Self::Fireworks => "/inference/v1/chat/completions",
113 Self::Mistral => "/v1/chat/completions",
114 Self::Custom { path, .. } => path,
115 }
116 }
117}
118
119/// Everything a call needs but its words. The key is a secret and is never logged; a caller storing one
120/// gives it the same at-rest protection its other secrets get. Held here only for the moment of a call.
121#[derive(Clone, Debug)]
122pub struct LlmConfig {
123 pub provider: Provider,
124 pub model: String, // the provider's own naming, e.g. `mistralai/mistral-large-latest`
125 pub api_key: String, // bearer key, never logged
126}
127
128fn json_decoder() -> DecoderConfig<
129 BTreeMap<UsrKindCode, UsrKind>,
130 BTreeMap<String, UsrKindId>,
131>
132{
133 DecoderConfig::json(None)
134}
135
136/// The request body for a single-turn completion: a system instruction, then the user's text.
137///
138/// Built through the daticle encoder rather than by hand, so a system prompt an operator typed and a
139/// post a reader wrote reach the model correctly quoted whatever they contain -- a quote, a newline, a
140/// backslash. `temperature` is low, because both callers want the model to tidy or judge, not to
141/// invent, and the same text should get much the same answer twice.
142pub fn chat_body(model: &str, system: &str, user: &str) -> Outcome<String> {
143 let msg = |role: &str, content: &str| {
144 let mut m = DaticleMap::new();
145 m.insert(dat!("role"), dat!(role.to_string()));
146 m.insert(dat!("content"), dat!(content.to_string()));
147 Dat::Map(m)
148 };
149 let mut body = DaticleMap::new();
150 body.insert(dat!("model"), dat!(model.to_string()));
151 body.insert(dat!("messages"), Dat::List(vec![
152 msg("system", system),
153 msg("user", user),
154 ]));
155 body.insert(dat!("temperature"), dat!(0.2f64));
156 Dat::Map(body).json()
157}
158
159/// The request body for a single-turn completion carrying one image beside the user's text.
160///
161/// The only thing that differs from [`chat_body`] is the user message's `content`: instead of a bare
162/// string it is the OpenAI multimodal array -- a `text` part and an `image_url` part -- so a vision
163/// model reads the words and the picture as one turn. The image travels inline as a `data:` URL,
164/// `data:<image_mime>;base64,<image_b64>`, which is what the OpenAI-compatible vision dialect expects and
165/// what lets a caller send bytes it holds without first hosting them somewhere fetchable. `image_mime` is
166/// the media type as the model wants to see it, e.g. `image/png` or `image/jpeg`; `image_b64` is the
167/// image already Base64-encoded, since encoding is the caller's to do and not this builder's to guess.
168/// Built through the daticle encoder for the same reason [`chat_body`] is: whatever a system prompt or a
169/// user text contains reaches the model correctly quoted.
170pub fn chat_body_vision(
171 model: &str,
172 system: &str,
173 user: &str,
174 image_mime: &str,
175 image_b64: &str,
176)
177 -> Outcome<String>
178{
179 let text_msg = |role: &str, content: &str| {
180 let mut m = DaticleMap::new();
181 m.insert(dat!("role"), dat!(role.to_string()));
182 m.insert(dat!("content"), dat!(content.to_string()));
183 Dat::Map(m)
184 };
185
186 // The user turn's content is an array of typed parts rather than a plain string.
187 let mut text_part = DaticleMap::new();
188 text_part.insert(dat!("type"), dat!("text".to_string()));
189 text_part.insert(dat!("text"), dat!(user.to_string()));
190
191 let mut url_holder = DaticleMap::new();
192 url_holder.insert(dat!("url"), dat!(fmt!("data:{};base64,{}", image_mime, image_b64)));
193
194 let mut image_part = DaticleMap::new();
195 image_part.insert(dat!("type"), dat!("image_url".to_string()));
196 image_part.insert(dat!("image_url"), Dat::Map(url_holder));
197
198 let mut user_msg = DaticleMap::new();
199 user_msg.insert(dat!("role"), dat!("user".to_string()));
200 user_msg.insert(dat!("content"), Dat::List(vec![
201 Dat::Map(text_part),
202 Dat::Map(image_part),
203 ]));
204
205 let mut body = DaticleMap::new();
206 body.insert(dat!("model"), dat!(model.to_string()));
207 body.insert(dat!("messages"), Dat::List(vec![
208 text_msg("system", system),
209 Dat::Map(user_msg),
210 ]));
211 body.insert(dat!("temperature"), dat!(0.2f64));
212 Dat::Map(body).json()
213}
214
215/// The assistant's text from a provider's reply, or the reason there is none.
216///
217/// A provider answers a good request with `{"choices":[{"message":{"content":"..."}}]}` and a bad one
218/// with `{"error":{"message":"..."}}` (or a bare `{"error":"..."}`); this reads the first and surfaces
219/// the second as an error rather than an empty string, so a caller can tell "the model said nothing"
220/// from "the model was never asked". An empty `choices` is the former and says so.
221pub fn chat_reply(json: &str) -> Outcome<String> {
222 let dat = res!(Dat::decode_string_with_config(json.to_string(), &json_decoder()));
223 let map = match &dat {
224 Dat::Map(m) => m,
225 other => return Err(err!(
226 "The LLM reply was not a JSON object but {:?}.", other.kind();
227 Network, Data, Mismatch)),
228 };
229
230 // An error reply is surfaced with its own message, since that is the useful thing to show.
231 if let Some(e) = map.get(&dat!("error")) {
232 let why = match e {
233 Dat::Str(s) => s.clone(),
234 Dat::Map(em) => match em.get(&dat!("message")) {
235 Some(Dat::Str(s)) => s.clone(),
236 _ => fmt!("{:?}", e),
237 },
238 _ => fmt!("{:?}", e),
239 };
240 return Err(err!("The LLM returned an error: {}", why; Network, Data));
241 }
242
243 let choices = match map.get(&dat!("choices")) {
244 Some(Dat::List(l)) => l,
245 _ => return Err(err!(
246 "The LLM reply carried no 'choices' list: {}", json;
247 Network, Data, Missing)),
248 };
249 let first = match choices.first() {
250 Some(Dat::Map(m)) => m,
251 _ => return Err(err!(
252 "The LLM returned no choices, so it said nothing.";
253 Network, Data, Missing)),
254 };
255 let message = match first.get(&dat!("message")) {
256 Some(Dat::Map(m)) => m,
257 _ => return Err(err!(
258 "The LLM choice carried no message: {}", json;
259 Network, Data, Missing)),
260 };
261 match message.get(&dat!("content")) {
262 Some(Dat::Str(s)) => Ok(s.clone()),
263 _ => Err(err!(
264 "The LLM message carried no text content: {}", json;
265 Network, Data, Missing)),
266 }
267}
268
269/// The thin wrapper around the two pure functions: build the body, `POST` it bearer-authenticated over
270/// TLS, read the reply. A non-2xx status is surfaced with the body the provider sent, since that body
271/// is where a provider says what was wrong with a key or a model name.
272pub async fn complete(
273 cfg: &LlmConfig,
274 system: &str,
275 user: &str,
276 tls: Arc<ClientConfig>,
277)
278 -> Outcome<String>
279{
280 let body = res!(chat_body(&cfg.model, system, user));
281 let auth = fmt!("Bearer {}", cfg.api_key);
282 let headers: &[(&str, &str)] = &[
283 ("Host", cfg.provider.host()),
284 ("Authorization", &auth),
285 ("Content-Type", "application/json"),
286 ("Accept", "application/json"),
287 ];
288 let resp = res!(https_request(
289 cfg.provider.host(),
290 443,
291 HttpMethod::POST,
292 cfg.provider.path(),
293 headers,
294 body.as_bytes(),
295 tls,
296 ).await);
297
298 let payload = String::from_utf8_lossy(&resp.body).to_string();
299 let status = match &resp.header.headline {
300 HttpHeadline::Response { status } => *status as u16,
301 _ => 0,
302 };
303 if !(200..300).contains(&status) {
304 return Err(err!(
305 "The LLM provider answered {} to a completion request: {}", status, payload;
306 Network, Data));
307 }
308 chat_reply(&payload)
309}
310
311/// [`complete`] for a vision endpoint: the same send seam and the same reply reader, but the body carries
312/// one image beside the text (see [`chat_body_vision`]). The reply dialect is unchanged -- a vision model
313/// answers with `choices[0].message.content` exactly as a text one does -- so [`chat_reply`] reads it.
314pub async fn complete_vision(
315 cfg: &LlmConfig,
316 system: &str,
317 user: &str,
318 image_mime: &str,
319 image_b64: &str,
320 tls: Arc<ClientConfig>,
321)
322 -> Outcome<String>
323{
324 let body = res!(chat_body_vision(&cfg.model, system, user, image_mime, image_b64));
325 let auth = fmt!("Bearer {}", cfg.api_key);
326 let headers: &[(&str, &str)] = &[
327 ("Host", cfg.provider.host()),
328 ("Authorization", &auth),
329 ("Content-Type", "application/json"),
330 ("Accept", "application/json"),
331 ];
332 let resp = res!(https_request(
333 cfg.provider.host(),
334 443,
335 HttpMethod::POST,
336 cfg.provider.path(),
337 headers,
338 body.as_bytes(),
339 tls,
340 ).await);
341
342 let payload = String::from_utf8_lossy(&resp.body).to_string();
343 let status = match &resp.header.headline {
344 HttpHeadline::Response { status } => *status as u16,
345 _ => 0,
346 };
347 if !(200..300).contains(&status) {
348 return Err(err!(
349 "The LLM provider answered {} to a vision completion request: {}", status, payload;
350 Network, Data));
351 }
352 chat_reply(&payload)
353}
354
355
356#[cfg(test)]
357mod tests {
358 use super::*;
359
360 /// A provider round-trips through its word, and an unknown word is refused rather than guessed.
361 #[test]
362 fn test_a_provider_names_itself_00() -> Outcome<()> {
363 for word in ["openrouter", "fireworks", "mistral"] {
364 let p = res!(Provider::of(word));
365 assert_eq!(p.as_str(), Some(word), "'{}' did not round-trip", word);
366 assert!(p.host().contains('.'), "'{}' has no host", word);
367 assert!(p.path().starts_with('/'), "'{}' has no path", word);
368 }
369 assert!(Provider::of("claude").is_err(), "an unknown provider was accepted");
370 Ok(())
371 }
372
373 /// The request body carries the model and both messages, and quotes what the text contains.
374 #[test]
375 fn test_the_body_quotes_its_text_01() -> Outcome<()> {
376 // A system prompt and a user text that between them hold every character a naive concatenation
377 // would break out of: a quote, a newline, a backslash, a brace.
378 let body = res!(chat_body(
379 "acme/model-1",
380 "You are a \"strict\" editor.\nFix typos only.",
381 "He said {hi} and\\or bye.",
382 ));
383 // It parses back as JSON, which a hand-built body with an unescaped quote would not.
384 let dat = res!(Dat::decode_string_with_config(body.clone(), &json_decoder()));
385 let map = match dat { Dat::Map(m) => m, _ => return Err(err!("not an object"; Test)) };
386 assert!(matches!(map.get(&dat!("model")), Some(Dat::Str(s)) if s == "acme/model-1"),
387 "model missing: {}", body);
388 let msgs = match map.get(&dat!("messages")) {
389 Some(Dat::List(l)) => l,
390 _ => return Err(err!("no messages list: {}", body; Test)),
391 };
392 assert_eq!(msgs.len(), 2, "expected system then user: {}", body);
393 // The roles are in order and the awkward text survived the round trip intact.
394 let role = |d: &Dat| match d { Dat::Map(m) => match m.get(&dat!("role")) {
395 Some(Dat::Str(s)) => s.clone(), _ => String::new() }, _ => String::new() };
396 assert_eq!(role(&msgs[0]), "system");
397 assert_eq!(role(&msgs[1]), "user");
398 Ok(())
399 }
400
401 /// A good reply yields its content; an empty `choices` says the model said nothing rather than
402 /// returning an empty string that reads like a valid answer.
403 #[test]
404 fn test_a_reply_yields_its_content_02() -> Outcome<()> {
405 let good = r#"{"choices":[{"message":{"role":"assistant","content":"Fixed text."}}]}"#;
406 assert_eq!(res!(chat_reply(good)), "Fixed text.");
407
408 let empty = r#"{"choices":[]}"#;
409 assert!(chat_reply(empty).is_err(), "an empty choices list should be an error");
410 Ok(())
411 }
412
413 /// A provider's error reply is surfaced with its message, in both the shapes providers send.
414 #[test]
415 fn test_an_error_reply_is_surfaced_03() -> Outcome<()> {
416 let nested = r#"{"error":{"message":"invalid api key","type":"auth"}}"#;
417 let e = fmt!("{}", chat_reply(nested).err().unwrap());
418 assert!(e.contains("invalid api key"), "the message was not surfaced: {}", e);
419
420 let bare = r#"{"error":"model not found"}"#;
421 let e2 = fmt!("{}", chat_reply(bare).err().unwrap());
422 assert!(e2.contains("model not found"), "the bare message was not surfaced: {}", e2);
423 Ok(())
424 }
425
426 /// A vision body keeps the text-only system message but turns the user `content` into the two-part
427 /// array a vision model reads, with the image inlined as a `data:` URL of the given mime and bytes.
428 #[test]
429 fn test_the_vision_body_carries_an_image_part_04() -> Outcome<()> {
430 let body = res!(chat_body_vision(
431 "acme/vision-1",
432 "You describe images.",
433 "What is in this picture?",
434 "image/png",
435 "aGVsbG8=", // "hello" in Base64, standing in for image bytes.
436 ));
437 let dat = res!(Dat::decode_string_with_config(body.clone(), &json_decoder()));
438 let map = match dat { Dat::Map(m) => m, _ => return Err(err!("not an object"; Test)) };
439 let msgs = match map.get(&dat!("messages")) {
440 Some(Dat::List(l)) => l,
441 _ => return Err(err!("no messages list: {}", body; Test)),
442 };
443 assert_eq!(msgs.len(), 2, "expected system then user: {}", body);
444
445 // The system turn is still a plain string, unchanged from the text-only shape.
446 let system = match &msgs[0] {
447 Dat::Map(m) => m,
448 _ => return Err(err!("system message not a map: {}", body; Test)),
449 };
450 assert!(matches!(system.get(&dat!("content")), Some(Dat::Str(_))),
451 "system content should be a bare string: {}", body);
452
453 // The user turn's content is the multimodal array of a text part and an image part.
454 let user = match &msgs[1] {
455 Dat::Map(m) => m,
456 _ => return Err(err!("user message not a map: {}", body; Test)),
457 };
458 let parts = match user.get(&dat!("content")) {
459 Some(Dat::List(l)) => l,
460 _ => return Err(err!("user content is not an array: {}", body; Test)),
461 };
462 assert_eq!(parts.len(), 2, "expected a text part and an image part: {}", body);
463
464 let part_type = |d: &Dat| match d { Dat::Map(m) => match m.get(&dat!("type")) {
465 Some(Dat::Str(s)) => s.clone(), _ => String::new() }, _ => String::new() };
466 assert_eq!(part_type(&parts[0]), "text", "first part should be text: {}", body);
467 assert_eq!(part_type(&parts[1]), "image_url", "second part should be image_url: {}", body);
468
469 // The image part nests `image_url.url` as a data URL of the given mime and Base64 payload.
470 let image = match &parts[1] { Dat::Map(m) => m, _ => return Err(err!("image part not a map"; Test)) };
471 let holder = match image.get(&dat!("image_url")) {
472 Some(Dat::Map(m)) => m,
473 _ => return Err(err!("image_url is not a nested object: {}", body; Test)),
474 };
475 match holder.get(&dat!("url")) {
476 Some(Dat::Str(s)) => assert_eq!(s, "data:image/png;base64,aGVsbG8=",
477 "the data URL was not built as expected: {}", body),
478 _ => return Err(err!("image_url carried no url string: {}", body; Test)),
479 }
480 Ok(())
481 }
482}