Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_sbj/src/text.rs

16.5 KiB, 1 run

created by r1870400018:22226, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The authoring text form: a document as JDAT text, which is what an author writes.
2//!
3//! A document reaches the wire as BDAT, and BDAT is not a thing anyone types. The source of a
4//! document is therefore its JDAT text form, which is what the fixtures of `SPEC.md` §7 carry in
5//! their `doc.jdat`, and what the compiler of the `sbj` binary reads. The text is the source and the
6//! bytes are the artefact, exactly as they are for the fixtures.
7//!
8//! Two of the v0 kind labels, `box` and `list`, are also JDAT's own kind labels, so a node written
9//! as `(box|{..})` would read back as a `Dat::Box` and a node written as `(list|[..])` as a
10//! `Dat::List`. Every node label therefore carries the prefix `sbj_`, and a heading is written
11//! `(sbj_heading|{..})`. Nothing of this reaches the wire: BDAT carries the `u16` kind code and no
12//! label at all, and a `UsrKindId` compares by code, so the label is the text form's business alone.
13//!
14//! A node of a kind the v0 vocabulary does not know (§4.5) is written `(sbj_k<code>|{..})`, e.g.
15//! `(sbj_k99|{..})`, since a decoder must be told which code a label names before it reads a byte of
16//! the document. A document may instead declare a label of its own choosing through [`KindDecl`],
17//! which is what the `--kind` option of the compiler passes.
18
19use crate::{
20 kinds::NodeKind,
21 limit,
22};
23
24use oxedyne_fe2o3_core::prelude::*;
25use oxedyne_fe2o3_jdat::{
26 prelude::*,
27 bdat::DecodeLimits,
28 string::{
29 dec::DecoderConfig,
30 enc::EncoderConfig,
31 },
32 usr::{
33 UsrKind,
34 UsrKindCode,
35 UsrKindId,
36 UsrKinds,
37 },
38};
39
40use std::collections::BTreeMap;
41
42/// The registry through which the JDAT text codec reads and writes node kinds.
43pub type Ukinds = UsrKinds<BTreeMap<UsrKindCode, UsrKind>, BTreeMap<String, UsrKindId>>;
44
45/// The nesting depth the text form of a tree at the node depth limit of §5 reaches.
46///
47/// The text decoder counts every bracket, brace and kindicle rather than every value, so a node
48/// costs more levels in the text than it does in the bytes: the kindicle naming the kind, the
49/// kindicle naming its payload map, the map itself, the kindicle naming its children list, and the
50/// list. This is an upper bound on the text depth of a tree that obeys the node depth limit of §5,
51/// which the validator enforces exactly. It is stated here rather than left to the decoder's default,
52/// because the limit is the format's: a document that nests to the ceiling §5 sets must read, and
53/// one that nests past it must be refused for that reason and not for a library's.
54pub const TEXT_DEPTH: usize = 6 * limit::DEPTH + 4;
55
56/// The greatest length, in bytes, of the text form of a document.
57///
58/// A tree region is at most 4 MiB (§5) and its text form is larger, since every daticle carries a
59/// kindicle the bytes do not, a string may escape one character into six, and the text is indented.
60/// Eight times the tree limit is beyond anything a document at the limit can reach in text, and it
61/// bounds what a decoder will read from a file nobody has vouched for.
62pub const TEXT_BYTES: usize = 8 * limit::TREE_BYTES;
63
64/// The limits the text form of a document is read under. See [`TEXT_DEPTH`] and [`TEXT_BYTES`].
65pub fn decode_limits() -> DecodeLimits {
66 DecodeLimits::new(TEXT_DEPTH, TEXT_BYTES)
67}
68
69/// The prefix every node label carries in the text form.
70pub const LABEL_PREFIX: &'static str = "sbj_";
71
72/// The prefix a node of a kind the v0 vocabulary does not know carries, followed by its code.
73pub const UNKNOWN_LABEL_PREFIX: &'static str = "sbj_k";
74
75/// The indent one level of the text form is written with.
76pub const INDENT: &'static str = " ";
77
78/// Every v0 node kind, in code order.
79pub const KINDS: [NodeKind; 13] = [
80 NodeKind::Doc,
81 NodeKind::Section,
82 NodeKind::Para,
83 NodeKind::Heading,
84 NodeKind::List,
85 NodeKind::Item,
86 NodeKind::Boxx,
87 NodeKind::Image,
88 NodeKind::Text,
89 NodeKind::Emph,
90 NodeKind::Link,
91 NodeKind::Code,
92 NodeKind::Quote,
93];
94
95/// One node kind the v0 vocabulary does not know: the label the text names it by, and its code.
96#[derive(Clone, Debug, PartialEq, Eq)]
97pub struct KindDecl {
98 /// The label the text form uses, e.g. `sbj_k99`.
99 pub label: String,
100 /// The wire code the label names.
101 pub code: u16,
102}
103
104/// The label a known node kind carries in the text form, e.g. `sbj_heading`.
105pub fn label(kind: NodeKind) -> String {
106 fmt!("{}{}", LABEL_PREFIX, kind.label())
107}
108
109/// The label a node of an unknown kind carries in the text form, e.g. `sbj_k99` (§4.5).
110pub fn unknown_label(code: u16) -> String {
111 fmt!("{}{}", UNKNOWN_LABEL_PREFIX, code)
112}
113
114/// The code an unknown-kind label names, or `None` if the label is not one.
115pub fn unknown_code(label: &str) -> Option<u16> {
116 let digits = match label.strip_prefix(UNKNOWN_LABEL_PREFIX) {
117 Some(digits) => digits,
118 None => return None,
119 };
120 if digits.is_empty() || !digits.chars().all(|c| c.is_ascii_digit()) {
121 return None;
122 }
123 match digits.parse::<u16>() {
124 Ok(code) => Some(code),
125 Err(_) => None,
126 }
127}
128
129/// The user kind id of a known node kind: its code, its label, and the shape of its payload.
130///
131/// The payload kind is declared because the JDAT text decoder reads a user kind that declares one
132/// and drops the payload of one that does not. Nothing of it reaches the wire, where BDAT writes the
133/// `u16` code and nothing else.
134pub fn ukid(kind: NodeKind) -> UsrKindId {
135 let payload = if kind.payload_is_str() {
136 Kind::Str
137 } else {
138 Kind::Map
139 };
140 UsrKindId::new(kind.code(), Some(&label(kind)), Some(payload))
141}
142
143/// The user kind id of a kind the v0 vocabulary does not know, whose payload §4.5 requires to be a
144/// map.
145pub fn unknown_ukid(decl: &KindDecl) -> UsrKindId {
146 UsrKindId::new(decl.code, Some(&decl.label), Some(Kind::Map))
147}
148
149/// Builds the registry: the thirteen v0 kinds, and the unknown kinds the caller declares.
150///
151/// A declaration naming a code the vocabulary already knows is refused, since a node of a known kind
152/// is written under its own label and a second label for it would give one document two texts.
153pub fn ukinds(decls: &[KindDecl]) -> Outcome<Ukinds> {
154 let mut uks = UsrKinds::new(BTreeMap::new(), BTreeMap::new());
155 for kind in KINDS {
156 res!(uks.add(ukid(kind)));
157 }
158 for decl in decls {
159 if let Ok(known) = NodeKind::from_code(decl.code) {
160 return Err(err!(
161 "The kind declaration '{}' names the code {}, which is the v0 kind '{}'. A known \
162 kind is written under its own label, '{}'.",
163 decl.label, decl.code, known.label(), label(known);
164 Invalid, Input, Conflict));
165 }
166 match uks.add(unknown_ukid(decl)) {
167 Ok(()) => (),
168 Err(e) => return Err(err!(e,
169 "The kind declaration '{} = {}' could not be registered.", decl.label, decl.code;
170 Invalid, Input)),
171 }
172 }
173 Ok(uks)
174}
175
176/// Reads a document tree from its JDAT text form.
177///
178/// The unknown kinds the text names by the `sbj_k<code>` convention are found by [`scan`] and need
179/// no declaring; any other label for an unknown kind must be declared in `decls`, since a decoder
180/// cannot guess which code a label it has never seen names.
181///
182/// The JDAT text decoder is recursive and generous with its frames, spending far more of a stack per
183/// level than the BDAT decoder does, so a caller reading a document that nests deeply should give the
184/// reading thread a stack to do it on, as the `sbj` binary does.
185pub fn decode(
186 src: &str,
187 decls: &[KindDecl],
188)
189 -> Outcome<Dat>
190{
191 let uks = res!(ukinds(&declarations(src, decls)));
192 let cfg = DecoderConfig::jdat(Some(uks)).with_limits(decode_limits());
193 match Dat::decode_string_with_config(src, &cfg) {
194 Ok(tree) => Ok(tree),
195 Err(e) => Err(err!(e,
196 "The source is not readable JDAT. A node is written as its kind label and its payload, \
197 e.g. (sbj_para|{{ (str|\"children\"): (list|[(sbj_text|(str|\"...\"))]) }}), and a kind \
198 the v0 vocabulary does not know is written (sbj_k<code>|{{..}}).";
199 Invalid, Input, Decode)),
200 }
201}
202
203/// Writes a document tree in JDAT text form.
204///
205/// Every kindicle is written out, including the ones JDAT would infer, so that the text says what
206/// the bytes say and nothing is left to a reader's guess: a `u8` reads as a `u8`, a list as a list,
207/// and a map as a map. It is what §3 asks of the bytes, asked of the text. A node of a kind the
208/// vocabulary does not know is written under the `sbj_k<code>` label, so that what is written here
209/// reads back through [`decode`] without a declaration.
210pub fn encode(tree: &Dat) -> Outcome<String> {
211 let mut decls = Vec::new();
212 collect_unknown(tree, &mut decls);
213 let uks = res!(ukinds(&decls));
214 let cfg = EncoderConfig::jdat_full_to_lines(Some(uks), INDENT);
215 let mut s = res!(tree.encode_string_with_config(&cfg));
216 s.push('\n');
217 Ok(s)
218}
219
220/// Reads a plain daticle, such as a key file, which carries no node kinds.
221pub fn decode_plain(src: &str) -> Outcome<Dat> {
222 let cfg = DecoderConfig::<
223 BTreeMap<UsrKindCode, UsrKind>,
224 BTreeMap<String, UsrKindId>,
225 >::jdat(None);
226 Dat::decode_string_with_config(src, &cfg)
227}
228
229/// Writes a plain daticle, such as a key file, in JDAT text form.
230pub fn encode_plain(dat: &Dat) -> Outcome<String> {
231 let cfg = EncoderConfig::<
232 BTreeMap<UsrKindCode, UsrKind>,
233 BTreeMap<String, UsrKindId>,
234 >::jdat_to_lines(None, INDENT);
235 let mut s = res!(dat.encode_string_with_config(&cfg));
236 s.push('\n');
237 Ok(s)
238}
239
240/// The declarations a source needs: the ones the caller gave, and the `sbj_k<code>` labels it uses.
241fn declarations(
242 src: &str,
243 decls: &[KindDecl],
244)
245 -> Vec<KindDecl>
246{
247 let mut all = decls.to_vec();
248 for decl in scan(src) {
249 // A caller's declaration wins, and a label already declared is not declared twice, since
250 // registering one code under two labels is refused by the registry.
251 if all.iter().any(|d| d.code == decl.code || d.label == decl.label) {
252 continue;
253 }
254 all.push(decl);
255 }
256 all
257}
258
259/// Finds the `sbj_k<code>` labels a source uses, so that an unknown kind needs no declaring (§4.5).
260///
261/// String literals are stepped over rather than read, so that a document whose prose happens to
262/// mention a label does not thereby declare a node kind.
263pub fn scan(src: &str) -> Vec<KindDecl> {
264 let mut out: Vec<KindDecl> = Vec::new();
265 let chars: Vec<char> = src.chars().collect();
266 let mut i = 0;
267 while i < chars.len() {
268 match chars[i] {
269 '"' => {
270 // Step over the string literal, honouring the backslash escape.
271 i += 1;
272 while i < chars.len() && chars[i] != '"' {
273 if chars[i] == '\\' {
274 i += 1;
275 }
276 i += 1;
277 }
278 i += 1;
279 },
280 '(' => {
281 // A kindicle: the label runs to the vertical bar that ends it.
282 i += 1;
283 let start = i;
284 while i < chars.len() && chars[i] != '|' && chars[i] != ')' && chars[i] != '"' {
285 i += 1;
286 }
287 let word: String = chars[start..i].iter().collect();
288 if let Some(code) = unknown_code(word.trim()) {
289 let decl = KindDecl {
290 label: word.trim().to_string(),
291 code,
292 };
293 if !out.contains(&decl) {
294 out.push(decl);
295 }
296 }
297 },
298 _ => i += 1,
299 }
300 }
301 out
302}
303
304/// Collects a declaration for every unknown kind code a tree carries, so that it can be written.
305fn collect_unknown(
306 dat: &Dat,
307 out: &mut Vec<KindDecl>,
308) {
309 match dat {
310 Dat::Usr(uid, payload) => {
311 if NodeKind::from_code(uid.code()).is_err() {
312 let decl = KindDecl {
313 label: unknown_label(uid.code()),
314 code: uid.code(),
315 };
316 if !out.contains(&decl) {
317 out.push(decl);
318 }
319 }
320 if let Some(boxd) = payload {
321 collect_unknown(boxd, out);
322 }
323 },
324 Dat::Map(map) => {
325 for (_, v) in map {
326 collect_unknown(v, out);
327 }
328 },
329 Dat::OrdMap(map) => {
330 for (_, v) in map {
331 collect_unknown(v, out);
332 }
333 },
334 Dat::List(list) => {
335 for item in list {
336 collect_unknown(item, out);
337 }
338 },
339 Dat::Box(boxd) => collect_unknown(boxd, out),
340 Dat::Opt(boxoptd) => {
341 if let Some(d) = &**boxoptd {
342 collect_unknown(d, out);
343 }
344 },
345 _ => (),
346 }
347}
348
349#[cfg(test)]
350mod tests {
351 use super::*;
352
353 /// The stack a thread is given before it reads a document.
354 ///
355 /// The JDAT text decoder spends a great deal of a stack on every level of a build with no
356 /// optimisation, and a test thread is given two megabytes, which a document of a few levels
357 /// exhausts. The format's limits do not move to suit a test, so the test moves.
358 const STACK_BYTES: usize = 64 * 1024 * 1024;
359
360 /// Runs a test on a thread with a stack that can hold what the text decoder spends.
361 fn on_a_stack<F>(f: F) -> Outcome<()>
362 where
363 F: FnOnce() -> Outcome<()> + Send + 'static,
364 {
365 let thread = match std::thread::Builder::new()
366 .name("sbj_text".to_string())
367 .stack_size(STACK_BYTES)
368 .spawn(f)
369 {
370 Ok(thread) => thread,
371 Err(e) => return Err(err!(e,
372 "Could not spawn the thread the document is read on."; Test, Init)),
373 };
374 match thread.join() {
375 Ok(outcome) => outcome,
376 Err(_) => Err(err!(
377 "The thread reading the document did not return."; Test, Panic)),
378 }
379 }
380
381 /// The tree of the `one_para` fixture, in text.
382 const ONE_PARA: &'static str = "\
383(sbj_doc|(map|{
384 (str|\"children\"): (list|[
385 (sbj_para|(map|{
386 (str|\"children\"): (list|[
387 (sbj_text|(str|\"One paragraph.\")),
388 ]),
389 })),
390 ]),
391 (str|\"lang\"): (str|\"en\"),
392 (str|\"title\"): (str|\"A document\"),
393}))
394";
395
396 /// A document whose one child is a kind the v0 vocabulary does not know, carrying a fallback.
397 const UNKNOWN_KIND: &'static str = "\
398(sbj_doc|(map|{
399 (str|\"children\"): (list|[
400 (sbj_k99|(map|{
401 (str|\"fallback\"): (list|[
402 (sbj_para|(map|{
403 (str|\"children\"): (list|[(sbj_text|(str|\"A stand-in.\"))]),
404 })),
405 ]),
406 })),
407 ]),
408 (str|\"lang\"): (str|\"en\"),
409 (str|\"title\"): (str|\"A document\"),
410}))
411";
412
413 #[test]
414 fn test_labels_are_prefixed_00() -> Outcome<()> {
415 // The two labels that collide with JDAT's own kinds are what the prefix is for.
416 assert_eq!(label(NodeKind::Boxx), "sbj_box");
417 assert_eq!(label(NodeKind::List), "sbj_list");
418 assert_eq!(label(NodeKind::Heading), "sbj_heading");
419 assert_eq!(unknown_label(99), "sbj_k99");
420 assert_eq!(unknown_code("sbj_k99"), Some(99));
421 assert_eq!(unknown_code("sbj_doc"), None);
422 assert_eq!(unknown_code("sbj_k"), None);
423 assert_eq!(unknown_code("sbj_k99x"), None);
424 Ok(())
425 }
426
427 #[test]
428 fn test_text_round_trip_01() -> Outcome<()> {
429 on_a_stack(|| {
430 let tree = res!(decode(ONE_PARA, &[]));
431 let text = res!(encode(&tree));
432 let again = res!(decode(&text, &[]));
433 assert_eq!(tree, again, "A tree did not survive its own text form.");
434 // And the text is stable: writing what was read gives the text back.
435 assert_eq!(text, res!(encode(&again)), "The text form is not stable.");
436 Ok(())
437 })
438 }
439
440 #[test]
441 fn test_unknown_kind_needs_no_declaration_02() -> Outcome<()> {
442 on_a_stack(|| {
443 let tree = res!(decode(UNKNOWN_KIND, &[]));
444 let text = res!(encode(&tree));
445 assert!(text.contains("sbj_k99"), "The unknown kind lost its label: {}", text);
446 assert_eq!(tree, res!(decode(&text, &[])),
447 "An unknown kind did not survive a round trip.");
448 Ok(())
449 })
450 }
451
452 #[test]
453 fn test_a_declared_label_is_read_03() -> Outcome<()> {
454 on_a_stack(|| {
455 // The label a document chooses for an unknown kind is declared, never guessed.
456 let src = "(sbj_alien|(map|{ (str|\"rows\"): (u8|1) }))";
457 assert!(decode(src, &[]).is_err(), "An undeclared label was read.");
458 let decls = vec![KindDecl { label: "sbj_alien".to_string(), code: 99 }];
459 let tree = res!(decode(src, &decls));
460 match &tree {
461 Dat::Usr(uid, _) => assert_eq!(uid.code(), 99),
462 d => return Err(err!("Expected a node, found a {:?}.", d.kind(); Test, Invalid)),
463 }
464 // Written back, it carries the conventional label, which needs no declaring.
465 let text = res!(encode(&tree));
466 assert!(text.contains("sbj_k99"), "The unknown kind was not written by code: {}", text);
467 Ok(())
468 })
469 }
470
471 #[test]
472 fn test_a_declaration_may_not_relabel_a_known_kind_04() -> Outcome<()> {
473 let decls = vec![KindDecl { label: "sbj_alien".to_string(), code: 3 }];
474 match ukinds(&decls) {
475 Ok(_) => Err(err!("A second label for the para kind was registered."; Test, Invalid)),
476 Err(e) => {
477 let msg = fmt!("{}", e);
478 assert!(msg.contains("para"), "The refusal should name the kind: {}", msg);
479 Ok(())
480 },
481 }
482 }
483
484 #[test]
485 fn test_the_scan_steps_over_strings_05() -> Outcome<()> {
486 // A label mentioned in prose is prose, not a declaration, and one naming a known code would
487 // otherwise collide with the vocabulary.
488 let decls = scan("(sbj_text|(str|\"a mention of (sbj_k3| and of sbj_k99 in a string\"))");
489 assert!(decls.is_empty(), "The scan read a label out of a string: {:?}", decls);
490 let decls = scan("(sbj_k20|(map|{}))");
491 assert_eq!(decls, vec![KindDecl { label: "sbj_k20".to_string(), code: 20 }]);
492 Ok(())
493 }
494}