Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_sbj/src/import.rs

37.0 KiB, 1 run

created by r1870400018:22214, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Importing Markdown: ordinary prose, mapped to the node vocabulary of `SPEC.md` §4.
2//!
3//! A document's source is its JDAT text form, and nobody writes prose that way. Most prose that
4//! exists is Markdown, so this reads the neutral document tree that
5//! [`markdown`](oxedyne_fe2o3_text::doc::markdown) produces and maps it to a `doc` tree. What comes out
6//! is a tree like any other: it is validated, canonically encoded, hashed and signed by the same
7//! code that does it for a document written by hand, because by then it is the same document.
8//!
9//! # Where the two vocabularies do not meet
10//!
11//! The v0 vocabulary is closed, and Markdown says four things it has no room for. None of them is
12//! a reason to grow the vocabulary, and each is handled by saying less rather than by saying it
13//! wrongly:
14//!
15//! - A **thematic break** is dropped. v0 has no kind for one, and there is nothing it can degrade to
16//! that means what it meant.
17//! - A **table** becomes a box of paragraphs, one to a row. v0 has no kind for a grid, and a box is
18//! what says "these things are one thing", which is as much of a table as v0 says.
19//! - An **image** becomes its alt text. SBJ addresses an image by the content hash of the blob
20//! (§4.2), and Markdown gives a path; nothing here can resolve one into the other, and an image
21//! node with an invented hash would address a blob that does not exist.
22//! - An **inline code span** becomes its characters. SBJ's `code` is flow content and a paragraph
23//! admits inline content only, so a code span cannot sit where Markdown puts it.
24//!
25//! Each of these loses something, and each loses it visibly: the words survive, and only the mark-up
26//! around them goes. A mapping that cannot be exact should be lossy in the direction of the prose.
27
28use crate::{
29 kinds::{
30 NodeKind,
31 ADDR_NAME,
32 KEY_CHILDREN,
33 KEY_TO,
34 },
35 text::ukid,
36};
37
38use oxedyne_fe2o3_jdat::prelude::*;
39use oxedyne_fe2o3_text::{
40 doc::{
41 self,
42 html,
43 markdown,
44 Block,
45 Inline,
46 Row,
47 },
48 unicode::norm,
49};
50
51use oxedyne_fe2o3_core::prelude::*;
52
53/// The language a document declares when the caller names none.
54pub const DEFAULT_LANG: &'static str = "en";
55
56/// The name a document is titled by when it has no title and the caller supplies no source name.
57pub const DEFAULT_STEM: &'static str = "document";
58
59/// What a degraded table's cells are divided by, once the grid that divided them is gone.
60///
61/// A table's rows become paragraphs (see [`block`]), so a cell boundary either survives in the prose
62/// or does not survive at all. A pipe is what a reader of plain text reads a column boundary as, and
63/// it survives being laid out: a tab or a run of spaces is whitespace, and whatever renders the
64/// paragraph is free to collapse it, which would run two columns into one word.
65const CELL_SEP: &'static str = " | ";
66
67/// What the caller knows that the Markdown does not.
68///
69/// A `doc` node requires a title and a language (§4.2) and Markdown carries neither, so both are
70/// settled here.
71#[derive(Clone, Debug)]
72pub struct Options {
73 /// The document's title, when the caller names one. Otherwise the first level 1 heading, and
74 /// failing that, the [`stem`](Options::stem).
75 pub title: Option<String>,
76 /// The document's language tag, BCP-47.
77 pub lang: String,
78 /// The name the source is known by, usually its file stem: the title of last resort.
79 pub stem: String,
80}
81
82impl Default for Options {
83
84 fn default() -> Self {
85 Self {
86 title: None,
87 lang: DEFAULT_LANG.to_string(),
88 stem: DEFAULT_STEM.to_string(),
89 }
90 }
91}
92
93/// Reads Markdown text and maps it to a document tree.
94///
95/// The outcome is an error only when the Markdown will not parse, which is when it breaks a limit
96/// the parser holds against a hostile document. The mapping itself does not fail: see [`from_doc`].
97pub fn from_markdown(
98 src: &str,
99 opts: &Options,
100)
101 -> Outcome<Dat>
102{
103 let md = res!(markdown::parse(src));
104 Ok(from_doc(&md, opts))
105}
106
107/// Reads HTML and maps it to a document tree.
108///
109/// The same mapping as [`from_markdown`], reached by a second road. Prose written in a form no
110/// reader here understands can often be got at through HTML, because the thing that understands it
111/// will export HTML -- which is how prose written in Typst reaches a document, its author's own
112/// macros already resolved by the tool that defined them.
113pub fn from_html(
114 src: &str,
115 opts: &Options,
116)
117 -> Outcome<Dat>
118{
119 let doc = res!(html::parse(src));
120 Ok(from_doc(&doc, opts))
121}
122
123/// Maps a parsed Markdown document to a document tree.
124///
125/// The mapping never fails. Markdown has no syntax errors, only text that means less than the author
126/// hoped, and the three things v0 has no room for are dropped or degraded rather than refused. What
127/// comes back is a tree the validator accepts, and whether it does is [`validate`](crate::validate)'s
128/// business rather than a promise made here.
129pub fn from_doc(
130 md: &doc::Doc,
131 opts: &Options,
132)
133 -> Dat
134{
135 node(
136 NodeKind::Doc,
137 vec![
138 ("title", Dat::Str(title(md, opts))),
139 ("lang", Dat::Str(clean(&opts.lang))),
140 ],
141 blocks(&md.blocks),
142 )
143}
144
145/// The document's title: the caller's, else the first level 1 heading, else the source's name.
146///
147/// A title of nothing but whitespace is no title, whichever of the three it came from, since a `doc`
148/// carries the field whether or not the author thought about it.
149fn title(
150 md: &doc::Doc,
151 opts: &Options,
152)
153 -> String
154{
155 if let Some(t) = &opts.title {
156 if !t.trim().is_empty() {
157 return clean(t);
158 }
159 }
160 // The most prominent heading, and not the first level 1. What level a piece is headed
161 // by says where the prose came from rather than what it says: an author writing Markdown heads a
162 // chapter with a level 1, and the same chapter exported from Typst arrives headed by a level 2,
163 // because the exporter keeps level 1 for the document it thinks it is making. Asking for level 1
164 // found no title in a whole shelf of books, and titled every one of them after its file.
165 if let Some(t) = md.top_heading() {
166 if !t.trim().is_empty() {
167 return clean(&t);
168 }
169 }
170 clean(&opts.stem)
171}
172
173// ┌───────────────────────────────────────────────────────────────────────────┐
174// │ BLOCKS │
175// └───────────────────────────────────────────────────────────────────────────┘
176
177/// Maps a run of blocks to the flow nodes they become, leaving out the ones that become nothing.
178fn blocks(blocks: &[Block]) -> Vec<Dat> {
179 let mut out = Vec::with_capacity(blocks.len());
180 for b in blocks {
181 if let Some(node) = block(b) {
182 out.push(node);
183 }
184 }
185 out
186}
187
188/// Maps one block, or `None` where the vocabulary has no room for it.
189fn block(b: &Block) -> Option<Dat> {
190 match b {
191 Block::Heading { level, content } => Some(node(
192 NodeKind::Heading,
193 vec![("level", Dat::U8(heading_level(*level)))],
194 inlines(content),
195 )),
196 Block::Para(content) => {
197 // Whitespace is not a paragraph. A blank one carries no words and renders as nothing, so
198 // carrying it would give one document two addresses for the same prose.
199 if doc::text_of(content).trim().is_empty() {
200 return None;
201 }
202 Some(node(NodeKind::Para, Vec::new(), inlines(content)))
203 },
204 Block::List { ordered, items } => {
205 let kids: Vec<Dat> = items.iter()
206 .map(|item| node(NodeKind::Item, Vec::new(), blocks(item)))
207 .collect();
208 // SPEC §4.2 marks a list `item+`, so a list of no items is refused by the validator. A
209 // list that lost every item is a list of nothing, which is what dropping it says.
210 if kids.is_empty() {
211 return None;
212 }
213 Some(node(NodeKind::List, vec![("ordered", Dat::Bool(*ordered))], kids))
214 },
215 Block::Code { lang, text } => {
216 let mut fields = Vec::with_capacity(2);
217 // The language is optional, and a fence that named none, or named nothing but
218 // whitespace, has none to declare.
219 if let Some(lang) = lang {
220 if !lang.trim().is_empty() {
221 fields.push(("lang", Dat::Str(clean(lang))));
222 }
223 }
224 fields.push(("text", Dat::Str(clean(text))));
225 Some(node(NodeKind::Code, fields, Vec::new()))
226 },
227 // A quote carries no `cite`, because Markdown gives none. Attribution is a thing the author
228 // writes inside the quotation, and inventing one from it would be a guess.
229 Block::Quote(quoted) => Some(node(NodeKind::Quote, Vec::new(), self::blocks(quoted))),
230 // A table degrades to a box of paragraphs, one to a row, each row's cells divided by
231 // CELL_SEP. v0 has no kind for a grid (§4.2) and the vocabulary is frozen, so no mapping
232 // keeps the grid; what any mapping can keep is every word, in the order it was written, and
233 // the rows it was written in. A box is flow content that holds flow content, so a run of
234 // paragraphs that belong together is the thing it is for: it says "these paragraphs are one
235 // thing", which is the most of a table v0 says.
236 //
237 // The header row is not marked out from the rest. It comes first, as it did, and that is all.
238 // Emphasising it would say the author wrote emphasis, and they wrote a header; the bold a
239 // browser puts on a header cell is a rendering convention and not something the prose said.
240 // This is the reason a quote invents no `cite`, applied a second time.
241 //
242 // The column alignments go with the grid. An alignment is a column's, there are no columns
243 // left to carry one, and §4.4's `align` style belongs to a whole node rather than to a slice
244 // down the middle of several: a row is not a column, so there is nowhere true to put it.
245 Block::Table { head, rows, .. } => {
246 let mut kids = Vec::with_capacity(rows.len() + 1);
247 if let Some(head) = head {
248 if let Some(para) = table_row(head) {
249 kids.push(para);
250 }
251 }
252 for row in rows {
253 if let Some(para) = table_row(row) {
254 kids.push(para);
255 }
256 }
257 // A table that said nothing renders as nothing, as a blank paragraph does.
258 if kids.is_empty() {
259 return None;
260 }
261 Some(node(NodeKind::Boxx, Vec::new(), kids))
262 },
263 // A thematic break is dropped: v0 has no kind for one (§4.2), and the vocabulary is frozen. A
264 // rule is a division between passages and nothing else, so there is no node it degrades to
265 // that keeps what it meant -- an empty paragraph or a box would each say something the author
266 // did not.
267 Block::Rule => None,
268 // A division degrades to a box of its blocks. A box is flow content holding flow content, which
269 // is what a division is, and it says "these blocks are one thing" without keeping the id and
270 // classes v0 has no vocabulary for (§4.4 names styles locally, not by class). An empty division
271 // says nothing and is dropped, as a blank paragraph is.
272 Block::Div { content, .. } => {
273 let kids = self::blocks(content);
274 if kids.is_empty() {
275 return None;
276 }
277 Some(node(NodeKind::Boxx, Vec::new(), kids))
278 },
279 }
280}
281
282/// One row of a table, as the paragraph it degrades to, or `None` where the row says nothing.
283///
284/// A cell holds inline content and a paragraph admits inline content, so a cell's inlines are carried
285/// across whole: the emphasis, the links and the words are all the row's, and only the cell walls go.
286fn table_row(row: &Row) -> Option<Dat> {
287 // A row of empty cells carries no words, and a paragraph of nothing but separators would say
288 // something the author did not. It is dropped, as a blank paragraph is.
289 if row.0.iter().all(|cell| cell.text_of().trim().is_empty()) {
290 return None;
291 }
292 let mut out = Vec::new();
293 for (n, cell) in row.0.iter().enumerate() {
294 if n > 0 {
295 push_text(&mut out, CELL_SEP);
296 }
297 out.extend(inlines(&cell.0));
298 }
299 Some(node(NodeKind::Para, Vec::new(), out))
300}
301
302/// A heading's level, held to the 1 to 6 the schema admits.
303///
304/// Markdown has no other level, so this only ever matters for a tree built by hand. A level outside
305/// the range is clamped rather than refused, since an import that dies on a heading has told an
306/// author nothing they can act on.
307fn heading_level(n: u8) -> u8 {
308 n.clamp(1, 6)
309}
310
311// ┌───────────────────────────────────────────────────────────────────────────┐
312// │ INLINES │
313// └───────────────────────────────────────────────────────────────────────────┘
314
315/// Maps a run of inlines to the inline nodes they become.
316fn inlines(content: &[Inline]) -> Vec<Dat> {
317 let mut out = Vec::with_capacity(content.len());
318 for item in content {
319 match item {
320 Inline::Text(s) => push_text(&mut out, s),
321 Inline::Emph { strong, content } => out.push(node(
322 NodeKind::Emph,
323 vec![("strong", Dat::Bool(*strong))],
324 inlines(content),
325 )),
326 Inline::Link { to, content } => out.push(node(
327 NodeKind::Link,
328 vec![(KEY_TO, address(to))],
329 inlines(content),
330 )),
331 // An image degrades to its alt text. An `image` node addresses its blob by content hash
332 // (§4.2) and Markdown gives a path, so nothing here can build one: resolving a path to a
333 // blob, hashing it and publishing it is the business of whatever puts blobs on the
334 // oxeweb, and a hash invented to fill the field would address nothing.
335 Inline::Image { alt, .. } => push_text(&mut out, alt),
336 // A code span degrades to its characters. SBJ's `code` is flow content and a paragraph
337 // admits inline content only, so a code node cannot sit where Markdown puts this. The
338 // characters are what the span says; the monospace is how it looked.
339 Inline::Code(s) => push_text(&mut out, s),
340 // A hard break is a newline in a text run. §3 rule 5 permits one in a string, which is
341 // what the `every_kind` fixture carries.
342 Inline::Break => out.push(text("\n".to_string())),
343 // A span degrades to its content, flattened into the line. v0 has no generic inline grouping
344 // kind (§4.2), and the id and classes it carries have no vocabulary here, so what it keeps is
345 // every word and every emphasis inside it, in order, exactly as a code span keeps its
346 // characters.
347 Inline::Span { content, .. } => out.extend(inlines(content)),
348 }
349 }
350 out
351}
352
353/// Pushes a text run, unless it would say nothing.
354fn push_text(
355 out: &mut Vec<Dat>,
356 s: &str,
357) {
358 let s = clean(s);
359 if !s.is_empty() {
360 out.push(text(s));
361 }
362}
363
364/// A link's typed address (§4.3).
365///
366/// Markdown gives a destination exactly as it was written, which is a name and not a content hash: a
367/// hash is 32 bytes an author never types, and a URL, a path or a NAMES name are all names as far as
368/// the format is concerned.
369fn address(to: &str) -> Dat {
370 create_dat_map(vec![
371 (Dat::Str(ADDR_NAME.to_string()), Dat::Str(clean(to))),
372 ])
373}
374
375// ┌───────────────────────────────────────────────────────────────────────────┐
376// │ NODES AND STRINGS │
377// └───────────────────────────────────────────────────────────────────────────┘
378
379/// Builds a node from its fields and its children.
380///
381/// An empty children list is left out entirely rather than written empty, since §3 rule 4 admits one
382/// encoding of a node with no children and that is the one without the key.
383fn node(
384 kind: NodeKind,
385 fields: Vec<(&str, Dat)>,
386 kids: Vec<Dat>,
387)
388 -> Dat
389{
390 let mut kv: Vec<(Dat, Dat)> = fields.into_iter()
391 .map(|(k, v)| (Dat::Str(k.to_string()), v))
392 .collect();
393 if !kids.is_empty() {
394 kv.push((Dat::Str(KEY_CHILDREN.to_string()), Dat::List(kids)));
395 }
396 Dat::Usr(ukid(kind), Some(Box::new(create_dat_map(kv))))
397}
398
399/// Builds a text run, the one node whose payload is the string itself.
400fn text(s: String) -> Dat {
401 Dat::Usr(ukid(NodeKind::Text), Some(Box::new(Dat::Str(s))))
402}
403
404/// A string as a document may carry it: Unicode NFC, and no control character but tab and newline.
405///
406/// This is §3 rule 5 applied at the door. Prose in the wild carries a stray carriage return or a
407/// vertical tab often enough, and text typed on one machine and pasted from another is decomposed
408/// often enough, that an importer which passed either through would refuse its own output at the
409/// moment of signing, naming a rule the author never heard of. Both are corrected here instead, and
410/// neither changes what the prose says: the composed and decomposed forms of a letter display
411/// identically, and a control character displays as nothing at all.
412fn clean(s: &str) -> String {
413 let stripped: String = s.chars()
414 .filter(|c| !c.is_control() || *c == '\t' || *c == '\n')
415 .collect();
416 norm::nfc(&stripped)
417}
418
419#[cfg(test)]
420mod tests {
421 use super::*;
422 use crate::{
423 kinds::children_of,
424 validate,
425 SCHEMA_DOC,
426 };
427
428 use oxedyne_fe2o3_text::doc::{
429 Align,
430 Cell,
431 };
432
433 /// Builds a document of the given blocks, and holds it to the validator.
434 ///
435 /// Every test goes through here, because a mapping that produces a tree the format refuses is a
436 /// mapping that is wrong, whatever else the test then checks about it.
437 fn imported(blocks: Vec<Block>) -> Outcome<Dat> {
438 let md = doc::Doc { blocks };
439 let tree = from_doc(&md, &Options::default());
440 res!(validate::validate(&tree, SCHEMA_DOC));
441 Ok(tree)
442 }
443
444 /// The kind code and payload of a node.
445 fn parts(d: &Dat) -> Outcome<(u16, &Dat)> {
446 match d {
447 Dat::Usr(uid, Some(payload)) => Ok((uid.code(), payload.as_ref())),
448 d => Err(err!("Expected a node, found a {:?}.", d.kind(); Test, Invalid)),
449 }
450 }
451
452 /// The kind of a node, which every test asserts before it looks inside one.
453 fn kind_of(d: &Dat) -> Outcome<NodeKind> {
454 let (code, _) = res!(parts(d));
455 NodeKind::from_code(code)
456 }
457
458 /// The children of a node.
459 fn kids(d: &Dat) -> Outcome<Vec<Dat>> {
460 let (_, payload) = res!(parts(d));
461 children_of(payload)
462 }
463
464 /// The `n`th child of a node.
465 fn kid(
466 d: &Dat,
467 n: usize,
468 )
469 -> Outcome<Dat>
470 {
471 let kids = res!(kids(d));
472 match kids.get(n) {
473 Some(kid) => Ok(kid.clone()),
474 None => Err(err!(
475 "The node carries {} children, and child {} was asked for.", kids.len(), n;
476 Test, Missing)),
477 }
478 }
479
480 /// One field of a node's payload map.
481 fn field(
482 d: &Dat,
483 name: &str,
484 )
485 -> Outcome<Dat>
486 {
487 let (_, payload) = res!(parts(d));
488 match payload {
489 Dat::Map(map) => match map.get(&dat!(name)) {
490 Some(v) => Ok(v.clone()),
491 None => Err(err!("The node carries no '{}' field.", name; Test, Missing)),
492 },
493 d => Err(err!("Expected a map payload, found a {:?}.", d.kind(); Test, Invalid)),
494 }
495 }
496
497 /// The string a text run carries.
498 fn text_of(d: &Dat) -> Outcome<String> {
499 match res!(parts(d)) {
500 (9, Dat::Str(s)) => Ok(s.clone()),
501 (code, payload) => Err(err!(
502 "Expected a text run, found kind {} carrying a {:?}.", code, payload.kind();
503 Test, Invalid)),
504 }
505 }
506
507 /// The words a node holds, every text run within it flattened wherever it sits: what a degraded
508 /// row says, all told, and no more of how it says it than [`doc::text_of`] keeps.
509 fn said(d: &Dat) -> Outcome<String> {
510 if let (9, Dat::Str(s)) = res!(parts(d)) {
511 return Ok(s.clone());
512 }
513 let mut s = String::new();
514 for kid in res!(kids(d)) {
515 s.push_str(&res!(said(&kid)));
516 }
517 Ok(s)
518 }
519
520 /// A run of literal text.
521 fn t(s: &str) -> Inline {
522 Inline::Text(s.to_string())
523 }
524
525 /// A paragraph of one run of literal text.
526 fn p(s: &str) -> Block {
527 Block::Para(vec![t(s)])
528 }
529
530 /// A cell of one run of literal text.
531 fn c(s: &str) -> Cell {
532 Cell(vec![t(s)])
533 }
534
535 #[test]
536 fn test_a_doc_carries_its_title_and_language_00() -> Outcome<()> {
537 let tree = res!(imported(vec![p("A paragraph.")]));
538 assert_eq!(res!(kind_of(&tree)), NodeKind::Doc);
539 // Both fields are required (§4.2), so both are always there, whatever the Markdown said.
540 assert_eq!(res!(field(&tree, "title")), dat!("document"));
541 assert_eq!(res!(field(&tree, "lang")), dat!("en"));
542 Ok(())
543 }
544
545 #[test]
546 fn test_a_heading_carries_its_level_and_its_words_01() -> Outcome<()> {
547 let tree = res!(imported(vec![
548 Block::Heading { level: 3, content: vec![t("A heading")] },
549 ]));
550 let heading = res!(kid(&tree, 0));
551 assert_eq!(res!(kind_of(&heading)), NodeKind::Heading);
552 assert_eq!(res!(field(&heading, "level")), Dat::U8(3));
553 assert_eq!(res!(text_of(&res!(kid(&heading, 0)))), "A heading");
554 Ok(())
555 }
556
557 #[test]
558 fn test_a_heading_level_is_held_to_the_schemas_range_02() -> Outcome<()> {
559 // The parser gives 1 to 6 and nothing else, so this is only ever a hand-built tree. It is
560 // clamped rather than refused, and the validator is what says the clamp worked.
561 let tree = res!(imported(vec![
562 Block::Heading { level: 0, content: vec![t("Too high")] },
563 Block::Heading { level: 9, content: vec![t("Too low")] },
564 ]));
565 assert_eq!(res!(field(&res!(kid(&tree, 0)), "level")), Dat::U8(1));
566 assert_eq!(res!(field(&res!(kid(&tree, 1)), "level")), Dat::U8(6));
567 Ok(())
568 }
569
570 #[test]
571 fn test_a_paragraph_carries_its_inlines_03() -> Outcome<()> {
572 let tree = res!(imported(vec![p("A paragraph.")]));
573 let para = res!(kid(&tree, 0));
574 assert_eq!(res!(kind_of(&para)), NodeKind::Para);
575 assert_eq!(res!(text_of(&res!(kid(&para, 0)))), "A paragraph.");
576 Ok(())
577 }
578
579 #[test]
580 fn test_an_empty_paragraph_is_skipped_04() -> Outcome<()> {
581 // A paragraph of nothing but whitespace renders as nothing, so it is nothing.
582 let tree = res!(imported(vec![p(" \t "), p("Real prose."), Block::Para(Vec::new())]));
583 assert_eq!(res!(kids(&tree)).len(), 1);
584 assert_eq!(res!(text_of(&res!(kid(&res!(kid(&tree, 0)), 0)))), "Real prose.");
585 Ok(())
586 }
587
588 #[test]
589 fn test_a_list_carries_its_items_05() -> Outcome<()> {
590 let tree = res!(imported(vec![
591 Block::List {
592 ordered: true,
593 items: vec![vec![p("One")], vec![p("Two")]],
594 },
595 ]));
596 let list = res!(kid(&tree, 0));
597 assert_eq!(res!(kind_of(&list)), NodeKind::List);
598 assert_eq!(res!(field(&list, "ordered")), Dat::Bool(true));
599 assert_eq!(res!(kids(&list)).len(), 2);
600 // A list holds items, and an item holds flow content: the paragraph is the item's child.
601 let item = res!(kid(&list, 0));
602 assert_eq!(res!(kind_of(&item)), NodeKind::Item);
603 let para = res!(kid(&item, 0));
604 assert_eq!(res!(kind_of(&para)), NodeKind::Para);
605 assert_eq!(res!(text_of(&res!(kid(&para, 0)))), "One");
606 Ok(())
607 }
608
609 #[test]
610 fn test_a_list_of_no_items_is_dropped_06() -> Outcome<()> {
611 // SPEC §4.2 marks a list `item+`, so an empty one would be refused by the validator, which is
612 // what running this through it proves.
613 let tree = res!(imported(vec![
614 Block::List { ordered: false, items: Vec::new() },
615 p("After."),
616 ]));
617 assert_eq!(res!(kids(&tree)).len(), 1);
618 assert_eq!(res!(kind_of(&res!(kid(&tree, 0)))), NodeKind::Para);
619 Ok(())
620 }
621
622 #[test]
623 fn test_a_nested_list_nests_07() -> Outcome<()> {
624 let tree = res!(imported(vec![
625 Block::List {
626 ordered: false,
627 items: vec![vec![
628 p("Outer"),
629 Block::List {
630 ordered: false,
631 items: vec![vec![p("Inner")]],
632 },
633 ]],
634 },
635 ]));
636 // doc > list > item > [para, list] > item > para > text.
637 let outer_item = res!(kid(&res!(kid(&tree, 0)), 0));
638 assert_eq!(res!(kids(&outer_item)).len(), 2);
639 let inner_list = res!(kid(&outer_item, 1));
640 assert_eq!(res!(kind_of(&inner_list)), NodeKind::List);
641 let inner_para = res!(kid(&res!(kid(&inner_list, 0)), 0));
642 assert_eq!(res!(text_of(&res!(kid(&inner_para, 0)))), "Inner");
643 Ok(())
644 }
645
646 #[test]
647 fn test_a_code_block_carries_its_language_and_its_text_08() -> Outcome<()> {
648 let tree = res!(imported(vec![
649 Block::Code {
650 lang: Some("rust".to_string()),
651 text: "fn main() {}\n".to_string(),
652 },
653 ]));
654 let code = res!(kid(&tree, 0));
655 assert_eq!(res!(kind_of(&code)), NodeKind::Code);
656 assert_eq!(res!(field(&code, "lang")), dat!("rust"));
657 // The line structure is preserved: a listing is one string, not a run of spans.
658 assert_eq!(res!(field(&code, "text")), dat!("fn main() {}\n"));
659 Ok(())
660 }
661
662 #[test]
663 fn test_a_code_block_naming_no_language_carries_none_09() -> Outcome<()> {
664 let tree = res!(imported(vec![
665 Block::Code { lang: None, text: "plain".to_string() },
666 Block::Code { lang: Some(" ".to_string()), text: "plain".to_string() },
667 ]));
668 // The field is optional (§4.2), and the canonical encoding of an absent optional is its
669 // absence, so a fence that named no language carries no key.
670 for n in 0..2 {
671 let code = res!(kid(&tree, n));
672 assert!(field(&code, "lang").is_err(), "A code block invented a language.");
673 assert_eq!(res!(field(&code, "text")), dat!("plain"));
674 }
675 Ok(())
676 }
677
678 #[test]
679 fn test_a_quote_carries_its_blocks_and_no_cite_10() -> Outcome<()> {
680 let tree = res!(imported(vec![Block::Quote(vec![p("A quoted line.")])]));
681 let quote = res!(kid(&tree, 0));
682 assert_eq!(res!(kind_of(&quote)), NodeKind::Quote);
683 // Markdown gives no attribution, so none is invented.
684 assert!(field(&quote, "cite").is_err(), "A quote invented an attribution.");
685 let para = res!(kid(&quote, 0));
686 assert_eq!(res!(kind_of(&para)), NodeKind::Para);
687 assert_eq!(res!(text_of(&res!(kid(&para, 0)))), "A quoted line.");
688 Ok(())
689 }
690
691 #[test]
692 fn test_emphasis_carries_its_strength_11() -> Outcome<()> {
693 let tree = res!(imported(vec![
694 Block::Para(vec![
695 Inline::Emph { strong: true, content: vec![t("loud")] },
696 Inline::Emph { strong: false, content: vec![t("quiet")] },
697 ]),
698 ]));
699 let para = res!(kid(&tree, 0));
700 let strong = res!(kid(&para, 0));
701 assert_eq!(res!(kind_of(&strong)), NodeKind::Emph);
702 assert_eq!(res!(field(&strong, "strong")), Dat::Bool(true));
703 assert_eq!(res!(text_of(&res!(kid(&strong, 0)))), "loud");
704 assert_eq!(res!(field(&res!(kid(&para, 1)), "strong")), Dat::Bool(false));
705 Ok(())
706 }
707
708 #[test]
709 fn test_a_link_carries_a_named_address_12() -> Outcome<()> {
710 let tree = res!(imported(vec![
711 Block::Para(vec![
712 Inline::Link {
713 to: "news.cricket".to_string(),
714 content: vec![t("a link")],
715 },
716 ]),
717 ]));
718 let link = res!(kid(&res!(kid(&tree, 0)), 0));
719 assert_eq!(res!(kind_of(&link)), NodeKind::Link);
720 // A destination as written is a name, never a content hash (§4.3).
721 assert_eq!(res!(field(&link, "to")), mapdat!{ "name" => dat!("news.cricket") });
722 assert_eq!(res!(text_of(&res!(kid(&link, 0)))), "a link");
723 Ok(())
724 }
725
726 #[test]
727 fn test_a_rule_is_dropped_13() -> Outcome<()> {
728 // v0 has no thematic break, and the vocabulary is frozen, so the rule goes and the prose stays.
729 let tree = res!(imported(vec![p("Before."), Block::Rule, p("After.")]));
730 let kids = res!(kids(&tree));
731 assert_eq!(kids.len(), 2, "A rule left something behind: {:?}", kids);
732 assert_eq!(res!(text_of(&res!(kid(&kids[0], 0)))), "Before.");
733 assert_eq!(res!(text_of(&res!(kid(&kids[1], 0)))), "After.");
734 Ok(())
735 }
736
737 #[test]
738 fn test_an_image_degrades_to_its_alt_text_14() -> Outcome<()> {
739 // An `image` node addresses a blob by content hash, and Markdown gives a path, so the words
740 // survive and the picture does not.
741 let tree = res!(imported(vec![
742 Block::Para(vec![
743 t("Look: "),
744 Inline::Image {
745 src: "tree.png".to_string(),
746 alt: "A diagram of a tree".to_string(),
747 },
748 ]),
749 ]));
750 let para = res!(kid(&tree, 0));
751 let alt = res!(kid(&para, 1));
752 assert_eq!(res!(kind_of(&alt)), NodeKind::Text);
753 assert_eq!(res!(text_of(&alt)), "A diagram of a tree");
754 Ok(())
755 }
756
757 #[test]
758 fn test_an_image_with_no_alt_text_says_nothing_15() -> Outcome<()> {
759 let tree = res!(imported(vec![
760 Block::Para(vec![
761 t("Words."),
762 Inline::Image { src: "tree.png".to_string(), alt: String::new() },
763 ]),
764 ]));
765 assert_eq!(res!(kids(&res!(kid(&tree, 0)))).len(), 1);
766 Ok(())
767 }
768
769 #[test]
770 fn test_an_inline_code_span_degrades_to_text_16() -> Outcome<()> {
771 // SBJ's `code` is flow content and a para admits inline content only, so a code node cannot
772 // sit here at all. That the tree validates is the whole point of the degrading.
773 let tree = res!(imported(vec![
774 Block::Para(vec![t("Run "), Inline::Code("cargo test".to_string()), t(" first.")]),
775 ]));
776 let para = res!(kid(&tree, 0));
777 let span = res!(kid(&para, 1));
778 assert_eq!(res!(kind_of(&span)), NodeKind::Text);
779 assert_eq!(res!(text_of(&span)), "cargo test");
780 Ok(())
781 }
782
783 #[test]
784 fn test_a_hard_break_is_a_newline_17() -> Outcome<()> {
785 let tree = res!(imported(vec![
786 Block::Para(vec![t("One line"), Inline::Break, t("and the next")]),
787 ]));
788 let para = res!(kid(&tree, 0));
789 let brk = res!(kid(&para, 1));
790 assert_eq!(res!(kind_of(&brk)), NodeKind::Text);
791 assert_eq!(res!(text_of(&brk)), "\n");
792 Ok(())
793 }
794
795 #[test]
796 fn test_the_title_is_the_callers_18() -> Outcome<()> {
797 let md = doc::Doc {
798 blocks: vec![Block::Heading { level: 1, content: vec![t("The heading")] }],
799 };
800 let opts = Options {
801 title: Some("The caller's title".to_string()),
802 ..Options::default()
803 };
804 let tree = from_doc(&md, &opts);
805 res!(validate::validate(&tree, SCHEMA_DOC));
806 assert_eq!(res!(field(&tree, "title")), dat!("The caller's title"));
807 Ok(())
808 }
809
810 #[test]
811 fn test_the_title_falls_back_to_the_first_heading_19() -> Outcome<()> {
812 let md = doc::Doc {
813 blocks: vec![
814 Block::Heading { level: 2, content: vec![t("Not the title")] },
815 Block::Heading {
816 level: 1,
817 content: vec![
818 t("A "),
819 Inline::Emph { strong: true, content: vec![t("loud")] },
820 t(" title"),
821 ],
822 },
823 Block::Heading { level: 1, content: vec![t("The second one")] },
824 ],
825 };
826 let tree = from_doc(&md, &Options::default());
827 res!(validate::validate(&tree, SCHEMA_DOC));
828 // The first level 1 heading, flattened: emphasis inside a title keeps its words.
829 assert_eq!(res!(field(&tree, "title")), dat!("A loud title"));
830 Ok(())
831 }
832
833 #[test]
834 fn test_the_title_falls_back_to_the_stem_20() -> Outcome<()> {
835 let md = doc::Doc { blocks: vec![p("No heading here.")] };
836 let opts = Options {
837 // A title of whitespace is no title, and neither is a level 2 heading.
838 title: Some(" ".to_string()),
839 stem: "the_file_name".to_string(),
840 ..Options::default()
841 };
842 let tree = from_doc(&md, &opts);
843 res!(validate::validate(&tree, SCHEMA_DOC));
844 assert_eq!(res!(field(&tree, "title")), dat!("the_file_name"));
845 Ok(())
846 }
847
848 #[test]
849 fn test_the_language_is_the_callers_21() -> Outcome<()> {
850 let md = doc::Doc { blocks: vec![p("Une phrase.")] };
851 let opts = Options {
852 lang: "fr".to_string(),
853 ..Options::default()
854 };
855 let tree = from_doc(&md, &opts);
856 res!(validate::validate(&tree, SCHEMA_DOC));
857 assert_eq!(res!(field(&tree, "lang")), dat!("fr"));
858 Ok(())
859 }
860
861 #[test]
862 fn test_a_string_is_normalised_and_stripped_22() -> Outcome<()> {
863 // §3 rule 5: the composed and decomposed forms of a letter display identically and hash
864 // differently, and a control character displays as nothing. Text that reaches the signer
865 // carrying either would be refused there, naming a rule the author never heard of.
866 let tree = res!(imported(vec![
867 Block::Para(vec![t("Cafe\u{0301}\r, said the \u{000B}sign.")]),
868 ]));
869 let s = res!(text_of(&res!(kid(&res!(kid(&tree, 0)), 0))));
870 assert_eq!(s, "Café, said the sign.");
871 res!(crate::canon::check(&tree));
872 Ok(())
873 }
874
875 #[test]
876 fn test_an_empty_document_validates_23() -> Outcome<()> {
877 // A doc is `flow*`, so nothing at all is a document, and an empty children list is not the
878 // canonical way to say so (§3 rule 4).
879 let tree = res!(imported(Vec::new()));
880 assert!(kids(&tree).is_ok());
881 assert_eq!(res!(kids(&tree)).len(), 0);
882 assert!(field(&tree, KEY_CHILDREN).is_err(), "An empty children list was written.");
883 res!(crate::canon::check(&tree));
884 Ok(())
885 }
886
887 #[test]
888 fn test_every_block_kind_at_once_validates_24() -> Outcome<()> {
889 // The whole vocabulary the importer reaches, in one document, held to the validator and to
890 // the canonical encoding rules that the signing path would hold it to.
891 let tree = res!(imported(vec![
892 Block::Heading { level: 1, content: vec![t("A title")] },
893 Block::Para(vec![
894 t("Prose with "),
895 Inline::Emph { strong: true, content: vec![t("emphasis")] },
896 t(", a "),
897 Inline::Link { to: "somewhere".to_string(), content: vec![t("link")] },
898 t(", a "),
899 Inline::Code("span".to_string()),
900 Inline::Break,
901 Inline::Image { src: "x.png".to_string(), alt: "a picture".to_string() },
902 ]),
903 Block::Rule,
904 Block::Table {
905 head: Some(Row(vec![c("Name"), c("Age")])),
906 rows: vec![Row(vec![c("Alice"), c("30")])],
907 cols: vec![Align::Start, Align::None],
908 },
909 Block::Quote(vec![
910 p("Quoted."),
911 Block::List { ordered: false, items: vec![vec![p("In a quote")]] },
912 ]),
913 Block::List {
914 ordered: true,
915 items: vec![
916 vec![p("One"), Block::Code { lang: None, text: "x".to_string() }],
917 vec![p("Two")],
918 ],
919 },
920 Block::Code { lang: Some("rust".to_string()), text: "fn main() {}".to_string() },
921 ]));
922 let stats = res!(validate::validate(&tree, SCHEMA_DOC));
923 assert!(stats.nodes > 15, "The document lost most of itself: {:?}", stats);
924 res!(crate::canon::check(&tree));
925 Ok(())
926 }
927
928 #[test]
929 fn test_an_imported_tree_signs_and_reads_back_25() -> Outcome<()> {
930 // The claim the importer makes is that what comes out is a document, and this is that claim
931 // rather than an assertion about it: the tree goes through the whole path a compile puts one
932 // through -- validated, canonically encoded, hashed, signed -- and is then read back the way
933 // a reader reads it, which verifies every one of those in turn.
934 let pair = res!(crate::key::KeyPair::generate());
935 let signer = res!(pair.signer());
936 let tree = res!(imported(vec![
937 Block::Heading { level: 1, content: vec![t("A title")] },
938 Block::Para(vec![
939 t("Prose, "),
940 Inline::Emph { strong: false, content: vec![t("emphasised")] },
941 t(", and a "),
942 Inline::Link { to: "news.cricket".to_string(), content: vec![t("link")] },
943 t("."),
944 ]),
945 Block::Rule,
946 Block::List { ordered: false, items: vec![vec![p("An item")]] },
947 Block::Quote(vec![p("A quoted line.")]),
948 Block::Code { lang: Some("rust".to_string()), text: "fn main() {}".to_string() },
949 ]));
950 let buf = res!(crate::doc::write(&tree, SCHEMA_DOC, &signer, 0));
951 let read = res!(crate::doc::read(&buf));
952 assert_eq!(read.tree(), &tree, "An imported document did not survive its own signing path.");
953 Ok(())
954 }
955
956 #[test]
957 fn test_a_table_degrades_to_a_box_of_rows_26() -> Outcome<()> {
958 // v0 has no kind for a grid and the vocabulary is frozen, so the grid goes and every word
959 // stays. That the box validates is the whole of the claim: a box is flow content that holds
960 // flow content, so a paragraph to a row is a thing a document may say.
961 let tree = res!(imported(vec![
962 Block::Table {
963 head: Some(Row(vec![c("Name"), c("Age")])),
964 rows: vec![
965 Row(vec![c("Alice"), c("30")]),
966 Row(vec![c("Bob"), c("25")]),
967 ],
968 cols: vec![Align::Start, Align::End],
969 },
970 ]));
971 let boxx = res!(kid(&tree, 0));
972 assert_eq!(res!(kind_of(&boxx)), NodeKind::Boxx);
973 // A paragraph to a row, the header's among them, and nothing lost but the walls between the
974 // cells.
975 assert_eq!(res!(kids(&boxx)).len(), 3);
976 for (n, expected) in ["Name | Age", "Alice | 30", "Bob | 25"].iter().enumerate() {
977 let para = res!(kid(&boxx, n));
978 assert_eq!(res!(kind_of(&para)), NodeKind::Para);
979 assert_eq!(&res!(said(&para)), expected);
980 }
981 res!(crate::canon::check(&tree));
982 Ok(())
983 }
984
985 #[test]
986 fn test_a_table_needs_no_header_and_keeps_its_markup_27() -> Outcome<()> {
987 // A cell's inlines are a paragraph's inlines, so what a cell holds sits where it sat.
988 let tree = res!(imported(vec![
989 Block::Table {
990 head: None,
991 rows: vec![Row(vec![
992 Cell(vec![Inline::Emph { strong: true, content: vec![t("loud")] }]),
993 Cell(vec![Inline::Link {
994 to: "news.cricket".to_string(),
995 content: vec![t("a link")],
996 }]),
997 ])],
998 cols: vec![Align::None, Align::None],
999 },
1000 ]));
1001 let boxx = res!(kid(&tree, 0));
1002 assert_eq!(res!(kind_of(&boxx)), NodeKind::Boxx);
1003 // A table with no header row is a table of its body, and the box holds the one row it has.
1004 assert_eq!(res!(kids(&boxx)).len(), 1);
1005 let para = res!(kid(&boxx, 0));
1006 assert_eq!(res!(kind_of(&para)), NodeKind::Para);
1007 assert_eq!(res!(kind_of(&res!(kid(&para, 0)))), NodeKind::Emph);
1008 assert_eq!(res!(text_of(&res!(kid(&para, 1)))), CELL_SEP);
1009 assert_eq!(res!(kind_of(&res!(kid(&para, 2)))), NodeKind::Link);
1010 assert_eq!(res!(said(&para)), "loud | a link");
1011 Ok(())
1012 }
1013
1014 #[test]
1015 fn test_a_table_that_says_nothing_is_dropped_28() -> Outcome<()> {
1016 // A row of empty cells is a blank paragraph by another name, and a box of no paragraphs
1017 // renders as nothing at all: the same rule an empty paragraph is held to.
1018 let tree = res!(imported(vec![
1019 Block::Table {
1020 head: Some(Row(vec![Cell(Vec::new()), Cell(Vec::new())])),
1021 rows: vec![Row(vec![c(" "), Cell(Vec::new())])],
1022 cols: vec![Align::None, Align::None],
1023 },
1024 p("After."),
1025 ]));
1026 let kids = res!(kids(&tree));
1027 assert_eq!(kids.len(), 1, "An empty table left something behind: {:?}", kids);
1028 assert_eq!(res!(kind_of(&kids[0])), NodeKind::Para);
1029 Ok(())
1030 }
1031}