Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/src/doc/mod.rs

17.4 KiB, 51 runs

created by r1870400018:14266, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A neutral document tree: what a piece of prose *is*, free of the syntax it was written in and the
2//! form it will be rendered to.
3//!
4//! A document is a sequence of [`Block`]s, and a line of prose a sequence of [`Inline`]s. The
5//! vocabulary is small and closed: a heading, a paragraph, a list, a quotation, a table, a run of
6//! text, a link. That is the whole of it.
7//!
8//! # The tree names nothing at either end
9//!
10//! It names no input syntax -- there is no `Block::Asterisks`, only [`Inline::Emph`] -- so a second
11//! front-end produces the same tree and every consumer of it keeps working. And it names no output
12//! format, so a caller walks it and makes of it whatever it likes: HTML, a signed document, a
13//! terminal rendering, an index.
14//!
15//! Both halves are load-bearing. A tree that admitted one syntax's spelling would make every consumer
16//! learn that syntax; a tree that admitted one format's constructs -- a raw HTML node, say -- would
17//! make every front-end learn that format. The tree is the narrow waist between the two, and it stays
18//! narrow by carrying meaning rather than markup.
19//!
20//! # Attributes are names, not meanings
21//!
22//! A [`Block::Div`] and an [`Inline::Span`] carry [`Attrs`] -- an id, classes, key-value pairs. The
23//! tree carries them and interprets none of them. `{.warning}` says a region is in the class
24//! `warning`; it does not say what `warning` looks like. That is the whole of how the tree can carry a
25//! named box or a styled span and still name no format: the name travels, and the meaning is supplied
26//! where the tree is rendered -- a stylesheet for HTML, a style table for a signed document -- never
27//! here. A tree that resolved `warning` to a colour would be an HTML tree, or an SBJ tree, and no
28//! longer the narrow waist between them.
29//!
30//! # Front-ends
31//!
32//! - [`markdown`] -- reads Markdown, the form most existing prose is written in.
33//! - [`djot`] -- reads Djot, which a prose author reaches for to name a box or a style the syntax of
34//! Markdown cannot.
35//! - [`html`] -- reads HTML, the form a typesetter exports prose to once it has resolved the author's
36//! own macros.
37//!
38//! # Outputs
39//!
40//! - [`html`] -- writes the tree out as HTML, for a browser to read.
41//!
42//! # Usage
43//!
44//! ```ignore
45//! use oxedyne_fe2o3_text::doc::markdown;
46//!
47//! let tree = res!(markdown::parse("# A heading\n\nA paragraph with *emphasis*.\n"));
48//! for block in &tree.blocks {
49//! // Walk the tree.
50//! }
51//! ```
52
53pub mod djot;
54pub mod html;
55pub mod markdown;
56pub mod policy;
57
58/// The attributes a [`Block::Div`] or an [`Inline::Span`] carries: an id, classes, and key-value
59/// pairs.
60///
61/// Opaque, on purpose. The tree holds the names and interprets none of them, which is what lets it
62/// carry a named box or a styled span while still naming no output format. See the module's own
63/// "Attributes are names, not meanings". `{#intro .warning .boxed k=v}` parses to an id `intro`, the
64/// classes `warning` and `boxed`, and the pair `(k, v)`; what any of those *mean* is the business of
65/// whatever renders the tree.
66#[derive(Clone, Debug, Default, PartialEq)]
67pub struct Attrs {
68 /// The id, where one was given. At most one; a second `{#...}` replaces the first, as Djot's does.
69 pub id: Option<String>,
70 /// The classes, in the order written.
71 pub classes: Vec<String>,
72 /// Key-value pairs, in the order written.
73 pub pairs: Vec<(String, String)>,
74}
75
76impl Attrs {
77
78 /// Whether these attributes name nothing at all -- no id, no class, no pair.
79 ///
80 /// A span or a div that carries empty attributes is one the syntax marked but named nothing on;
81 /// a consumer may treat it as the bare content, since there is nothing to render from an empty
82 /// set.
83 pub fn is_empty(&self) -> bool {
84 self.id.is_none() && self.classes.is_empty() && self.pairs.is_empty()
85 }
86}
87
88/// A document: the blocks it is made of, in the order they were written.
89#[derive(Clone, Debug, Default, PartialEq)]
90pub struct Doc {
91 /// The document's blocks, in reading order.
92 pub blocks: Vec<Block>,
93}
94
95impl Doc {
96
97 /// An empty document.
98 pub fn new() -> Self {
99 Self { blocks: Vec::new() }
100 }
101
102 /// The text of the document's first heading of the given level, if it has one.
103 ///
104 /// A convenience for the common case of a document whose title is its opening heading. Returns the
105 /// heading's text with every inline flattened, so emphasis inside a title does not lose its words.
106 pub fn first_heading(&self, level: u8) -> Option<String> {
107 for block in &self.blocks {
108 if let Block::Heading { level: l, content } = block {
109 if *l == level {
110 return Some(text_of(content));
111 }
112 }
113 }
114 None
115 }
116
117 /// The text of the document's most prominent heading: the first at the shallowest level it has.
118 ///
119 /// What a caller after the document's own idea of its title wants. Naming a level would ask the
120 /// wrong question, because which level a piece is headed by says where the prose came from rather
121 /// than what it says: an author writing Markdown heads a chapter with a level 1, and the same
122 /// chapter exported from Typst arrives headed by a level 2, since the exporter keeps level 1 for
123 /// the document it thinks it is making. A caller asking for level 1 finds no title at all in the
124 /// second, and titles the chapter after its file.
125 ///
126 /// Taking the first heading of any level would be wrong the other way, since a piece may carry a
127 /// lesser heading above its title. The shallowest level present is the prominent one, whatever
128 /// number it happens to wear, and among equals the first wins.
129 pub fn top_heading(&self) -> Option<String> {
130 let mut top: Option<(u8, &Vec<Inline>)> = None;
131 for block in &self.blocks {
132 if let Block::Heading { level, content } = block {
133 match top {
134 // Strictly shallower, so a later heading of an equal level never displaces an
135 // earlier one.
136 Some((best, _)) if *level >= best => {},
137 _ => top = Some((*level, content)),
138 }
139 }
140 }
141 top.map(|(_, content)| text_of(content))
142 }
143
144 /// The number of words in the document's prose.
145 ///
146 /// Counts what a reader reads: headings, paragraphs, lists, quotations, divisions and the cells of
147 /// a table. A code block is left out, because a listing is scanned rather than read, and counting
148 /// one at prose speed puts minutes on a piece that no reader spends.
149 pub fn word_count(&self) -> usize {
150 count_blocks(&self.blocks)
151 }
152}
153
154/// A block-level element: the things a document is a sequence of.
155#[derive(Clone, Debug, PartialEq)]
156pub enum Block {
157 /// A heading, of a level from 1 to 6.
158 Heading {
159 /// The heading's level, 1 being the most prominent.
160 level: u8,
161 /// The heading's inline content.
162 content: Vec<Inline>,
163 },
164 /// A paragraph of inline content.
165 Para(Vec<Inline>),
166 /// An ordered or unordered list.
167 List {
168 /// Whether the list is numbered.
169 ordered: bool,
170 /// The items, each a sequence of blocks, so an item may hold a paragraph, a nested list, or more.
171 items: Vec<Vec<Block>>,
172 },
173 /// A run of source code, preserved exactly as written.
174 Code {
175 /// The language the fence named, if it named one.
176 lang: Option<String>,
177 /// The code itself, its line structure intact.
178 text: String,
179 },
180 /// A block quotation, itself a sequence of blocks.
181 Quote(Vec<Block>),
182 /// A table: a header row where there is one, the rows of the body, and the columns they are laid
183 /// out in.
184 ///
185 /// The table's words reach a summary or an index through its cells, each of which flattens with
186 /// [`Cell::text_of`] as any other run of inlines does.
187 Table {
188 /// The header row, where the table names its columns. A table need not: a grid of figures is
189 /// a table whether or not anything stands at the head of it.
190 head: Option<Row>,
191 /// The rows of the body, in reading order.
192 rows: Vec<Row>,
193 /// The columns, one entry to each, so a row's nth cell is aligned by the nth entry.
194 cols: Vec<Align>,
195 },
196 /// A thematic break: a division between passages.
197 Rule,
198 /// A named or attributed division: a box the prose itself asked for.
199 ///
200 /// The construct a prose front-end reaches for to name a region -- an aside, a warning, a figure
201 /// -- without saying, or knowing, what that region looks like. Markdown cannot write one; Djot's
202 /// `:::` can. The [`Attrs`] name it and the content is a document in its own right, so a division
203 /// may hold paragraphs, lists, or further divisions.
204 Div {
205 /// What names the division: its id, classes and pairs.
206 attrs: Attrs,
207 /// The blocks the division holds.
208 content: Vec<Block>,
209 },
210}
211
212/// One row of a [`Block::Table`]: the cells it holds, in reading order.
213#[derive(Clone, Debug, Default, PartialEq)]
214pub struct Row(pub Vec<Cell>);
215
216impl Row {
217
218 /// The row's plain text: every cell's words, a space between each.
219 pub fn text_of(&self) -> String {
220 let mut s = String::new();
221 for (i, cell) in self.0.iter().enumerate() {
222 if i > 0 {
223 s.push(' ');
224 }
225 s.push_str(&cell.text_of());
226 }
227 s
228 }
229}
230
231/// One cell of a [`Row`]: the inline content it holds.
232///
233/// A cell holds inlines and not blocks. A cell is a phrase -- a name, a figure, a link -- and a tree
234/// that admitted a list or a quotation here would promise every consumer a cell it must lay out as a
235/// document of its own. That is a promise no front-end this tree has can keep, and a tree should not
236/// make one on their behalf.
237#[derive(Clone, Debug, Default, PartialEq)]
238pub struct Cell(pub Vec<Inline>);
239
240impl Cell {
241
242 /// The cell's plain text, its every inline flattened. See [`text_of`].
243 pub fn text_of(&self) -> String {
244 text_of(&self.0)
245 }
246}
247
248/// How a column's cells sit within the width they are given.
249///
250/// The sides are named `Start` and `End`, and are never named `Left` and `Right`. The tree does not
251/// know left from right, because it does not know which way its text runs: this crate ships
252/// [`bidi`](crate::unicode::bidi) precisely because the prose it carries may run right to left, and a
253/// column aligned to the start of the line is then on the *right* of the page. `Start` is the side the
254/// text begins on, whichever side that is, and the consumer -- which knows the direction it is laying
255/// out in, and is the only thing that does -- is where the two meet.
256///
257/// A tree that said `Left` would be wrong for half the world's prose, and would be wrong silently: the
258/// table would lay out, and lay out backwards. This is worth leaving alone.
259#[derive(Clone, Copy, Debug, Default, PartialEq)]
260pub enum Align {
261 /// No alignment given, which is most columns: the consumer's own default stands.
262 #[default]
263 None,
264 /// Aligned to the side the text begins on.
265 Start,
266 /// Centred within the column.
267 Centre,
268 /// Aligned to the side the text ends on.
269 End,
270}
271
272/// An inline element: the things a line of prose is a sequence of.
273#[derive(Clone, Debug, PartialEq)]
274pub enum Inline {
275 /// A run of literal text.
276 Text(String),
277 /// Emphasised content.
278 Emph {
279 /// Whether the emphasis is strong (bold) rather than ordinary (italic).
280 strong: bool,
281 /// The emphasised content.
282 content: Vec<Inline>,
283 },
284 /// A link to a destination.
285 Link {
286 /// Where the link points, exactly as written: a URL, a path, or any other name.
287 to: String,
288 /// The link's own content, which is what a reader sees.
289 content: Vec<Inline>,
290 },
291 /// An image, by its source and the text that stands for it.
292 Image {
293 /// Where the image is, exactly as written.
294 src: String,
295 /// The text that stands in for the image.
296 alt: String,
297 },
298 /// A span of code within a line.
299 Code(String),
300 /// A run of inline content the prose named or attributed.
301 ///
302 /// The inline counterpart to [`Block::Div`]: `[text]{.highlight}` marks a span the way `:::` marks
303 /// a division. The [`Attrs`] name it and interpret nothing.
304 Span {
305 /// What names the span.
306 attrs: Attrs,
307 /// The span's content.
308 content: Vec<Inline>,
309 },
310 /// A break the author asked for within a paragraph.
311 ///
312 /// Only ever a *hard* break. Where an author's editor wrapped a line is not a break the author
313 /// asked for, so a front-end resolves such a wrap to a space in the surrounding [`Inline::Text`]
314 /// and never emits this. A consumer may therefore honour this as a break unconditionally, and
315 /// needs no rule of its own about whitespace.
316 Break,
317}
318
319/// The plain text of a run of inlines, with every element flattened to its words.
320///
321/// Emphasis and links contribute their content, an image its alt text, a code span its code, and a
322/// hard break a single space. What is left is what the passage says, with nothing of how it is marked
323/// up -- which is what a title, a summary or an index wants.
324pub fn text_of(content: &[Inline]) -> String {
325 let mut s = String::new();
326 for item in content {
327 match item {
328 Inline::Text(t) => s.push_str(t),
329 Inline::Emph { content, .. } => s.push_str(&text_of(content)),
330 Inline::Link { content, .. } => s.push_str(&text_of(content)),
331 Inline::Image { alt, .. } => s.push_str(alt),
332 Inline::Code(c) => s.push_str(c),
333 Inline::Span { content, .. } => s.push_str(&text_of(content)),
334 Inline::Break => s.push(' '),
335 }
336 }
337 s
338}
339
340/// The number of words in a run of blocks, descending into those that hold blocks of their own.
341fn count_blocks(blocks: &[Block]) -> usize {
342 let mut n = 0;
343 for block in blocks {
344 n += match block {
345 Block::Heading { content, .. } => count_inlines(content),
346 Block::Para(content) => count_inlines(content),
347 Block::List { items, .. } => items.iter().map(|item| count_blocks(item)).sum(),
348 Block::Quote(content) => count_blocks(content),
349 Block::Div { content, .. } => count_blocks(content),
350 Block::Table { head, rows, .. } => head.iter().chain(rows)
351 .map(|row| row.0.iter().map(|cell| count_inlines(&cell.0)).sum::<usize>())
352 .sum(),
353 // A listing is scanned rather than read, and a rule holds no words. See `Doc::word_count`.
354 Block::Code { .. } | Block::Rule => 0,
355 };
356 }
357 n
358}
359
360/// The number of words in a run of inlines, flattened.
361///
362/// A word is a segment holding at least one alphanumeric character, which is what separates the words
363/// from the runs of space and punctuation that [`words`](crate::unicode::segment::words) returns
364/// between them.
365fn count_inlines(content: &[Inline]) -> usize {
366 crate::unicode::segment::words(&text_of(content))
367 .iter()
368 .filter(|w| w.chars().any(char::is_alphanumeric))
369 .count()
370}
371
372#[cfg(test)]
373mod tests {
374 use super::*;
375
376 use oxedyne_fe2o3_core::prelude::*;
377
378 #[test]
379 fn test_the_plain_text_of_a_run_flattens_every_inline_00() -> Outcome<()> {
380 // Every inline contributes its words and none of its markup, so a title reads as it was written.
381 let content = vec![
382 Inline::Text("A ".to_string()),
383 Inline::Emph {
384 strong: true,
385 content: vec![Inline::Text("loud".to_string())],
386 },
387 Inline::Text(" ".to_string()),
388 Inline::Link {
389 to: "somewhere".to_string(),
390 content: vec![Inline::Text("link".to_string())],
391 },
392 ];
393 assert_eq!(text_of(&content), "A loud link");
394 Ok(())
395 }
396
397 /// A table says what its cells say, so a summary or an index that walks the tree finds a table's
398 /// words where it finds every other block's.
399 #[test]
400 fn test_a_table_contributes_the_words_of_its_cells_01() -> Outcome<()> {
401 let head = Row(vec![
402 Cell(vec![Inline::Text("Name".to_string())]),
403 Cell(vec![Inline::Text("Age".to_string())]),
404 ]);
405 let row = Row(vec![
406 Cell(vec![
407 Inline::Emph {
408 strong: true,
409 content: vec![Inline::Text("Alice".to_string())],
410 },
411 ]),
412 Cell(vec![Inline::Text("30".to_string())]),
413 ]);
414 // A cell flattens as any other run of inlines does, and a row is its cells.
415 assert_eq!(head.0[0].text_of(), "Name");
416 assert_eq!(head.text_of(), "Name Age");
417 assert_eq!(row.text_of(), "Alice 30");
418 let table = Block::Table {
419 head: Some(head),
420 rows: vec![row],
421 cols: vec![Align::Start, Align::End],
422 };
423 match &table {
424 Block::Table { head, rows, cols } => {
425 match head {
426 Some(head) => assert_eq!(head.text_of(), "Name Age"),
427 None => panic!("the table lost its header row"),
428 }
429 assert_eq!(rows[0].text_of(), "Alice 30");
430 assert_eq!(cols.len(), 2);
431 }
432 other => panic!("expected a table, got {:?}", other),
433 }
434 Ok(())
435 }
436
437 /// A column nobody aligned is aligned by nothing, which is what a consumer's own default is for.
438 #[test]
439 fn test_an_alignment_defaults_to_none_02() -> Outcome<()> {
440 assert_eq!(Align::default(), Align::None);
441 Ok(())
442 }
443
444 /// The count is of words and not of the punctuation and spaces between them, and it reaches the
445 /// words a nested block holds.
446 #[test]
447 fn test_a_document_counts_the_words_a_reader_reads_03() -> Outcome<()> {
448 let text = |s: &str| vec![Inline::Text(s.to_string())];
449 let doc = Doc {
450 blocks: vec![
451 Block::Heading { level: 1, content: text("A short title") }, // 3
452 Block::Para(text("Four words, one comma.")), // 4
453 Block::Quote(vec![Block::Para(text("Two words"))]), // 2
454 Block::List {
455 ordered: false,
456 items: vec![
457 vec![Block::Para(text("One"))], // 1
458 vec![Block::Para(text("Another two"))], // 2
459 ],
460 },
461 Block::Rule,
462 ],
463 };
464 assert_eq!(doc.word_count(), 12);
465 Ok(())
466 }
467
468 /// A listing is scanned rather than read, so it is not counted: a post whose bulk is code would
469 /// otherwise be given minutes no reader spends on it.
470 #[test]
471 fn test_a_code_block_is_not_counted_04() -> Outcome<()> {
472 let doc = Doc {
473 blocks: vec![
474 Block::Para(vec![Inline::Text("Three words here".to_string())]),
475 Block::Code {
476 lang: Some("rust".to_string()),
477 text: "let a = 1; let b = 2; let c = 3;".to_string(),
478 },
479 ],
480 };
481 assert_eq!(doc.word_count(), 3);
482 Ok(())
483 }
484}