Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/tests/invisible.rs

7.0 KiB, 1 run

created by r1870400018:67174, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `unicode::property::is_invisible`: the characters a reader cannot see and a program reads.
2//!
3//! Written for the Daimond steering lint (QA A-F3, 2026-10-04), which let a soft hyphen, a
4//! variation selector and a run of tag characters through a hand-made list of ranges. The cases
5//! are named by code point so a failure says which character got in.
6
7use oxedyne_fe2o3_text::unicode::{
8 norm,
9 property::{
10 is_invisible,
11 CharClass,
12 },
13};
14
15use oxedyne_fe2o3_core::prelude::*;
16
17
18// Every code point, surrogates left out because a char cannot hold one.
19fn every_char() -> impl Iterator<Item = char> {
20 (0u32..=0x10FFFF).filter_map(char::from_u32)
21}
22
23fn class(query: &str) -> Outcome<CharClass> {
24 CharClass::parse(query)
25}
26
27#[test]
28fn test_a_character_that_draws_nothing_is_invisible() {
29 for (cp, name) in [
30 // Controls, C0 and C1.
31 (0x0000u32, "NUL"),
32 (0x0007, "BEL"),
33 (0x0009, "TAB"),
34 (0x000A, "LF"),
35 (0x000D, "CR"),
36 (0x001B, "ESC"),
37 (0x007F, "DEL"),
38 (0x0080, "PAD"),
39 (0x0085, "NEL"),
40 (0x009F, "APC"),
41 // The probes of the QA run.
42 (0x00AD, "SOFT HYPHEN"),
43 (0x034F, "COMBINING GRAPHEME JOINER"),
44 (0x061C, "ARABIC LETTER MARK"),
45 (0x180E, "MONGOLIAN VOWEL SEPARATOR"),
46 (0xFE0F, "VARIATION SELECTOR-16"),
47 (0xE0020, "TAG SPACE"),
48 // Zero width and direction.
49 (0x200B, "ZERO WIDTH SPACE"),
50 (0x200C, "ZERO WIDTH NON-JOINER"),
51 (0x200D, "ZERO WIDTH JOINER"),
52 (0x200E, "LEFT-TO-RIGHT MARK"),
53 (0x200F, "RIGHT-TO-LEFT MARK"),
54 (0x202A, "LEFT-TO-RIGHT EMBEDDING"),
55 (0x202E, "RIGHT-TO-LEFT OVERRIDE"),
56 (0x2066, "LEFT-TO-RIGHT ISOLATE"),
57 (0x2069, "POP DIRECTIONAL ISOLATE"),
58 (0x2060, "WORD JOINER"),
59 (0x2061, "FUNCTION APPLICATION"),
60 (0x2064, "INVISIBLE PLUS"),
61 (0xFEFF, "BYTE ORDER MARK"),
62 // Line and paragraph separators.
63 (0x2028, "LINE SEPARATOR"),
64 (0x2029, "PARAGRAPH SEPARATOR"),
65 // Variation selectors, both blocks, and the Mongolian free ones.
66 (0xFE00, "VARIATION SELECTOR-1"),
67 (0xE0100, "VARIATION SELECTOR-17"),
68 (0xE01EF, "VARIATION SELECTOR-256"),
69 (0x180B, "MONGOLIAN FREE VARIATION SELECTOR ONE"),
70 // The tag block, whole.
71 (0xE0000, "TAG block start, unassigned"),
72 (0xE0001, "LANGUAGE TAG"),
73 (0xE0041, "TAG LATIN CAPITAL LETTER A"),
74 (0xE007E, "TAG TILDE"),
75 (0xE007F, "CANCEL TAG"),
76 // Fillers that stand for a letter and draw none.
77 (0x115F, "HANGUL CHOSEONG FILLER"),
78 (0x1160, "HANGUL JUNGSEONG FILLER"),
79 (0x17B4, "KHMER VOWEL INHERENT AQ"),
80 (0x3164, "HANGUL FILLER"),
81 (0xFFA0, "HALFWIDTH HANGUL FILLER"),
82 // Other format characters.
83 (0x0600, "ARABIC NUMBER SIGN"),
84 (0xFFF9, "INTERLINEAR ANNOTATION ANCHOR"),
85 (0x1BCA0, "SHORTHAND FORMAT LETTER OVERLAP"),
86 (0x1D173, "MUSICAL SYMBOL BEGIN BEAM"),
87 // Private use, three planes.
88 (0xE000, "PRIVATE USE, BMP"),
89 (0xF8FF, "PRIVATE USE, last of BMP"),
90 (0xF0000, "PRIVATE USE, plane 15"),
91 (0x10FFFD, "PRIVATE USE, plane 16"),
92 // Unassigned and not characters.
93 (0x0378, "UNASSIGNED, Greek block"),
94 (0xE0080, "UNASSIGNED, past the tag block"),
95 (0xFFFE, "NONCHARACTER"),
96 (0xFFFF, "NONCHARACTER"),
97 (0x2FFFF, "NONCHARACTER"),
98 ] {
99 match char::from_u32(cp) {
100 Some(c) => assert!(is_invisible(c), "U+{:04X} {} is let through", cp, name),
101 None => panic!("U+{:04X} {} is not a char", cp, name),
102 }
103 }
104}
105
106#[test]
107fn test_a_character_that_draws_something_is_not_invisible() {
108 for (c, name) in [
109 ('a', "a"),
110 ('Z', "Z"),
111 ('7', "7"),
112 ('#', "number sign"),
113 (' ', "space"),
114 ('-', "hyphen-minus"),
115 ('\u{00E9}', "e acute"),
116 ('\u{00DF}', "sharp s"),
117 ('\u{03BB}', "Greek lambda"),
118 ('\u{044F}', "Cyrillic ya"),
119 ('\u{0301}', "combining acute, which a reader sees on its letter"),
120 ('\u{4E2D}', "CJK ideograph"),
121 ('\u{3042}', "hiragana a"),
122 ('\u{D55C}', "Hangul han"),
123 ('\u{3002}', "ideographic full stop"),
124 // A CJK writer's commas and colons are fullwidth forms and must stay writable.
125 ('\u{FF0C}', "fullwidth comma"),
126 ('\u{FF1B}', "fullwidth semicolon"),
127 ('\u{FF1A}', "fullwidth colon"),
128 ('\u{FF21}', "fullwidth A, visible and folded by NFKC instead"),
129 ('\u{FF71}', "halfwidth katakana a"),
130 // Spaces that draw as space.
131 ('\u{00A0}', "no-break space"),
132 ('\u{2003}', "em space"),
133 ('\u{3000}', "ideographic space"),
134 // Symbols and emoji, without their selectors.
135 ('\u{1F44D}', "thumbs up"),
136 ('\u{2764}', "heavy black heart"),
137 ('\u{2026}', "horizontal ellipsis"),
138 ('\u{2122}', "trade mark"),
139 ('\u{1D400}', "mathematical bold A"),
140 ] {
141 assert!(!is_invisible(c), "U+{:04X} {} is refused", c as u32, name);
142 }
143}
144
145#[test]
146fn test_whatever_the_categories_call_hidden_is_invisible() -> Outcome<()> {
147 // The four categories of the brief, and the two separators, taken from the tables rather than
148 // from a list written here, so a character the list forgot cannot slip through.
149 for cat in ["gc=Cc", "gc=Cf", "gc=Co", "gc=Cn", "gc=Zl", "gc=Zp"] {
150 let set = res!(class(cat));
151 let mut n = 0usize;
152 for c in every_char().filter(|c| set.contains(*c)) {
153 n += 1;
154 assert!(is_invisible(c), "U+{:04X} of {} is let through", c as u32, cat);
155 }
156 assert!(n > 0, "{} holds no character, so nothing was tested", cat);
157 }
158 Ok(())
159}
160
161#[test]
162fn test_every_default_ignorable_and_variation_selector_is_invisible() -> Outcome<()> {
163 for prop in ["Default_Ignorable_Code_Point", "Variation_Selector", "Bidi_Control", "Join_Control"] {
164 let set = res!(class(prop));
165 let mut n = 0usize;
166 for c in every_char().filter(|c| set.contains(*c)) {
167 n += 1;
168 assert!(is_invisible(c), "U+{:04X} of {} is let through", c as u32, prop);
169 }
170 assert!(n > 0, "{} holds no character, so nothing was tested", prop);
171 }
172 // The tag block in full, assigned or not.
173 for cp in 0xE0000u32..=0xE007F {
174 if let Some(c) = char::from_u32(cp) {
175 assert!(is_invisible(c), "tag U+{:04X} is let through", cp);
176 }
177 }
178 Ok(())
179}
180
181#[test]
182fn test_the_invisible_set_is_small_and_leaves_the_writing_alone() -> Outcome<()> {
183 // A guard against the predicate growing past the intent: no letter, mark, number, punctuation
184 // or symbol is invisible, in any script.
185 for cat in ["L", "M", "N", "P", "S"] {
186 let set = res!(class(cat));
187 for c in every_char().filter(|c| set.contains(*c)) {
188 // The fillers are letters that draw nothing, and the variation selectors are marks.
189 let dropped = matches!(c as u32,
190 0x115F | 0x1160 | 0x17B4 | 0x17B5 | 0x3164 | 0xFFA0
191 | 0x034F | 0x180B..=0x180D | 0x180F | 0xFE00..=0xFE0F | 0xE0100..=0xE01EF);
192 assert_eq!(dropped, is_invisible(c), "U+{:04X} of {}", c as u32, cat);
193 }
194 }
195 Ok(())
196}
197
198#[test]
199fn test_folding_a_visible_character_never_makes_it_invisible() {
200 // The lint judges the text as written and then judges the NFKC fold of it for the word list.
201 // That is sound only if the fold cannot bring a hidden character out of a visible one.
202 let mut folded = 0usize;
203 for c in every_char().filter(|c| !is_invisible(*c)) {
204 let s = c.to_string();
205 let f = norm::nfkc(&s);
206 if f != s {
207 folded += 1;
208 }
209 assert!(!f.chars().any(is_invisible),
210 "U+{:04X} folds to a hidden character: {:?}", c as u32, f);
211 }
212 assert!(folded > 3000, "only {} characters changed under NFKC, so the table did not load", folded);
213}