oxedyne/fe2o3/fe2o3_text/tests/invisible.rs
7.0 KiB, 1 run
created by r1870400018:67174, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! `unicode::property::is_invisible`: the characters a reader cannot see and a program reads. |
| 2 | //! |
| 3 | //! Written for the Daimond steering lint (QA A-F3, 2026-10-04), which let a soft hyphen, a |
| 4 | //! variation selector and a run of tag characters through a hand-made list of ranges. The cases |
| 5 | //! are named by code point so a failure says which character got in. |
| 6 | |
| 7 | use oxedyne_fe2o3_text::unicode::{ |
| 8 | norm, |
| 9 | property::{ |
| 10 | is_invisible, |
| 11 | CharClass, |
| 12 | }, |
| 13 | }; |
| 14 | |
| 15 | use oxedyne_fe2o3_core::prelude::*; |
| 16 | |
| 17 | |
| 18 | // Every code point, surrogates left out because a char cannot hold one. |
| 19 | fn every_char() -> impl Iterator<Item = char> { |
| 20 | (0u32..=0x10FFFF).filter_map(char::from_u32) |
| 21 | } |
| 22 | |
| 23 | fn class(query: &str) -> Outcome<CharClass> { |
| 24 | CharClass::parse(query) |
| 25 | } |
| 26 | |
| 27 | #[test] |
| 28 | fn test_a_character_that_draws_nothing_is_invisible() { |
| 29 | for (cp, name) in [ |
| 30 | // Controls, C0 and C1. |
| 31 | (0x0000u32, "NUL"), |
| 32 | (0x0007, "BEL"), |
| 33 | (0x0009, "TAB"), |
| 34 | (0x000A, "LF"), |
| 35 | (0x000D, "CR"), |
| 36 | (0x001B, "ESC"), |
| 37 | (0x007F, "DEL"), |
| 38 | (0x0080, "PAD"), |
| 39 | (0x0085, "NEL"), |
| 40 | (0x009F, "APC"), |
| 41 | // The probes of the QA run. |
| 42 | (0x00AD, "SOFT HYPHEN"), |
| 43 | (0x034F, "COMBINING GRAPHEME JOINER"), |
| 44 | (0x061C, "ARABIC LETTER MARK"), |
| 45 | (0x180E, "MONGOLIAN VOWEL SEPARATOR"), |
| 46 | (0xFE0F, "VARIATION SELECTOR-16"), |
| 47 | (0xE0020, "TAG SPACE"), |
| 48 | // Zero width and direction. |
| 49 | (0x200B, "ZERO WIDTH SPACE"), |
| 50 | (0x200C, "ZERO WIDTH NON-JOINER"), |
| 51 | (0x200D, "ZERO WIDTH JOINER"), |
| 52 | (0x200E, "LEFT-TO-RIGHT MARK"), |
| 53 | (0x200F, "RIGHT-TO-LEFT MARK"), |
| 54 | (0x202A, "LEFT-TO-RIGHT EMBEDDING"), |
| 55 | (0x202E, "RIGHT-TO-LEFT OVERRIDE"), |
| 56 | (0x2066, "LEFT-TO-RIGHT ISOLATE"), |
| 57 | (0x2069, "POP DIRECTIONAL ISOLATE"), |
| 58 | (0x2060, "WORD JOINER"), |
| 59 | (0x2061, "FUNCTION APPLICATION"), |
| 60 | (0x2064, "INVISIBLE PLUS"), |
| 61 | (0xFEFF, "BYTE ORDER MARK"), |
| 62 | // Line and paragraph separators. |
| 63 | (0x2028, "LINE SEPARATOR"), |
| 64 | (0x2029, "PARAGRAPH SEPARATOR"), |
| 65 | // Variation selectors, both blocks, and the Mongolian free ones. |
| 66 | (0xFE00, "VARIATION SELECTOR-1"), |
| 67 | (0xE0100, "VARIATION SELECTOR-17"), |
| 68 | (0xE01EF, "VARIATION SELECTOR-256"), |
| 69 | (0x180B, "MONGOLIAN FREE VARIATION SELECTOR ONE"), |
| 70 | // The tag block, whole. |
| 71 | (0xE0000, "TAG block start, unassigned"), |
| 72 | (0xE0001, "LANGUAGE TAG"), |
| 73 | (0xE0041, "TAG LATIN CAPITAL LETTER A"), |
| 74 | (0xE007E, "TAG TILDE"), |
| 75 | (0xE007F, "CANCEL TAG"), |
| 76 | // Fillers that stand for a letter and draw none. |
| 77 | (0x115F, "HANGUL CHOSEONG FILLER"), |
| 78 | (0x1160, "HANGUL JUNGSEONG FILLER"), |
| 79 | (0x17B4, "KHMER VOWEL INHERENT AQ"), |
| 80 | (0x3164, "HANGUL FILLER"), |
| 81 | (0xFFA0, "HALFWIDTH HANGUL FILLER"), |
| 82 | // Other format characters. |
| 83 | (0x0600, "ARABIC NUMBER SIGN"), |
| 84 | (0xFFF9, "INTERLINEAR ANNOTATION ANCHOR"), |
| 85 | (0x1BCA0, "SHORTHAND FORMAT LETTER OVERLAP"), |
| 86 | (0x1D173, "MUSICAL SYMBOL BEGIN BEAM"), |
| 87 | // Private use, three planes. |
| 88 | (0xE000, "PRIVATE USE, BMP"), |
| 89 | (0xF8FF, "PRIVATE USE, last of BMP"), |
| 90 | (0xF0000, "PRIVATE USE, plane 15"), |
| 91 | (0x10FFFD, "PRIVATE USE, plane 16"), |
| 92 | // Unassigned and not characters. |
| 93 | (0x0378, "UNASSIGNED, Greek block"), |
| 94 | (0xE0080, "UNASSIGNED, past the tag block"), |
| 95 | (0xFFFE, "NONCHARACTER"), |
| 96 | (0xFFFF, "NONCHARACTER"), |
| 97 | (0x2FFFF, "NONCHARACTER"), |
| 98 | ] { |
| 99 | match char::from_u32(cp) { |
| 100 | Some(c) => assert!(is_invisible(c), "U+{:04X} {} is let through", cp, name), |
| 101 | None => panic!("U+{:04X} {} is not a char", cp, name), |
| 102 | } |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | #[test] |
| 107 | fn test_a_character_that_draws_something_is_not_invisible() { |
| 108 | for (c, name) in [ |
| 109 | ('a', "a"), |
| 110 | ('Z', "Z"), |
| 111 | ('7', "7"), |
| 112 | ('#', "number sign"), |
| 113 | (' ', "space"), |
| 114 | ('-', "hyphen-minus"), |
| 115 | ('\u{00E9}', "e acute"), |
| 116 | ('\u{00DF}', "sharp s"), |
| 117 | ('\u{03BB}', "Greek lambda"), |
| 118 | ('\u{044F}', "Cyrillic ya"), |
| 119 | ('\u{0301}', "combining acute, which a reader sees on its letter"), |
| 120 | ('\u{4E2D}', "CJK ideograph"), |
| 121 | ('\u{3042}', "hiragana a"), |
| 122 | ('\u{D55C}', "Hangul han"), |
| 123 | ('\u{3002}', "ideographic full stop"), |
| 124 | // A CJK writer's commas and colons are fullwidth forms and must stay writable. |
| 125 | ('\u{FF0C}', "fullwidth comma"), |
| 126 | ('\u{FF1B}', "fullwidth semicolon"), |
| 127 | ('\u{FF1A}', "fullwidth colon"), |
| 128 | ('\u{FF21}', "fullwidth A, visible and folded by NFKC instead"), |
| 129 | ('\u{FF71}', "halfwidth katakana a"), |
| 130 | // Spaces that draw as space. |
| 131 | ('\u{00A0}', "no-break space"), |
| 132 | ('\u{2003}', "em space"), |
| 133 | ('\u{3000}', "ideographic space"), |
| 134 | // Symbols and emoji, without their selectors. |
| 135 | ('\u{1F44D}', "thumbs up"), |
| 136 | ('\u{2764}', "heavy black heart"), |
| 137 | ('\u{2026}', "horizontal ellipsis"), |
| 138 | ('\u{2122}', "trade mark"), |
| 139 | ('\u{1D400}', "mathematical bold A"), |
| 140 | ] { |
| 141 | assert!(!is_invisible(c), "U+{:04X} {} is refused", c as u32, name); |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | #[test] |
| 146 | fn test_whatever_the_categories_call_hidden_is_invisible() -> Outcome<()> { |
| 147 | // The four categories of the brief, and the two separators, taken from the tables rather than |
| 148 | // from a list written here, so a character the list forgot cannot slip through. |
| 149 | for cat in ["gc=Cc", "gc=Cf", "gc=Co", "gc=Cn", "gc=Zl", "gc=Zp"] { |
| 150 | let set = res!(class(cat)); |
| 151 | let mut n = 0usize; |
| 152 | for c in every_char().filter(|c| set.contains(*c)) { |
| 153 | n += 1; |
| 154 | assert!(is_invisible(c), "U+{:04X} of {} is let through", c as u32, cat); |
| 155 | } |
| 156 | assert!(n > 0, "{} holds no character, so nothing was tested", cat); |
| 157 | } |
| 158 | Ok(()) |
| 159 | } |
| 160 | |
| 161 | #[test] |
| 162 | fn test_every_default_ignorable_and_variation_selector_is_invisible() -> Outcome<()> { |
| 163 | for prop in ["Default_Ignorable_Code_Point", "Variation_Selector", "Bidi_Control", "Join_Control"] { |
| 164 | let set = res!(class(prop)); |
| 165 | let mut n = 0usize; |
| 166 | for c in every_char().filter(|c| set.contains(*c)) { |
| 167 | n += 1; |
| 168 | assert!(is_invisible(c), "U+{:04X} of {} is let through", c as u32, prop); |
| 169 | } |
| 170 | assert!(n > 0, "{} holds no character, so nothing was tested", prop); |
| 171 | } |
| 172 | // The tag block in full, assigned or not. |
| 173 | for cp in 0xE0000u32..=0xE007F { |
| 174 | if let Some(c) = char::from_u32(cp) { |
| 175 | assert!(is_invisible(c), "tag U+{:04X} is let through", cp); |
| 176 | } |
| 177 | } |
| 178 | Ok(()) |
| 179 | } |
| 180 | |
| 181 | #[test] |
| 182 | fn test_the_invisible_set_is_small_and_leaves_the_writing_alone() -> Outcome<()> { |
| 183 | // A guard against the predicate growing past the intent: no letter, mark, number, punctuation |
| 184 | // or symbol is invisible, in any script. |
| 185 | for cat in ["L", "M", "N", "P", "S"] { |
| 186 | let set = res!(class(cat)); |
| 187 | for c in every_char().filter(|c| set.contains(*c)) { |
| 188 | // The fillers are letters that draw nothing, and the variation selectors are marks. |
| 189 | let dropped = matches!(c as u32, |
| 190 | 0x115F | 0x1160 | 0x17B4 | 0x17B5 | 0x3164 | 0xFFA0 |
| 191 | | 0x034F | 0x180B..=0x180D | 0x180F | 0xFE00..=0xFE0F | 0xE0100..=0xE01EF); |
| 192 | assert_eq!(dropped, is_invisible(c), "U+{:04X} of {}", c as u32, cat); |
| 193 | } |
| 194 | } |
| 195 | Ok(()) |
| 196 | } |
| 197 | |
| 198 | #[test] |
| 199 | fn test_folding_a_visible_character_never_makes_it_invisible() { |
| 200 | // The lint judges the text as written and then judges the NFKC fold of it for the word list. |
| 201 | // That is sound only if the fold cannot bring a hidden character out of a visible one. |
| 202 | let mut folded = 0usize; |
| 203 | for c in every_char().filter(|c| !is_invisible(*c)) { |
| 204 | let s = c.to_string(); |
| 205 | let f = norm::nfkc(&s); |
| 206 | if f != s { |
| 207 | folded += 1; |
| 208 | } |
| 209 | assert!(!f.chars().any(is_invisible), |
| 210 | "U+{:04X} folds to a hidden character: {:?}", c as u32, f); |
| 211 | } |
| 212 | assert!(folded > 3000, "only {} characters changed under NFKC, so the table did not load", folded); |
| 213 | } |