diff --git a/core/unicode/generated.odin b/core/unicode/generated.odin index d35aa020c..85d4e9e8b 100644 --- a/core/unicode/generated.odin +++ b/core/unicode/generated.odin @@ -978,6 +978,445 @@ lu_ranges := Range{ ranges_32 = lu_ranges32[:], } +@(rodata) +mc_singles16 := [?]u16{ + 0x0903, 0x093B, 0x09D7, 0x0A03, 0x0A83, 0x0AC9, 0x0B3E, 0x0B40, 0x0B57, 0x0BD7, + 0x0CBE, 0x0CF3, 0x0D57, 0x0F7F, 0x1031, 0x1038, 0x108F, 0x1715, 0x1734, 0x17B6, + 0x1A55, 0x1A57, 0x1A61, 0x1B04, 0x1B35, 0x1B3B, 0x1B82, 0x1BA1, 0x1BAA, 0x1BE7, + 0x1BEE, 0x1CE1, 0x1CF7, 0xA827, 0xA983, 0xAA4D, 0xAA7B, 0xAA7D, 0xAAEB, 0xAAF5, + 0xABEC, +} + +@(rodata) +mc_ranges16 := [?]u16{ + 0x093E, 0x0940, + 0x0949, 0x094C, + 0x094E, 0x094F, + 0x0982, 0x0983, + 0x09BE, 0x09C0, + 0x09C7, 0x09C8, + 0x09CB, 0x09CC, + 0x0A3E, 0x0A40, + 0x0ABE, 0x0AC0, + 0x0ACB, 0x0ACC, + 0x0B02, 0x0B03, + 0x0B47, 0x0B48, + 0x0B4B, 0x0B4C, + 0x0BBE, 0x0BBF, + 0x0BC1, 0x0BC2, + 0x0BC6, 0x0BC8, + 0x0BCA, 0x0BCC, + 0x0C01, 0x0C03, + 0x0C41, 0x0C44, + 0x0C82, 0x0C83, + 0x0CC0, 0x0CC4, + 0x0CC7, 0x0CC8, + 0x0CCA, 0x0CCB, + 0x0CD5, 0x0CD6, + 0x0D02, 0x0D03, + 0x0D3E, 0x0D40, + 0x0D46, 0x0D48, + 0x0D4A, 0x0D4C, + 0x0D82, 0x0D83, + 0x0DCF, 0x0DD1, + 0x0DD8, 0x0DDF, + 0x0DF2, 0x0DF3, + 0x0F3E, 0x0F3F, + 0x102B, 0x102C, + 0x103B, 0x103C, + 0x1056, 0x1057, + 0x1062, 0x1064, + 0x1067, 0x106D, + 0x1083, 0x1084, + 0x1087, 0x108C, + 0x109A, 0x109C, + 0x17BE, 0x17C5, + 0x17C7, 0x17C8, + 0x1923, 0x1926, + 0x1929, 0x192B, + 0x1930, 0x1931, + 0x1933, 0x1938, + 0x1A19, 0x1A1A, + 0x1A63, 0x1A64, + 0x1A6D, 0x1A72, + 0x1B3D, 0x1B41, + 0x1B43, 0x1B44, + 0x1BA6, 0x1BA7, + 0x1BEA, 0x1BEC, + 0x1BF2, 0x1BF3, + 0x1C24, 0x1C2B, + 0x1C34, 0x1C35, + 0x302E, 0x302F, + 0xA823, 0xA824, + 0xA880, 0xA881, + 0xA8B4, 0xA8C3, + 0xA952, 0xA953, + 0xA9B4, 0xA9B5, + 0xA9BA, 0xA9BB, + 0xA9BE, 0xA9C0, + 0xAA2F, 0xAA30, + 0xAA33, 0xAA34, + 0xAAEE, 0xAAEF, + 0xABE3, 0xABE4, + 0xABE6, 0xABE7, + 0xABE9, 0xABEA, +} + +@(rodata) +mc_singles32 := [?]i32{ + 0x11000, 0x11002, 0x11082, 0x1112C, 0x11182, 0x111CE, 0x11235, 0x11357, + 0x113C2, 0x113C5, 0x113CF, 0x11445, 0x114B9, 0x114C1, 0x115BE, 0x1163E, + 0x116AC, 0x116B6, 0x1171E, 0x11726, 0x11838, 0x1193D, 0x11940, 0x11942, + 0x119E4, 0x11A39, 0x11A97, 0x11B61, 0x11B65, 0x11B67, 0x11C2F, 0x11C3E, + 0x11CA9, 0x11CB1, 0x11CB4, 0x11D96, 0x11F03, 0x11F41, +} + +@(rodata) +mc_ranges32 := [?]i32{ + 0x110B0, 0x110B2, + 0x110B7, 0x110B8, + 0x11145, 0x11146, + 0x111B3, 0x111B5, + 0x111BF, 0x111C0, + 0x1122C, 0x1122E, + 0x11232, 0x11233, + 0x112E0, 0x112E2, + 0x11302, 0x11303, + 0x1133E, 0x1133F, + 0x11341, 0x11344, + 0x11347, 0x11348, + 0x1134B, 0x1134D, + 0x11362, 0x11363, + 0x113B8, 0x113BA, + 0x113C7, 0x113CA, + 0x113CC, 0x113CD, + 0x11435, 0x11437, + 0x11440, 0x11441, + 0x114B0, 0x114B2, + 0x114BB, 0x114BE, + 0x115AF, 0x115B1, + 0x115B8, 0x115BB, + 0x11630, 0x11632, + 0x1163B, 0x1163C, + 0x116AE, 0x116AF, + 0x11720, 0x11721, + 0x1182C, 0x1182E, + 0x11930, 0x11935, + 0x11937, 0x11938, + 0x119D1, 0x119D3, + 0x119DC, 0x119DF, + 0x11A57, 0x11A58, + 0x11D8A, 0x11D8E, + 0x11D93, 0x11D94, + 0x11EF5, 0x11EF6, + 0x11F34, 0x11F35, + 0x11F3E, 0x11F3F, + 0x1612A, 0x1612C, + 0x16F51, 0x16F87, + 0x16FF0, 0x16FF1, + 0x1D165, 0x1D166, + 0x1D16D, 0x1D172, +} + +mc_ranges := Range{ + single_16 = mc_singles16[:], + ranges_16 = mc_ranges16[:], + single_32 = mc_singles32[:], + ranges_32 = mc_ranges32[:], +} + +@(rodata) +me_singles16 := [?]u16{ + 0x1ABE, +} + +@(rodata) +me_ranges16 := [?]u16{ + 0x0488, 0x0489, + 0x20DD, 0x20E0, + 0x20E2, 0x20E4, + 0xA670, 0xA672, +} + +me_ranges := Range{ + single_16 = me_singles16[:], + ranges_16 = me_ranges16[:], +} + +@(rodata) +mn_singles16 := [?]u16{ + 0x05BF, 0x05C7, 0x0670, 0x0711, 0x07FD, 0x093A, 0x093C, 0x094D, 0x0981, 0x09BC, + 0x09CD, 0x09FE, 0x0A3C, 0x0A51, 0x0A75, 0x0ABC, 0x0ACD, 0x0B01, 0x0B3C, 0x0B3F, + 0x0B4D, 0x0B82, 0x0BC0, 0x0BCD, 0x0C00, 0x0C04, 0x0C3C, 0x0C81, 0x0CBC, 0x0CBF, + 0x0CC6, 0x0D4D, 0x0D81, 0x0DCA, 0x0DD6, 0x0E31, 0x0EB1, 0x0F35, 0x0F37, 0x0F39, + 0x0FC6, 0x1082, 0x108D, 0x109D, 0x17C6, 0x17DD, 0x180F, 0x18A9, 0x1932, 0x1A1B, + 0x1A56, 0x1A60, 0x1A62, 0x1A7F, 0x1B34, 0x1B3C, 0x1B42, 0x1BE6, 0x1BED, 0x1CED, + 0x1CF4, 0x20E1, 0x2D7F, 0xA66F, 0xA802, 0xA806, 0xA80B, 0xA82C, 0xA8FF, 0xA9B3, + 0xA9E5, 0xAA43, 0xAA4C, 0xAA7C, 0xAAB0, 0xAAC1, 0xAAF6, 0xABE5, 0xABE8, 0xABED, + 0xFB1E, +} + +@(rodata) +mn_ranges16 := [?]u16{ + 0x0300, 0x036F, + 0x0483, 0x0487, + 0x0591, 0x05BD, + 0x05C1, 0x05C2, + 0x05C4, 0x05C5, + 0x0610, 0x061A, + 0x064B, 0x065F, + 0x06D6, 0x06DC, + 0x06DF, 0x06E4, + 0x06E7, 0x06E8, + 0x06EA, 0x06ED, + 0x0730, 0x074A, + 0x07A6, 0x07B0, + 0x07EB, 0x07F3, + 0x0816, 0x0819, + 0x081B, 0x0823, + 0x0825, 0x0827, + 0x0829, 0x082D, + 0x0859, 0x085B, + 0x0897, 0x089F, + 0x08CA, 0x08E1, + 0x08E3, 0x0902, + 0x0941, 0x0948, + 0x0951, 0x0957, + 0x0962, 0x0963, + 0x09C1, 0x09C4, + 0x09E2, 0x09E3, + 0x0A01, 0x0A02, + 0x0A41, 0x0A42, + 0x0A47, 0x0A48, + 0x0A4B, 0x0A4D, + 0x0A70, 0x0A71, + 0x0A81, 0x0A82, + 0x0AC1, 0x0AC5, + 0x0AC7, 0x0AC8, + 0x0AE2, 0x0AE3, + 0x0AFA, 0x0AFF, + 0x0B41, 0x0B44, + 0x0B55, 0x0B56, + 0x0B62, 0x0B63, + 0x0C3E, 0x0C40, + 0x0C46, 0x0C48, + 0x0C4A, 0x0C4D, + 0x0C55, 0x0C56, + 0x0C62, 0x0C63, + 0x0CCC, 0x0CCD, + 0x0CE2, 0x0CE3, + 0x0D00, 0x0D01, + 0x0D3B, 0x0D3C, + 0x0D41, 0x0D44, + 0x0D62, 0x0D63, + 0x0DD2, 0x0DD4, + 0x0E34, 0x0E3A, + 0x0E47, 0x0E4E, + 0x0EB4, 0x0EBC, + 0x0EC8, 0x0ECE, + 0x0F18, 0x0F19, + 0x0F71, 0x0F7E, + 0x0F80, 0x0F84, + 0x0F86, 0x0F87, + 0x0F8D, 0x0F97, + 0x0F99, 0x0FBC, + 0x102D, 0x1030, + 0x1032, 0x1037, + 0x1039, 0x103A, + 0x103D, 0x103E, + 0x1058, 0x1059, + 0x105E, 0x1060, + 0x1071, 0x1074, + 0x1085, 0x1086, + 0x135D, 0x135F, + 0x1712, 0x1714, + 0x1732, 0x1733, + 0x1752, 0x1753, + 0x1772, 0x1773, + 0x17B4, 0x17B5, + 0x17B7, 0x17BD, + 0x17C9, 0x17D3, + 0x180B, 0x180D, + 0x1885, 0x1886, + 0x1920, 0x1922, + 0x1927, 0x1928, + 0x1939, 0x193B, + 0x1A17, 0x1A18, + 0x1A58, 0x1A5E, + 0x1A65, 0x1A6C, + 0x1A73, 0x1A7C, + 0x1AB0, 0x1ABD, + 0x1ABF, 0x1ADD, + 0x1AE0, 0x1AEB, + 0x1B00, 0x1B03, + 0x1B36, 0x1B3A, + 0x1B6B, 0x1B73, + 0x1B80, 0x1B81, + 0x1BA2, 0x1BA5, + 0x1BA8, 0x1BA9, + 0x1BAB, 0x1BAD, + 0x1BE8, 0x1BE9, + 0x1BEF, 0x1BF1, + 0x1C2C, 0x1C33, + 0x1C36, 0x1C37, + 0x1CD0, 0x1CD2, + 0x1CD4, 0x1CE0, + 0x1CE2, 0x1CE8, + 0x1CF8, 0x1CF9, + 0x1DC0, 0x1DFF, + 0x20D0, 0x20DC, + 0x20E5, 0x20F0, + 0x2CEF, 0x2CF1, + 0x2DE0, 0x2DFF, + 0x302A, 0x302D, + 0x3099, 0x309A, + 0xA674, 0xA67D, + 0xA69E, 0xA69F, + 0xA6F0, 0xA6F1, + 0xA825, 0xA826, + 0xA8C4, 0xA8C5, + 0xA8E0, 0xA8F1, + 0xA926, 0xA92D, + 0xA947, 0xA951, + 0xA980, 0xA982, + 0xA9B6, 0xA9B9, + 0xA9BC, 0xA9BD, + 0xAA29, 0xAA2E, + 0xAA31, 0xAA32, + 0xAA35, 0xAA36, + 0xAAB2, 0xAAB4, + 0xAAB7, 0xAAB8, + 0xAABE, 0xAABF, + 0xAAEC, 0xAAED, + 0xFE00, 0xFE0F, + 0xFE20, 0xFE2F, +} + +@(rodata) +mn_singles32 := [?]i32{ + 0x101FD, 0x102E0, 0x10A3F, 0x11001, 0x11070, 0x110C2, 0x11173, 0x111CF, + 0x11234, 0x1123E, 0x11241, 0x112DF, 0x11340, 0x113CE, 0x113D0, 0x113D2, + 0x11446, 0x1145E, 0x114BA, 0x1163D, 0x116AB, 0x116AD, 0x116B7, 0x1171D, + 0x1171F, 0x1193E, 0x11943, 0x119E0, 0x11A47, 0x11B60, 0x11B66, 0x11C3F, + 0x11D3A, 0x11D47, 0x11D95, 0x11D97, 0x11F40, 0x11F42, 0x11F5A, 0x13440, + 0x16F4F, 0x16FE4, 0x1DA75, 0x1DA84, 0x1E08F, 0x1E2AE, 0x1E6E3, 0x1E6E6, + 0x1E6F5, +} + +@(rodata) +mn_ranges32 := [?]i32{ + 0x10376, 0x1037A, + 0x10A01, 0x10A03, + 0x10A05, 0x10A06, + 0x10A0C, 0x10A0F, + 0x10A38, 0x10A3A, + 0x10AE5, 0x10AE6, + 0x10D24, 0x10D27, + 0x10D69, 0x10D6D, + 0x10EAB, 0x10EAC, + 0x10EFA, 0x10EFF, + 0x10F46, 0x10F50, + 0x10F82, 0x10F85, + 0x11038, 0x11046, + 0x11073, 0x11074, + 0x1107F, 0x11081, + 0x110B3, 0x110B6, + 0x110B9, 0x110BA, + 0x11100, 0x11102, + 0x11127, 0x1112B, + 0x1112D, 0x11134, + 0x11180, 0x11181, + 0x111B6, 0x111BE, + 0x111C9, 0x111CC, + 0x1122F, 0x11231, + 0x11236, 0x11237, + 0x112E3, 0x112EA, + 0x11300, 0x11301, + 0x1133B, 0x1133C, + 0x11366, 0x1136C, + 0x11370, 0x11374, + 0x113BB, 0x113C0, + 0x113E1, 0x113E2, + 0x11438, 0x1143F, + 0x11442, 0x11444, + 0x114B3, 0x114B8, + 0x114BF, 0x114C0, + 0x114C2, 0x114C3, + 0x115B2, 0x115B5, + 0x115BC, 0x115BD, + 0x115BF, 0x115C0, + 0x115DC, 0x115DD, + 0x11633, 0x1163A, + 0x1163F, 0x11640, + 0x116B0, 0x116B5, + 0x11722, 0x11725, + 0x11727, 0x1172B, + 0x1182F, 0x11837, + 0x11839, 0x1183A, + 0x1193B, 0x1193C, + 0x119D4, 0x119D7, + 0x119DA, 0x119DB, + 0x11A01, 0x11A0A, + 0x11A33, 0x11A38, + 0x11A3B, 0x11A3E, + 0x11A51, 0x11A56, + 0x11A59, 0x11A5B, + 0x11A8A, 0x11A96, + 0x11A98, 0x11A99, + 0x11B62, 0x11B64, + 0x11C30, 0x11C36, + 0x11C38, 0x11C3D, + 0x11C92, 0x11CA7, + 0x11CAA, 0x11CB0, + 0x11CB2, 0x11CB3, + 0x11CB5, 0x11CB6, + 0x11D31, 0x11D36, + 0x11D3C, 0x11D3D, + 0x11D3F, 0x11D45, + 0x11D90, 0x11D91, + 0x11EF3, 0x11EF4, + 0x11F00, 0x11F01, + 0x11F36, 0x11F3A, + 0x13447, 0x13455, + 0x1611E, 0x16129, + 0x1612D, 0x1612F, + 0x16AF0, 0x16AF4, + 0x16B30, 0x16B36, + 0x16F8F, 0x16F92, + 0x1BC9D, 0x1BC9E, + 0x1CF00, 0x1CF2D, + 0x1CF30, 0x1CF46, + 0x1D167, 0x1D169, + 0x1D17B, 0x1D182, + 0x1D185, 0x1D18B, + 0x1D1AA, 0x1D1AD, + 0x1D242, 0x1D244, + 0x1DA00, 0x1DA36, + 0x1DA3B, 0x1DA6C, + 0x1DA9B, 0x1DA9F, + 0x1DAA1, 0x1DAAF, + 0x1E000, 0x1E006, + 0x1E008, 0x1E018, + 0x1E01B, 0x1E021, + 0x1E023, 0x1E024, + 0x1E026, 0x1E02A, + 0x1E130, 0x1E136, + 0x1E2EC, 0x1E2EF, + 0x1E4EC, 0x1E4EF, + 0x1E5EE, 0x1E5EF, + 0x1E6EE, 0x1E6EF, + 0x1E8D0, 0x1E8D6, + 0x1E944, 0x1E94A, + 0xE0100, 0xE01EF, +} + +mn_ranges := Range{ + single_16 = mn_singles16[:], + ranges_16 = mn_ranges16[:], + single_32 = mn_singles32[:], + ranges_32 = mn_ranges32[:], +} + @(rodata) nd_ranges16 := [?]u16{ 0x0030, 0x0039, @@ -1743,6 +2182,21 @@ so_ranges := Range{ ranges_32 = so_ranges32[:], } +@(rodata) +zs_singles16 := [?]u16{ + 0x0020, 0x00A0, 0x1680, 0x202F, 0x205F, 0x3000, +} + +@(rodata) +zs_ranges16 := [?]u16{ + 0x2000, 0x200A, +} + +zs_ranges := Range{ + single_16 = zs_singles16[:], + ranges_16 = zs_ranges16[:], +} + @(rodata) extra_digits_singles16 := [?]u16{ 0x00B9, 0x19DA, 0x2070, 0x24EA, 0x24FF, diff --git a/core/unicode/letter.odin b/core/unicode/letter.odin index c6df96f3f..92aee9a68 100644 --- a/core/unicode/letter.odin +++ b/core/unicode/letter.odin @@ -197,6 +197,15 @@ is_graphic :: proc(r: rune) -> bool { if u32(r) <= MAX_LATIN1 { return char_properties[u8(r)]&pg != 0 } + + if is_letter(r) || is_number(r) || is_punct(r) || is_symbol(r) || in_range(r, zs_ranges) { + return true + } + + if in_range(r, mc_ranges) || in_range(r, me_ranges) || in_range(r, mn_ranges) { + return true + } + return false } diff --git a/core/unicode/tools/generate_unicode.odin b/core/unicode/tools/generate_unicode.odin index 27cee0012..f2d07442a 100644 --- a/core/unicode/tools/generate_unicode.odin +++ b/core/unicode/tools/generate_unicode.odin @@ -278,9 +278,9 @@ main :: proc() { // .Lo, // Other_Letter, other letters, including syllables and ideographs // .Lt, // Titlecase_Letter, a digraph encoded as a single character, with first part uppercase // .Lu, // Uppercase_Letter, an uppercase letter - .Mc, // Spacing_Mark, a spacing combining mark (positive advance width) - .Me, // Enclosing_Mark, an enclosing combining mark - .Mn, // Nonspacing_Mark, a nonspacing combining mark (zero advance width) + // .Mc, // Spacing_Mark, a spacing combining mark (positive advance width) + // .Me, // Enclosing_Mark, an enclosing combining mark + // .Mn, // Nonspacing_Mark, a nonspacing combining mark (zero advance width) //.Nd, // Decimal_Number, a decimal digit //.Nl, // Letter_Number, a letterlike numeric character //.No, // Other_Number, a numeric character of other type @@ -295,9 +295,9 @@ main :: proc() { // .Sk, // Modifier_Symbol, a non-letterlike modifier symbol // .Sm, // Math_Symbol, a symbol of mathematical use // .So, // Other_Symbol, a symbol of other type - .Zl, // Line_Separator, U+2028 LINE SEPARATOR only - .Zp, // Paragraph_Separator, U+2029 PARAGRAPH SEPARATOR only - .Zs, // Space_Separator, a space character (of various non-zero widths) + .Zl, // Line_Separator, U+2028 LINE SEPARATOR only + .Zp, // Paragraph_Separator, U+2029 PARAGRAPH SEPARATOR only + //.Zs, // Space_Separator, a space character (of various non-zero widths) } write_loop : for gc, i in general_category_ranges {