unicode: switch to uucode grapheme break to match unicode 16 spec

This commit is contained in:
Jacob Sandlund
2025-11-23 19:16:16 -05:00
parent 97926ca307
commit 97aff0743e
8 changed files with 152 additions and 198 deletions

View File

@@ -4,57 +4,18 @@ const assert = std.debug.assert;
const uucode = @import("uucode");
const lut = @import("lut.zig");
const Properties = @import("props.zig").Properties;
const GraphemeBoundaryClass = @import("props.zig").GraphemeBoundaryClass;
/// Gets the grapheme boundary class for a codepoint.
/// The use case for this is only in generating lookup tables.
fn graphemeBoundaryClass(cp: u21) GraphemeBoundaryClass {
if (cp > uucode.config.max_code_point) return .invalid;
return switch (uucode.get(.grapheme_break, cp)) {
.extended_pictographic => .extended_pictographic,
.l => .L,
.v => .V,
.t => .T,
.lv => .LV,
.lvt => .LVT,
.prepend => .prepend,
.zwj => .zwj,
.spacing_mark => .spacing_mark,
.regional_indicator => .regional_indicator,
.emoji_modifier => .emoji_modifier,
.emoji_modifier_base => .extended_pictographic_base,
.zwnj,
.indic_conjunct_break_extend,
.indic_conjunct_break_linker,
=> .extend,
// This is obviously not INVALID invalid, there is SOME grapheme
// boundary class for every codepoint. But we don't care about
// anything that doesn't fit into the above categories. Also note
// that `indic_conjunct_break_consonant` is `other` in
// 'GraphemeBreakProperty.txt' (it's missing).
.other,
.indic_conjunct_break_consonant,
.cr,
.lf,
.control,
=> .invalid,
};
}
pub fn get(cp: u21) Properties {
if (cp > uucode.config.max_code_point) return .{
.width = 1,
.grapheme_boundary_class = .invalid,
.grapheme_break = .other,
.emoji_vs_text = false,
.emoji_vs_emoji = false,
};
return .{
.width = uucode.get(.width, cp),
.grapheme_boundary_class = graphemeBoundaryClass(cp),
.grapheme_break = uucode.get(.grapheme_break_no_control, cp),
.emoji_vs_text = uucode.get(.is_emoji_vs_text, cp),
.emoji_vs_emoji = uucode.get(.is_emoji_vs_emoji, cp),
};