diff --git a/include/ghostty/vt/unicode.h b/include/ghostty/vt/unicode.h index 9f1f114a9..e4d62c818 100644 --- a/include/ghostty/vt/unicode.h +++ b/include/ghostty/vt/unicode.h @@ -17,13 +17,17 @@ * * Use ghostty_unicode_codepoint_width() to determine how many terminal * grid cells a codepoint occupies, using the exact same width table the - * terminal itself uses when laying out printed text. This is useful for - * predicting column layout of text that has not yet been written to the - * terminal, such as IME preedit (composition) overlays. + * terminal itself uses when laying out printed text. Use + * ghostty_unicode_grapheme_width() to segment and measure full grapheme + * clusters with the same rules the terminal uses when mode 2027 is + * enabled. These functions are useful for predicting column layout of + * text that has not yet been written to the terminal, such as IME + * preedit (composition) overlays. * * @{ */ +#include #include #include @@ -51,11 +55,11 @@ extern "C" { * - Returns 1 for everything else, including invalid codepoints * beyond U+10FFFF (this function is total; it never fails). * - * This operates on a single codepoint only and therefore cannot - * account for grapheme-cluster-level width rules (VS16 emoji - * presentation, combining sequences, etc.). Callers wanting - * cluster-accurate widths must segment text into grapheme clusters - * themselves and combine per-codepoint widths. + * This operates on a single codepoint only and therefore cannot account + * for grapheme-cluster-level width rules (VS16 emoji presentation, + * combining sequences, etc.). For cluster-accurate widths, use + * ghostty_unicode_grapheme_width(). Summing per-codepoint widths is only + * correct when mode 2027 (grapheme clustering) is disabled. * * This function is pure, allocates nothing, and is thread-safe. * @@ -64,6 +68,83 @@ extern "C" { */ GHOSTTY_API uint8_t ghostty_unicode_codepoint_width(uint32_t cp); +/** + * Measures the terminal display width of the first grapheme cluster in a + * sequence of Unicode codepoints. + * + * This uses the exact same grapheme segmentation and cluster width rules + * the terminal itself uses when printing text with grapheme clustering + * enabled (mode 2027), so callers can predict column layout (e.g. IME + * preedit overlays) that exactly matches what the terminal will do when + * the text is actually written to it. Unlike + * ghostty_unicode_codepoint_width(), this accounts for cluster-level + * rules: emoji variation selectors, ZWJ sequences, combining marks, and + * skin tone modifiers. + * + * Reads codepoints from cps until the terminal would consider the + * grapheme cluster complete, stores the cluster's total width in cells + * (0, 1, or 2) into width (which may be NULL if only segmentation is + * desired), and returns the number of codepoints consumed. Returns 0 if + * and only if len is 0; otherwise consumes at least one codepoint. Measure + * a whole string by calling in a loop: + * + * @code + * size_t total = 0; + * for (size_t i = 0; i < len;) { + * uint8_t width; + * i += ghostty_unicode_grapheme_width(cps + i, len - i, &width); + * total += width; + * } + * @endcode + * + * This is not a streaming API. The provided sequence must contain a + * complete first grapheme cluster, or the logical end of the string. If + * input arrives in chunks, keep buffering while this function consumes all + * available codepoints (return value == len) and the stream may still + * continue; a later codepoint could still extend the cluster and change + * its width. + * + * Width semantics, matching the terminal with mode 2027 enabled: + * - The cluster starts at the width of its first codepoint, as returned by + * ghostty_unicode_codepoint_width(). + * - VS16 (U+FE0F) forces the cluster wide (2) and VS15 (U+FE0E) forces it + * narrow (1), but only when the immediately preceding codepoint in the + * cluster is a valid emoji variation sequence base (per Unicode + * emoji-variation-sequences.txt). Invalid variation selectors are + * ignored entirely. + * - Any other continuation codepoint that contributes to grapheme width + * forces the cluster wide (2). Note this means cluster width is NOT the + * maximum of per-codepoint widths: some continuation marks have narrow + * codepoint width yet still widen the cluster. + * + * Mode dependence: this models mode 2027 (grapheme clustering) enabled, + * which is Ghostty's recommended configuration. When mode 2027 is + * disabled, clusters never combine and variation selectors never change + * width; predict layout in that case by summing + * ghostty_unicode_codepoint_width() over each codepoint instead. + * + * Edge cases: + * - Codepoints beyond U+10FFFF consume one codepoint, have width 1, and + * are always cluster boundaries. This function is total; it never fails. + * - Control characters (C0/C1, CR, LF) are never printed through the + * terminal's text path; passing them here returns an unspecified (but + * stable and bounded) result. + * - A cluster whose first codepoint is zero-width (e.g. a lone combining + * mark) is malformed at a cell start; the terminal may attach it to + * earlier screen content. This function reports the fold result for the + * sequence in isolation (typically 0). + * + * This function is pure, allocates nothing, and is thread-safe. + * + * @param cps Pointer to codepoints (may be NULL only when len is 0) + * @param len Number of codepoints available + * @param width Out: cluster display width in cells (0-2); may be NULL + * @return Number of codepoints in the first grapheme cluster + */ +GHOSTTY_API size_t ghostty_unicode_grapheme_width(const uint32_t *cps, + size_t len, + uint8_t *width); + #ifdef __cplusplus } #endif diff --git a/src/lib_vt.zig b/src/lib_vt.zig index ad78190e4..5af267f55 100644 --- a/src/lib_vt.zig +++ b/src/lib_vt.zig @@ -138,6 +138,7 @@ pub const unicode = struct { const unicode_pkg = @import("unicode/main.zig"); pub const codepointWidth = unicode_pkg.codepointWidth; + pub const graphemeWidth = unicode_pkg.graphemeWidth; }; comptime { @@ -195,6 +196,7 @@ comptime { @export(&c.paste_is_safe, .{ .name = "ghostty_paste_is_safe" }); @export(&c.paste_encode, .{ .name = "ghostty_paste_encode" }); @export(&c.unicode_codepoint_width, .{ .name = "ghostty_unicode_codepoint_width" }); + @export(&c.unicode_grapheme_width, .{ .name = "ghostty_unicode_grapheme_width" }); @export(&c.size_report_encode, .{ .name = "ghostty_size_report_encode" }); @export(&c.style_default, .{ .name = "ghostty_style_default" }); @export(&c.style_is_default, .{ .name = "ghostty_style_is_default" }); diff --git a/src/terminal/Terminal.zig b/src/terminal/Terminal.zig index 30a1c93fd..ce066692b 100644 --- a/src/terminal/Terminal.zig +++ b/src/terminal/Terminal.zig @@ -406,33 +406,8 @@ pub fn print(self: *Terminal, c: u21) !void { // If we can NOT break, this means that "c" is part of a grapheme // with the previous char. if (!grapheme_break) { - var desired_wide: enum { no_change, wide, narrow } = .no_change; - - // If this is an emoji variation selector then we need to modify - // the cell width accordingly. VS16 makes the character wide and - // VS15 makes it narrow. - if (c == 0xFE0F or c == 0xFE0E) { - const prev_props = unicode.table.get(previous_codepoint); - // Check if it is a valid variation sequence in - // emoji-variation-sequences.txt, and if not, ignore the char. - if (!prev_props.emoji_vs_base) return; - - switch (c) { - 0xFE0F => desired_wide = .wide, - 0xFE0E => desired_wide = .narrow, - else => unreachable, - } - } else if (!unicode.table.get(c).width_zero_in_grapheme) { - // If we have a code point that contributes to the width of a - // grapheme, it necessarily means that we're at least at width - // 2, since the first code point must be at least width 1 to - // start. (Note that Prepend code points could effectively mean - // the first code point should be width 0, but we don't handle - // that yet.) - desired_wide = .wide; - } - - switch (desired_wide) { + switch (unicode.graphemeWidthEffect(previous_codepoint, c)) { + .ignore => return, .wide => wide: { if (prev.cell.wide == .wide) break :wide; @@ -549,7 +524,7 @@ pub fn print(self: *Terminal, c: u21) !void { break :narrow; }, - else => {}, + .no_change => {}, } log.debug("c={X} grapheme attach to left={} primary_cp={X}", .{ @@ -3690,6 +3665,44 @@ test "Terminal: print multicodepoint grapheme, disabled mode 2027" { try testing.expect(t.isDirty(.{ .screen = .{ .x = 0, .y = 0 } })); } +// Terminal.print receives one codepoint at a time, so it can't use +// unicode.graphemeWidth directly; that API requires a complete buffered +// cluster or string end. This keeps the streaming printer's cursor advance +// in sync with the buffered measurement API for representative clusters. +fn expectGraphemeWidthParity(cps: []const u21) !void { + var t = try init(testing.allocator, .{ .cols = 80, .rows = 5 }); + defer t.deinit(testing.allocator); + + t.modes.set(.grapheme_cluster, true); + + var expected: usize = 0; + var i: usize = 0; + while (i < cps.len) { + const result = unicode.graphemeWidth(u21, cps[i..]); + try testing.expect(result.len > 0); + i += result.len; + expected += result.width; + } + + for (cps) |cp| try t.print(cp); + try testing.expectEqual(@as(usize, 0), t.screens.active.cursor.y); + try testing.expectEqual(expected, t.screens.active.cursor.x); +} + +test "Terminal: graphemeWidth parity" { + try expectGraphemeWidthParity(&.{ 0x2764, 0xFE0F }); + try expectGraphemeWidthParity(&.{ 'x', 0xFE0F, 0xFE0F }); + try expectGraphemeWidthParity(&.{ 0x231A, 0xFE0E, 0xFE0F }); + try expectGraphemeWidthParity(&.{ 0x1F3F4, 0x200D, 0x2620, 0xFE0F }); + try expectGraphemeWidthParity(&.{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 }); + try expectGraphemeWidthParity(&.{ 0x23, 0xFE0F, 0x20E3 }); + try expectGraphemeWidthParity(&.{ '1', 0x20E3 }); + try expectGraphemeWidthParity(&.{ 0x1F44B, 0x1F3FF }); + try expectGraphemeWidthParity(&.{ 0x1F1E6, 0x1F1E7, 0x1F1E8 }); + try expectGraphemeWidthParity(&.{ 'a', 'b' }); + try expectGraphemeWidthParity(&.{ 0x0301, 0x0302 }); +} + test "Terminal: VS16 doesn't make character with 2027 disabled" { var t = try init(testing.allocator, .{ .rows = 5, .cols = 5 }); defer t.deinit(testing.allocator); diff --git a/src/terminal/c/main.zig b/src/terminal/c/main.zig index 1344351af..ba04209c7 100644 --- a/src/terminal/c/main.zig +++ b/src/terminal/c/main.zig @@ -200,6 +200,7 @@ pub const terminal_point_from_grid_ref = terminal.point_from_grid_ref; pub const type_json = types.get_json; pub const unicode_codepoint_width = unicode.codepoint_width; +pub const unicode_grapheme_width = unicode.grapheme_width; pub const grid_ref_cell = grid_ref.grid_ref_cell; pub const grid_ref_row = grid_ref.grid_ref_row; diff --git a/src/terminal/c/unicode.zig b/src/terminal/c/unicode.zig index 716ff1e23..144513389 100644 --- a/src/terminal/c/unicode.zig +++ b/src/terminal/c/unicode.zig @@ -7,6 +7,25 @@ pub fn codepoint_width(cp: u32) callconv(lib.calling_conv) u8 { return unicode_pkg.codepointWidth(@intCast(cp)); } +pub fn grapheme_width( + cps: ?[*]const u32, + len: usize, + width: ?*u8, +) callconv(lib.calling_conv) usize { + if (len == 0) { + if (width) |ptr| ptr.* = 0; + return 0; + } + + const ptr = cps orelse { + if (width) |out| out.* = 0; + return 0; + }; + const result = unicode_pkg.graphemeWidth(u32, ptr[0..len]); + if (width) |out| out.* = result.width; + return result.len; +} + test "codepoint_width narrow" { const testing = std.testing; try testing.expectEqual(1, codepoint_width('a')); @@ -27,3 +46,40 @@ test "codepoint_width out of range" { try testing.expectEqual(1, codepoint_width(0x110000)); try testing.expectEqual(1, codepoint_width(std.math.maxInt(u32))); } + +test "grapheme_width empty" { + const testing = std.testing; + + var width: u8 = 42; + try testing.expectEqual(@as(usize, 0), grapheme_width(null, 0, &width)); + try testing.expectEqual(@as(u8, 0), width); +} + +test "grapheme_width null width" { + const testing = std.testing; + + const cps = [_]u32{ 0x2764, 0xFE0F }; + try testing.expectEqual(@as(usize, 2), grapheme_width(cps[0..].ptr, cps.len, null)); +} + +test "grapheme_width out of range" { + const testing = std.testing; + + var width: u8 = 0; + const invalid_first = [_]u32{ 0x110000, 0x0301 }; + try testing.expectEqual(@as(usize, 1), grapheme_width(invalid_first[0..].ptr, invalid_first.len, &width)); + try testing.expectEqual(@as(u8, 1), width); + + const invalid_second = [_]u32{ 'a', 0x110000 }; + try testing.expectEqual(@as(usize, 1), grapheme_width(invalid_second[0..].ptr, invalid_second.len, &width)); + try testing.expectEqual(@as(u8, 1), width); +} + +test "grapheme_width emoji sequence" { + const testing = std.testing; + + var width: u8 = 0; + const cps = [_]u32{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 }; + try testing.expectEqual(@as(usize, cps.len), grapheme_width(cps[0..].ptr, cps.len, &width)); + try testing.expectEqual(@as(u8, 2), width); +} diff --git a/src/unicode/grapheme.zig b/src/unicode/grapheme.zig index d233dec72..73782dd6e 100644 --- a/src/unicode/grapheme.zig +++ b/src/unicode/grapheme.zig @@ -2,6 +2,31 @@ const std = @import("std"); const table = @import("props_table.zig").table; const uucode = @import("uucode"); +/// Width change requested by a codepoint that continues a grapheme cluster. +pub const GraphemeWidthEffect = enum { + /// Do not append the codepoint to the cluster and leave break state as it + /// was before seeing it. + ignore, + + /// Append the codepoint but leave the current cluster width unchanged. + no_change, + + /// Make the cluster occupy two terminal cells. + wide, + + /// Make the cluster occupy one terminal cell. + narrow, +}; + +/// Result of measuring the first grapheme cluster in a codepoint slice. +pub const GraphemeWidth = struct { + /// Number of codepoints consumed from the input slice. + len: usize, + + /// Display width in terminal cells. + width: u2, +}; + /// Determines if there is a grapheme break between two codepoints. This /// must be called sequentially maintaining the state between calls. /// @@ -21,6 +46,99 @@ pub fn graphemeBreak(cp1: u21, cp2: u21, state: *uucode.grapheme.BreakState) boo return value.result; } +/// Returns the width effect of appending cp after prev within a grapheme. +/// +/// This is the shared width-decision kernel for the streaming terminal +/// printer and for graphemeWidth. It assumes graphemeBreak has already said +/// there is no break between prev and cp; it does not perform segmentation. +/// +/// The .ignore result is important for invalid emoji variation selectors. The +/// terminal does not store those selectors in the cell, so callers must also +/// restore their grapheme break state and leave prev unchanged when they see +/// .ignore. +pub inline fn graphemeWidthEffect(prev: u21, cp: u21) GraphemeWidthEffect { + // Emoji variation selectors modify the width of a valid base: + // VS16 makes the grapheme wide and VS15 makes it narrow. Check that + // prev forms a valid variation sequence in emoji-variation-sequences.txt; + // if it does not, ignore the selector entirely. + if (cp == 0xFE0F or cp == 0xFE0E) { + const prev_props = table.get(prev); + if (!prev_props.emoji_vs_base) return .ignore; + + return switch (cp) { + 0xFE0F => .wide, + 0xFE0E => .narrow, + else => unreachable, + }; + } + + // If a code point contributes to the width of a grapheme, the whole + // grapheme is at least width 2 because the first code point must be at + // least width 1 to start. Prepend code points could effectively mean + // the first code point should be width 0, but we don't handle that yet. + if (!table.get(cp).width_zero_in_grapheme) return .wide; + + return .no_change; +} + +/// Measures the first grapheme cluster in cps using the same segmentation and +/// width rules as Terminal.print with mode 2027. +/// +/// This is not a streaming API: cps must contain a complete first grapheme +/// cluster or the logical end of the string. If bytes/codepoints arrive in +/// chunks, keep buffering when this consumes all available codepoints and more +/// input may still arrive. +/// +/// For codepoint types wider than u21, values greater than U+10FFFF are +/// accepted so FFI-facing callers can use u32 input without trapping. An +/// invalid value consumes one codepoint at width 1 when it starts the slice, +/// and terminates the current cluster when it appears later. For u21 callers +/// these checks are comptime-dead. +pub fn graphemeWidth(comptime T: type, cps: []const T) GraphemeWidth { + const check_invalid = comptime @bitSizeOf(T) > @bitSizeOf(u21); + + if (cps.len == 0) return .{ .len = 0, .width = 0 }; + + // The C API accepts u32 codepoints, so it can receive values outside + // Unicode's range. Guard before narrowing to u21; native u21 callers + // skip this path at comptime. + if (check_invalid and invalidCodepoint(cps[0])) return .{ .len = 1, .width = 1 }; + + var len: usize = 1; + var width = table.get(@as(u21, @intCast(cps[0]))).width; + var prev: u21 = @intCast(cps[0]); + var state: uucode.grapheme.BreakState = .default; + + while (len < cps.len) : (len += 1) { + // Treat invalid u32 input as a boundary so a valid prefix cluster can + // still be returned without attempting to narrow the invalid value. + if (check_invalid and invalidCodepoint(cps[len])) break; + + const cp: u21 = @intCast(cps[len]); + const state_before = state; + if (graphemeBreak(prev, cp, &state)) break; + + switch (graphemeWidthEffect(prev, cp)) { + .ignore => state = state_before, + .no_change => prev = cp, + .wide => { + width = 2; + prev = cp; + }, + .narrow => { + width = 1; + prev = cp; + }, + } + } + + return .{ .len = len, .width = width }; +} + +inline fn invalidCodepoint(cp: anytype) bool { + return cp > 0x10FFFF; +} + /// This is all the structures and data for the precomputed lookup table /// for all possible permutations of state and grapheme break properties. /// Precomputation requires 2^13 keys of 4 bit values so the whole table is @@ -178,3 +296,70 @@ test "long emoji zwj sequences" { try std.testing.expect(cp1 == 0x1F466); // 👦 try std.testing.expect(graphemeBreak(cp1, cp2, &state)); // break } + +test "grapheme width: variation selectors" { + const testing = std.testing; + + try testing.expectEqual(GraphemeWidthEffect.wide, graphemeWidthEffect(0x2764, 0xFE0F)); + try testing.expectEqual(GraphemeWidthEffect.narrow, graphemeWidthEffect(0x23, 0xFE0E)); + try testing.expectEqual(GraphemeWidthEffect.ignore, graphemeWidthEffect('x', 0xFE0F)); + + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x2764, 0xFE0F })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x23, 0xFE0F })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 'x', 0xFE0F })); + try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 1 }, graphemeWidth(u21, &.{ 'x', 0xFE0F, 0xFE0F })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 0x23, 0xFE0E })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 0x231A, 0xFE0E })); + try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 1 }, graphemeWidth(u21, &.{ 0x231A, 0xFE0E, 0xFE0F })); + try testing.expectEqual(GraphemeWidth{ .len = 4, .width = 2 }, graphemeWidth(u21, &.{ 0x1F3F4, 0x200D, 0x2620, 0xFE0F })); +} + +test "grapheme width: emoji sequences" { + const testing = std.testing; + + try testing.expectEqual(GraphemeWidth{ .len = 5, .width = 2 }, graphemeWidth(u21, &.{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 })); + try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 2 }, graphemeWidth(u21, &.{ 0x23, 0xFE0F, 0x20E3 })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ '1', 0x20E3 })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x1F44B, 0x1F3FF })); +} + +test "grapheme width: spacing marks can widen narrow clusters" { + const testing = std.testing; + + var mark: ?u21 = null; + for (0..0x110000) |cp_usize| { + const cp: u21 = @intCast(cp_usize); + const props = table.get(cp); + if (props.width != 1 or props.width_zero_in_grapheme) continue; + + var state: uucode.grapheme.BreakState = .default; + if (!graphemeBreak('a', cp, &state)) { + mark = cp; + break; + } + } + + try testing.expect(mark != null); + const cp = mark.?; + try testing.expectEqual(@as(u2, 1), table.get(cp).width); + try testing.expect(!table.get(cp).width_zero_in_grapheme); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 'a', cp })); +} + +test "grapheme width: segmentation" { + const testing = std.testing; + + try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u21, &.{'a'})); + try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u21, &.{ 'a', 'b' })); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x1F1E6, 0x1F1E7, 0x1F1E8 })); + try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 2 }, graphemeWidth(u21, &.{0x1F1E8})); + try testing.expectEqual(GraphemeWidth{ .len = 0, .width = 0 }, graphemeWidth(u21, &.{})); + try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 0 }, graphemeWidth(u21, &.{ 0x0301, 0x0302 })); +} + +test "grapheme width: u32 invalid codepoints stand alone" { + const testing = std.testing; + + try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u32, &.{ 0x110000, 0x0301 })); + try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u32, &.{ 'a', 0x110000 })); +} diff --git a/src/unicode/main.zig b/src/unicode/main.zig index 1e8466285..fb18b7d13 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -3,7 +3,11 @@ pub const lut = @import("lut.zig"); const grapheme = @import("grapheme.zig"); pub const table = @import("props_table.zig").table; pub const Properties = @import("props.zig").Properties; +pub const GraphemeWidthEffect = grapheme.GraphemeWidthEffect; +pub const GraphemeWidth = grapheme.GraphemeWidth; pub const graphemeBreak = grapheme.graphemeBreak; +pub const graphemeWidth = grapheme.graphemeWidth; +pub const graphemeWidthEffect = grapheme.graphemeWidthEffect; /// Returns the terminal display width of a codepoint in terminal /// grid cells: 0, 1, or 2. @@ -16,8 +20,9 @@ pub const graphemeBreak = grapheme.graphemeBreak; /// /// This operates on a single codepoint and cannot account for /// grapheme-cluster-level width rules (VS16, combining sequences); -/// callers needing cluster-accurate widths must segment into -/// grapheme clusters and combine per-codepoint widths. +/// callers needing cluster-accurate widths should use graphemeWidth(). +/// Summing per-codepoint widths is only correct when mode 2027 is +/// disabled. pub fn codepointWidth(cp: u21) u2 { return table.get(cp).width; }