mirrors/ghostty
1
0
Fork 0
mirror of https://github.com/ghostty-org/ghostty.git synced 2026-08-02 21:59:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib-vt: add unicode grapheme width API

Browse Source 
Embedders that render text outside the terminal grid need to predict
how many cells text will occupy once it is written to the terminal.
The existing codepoint width API exposes the table used by print, but
that is not enough for mode 2027 grapheme clustering: VS15/VS16, ZWJ
sequences, skin tone modifiers, and other continuation codepoints can
change the width of the whole cluster.

This exposes a single segment-and-measure API so callers use Ghostty
segmentation and width folding together:

    uint8_t width;
    size_t n = ghostty_unicode_grapheme_width(cps, len, &width);

From the Zig module:

    const vt = @import("ghostty-vt");
    const result = vt.unicode.graphemeWidth(u21, cps);

Callers loop until their string is consumed. The API is intentionally
not streaming: input must contain a complete first cluster or the
logical string end, so chunked readers should keep buffering when the
function consumes all available codepoints and more may arrive.

The terminal hot path now shares the width-decision func with the
API, the helper is inline and preserves the old branch structure. So
this doesn't change codegen at all.
This commit is contained in:
Mitchell Hashimoto
2026-07-04 14:02:16 -07:00
parent 61ce641fca
commit 65e61282a6
7 changed files with 381 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/lib_vt.zig
View File

@@ -138,6 +138,7 @@ pub const unicode = struct {
const unicode_pkg = @import("unicode/main.zig");
pub const codepointWidth = unicode_pkg.codepointWidth;
pub const graphemeWidth = unicode_pkg.graphemeWidth;
};
comptime {
@@ -195,6 +196,7 @@ comptime {
@export(&c.paste_is_safe, .{ .name = "ghostty_paste_is_safe" });
@export(&c.paste_encode, .{ .name = "ghostty_paste_encode" });
@export(&c.unicode_codepoint_width, .{ .name = "ghostty_unicode_codepoint_width" });
@export(&c.unicode_grapheme_width, .{ .name = "ghostty_unicode_grapheme_width" });
@export(&c.size_report_encode, .{ .name = "ghostty_size_report_encode" });
@export(&c.style_default, .{ .name = "ghostty_style_default" });
@export(&c.style_is_default, .{ .name = "ghostty_style_is_default" });

69
src/terminal/Terminal.zig
View File

@@ -406,33 +406,8 @@ pub fn print(self: *Terminal, c: u21) !void {
// If we can NOT break, this means that "c" is part of a grapheme
// with the previous char.
if (!grapheme_break) {
var desired_wide: enum { no_change, wide, narrow } = .no_change;
// If this is an emoji variation selector then we need to modify
// the cell width accordingly. VS16 makes the character wide and
// VS15 makes it narrow.
if (c == 0xFE0F or c == 0xFE0E) {
const prev_props = unicode.table.get(previous_codepoint);
// Check if it is a valid variation sequence in
// emoji-variation-sequences.txt, and if not, ignore the char.
if (!prev_props.emoji_vs_base) return;
switch (c) {
0xFE0F => desired_wide = .wide,
0xFE0E => desired_wide = .narrow,
else => unreachable,
}
} else if (!unicode.table.get(c).width_zero_in_grapheme) {
// If we have a code point that contributes to the width of a
// grapheme, it necessarily means that we're at least at width
// 2, since the first code point must be at least width 1 to
// start. (Note that Prepend code points could effectively mean
// the first code point should be width 0, but we don't handle
// that yet.)
desired_wide = .wide;
}
switch (desired_wide) {
switch (unicode.graphemeWidthEffect(previous_codepoint, c)) {
.ignore => return,
.wide => wide: {
if (prev.cell.wide == .wide) break :wide;
@@ -549,7 +524,7 @@ pub fn print(self: *Terminal, c: u21) !void {
break :narrow;
},
else => {},
.no_change => {},
}
log.debug("c={X} grapheme attach to left={} primary_cp={X}", .{
@@ -3690,6 +3665,44 @@ test "Terminal: print multicodepoint grapheme, disabled mode 2027" {
try testing.expect(t.isDirty(.{ .screen = .{ .x = 0, .y = 0 } }));
}
// Terminal.print receives one codepoint at a time, so it can't use
// unicode.graphemeWidth directly; that API requires a complete buffered
// cluster or string end. This keeps the streaming printer's cursor advance
// in sync with the buffered measurement API for representative clusters.
fn expectGraphemeWidthParity(cps: []const u21) !void {
var t = try init(testing.allocator, .{ .cols = 80, .rows = 5 });
defer t.deinit(testing.allocator);
t.modes.set(.grapheme_cluster, true);
var expected: usize = 0;
var i: usize = 0;
while (i < cps.len) {
const result = unicode.graphemeWidth(u21, cps[i..]);
try testing.expect(result.len > 0);
i += result.len;
expected += result.width;
}
for (cps) |cp| try t.print(cp);
try testing.expectEqual(@as(usize, 0), t.screens.active.cursor.y);
try testing.expectEqual(expected, t.screens.active.cursor.x);
}
test "Terminal: graphemeWidth parity" {
try expectGraphemeWidthParity(&.{ 0x2764, 0xFE0F });
try expectGraphemeWidthParity(&.{ 'x', 0xFE0F, 0xFE0F });
try expectGraphemeWidthParity(&.{ 0x231A, 0xFE0E, 0xFE0F });
try expectGraphemeWidthParity(&.{ 0x1F3F4, 0x200D, 0x2620, 0xFE0F });
try expectGraphemeWidthParity(&.{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 });
try expectGraphemeWidthParity(&.{ 0x23, 0xFE0F, 0x20E3 });
try expectGraphemeWidthParity(&.{ '1', 0x20E3 });
try expectGraphemeWidthParity(&.{ 0x1F44B, 0x1F3FF });
try expectGraphemeWidthParity(&.{ 0x1F1E6, 0x1F1E7, 0x1F1E8 });
try expectGraphemeWidthParity(&.{ 'a', 'b' });
try expectGraphemeWidthParity(&.{ 0x0301, 0x0302 });
}
test "Terminal: VS16 doesn't make character with 2027 disabled" {
var t = try init(testing.allocator, .{ .rows = 5, .cols = 5 });
defer t.deinit(testing.allocator);

1
src/terminal/c/main.zig
View File

@@ -200,6 +200,7 @@ pub const terminal_point_from_grid_ref = terminal.point_from_grid_ref;
pub const type_json = types.get_json;
pub const unicode_codepoint_width = unicode.codepoint_width;
pub const unicode_grapheme_width = unicode.grapheme_width;
pub const grid_ref_cell = grid_ref.grid_ref_cell;
pub const grid_ref_row = grid_ref.grid_ref_row;

56
src/terminal/c/unicode.zig
View File

@@ -7,6 +7,25 @@ pub fn codepoint_width(cp: u32) callconv(lib.calling_conv) u8 {
return unicode_pkg.codepointWidth(@intCast(cp));
}
pub fn grapheme_width(
cps: ?[*]const u32,
len: usize,
width: ?*u8,
) callconv(lib.calling_conv) usize {
if (len == 0) {
if (width) |ptr| ptr.* = 0;
return 0;
}
const ptr = cps orelse {
if (width) |out| out.* = 0;
return 0;
};
const result = unicode_pkg.graphemeWidth(u32, ptr[0..len]);
if (width) |out| out.* = result.width;
return result.len;
}
test "codepoint_width narrow" {
const testing = std.testing;
try testing.expectEqual(1, codepoint_width('a'));
@@ -27,3 +46,40 @@ test "codepoint_width out of range" {
try testing.expectEqual(1, codepoint_width(0x110000));
try testing.expectEqual(1, codepoint_width(std.math.maxInt(u32)));
}
test "grapheme_width empty" {
const testing = std.testing;
var width: u8 = 42;
try testing.expectEqual(@as(usize, 0), grapheme_width(null, 0, &width));
try testing.expectEqual(@as(u8, 0), width);
}
test "grapheme_width null width" {
const testing = std.testing;
const cps = [_]u32{ 0x2764, 0xFE0F };
try testing.expectEqual(@as(usize, 2), grapheme_width(cps[0..].ptr, cps.len, null));
}
test "grapheme_width out of range" {
const testing = std.testing;
var width: u8 = 0;
const invalid_first = [_]u32{ 0x110000, 0x0301 };
try testing.expectEqual(@as(usize, 1), grapheme_width(invalid_first[0..].ptr, invalid_first.len, &width));
try testing.expectEqual(@as(u8, 1), width);
const invalid_second = [_]u32{ 'a', 0x110000 };
try testing.expectEqual(@as(usize, 1), grapheme_width(invalid_second[0..].ptr, invalid_second.len, &width));
try testing.expectEqual(@as(u8, 1), width);
}
test "grapheme_width emoji sequence" {
const testing = std.testing;
var width: u8 = 0;
const cps = [_]u32{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 };
try testing.expectEqual(@as(usize, cps.len), grapheme_width(cps[0..].ptr, cps.len, &width));
try testing.expectEqual(@as(u8, 2), width);
}

185
src/unicode/grapheme.zig
View File

@@ -2,6 +2,31 @@ const std = @import("std");
const table = @import("props_table.zig").table;
const uucode = @import("uucode");
/// Width change requested by a codepoint that continues a grapheme cluster.
pub const GraphemeWidthEffect = enum {
/// Do not append the codepoint to the cluster and leave break state as it
/// was before seeing it.
ignore,
/// Append the codepoint but leave the current cluster width unchanged.
no_change,
/// Make the cluster occupy two terminal cells.
wide,
/// Make the cluster occupy one terminal cell.
narrow,
};
/// Result of measuring the first grapheme cluster in a codepoint slice.
pub const GraphemeWidth = struct {
/// Number of codepoints consumed from the input slice.
len: usize,
/// Display width in terminal cells.
width: u2,
};
/// Determines if there is a grapheme break between two codepoints. This
/// must be called sequentially maintaining the state between calls.
///
@@ -21,6 +46,99 @@ pub fn graphemeBreak(cp1: u21, cp2: u21, state: *uucode.grapheme.BreakState) boo
return value.result;
}
/// Returns the width effect of appending cp after prev within a grapheme.
///
/// This is the shared width-decision kernel for the streaming terminal
/// printer and for graphemeWidth. It assumes graphemeBreak has already said
/// there is no break between prev and cp; it does not perform segmentation.
///
/// The .ignore result is important for invalid emoji variation selectors. The
/// terminal does not store those selectors in the cell, so callers must also
/// restore their grapheme break state and leave prev unchanged when they see
/// .ignore.
pub inline fn graphemeWidthEffect(prev: u21, cp: u21) GraphemeWidthEffect {
// Emoji variation selectors modify the width of a valid base:
// VS16 makes the grapheme wide and VS15 makes it narrow. Check that
// prev forms a valid variation sequence in emoji-variation-sequences.txt;
// if it does not, ignore the selector entirely.
if (cp == 0xFE0F or cp == 0xFE0E) {
const prev_props = table.get(prev);
if (!prev_props.emoji_vs_base) return .ignore;
return switch (cp) {
0xFE0F => .wide,
0xFE0E => .narrow,
else => unreachable,
};
}
// If a code point contributes to the width of a grapheme, the whole
// grapheme is at least width 2 because the first code point must be at
// least width 1 to start. Prepend code points could effectively mean
// the first code point should be width 0, but we don't handle that yet.
if (!table.get(cp).width_zero_in_grapheme) return .wide;
return .no_change;
}
/// Measures the first grapheme cluster in cps using the same segmentation and
/// width rules as Terminal.print with mode 2027.
///
/// This is not a streaming API: cps must contain a complete first grapheme
/// cluster or the logical end of the string. If bytes/codepoints arrive in
/// chunks, keep buffering when this consumes all available codepoints and more
/// input may still arrive.
///
/// For codepoint types wider than u21, values greater than U+10FFFF are
/// accepted so FFI-facing callers can use u32 input without trapping. An
/// invalid value consumes one codepoint at width 1 when it starts the slice,
/// and terminates the current cluster when it appears later. For u21 callers
/// these checks are comptime-dead.
pub fn graphemeWidth(comptime T: type, cps: []const T) GraphemeWidth {
const check_invalid = comptime @bitSizeOf(T) > @bitSizeOf(u21);
if (cps.len == 0) return .{ .len = 0, .width = 0 };
// The C API accepts u32 codepoints, so it can receive values outside
// Unicode's range. Guard before narrowing to u21; native u21 callers
// skip this path at comptime.
if (check_invalid and invalidCodepoint(cps[0])) return .{ .len = 1, .width = 1 };
var len: usize = 1;
var width = table.get(@as(u21, @intCast(cps[0]))).width;
var prev: u21 = @intCast(cps[0]);
var state: uucode.grapheme.BreakState = .default;
while (len < cps.len) : (len += 1) {
// Treat invalid u32 input as a boundary so a valid prefix cluster can
// still be returned without attempting to narrow the invalid value.
if (check_invalid and invalidCodepoint(cps[len])) break;
const cp: u21 = @intCast(cps[len]);
const state_before = state;
if (graphemeBreak(prev, cp, &state)) break;
switch (graphemeWidthEffect(prev, cp)) {
.ignore => state = state_before,
.no_change => prev = cp,
.wide => {
width = 2;
prev = cp;
},
.narrow => {
width = 1;
prev = cp;
},
}
}
return .{ .len = len, .width = width };
}
inline fn invalidCodepoint(cp: anytype) bool {
return cp > 0x10FFFF;
}
/// This is all the structures and data for the precomputed lookup table
/// for all possible permutations of state and grapheme break properties.
/// Precomputation requires 2^13 keys of 4 bit values so the whole table is
@@ -178,3 +296,70 @@ test "long emoji zwj sequences" {
try std.testing.expect(cp1 == 0x1F466); // 👦
try std.testing.expect(graphemeBreak(cp1, cp2, &state)); // break
}
test "grapheme width: variation selectors" {
const testing = std.testing;
try testing.expectEqual(GraphemeWidthEffect.wide, graphemeWidthEffect(0x2764, 0xFE0F));
try testing.expectEqual(GraphemeWidthEffect.narrow, graphemeWidthEffect(0x23, 0xFE0E));
try testing.expectEqual(GraphemeWidthEffect.ignore, graphemeWidthEffect('x', 0xFE0F));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x2764, 0xFE0F }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x23, 0xFE0F }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 'x', 0xFE0F }));
try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 1 }, graphemeWidth(u21, &.{ 'x', 0xFE0F, 0xFE0F }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 0x23, 0xFE0E }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ 0x231A, 0xFE0E }));
try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 1 }, graphemeWidth(u21, &.{ 0x231A, 0xFE0E, 0xFE0F }));
try testing.expectEqual(GraphemeWidth{ .len = 4, .width = 2 }, graphemeWidth(u21, &.{ 0x1F3F4, 0x200D, 0x2620, 0xFE0F }));
}
test "grapheme width: emoji sequences" {
const testing = std.testing;
try testing.expectEqual(GraphemeWidth{ .len = 5, .width = 2 }, graphemeWidth(u21, &.{ 0x1F468, 0x200D, 0x1F469, 0x200D, 0x1F467 }));
try testing.expectEqual(GraphemeWidth{ .len = 3, .width = 2 }, graphemeWidth(u21, &.{ 0x23, 0xFE0F, 0x20E3 }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 1 }, graphemeWidth(u21, &.{ '1', 0x20E3 }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x1F44B, 0x1F3FF }));
}
test "grapheme width: spacing marks can widen narrow clusters" {
const testing = std.testing;
var mark: ?u21 = null;
for (0..0x110000) |cp_usize| {
const cp: u21 = @intCast(cp_usize);
const props = table.get(cp);
if (props.width != 1 or props.width_zero_in_grapheme) continue;
var state: uucode.grapheme.BreakState = .default;
if (!graphemeBreak('a', cp, &state)) {
mark = cp;
break;
}
}
try testing.expect(mark != null);
const cp = mark.?;
try testing.expectEqual(@as(u2, 1), table.get(cp).width);
try testing.expect(!table.get(cp).width_zero_in_grapheme);
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 'a', cp }));
}
test "grapheme width: segmentation" {
const testing = std.testing;
try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u21, &.{'a'}));
try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u21, &.{ 'a', 'b' }));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 2 }, graphemeWidth(u21, &.{ 0x1F1E6, 0x1F1E7, 0x1F1E8 }));
try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 2 }, graphemeWidth(u21, &.{0x1F1E8}));
try testing.expectEqual(GraphemeWidth{ .len = 0, .width = 0 }, graphemeWidth(u21, &.{}));
try testing.expectEqual(GraphemeWidth{ .len = 2, .width = 0 }, graphemeWidth(u21, &.{ 0x0301, 0x0302 }));
}
test "grapheme width: u32 invalid codepoints stand alone" {
const testing = std.testing;
try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u32, &.{ 0x110000, 0x0301 }));
try testing.expectEqual(GraphemeWidth{ .len = 1, .width = 1 }, graphemeWidth(u32, &.{ 'a', 0x110000 }));
}

9
src/unicode/main.zig
View File

@@ -3,7 +3,11 @@ pub const lut = @import("lut.zig");
const grapheme = @import("grapheme.zig");
pub const table = @import("props_table.zig").table;
pub const Properties = @import("props.zig").Properties;
pub const GraphemeWidthEffect = grapheme.GraphemeWidthEffect;
pub const GraphemeWidth = grapheme.GraphemeWidth;
pub const graphemeBreak = grapheme.graphemeBreak;
pub const graphemeWidth = grapheme.graphemeWidth;
pub const graphemeWidthEffect = grapheme.graphemeWidthEffect;
/// Returns the terminal display width of a codepoint in terminal
/// grid cells: 0, 1, or 2.
@@ -16,8 +20,9 @@ pub const graphemeBreak = grapheme.graphemeBreak;
///
/// This operates on a single codepoint and cannot account for
/// grapheme-cluster-level width rules (VS16, combining sequences);
/// callers needing cluster-accurate widths must segment into
/// grapheme clusters and combine per-codepoint widths.
/// callers needing cluster-accurate widths should use graphemeWidth().
/// Summing per-codepoint widths is only correct when mode 2027 is
/// disabled.
pub fn codepointWidth(cp: u21) u2 {
return table.get(cp).width;
}