This Fixes #5939 , where many characters like flags are reported with width 1 even though their width is 2. Country glags need additional special case also since RI letters are EAW neutral and not Wide.

i was working on this when i stumbled on this bug, so i did bunch of different characters to see which ones cause issue, hence test has many different options. I could make it either simpler or split it, as needed.
This commit is contained in:
MoonlightChase
2026-08-26 13:31:43 +02:00
parent cef4fb01a4
commit 556bdb0056
2 changed files with 72 additions and 10 deletions

View File

@@ -46,7 +46,6 @@ Grapheme_Iterator :: struct {
last_rune: rune,
last_rune_breaks_forward: bool,
last_width: int,
last_grapheme_count: int,
bypass_next_rune: bool,
@@ -58,6 +57,9 @@ Grapheme_Iterator :: struct {
current_grapheme: Grapheme,
continue_grapheme: bool,
current_cluster_width: int,
current_cluster_ri_count: int,
}
@@ -141,28 +143,45 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
it.curr_offset += this_rune_width
defer {
// "Break at the start and end of text, unless the text is empty."
//
// GB1: sot ÷ Any
// GB2: Any ÷ eot
if it.rune_count == 0 && it.grapheme_count == 0 {
it.grapheme_count += 1
}
this_rune_width = normalized_east_asian_width(this_rune)
if it.grapheme_count > it.last_grapheme_count {
it.width += normalized_east_asian_width(this_rune)
// new cluster is starting: close out the previous one first.
if it.continue_grapheme {
// completd regional indicator pair rendered as one
// 2-cell flag in terminals, even though each RI letter
// is individually EAW Neutral
if it.current_cluster_ri_count == 2 {
it.current_cluster_width = max(it.current_cluster_width, 2)
}
it.width += it.current_cluster_width
grapheme = it.current_grapheme
grapheme.text = it.str[it.current_grapheme.byte_index:byte_index]
grapheme.width = it.current_cluster_width
text = grapheme.text
ok = true
}
it.current_grapheme = Grapheme{byte_index = byte_index, rune_index = it.rune_count, width = it.width - it.last_width}
it.current_grapheme = Grapheme {
byte_index = byte_index,
rune_index = it.rune_count,
}
it.continue_grapheme = true
it.last_grapheme_count = it.grapheme_count
it.last_width = it.width
it.current_cluster_width = this_rune_width
it.current_cluster_ri_count = is_regional_indicator(this_rune) ? 1 : 0
} else {
// Continuation rune: fold its width in instead of dropping it.
it.current_cluster_width = max(it.current_cluster_width, this_rune_width)
if is_regional_indicator(this_rune) {
it.current_cluster_ri_count += 1
}
}
it.last_rune = this_rune
@@ -393,8 +412,13 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
// Flush the remaining grapheme - the loop only flushes when
// a new grapheme is encountered.
if !ok && it.continue_grapheme {
if it.current_cluster_ri_count == 2 {
it.current_cluster_width = max(it.current_cluster_width, 2)
}
it.width += it.current_cluster_width
grapheme = it.current_grapheme
grapheme.text = it.str[it.current_grapheme.byte_index:]
grapheme.width = it.current_cluster_width
text = grapheme.text
ok = true
it.continue_grapheme = false

View File

@@ -138,6 +138,44 @@ test_width :: proc(t: ^testing.T) {
}
}
@test
test_grapheme_width_continuation_runes :: proc(t: ^testing.T) {
Grapheme_Width_Test_Case :: struct {
str: string,
expected_grapheme_count: int,
expected_total_width: int,
expected_widths: []int,
}
cases :: []Grapheme_Width_Test_Case{
{"\U0001F3F3\uFE0F\u200D\U0001F308", 1, 2, {2}},
{"\U0001F1FA\U0001F1F8", 1, 2, {2}},
{"\U0001F1FA", 1, 1, {1}},
{"\U0001F1FA\U0001F1F8\U0001F1EE\U0001F1EA", 2, 4, {2, 2}},
{"\U0001F1FA\U0001F1F8\U0001F1EC", 2, 3, {2, 1}},
}
for c in cases {
// Test utf8.grapheme_width
graphemes, _, width := utf8.grapheme_count(c.str)
testing.expectf(t, graphemes == c.expected_grapheme_count, "%q: expected %d graphemes, got %d", c.str, c.expected_grapheme_count, graphemes)
testing.expectf(t, width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, width)
// Test utf8.decode_grapheme_clusters
clusters, cluster_count, _, total_width := utf8.decode_grapheme_clusters(c.str)
defer delete(clusters)
testing.expectf(t, cluster_count == c.expected_grapheme_count, "%q: expected %d clusters, got %d", c.str, c.expected_grapheme_count, cluster_count)
testing.expectf(t, total_width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, total_width)
if testing.expectf(t, len(clusters) == len(c.expected_widths), "%q: expected %d cluster widths, got %d", c.str, len(c.expected_widths), len(clusters)) {
for expected_w, i in c.expected_widths {
testing.expectf(t, clusters[i].width == expected_w, "%q cluster %d: expected width %d, got %d", c.str, i, expected_w, clusters[i].width)
}
}
}
}
@test
test_grapheme_cluster_text :: proc(t: ^testing.T) {