diff --git a/core/unicode/utf8/grapheme.odin b/core/unicode/utf8/grapheme.odin index c4ef3767e..421fb3205 100644 --- a/core/unicode/utf8/grapheme.odin +++ b/core/unicode/utf8/grapheme.odin @@ -46,7 +46,6 @@ Grapheme_Iterator :: struct { last_rune: rune, last_rune_breaks_forward: bool, - last_width: int, last_grapheme_count: int, bypass_next_rune: bool, @@ -58,6 +57,9 @@ Grapheme_Iterator :: struct { current_grapheme: Grapheme, continue_grapheme: bool, + + current_cluster_width: int, + current_cluster_ri_count: int, } @@ -141,28 +143,45 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe it.curr_offset += this_rune_width defer { - // "Break at the start and end of text, unless the text is empty." - // - // GB1: sot ÷ Any - // GB2: Any ÷ eot + if it.rune_count == 0 && it.grapheme_count == 0 { it.grapheme_count += 1 } + this_rune_width = normalized_east_asian_width(this_rune) + if it.grapheme_count > it.last_grapheme_count { - it.width += normalized_east_asian_width(this_rune) + // new cluster is starting: close out the previous one first. if it.continue_grapheme { + // completd regional indicator pair rendered as one + // 2-cell flag in terminals, even though each RI letter + // is individually EAW Neutral + if it.current_cluster_ri_count == 2 { + it.current_cluster_width = max(it.current_cluster_width, 2) + } + it.width += it.current_cluster_width grapheme = it.current_grapheme grapheme.text = it.str[it.current_grapheme.byte_index:byte_index] + grapheme.width = it.current_cluster_width text = grapheme.text ok = true } - it.current_grapheme = Grapheme{byte_index = byte_index, rune_index = it.rune_count, width = it.width - it.last_width} + + it.current_grapheme = Grapheme { + byte_index = byte_index, + rune_index = it.rune_count, + } it.continue_grapheme = true - - it.last_grapheme_count = it.grapheme_count - it.last_width = it.width + + it.current_cluster_width = this_rune_width + it.current_cluster_ri_count = is_regional_indicator(this_rune) ? 1 : 0 + } else { + // Continuation rune: fold its width in instead of dropping it. + it.current_cluster_width = max(it.current_cluster_width, this_rune_width) + if is_regional_indicator(this_rune) { + it.current_cluster_ri_count += 1 + } } it.last_rune = this_rune @@ -393,8 +412,13 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe // Flush the remaining grapheme - the loop only flushes when // a new grapheme is encountered. if !ok && it.continue_grapheme { + if it.current_cluster_ri_count == 2 { + it.current_cluster_width = max(it.current_cluster_width, 2) + } + it.width += it.current_cluster_width grapheme = it.current_grapheme grapheme.text = it.str[it.current_grapheme.byte_index:] + grapheme.width = it.current_cluster_width text = grapheme.text ok = true it.continue_grapheme = false diff --git a/tests/core/unicode/test_core_unicode.odin b/tests/core/unicode/test_core_unicode.odin index 874f4eebe..31cf2dc73 100644 --- a/tests/core/unicode/test_core_unicode.odin +++ b/tests/core/unicode/test_core_unicode.odin @@ -138,6 +138,44 @@ test_width :: proc(t: ^testing.T) { } } +@test +test_grapheme_width_continuation_runes :: proc(t: ^testing.T) { + Grapheme_Width_Test_Case :: struct { + str: string, + expected_grapheme_count: int, + expected_total_width: int, + expected_widths: []int, + } + + cases :: []Grapheme_Width_Test_Case{ + {"\U0001F3F3\uFE0F\u200D\U0001F308", 1, 2, {2}}, + {"\U0001F1FA\U0001F1F8", 1, 2, {2}}, + {"\U0001F1FA", 1, 1, {1}}, + {"\U0001F1FA\U0001F1F8\U0001F1EE\U0001F1EA", 2, 4, {2, 2}}, + {"\U0001F1FA\U0001F1F8\U0001F1EC", 2, 3, {2, 1}}, + } + + for c in cases { + // Test utf8.grapheme_width + graphemes, _, width := utf8.grapheme_count(c.str) + testing.expectf(t, graphemes == c.expected_grapheme_count, "%q: expected %d graphemes, got %d", c.str, c.expected_grapheme_count, graphemes) + testing.expectf(t, width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, width) + + // Test utf8.decode_grapheme_clusters + clusters, cluster_count, _, total_width := utf8.decode_grapheme_clusters(c.str) + defer delete(clusters) + + testing.expectf(t, cluster_count == c.expected_grapheme_count, "%q: expected %d clusters, got %d", c.str, c.expected_grapheme_count, cluster_count) + testing.expectf(t, total_width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, total_width) + + if testing.expectf(t, len(clusters) == len(c.expected_widths), "%q: expected %d cluster widths, got %d", c.str, len(c.expected_widths), len(clusters)) { + for expected_w, i in c.expected_widths { + testing.expectf(t, clusters[i].width == expected_w, "%q cluster %d: expected width %d, got %d", c.str, i, expected_w, clusters[i].width) + } + } + } +} + @test test_grapheme_cluster_text :: proc(t: ^testing.T) {