mirror of
https://github.com/odin-lang/Odin.git
synced 2026-09-05 11:50:44 +00:00
This Fixes #5939 , where many characters like flags are reported with width 1 even though their width is 2. Country glags need additional special case also since RI letters are EAW neutral and not Wide.
i was working on this when i stumbled on this bug, so i did bunch of different characters to see which ones cause issue, hence test has many different options. I could make it either simpler or split it, as needed.
This commit is contained in:
@@ -46,7 +46,6 @@ Grapheme_Iterator :: struct {
|
||||
last_rune: rune,
|
||||
last_rune_breaks_forward: bool,
|
||||
|
||||
last_width: int,
|
||||
last_grapheme_count: int,
|
||||
|
||||
bypass_next_rune: bool,
|
||||
@@ -58,6 +57,9 @@ Grapheme_Iterator :: struct {
|
||||
|
||||
current_grapheme: Grapheme,
|
||||
continue_grapheme: bool,
|
||||
|
||||
current_cluster_width: int,
|
||||
current_cluster_ri_count: int,
|
||||
}
|
||||
|
||||
|
||||
@@ -141,28 +143,45 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
|
||||
it.curr_offset += this_rune_width
|
||||
|
||||
defer {
|
||||
// "Break at the start and end of text, unless the text is empty."
|
||||
//
|
||||
// GB1: sot ÷ Any
|
||||
// GB2: Any ÷ eot
|
||||
|
||||
if it.rune_count == 0 && it.grapheme_count == 0 {
|
||||
it.grapheme_count += 1
|
||||
}
|
||||
|
||||
this_rune_width = normalized_east_asian_width(this_rune)
|
||||
|
||||
if it.grapheme_count > it.last_grapheme_count {
|
||||
it.width += normalized_east_asian_width(this_rune)
|
||||
// new cluster is starting: close out the previous one first.
|
||||
if it.continue_grapheme {
|
||||
// completd regional indicator pair rendered as one
|
||||
// 2-cell flag in terminals, even though each RI letter
|
||||
// is individually EAW Neutral
|
||||
if it.current_cluster_ri_count == 2 {
|
||||
it.current_cluster_width = max(it.current_cluster_width, 2)
|
||||
}
|
||||
it.width += it.current_cluster_width
|
||||
grapheme = it.current_grapheme
|
||||
grapheme.text = it.str[it.current_grapheme.byte_index:byte_index]
|
||||
grapheme.width = it.current_cluster_width
|
||||
text = grapheme.text
|
||||
ok = true
|
||||
}
|
||||
it.current_grapheme = Grapheme{byte_index = byte_index, rune_index = it.rune_count, width = it.width - it.last_width}
|
||||
|
||||
it.current_grapheme = Grapheme {
|
||||
byte_index = byte_index,
|
||||
rune_index = it.rune_count,
|
||||
}
|
||||
it.continue_grapheme = true
|
||||
|
||||
|
||||
it.last_grapheme_count = it.grapheme_count
|
||||
it.last_width = it.width
|
||||
|
||||
it.current_cluster_width = this_rune_width
|
||||
it.current_cluster_ri_count = is_regional_indicator(this_rune) ? 1 : 0
|
||||
} else {
|
||||
// Continuation rune: fold its width in instead of dropping it.
|
||||
it.current_cluster_width = max(it.current_cluster_width, this_rune_width)
|
||||
if is_regional_indicator(this_rune) {
|
||||
it.current_cluster_ri_count += 1
|
||||
}
|
||||
}
|
||||
|
||||
it.last_rune = this_rune
|
||||
@@ -393,8 +412,13 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
|
||||
// Flush the remaining grapheme - the loop only flushes when
|
||||
// a new grapheme is encountered.
|
||||
if !ok && it.continue_grapheme {
|
||||
if it.current_cluster_ri_count == 2 {
|
||||
it.current_cluster_width = max(it.current_cluster_width, 2)
|
||||
}
|
||||
it.width += it.current_cluster_width
|
||||
grapheme = it.current_grapheme
|
||||
grapheme.text = it.str[it.current_grapheme.byte_index:]
|
||||
grapheme.width = it.current_cluster_width
|
||||
text = grapheme.text
|
||||
ok = true
|
||||
it.continue_grapheme = false
|
||||
|
||||
@@ -138,6 +138,44 @@ test_width :: proc(t: ^testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
@test
|
||||
test_grapheme_width_continuation_runes :: proc(t: ^testing.T) {
|
||||
Grapheme_Width_Test_Case :: struct {
|
||||
str: string,
|
||||
expected_grapheme_count: int,
|
||||
expected_total_width: int,
|
||||
expected_widths: []int,
|
||||
}
|
||||
|
||||
cases :: []Grapheme_Width_Test_Case{
|
||||
{"\U0001F3F3\uFE0F\u200D\U0001F308", 1, 2, {2}},
|
||||
{"\U0001F1FA\U0001F1F8", 1, 2, {2}},
|
||||
{"\U0001F1FA", 1, 1, {1}},
|
||||
{"\U0001F1FA\U0001F1F8\U0001F1EE\U0001F1EA", 2, 4, {2, 2}},
|
||||
{"\U0001F1FA\U0001F1F8\U0001F1EC", 2, 3, {2, 1}},
|
||||
}
|
||||
|
||||
for c in cases {
|
||||
// Test utf8.grapheme_width
|
||||
graphemes, _, width := utf8.grapheme_count(c.str)
|
||||
testing.expectf(t, graphemes == c.expected_grapheme_count, "%q: expected %d graphemes, got %d", c.str, c.expected_grapheme_count, graphemes)
|
||||
testing.expectf(t, width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, width)
|
||||
|
||||
// Test utf8.decode_grapheme_clusters
|
||||
clusters, cluster_count, _, total_width := utf8.decode_grapheme_clusters(c.str)
|
||||
defer delete(clusters)
|
||||
|
||||
testing.expectf(t, cluster_count == c.expected_grapheme_count, "%q: expected %d clusters, got %d", c.str, c.expected_grapheme_count, cluster_count)
|
||||
testing.expectf(t, total_width == c.expected_total_width, "%q: expected total width %d, got %d", c.str, c.expected_total_width, total_width)
|
||||
|
||||
if testing.expectf(t, len(clusters) == len(c.expected_widths), "%q: expected %d cluster widths, got %d", c.str, len(c.expected_widths), len(clusters)) {
|
||||
for expected_w, i in c.expected_widths {
|
||||
testing.expectf(t, clusters[i].width == expected_w, "%q cluster %d: expected width %d, got %d", c.str, i, expected_w, clusters[i].width)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@test
|
||||
test_grapheme_cluster_text :: proc(t: ^testing.T) {
|
||||
|
||||
|
||||
Reference in New Issue
Block a user