diff --git a/core/unicode/utf8/grapheme.odin b/core/unicode/utf8/grapheme.odin index d8f6557b6..bcd234972 100644 --- a/core/unicode/utf8/grapheme.odin +++ b/core/unicode/utf8/grapheme.odin @@ -54,6 +54,9 @@ Grapheme_Iterator :: struct { current_sequence: Grapheme_Cluster_Sequence, continue_sequence: bool, + + current_grapheme: Grapheme, + continue_grapheme: bool, } @@ -147,13 +150,13 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe if it.grapheme_count > it.last_grapheme_count { it.width += normalized_east_asian_width(this_rune) - grapheme = Grapheme{ - byte_index, - it.rune_count, - it.width - it.last_width, + if it.continue_grapheme { + grapheme = it.current_grapheme + text = it.str[it.current_grapheme.byte_index:byte_index] + ok = true } - text = it.str[byte_index:][:grapheme.width] - ok = true + it.current_grapheme = Grapheme{byte_index, it.rune_count, it.width - it.last_width} + it.continue_grapheme = true it.last_grapheme_count = it.grapheme_count @@ -385,5 +388,14 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe it.grapheme_count += 1 } + // Flush the remaining grapheme - the loop only flushes when + // a new grapheme is encountered. + if !ok && it.continue_grapheme { + grapheme = it.current_grapheme + text = it.str[it.current_grapheme.byte_index:] + ok = true + it.continue_grapheme = false + } + return -} \ No newline at end of file +} diff --git a/tests/core/unicode/test_core_unicode.odin b/tests/core/unicode/test_core_unicode.odin index 30a40b30b..874f4eebe 100644 --- a/tests/core/unicode/test_core_unicode.odin +++ b/tests/core/unicode/test_core_unicode.odin @@ -9,6 +9,11 @@ Test_Case :: struct { expected_clusters: int, } +Text_Test_Case :: struct { + str: string, + expected_output: []string, +} + run_test_cases :: proc(t: ^testing.T, test_cases: []Test_Case, loc := #caller_location) { failed := 0 for c, i in test_cases { @@ -132,3 +137,32 @@ test_width :: proc(t: ^testing.T) { testing.expect_value(t, width, 50) } } + +@test +test_grapheme_cluster_text :: proc(t: ^testing.T) { + + cases :: []Text_Test_Case { + {"abc", {"a", "b", "c"}}, + {"é", {"é"}}, + {"中", {"中"}}, + {"\U0001F1FA\U0001F1F8", {"\U0001F1FA\U0001F1F8"}}, + {"\U0001F1FA\U0001F1F8\U0001F1EE\U0001F1EA", {"\U0001F1FA\U0001F1F8", "\U0001F1EE\U0001F1EA"}}, + {"\U0001F468‍\U0001F469‍\U0001F467‍\U0001F466", {"\U0001F468‍\U0001F469‍\U0001F467‍\U0001F466"}}, + {"\U0001F44D\U0001F3FD", {"\U0001F44D\U0001F3FD"}}, + {"a\r\nb", {"a", "\r\n", "b"}}, + } + + for c in cases { + it := utf8.decode_grapheme_iterator_make(c.str) + i := 0 + for text, grapheme in utf8.decode_grapheme_iterate(&it) { + if !testing.expectf(t, i < len(c.expected_output), "%q: expected %d clusters, got at least %d", c.str, len(c.expected_output), i + 1) { + break + } + testing.expectf(t, text == c.expected_output[i], "%q cluster %d: expected text %q, got %q", c.str, i, c.expected_output[i], text) + testing.expectf(t, text == c.str[grapheme.byte_index:][:len(text)], "%q cluster %d: text does not start at byte_index %d", c.str, i, grapheme.byte_index) + i += 1 + } + testing.expectf(t, i == len(c.expected_output), "%q: expected %d clusters, got %d", c.str, len(c.expected_output), i) + } +}