Merge pull request #7003 from jpgleeson/jpgleeson/6912-truncated-text

[core:unicode/utf8]: Fix truncated text for grapheme clusters

core/unicode/utf8/grapheme.odin

@@ -54,6 +54,9 @@ Grapheme_Iterator :: struct {
 
 	current_sequence:           Grapheme_Cluster_Sequence,
 	continue_sequence:          bool,
+
+	current_grapheme:           Grapheme,
+	continue_grapheme:          bool,
 }
 
 
@@ -147,13 +150,13 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
 
 			if it.grapheme_count > it.last_grapheme_count {
 				it.width += normalized_east_asian_width(this_rune)
-				grapheme = Grapheme{
-					byte_index,
-					it.rune_count,
-					it.width - it.last_width,
+				if it.continue_grapheme {
+					grapheme = it.current_grapheme
+					text = it.str[it.current_grapheme.byte_index:byte_index]
+					ok = true
 				}
-				text = it.str[byte_index:][:grapheme.width]
-				ok = true
+				it.current_grapheme = Grapheme{byte_index, it.rune_count, it.width - it.last_width}
+				it.continue_grapheme = true
 
 
 				it.last_grapheme_count = it.grapheme_count
@@ -385,5 +388,14 @@ decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, graphe
 		it.grapheme_count += 1
 	}
 
+	// Flush the remaining grapheme - the loop only flushes when
+	// a new grapheme is encountered.
+	if !ok && it.continue_grapheme {
+		grapheme = it.current_grapheme
+		text = it.str[it.current_grapheme.byte_index:]
+		ok = true
+		it.continue_grapheme = false
+	}
+
 	return
-}
+}

tests/core/unicode/test_core_unicode.odin

@@ -9,6 +9,11 @@ Test_Case :: struct {
 	expected_clusters: int,
 }
 
+Text_Test_Case :: struct {
+	str: string,
+	expected_output: []string,
+}
+
 run_test_cases :: proc(t: ^testing.T, test_cases: []Test_Case, loc := #caller_location) {
 	failed := 0
 	for c, i in test_cases {
@@ -132,3 +137,32 @@ test_width :: proc(t: ^testing.T) {
 		testing.expect_value(t, width, 50)
 	}
 }
+
+@test
+test_grapheme_cluster_text :: proc(t: ^testing.T) {
+
+	cases :: []Text_Test_Case {
+		{"abc", {"a", "b", "c"}},
+		{"é", {"é"}},
+		{"中", {"中"}},
+		{"\U0001F1FA\U0001F1F8", {"\U0001F1FA\U0001F1F8"}},
+		{"\U0001F1FA\U0001F1F8\U0001F1EE\U0001F1EA", {"\U0001F1FA\U0001F1F8", "\U0001F1EE\U0001F1EA"}},
+		{"\U0001F468‍\U0001F469‍\U0001F467‍\U0001F466", {"\U0001F468‍\U0001F469‍\U0001F467‍\U0001F466"}},
+		{"\U0001F44D\U0001F3FD", {"\U0001F44D\U0001F3FD"}},
+		{"a\r\nb", {"a", "\r\n", "b"}},
+	}
+
+	for c in cases {
+		it := utf8.decode_grapheme_iterator_make(c.str)
+		i := 0
+		for text, grapheme in utf8.decode_grapheme_iterate(&it) {
+			if !testing.expectf(t, i < len(c.expected_output), "%q: expected %d clusters, got at least %d", c.str, len(c.expected_output), i + 1) {
+				break
+			}
+			testing.expectf(t, text == c.expected_output[i], "%q cluster %d: expected text %q, got %q", c.str, i, c.expected_output[i], text)
+			testing.expectf(t, text == c.str[grapheme.byte_index:][:len(text)], "%q cluster %d: text does not start at byte_index %d", c.str, i, grapheme.byte_index)
+			i += 1
+		}
+		testing.expectf(t, i == len(c.expected_output), "%q: expected %d clusters, got %d", c.str, len(c.expected_output), i)
+	}
+}