terminal: scalar UTF-8 decode consumes partial sequences cut off by ESC

The scalar fallback of utf8DecodeUntilControlSeq (used when SIMD is
disabled, e.g. wasm builds) treated a valid-so-far but incomplete
UTF-8 sequence at the end of its decode region as pending input in
all cases: it stopped without consuming the bytes so a future chunk
could complete the sequence. That is correct when the region ends
at the end of the input, but the region can also be bounded by an
ESC byte. In that case the sequence can never be completed (the
next byte is already known to be ESC), and the SIMD implementation,
via simdutf, replaces the ill-formed prefix with U+FFFD and
consumes up to the ESC. The two implementations disagreed on both
the consumed count and the decoded output for inputs like
"\xC2\x1B[0m".

The divergence is invisible at the stream level (the pending bytes
take the scalar nextUtf8 path which also emits a replacement
character once it sees the ESC) but it means the scalar decoder is
not a faithful reference for the SIMD one.

This makes the scalar decoder treat a partial sequence bounded by
an ESC as a maximal subpart per Unicode 3-7: one U+FFFD, consumed
through the end of the region. Truncation at the true end of input
still leaves the bytes pending. Also adds a differential fuzz test
that runs 10k random mixtures of ASCII, escapes, controls, and
valid/invalid UTF-8 through both implementations and requires
identical results, which is what caught this.

src/simd/vt.zig

@@ -80,13 +80,18 @@ fn utf8DecodeUntilControlSeqScalar(
         var valid: usize = 1; // lead byte is valid
         for (0..seq.len - 1) |ci| {
             if (decode_offset + valid >= decode.len) {
-                // Truncated at end of buffer: treat as incomplete
-                // input that may be completed later. Stop decoding
-                // without consuming these bytes.
-                return .{
+                // The sequence is cut off by the end of the decode
+                // region. If the region ends at the true end of the
+                // input then it may be completed by future input, so
+                // stop without consuming these bytes. If the region
+                // was bounded by an ESC then the sequence can never
+                // be completed; the valid-so-far prefix is a maximal
+                // subpart which maps to a single U+FFFD below.
+                if (decode.len == input.len) return .{
                     .consumed = decode_offset,
                     .decoded = decode_count,
                 };
+                break;
             }
             const cb = decode[decode_offset + valid];
             if (cb < seq.ranges[ci][0] or cb > seq.ranges[ci][1]) {
@@ -148,6 +153,58 @@ fn utf8SeqInfo(lead: u8) Utf8SeqInfo {
     };
 }
 
+// Differential test: the SIMD implementation must agree with the
+// scalar implementation on any input. Exercises random mixtures of
+// ASCII, escapes, controls, valid and invalid UTF-8, at various
+// lengths (including chunk-boundary straddling cases).
+test "decode simd matches scalar" {
+    if (comptime !options.simd) return error.SkipZigTest;
+
+    const testing = std.testing;
+    var prng = std.Random.DefaultPrng.init(0xf00dface);
+    const rand = prng.random();
+
+    var input: [257]u8 = undefined;
+    var out_simd: [input.len]u32 = undefined;
+    var out_scalar: [input.len]u32 = undefined;
+
+    for (0..10_000) |_| {
+        const len = rand.intRangeAtMost(usize, 0, input.len);
+        const style = rand.intRangeAtMost(u8, 0, 2);
+        for (input[0..len]) |*b| {
+            b.* = switch (style) {
+                // Mostly ASCII with occasional specials.
+                0 => switch (rand.intRangeAtMost(u8, 0, 20)) {
+                    0 => 0x1B,
+                    1 => rand.intRangeAtMost(u8, 0, 0x1F),
+                    2 => rand.int(u8),
+                    else => rand.intRangeAtMost(u8, 0x20, 0x7E),
+                },
+                // Heavy multi-byte/invalid UTF-8.
+                1 => switch (rand.intRangeAtMost(u8, 0, 3)) {
+                    0 => rand.intRangeAtMost(u8, 0x80, 0xBF),
+                    1 => rand.intRangeAtMost(u8, 0xC0, 0xFF),
+                    2 => 0x1B,
+                    else => rand.intRangeAtMost(u8, 0x20, 0x7E),
+                },
+                // Fully random bytes.
+                else => rand.int(u8),
+            };
+        }
+
+        const res_simd = utf8DecodeUntilControlSeq(input[0..len], &out_simd);
+        const res_scalar = utf8DecodeUntilControlSeqScalar(input[0..len], &out_scalar);
+        errdefer std.debug.print("input={x}\n", .{input[0..len]});
+        try testing.expectEqual(res_scalar.consumed, res_simd.consumed);
+        try testing.expectEqual(res_scalar.decoded, res_simd.decoded);
+        try testing.expectEqualSlices(
+            u32,
+            out_scalar[0..res_scalar.decoded],
+            out_simd[0..res_simd.decoded],
+        );
+    }
+}
+
 test "decode no escape" {
     const testing = std.testing;
 
@@ -399,6 +456,35 @@ test "decode valid multibyte surrounded by invalid" {
     }
 }
 
+test "decode partial UTF-8 before escape" {
+    const testing = std.testing;
+
+    // A valid-so-far but incomplete sequence cut off by an ESC can
+    // never be completed, so it is consumed and replaced by a single
+    // U+FFFD (maximal subpart) rather than left pending. Only
+    // sequences cut off by the true end of input are left pending.
+    var output: [64]u32 = undefined;
+
+    // 2-byte lead cut off by ESC.
+    {
+        const str = "hi\xc2\x1b[0m";
+        const result = utf8DecodeUntilControlSeq(str, &output);
+        try testing.expectEqual(@as(usize, 3), result.consumed);
+        try testing.expectEqual(@as(usize, 3), result.decoded);
+        try testing.expectEqual(@as(u32, 0xFFFD), output[2]);
+    }
+
+    // 3-byte lead plus one valid continuation cut off by ESC:
+    // the whole prefix is one maximal subpart, one U+FFFD.
+    {
+        const str = "\xe0\xa0\x1bX";
+        const result = utf8DecodeUntilControlSeq(str, &output);
+        try testing.expectEqual(@as(usize, 2), result.consumed);
+        try testing.expectEqual(@as(usize, 1), result.decoded);
+        try testing.expectEqual(@as(u32, 0xFFFD), output[0]);
+    }
+}
+
 test "decode invalid byte before escape" {
     const testing = std.testing;