From cb4c49fbf206ce0c474493a1354629ebba43e2b9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 6 Jul 2026 12:25:57 -0700 Subject: [PATCH] terminal: scalar UTF-8 decode consumes partial sequences cut off by ESC The scalar fallback of utf8DecodeUntilControlSeq (used when SIMD is disabled, e.g. wasm builds) treated a valid-so-far but incomplete UTF-8 sequence at the end of its decode region as pending input in all cases: it stopped without consuming the bytes so a future chunk could complete the sequence. That is correct when the region ends at the end of the input, but the region can also be bounded by an ESC byte. In that case the sequence can never be completed (the next byte is already known to be ESC), and the SIMD implementation, via simdutf, replaces the ill-formed prefix with U+FFFD and consumes up to the ESC. The two implementations disagreed on both the consumed count and the decoded output for inputs like "\xC2\x1B[0m". The divergence is invisible at the stream level (the pending bytes take the scalar nextUtf8 path which also emits a replacement character once it sees the ESC) but it means the scalar decoder is not a faithful reference for the SIMD one. This makes the scalar decoder treat a partial sequence bounded by an ESC as a maximal subpart per Unicode 3-7: one U+FFFD, consumed through the end of the region. Truncation at the true end of input still leaves the bytes pending. Also adds a differential fuzz test that runs 10k random mixtures of ASCII, escapes, controls, and valid/invalid UTF-8 through both implementations and requires identical results, which is what caught this. --- src/simd/vt.zig | 94 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/src/simd/vt.zig b/src/simd/vt.zig index 4230665f4..807285f71 100644 --- a/src/simd/vt.zig +++ b/src/simd/vt.zig @@ -80,13 +80,18 @@ fn utf8DecodeUntilControlSeqScalar( var valid: usize = 1; // lead byte is valid for (0..seq.len - 1) |ci| { if (decode_offset + valid >= decode.len) { - // Truncated at end of buffer: treat as incomplete - // input that may be completed later. Stop decoding - // without consuming these bytes. - return .{ + // The sequence is cut off by the end of the decode + // region. If the region ends at the true end of the + // input then it may be completed by future input, so + // stop without consuming these bytes. If the region + // was bounded by an ESC then the sequence can never + // be completed; the valid-so-far prefix is a maximal + // subpart which maps to a single U+FFFD below. + if (decode.len == input.len) return .{ .consumed = decode_offset, .decoded = decode_count, }; + break; } const cb = decode[decode_offset + valid]; if (cb < seq.ranges[ci][0] or cb > seq.ranges[ci][1]) { @@ -148,6 +153,58 @@ fn utf8SeqInfo(lead: u8) Utf8SeqInfo { }; } +// Differential test: the SIMD implementation must agree with the +// scalar implementation on any input. Exercises random mixtures of +// ASCII, escapes, controls, valid and invalid UTF-8, at various +// lengths (including chunk-boundary straddling cases). +test "decode simd matches scalar" { + if (comptime !options.simd) return error.SkipZigTest; + + const testing = std.testing; + var prng = std.Random.DefaultPrng.init(0xf00dface); + const rand = prng.random(); + + var input: [257]u8 = undefined; + var out_simd: [input.len]u32 = undefined; + var out_scalar: [input.len]u32 = undefined; + + for (0..10_000) |_| { + const len = rand.intRangeAtMost(usize, 0, input.len); + const style = rand.intRangeAtMost(u8, 0, 2); + for (input[0..len]) |*b| { + b.* = switch (style) { + // Mostly ASCII with occasional specials. + 0 => switch (rand.intRangeAtMost(u8, 0, 20)) { + 0 => 0x1B, + 1 => rand.intRangeAtMost(u8, 0, 0x1F), + 2 => rand.int(u8), + else => rand.intRangeAtMost(u8, 0x20, 0x7E), + }, + // Heavy multi-byte/invalid UTF-8. + 1 => switch (rand.intRangeAtMost(u8, 0, 3)) { + 0 => rand.intRangeAtMost(u8, 0x80, 0xBF), + 1 => rand.intRangeAtMost(u8, 0xC0, 0xFF), + 2 => 0x1B, + else => rand.intRangeAtMost(u8, 0x20, 0x7E), + }, + // Fully random bytes. + else => rand.int(u8), + }; + } + + const res_simd = utf8DecodeUntilControlSeq(input[0..len], &out_simd); + const res_scalar = utf8DecodeUntilControlSeqScalar(input[0..len], &out_scalar); + errdefer std.debug.print("input={x}\n", .{input[0..len]}); + try testing.expectEqual(res_scalar.consumed, res_simd.consumed); + try testing.expectEqual(res_scalar.decoded, res_simd.decoded); + try testing.expectEqualSlices( + u32, + out_scalar[0..res_scalar.decoded], + out_simd[0..res_simd.decoded], + ); + } +} + test "decode no escape" { const testing = std.testing; @@ -399,6 +456,35 @@ test "decode valid multibyte surrounded by invalid" { } } +test "decode partial UTF-8 before escape" { + const testing = std.testing; + + // A valid-so-far but incomplete sequence cut off by an ESC can + // never be completed, so it is consumed and replaced by a single + // U+FFFD (maximal subpart) rather than left pending. Only + // sequences cut off by the true end of input are left pending. + var output: [64]u32 = undefined; + + // 2-byte lead cut off by ESC. + { + const str = "hi\xc2\x1b[0m"; + const result = utf8DecodeUntilControlSeq(str, &output); + try testing.expectEqual(@as(usize, 3), result.consumed); + try testing.expectEqual(@as(usize, 3), result.decoded); + try testing.expectEqual(@as(u32, 0xFFFD), output[2]); + } + + // 3-byte lead plus one valid continuation cut off by ESC: + // the whole prefix is one maximal subpart, one U+FFFD. + { + const str = "\xe0\xa0\x1bX"; + const result = utf8DecodeUntilControlSeq(str, &output); + try testing.expectEqual(@as(usize, 2), result.consumed); + try testing.expectEqual(@as(usize, 1), result.decoded); + try testing.expectEqual(@as(u32, 0xFFFD), output[0]); + } +} + test "decode invalid byte before escape" { const testing = std.testing;