diff --git a/src/simd/vt.zig b/src/simd/vt.zig index 4230665f4..807285f71 100644 --- a/src/simd/vt.zig +++ b/src/simd/vt.zig @@ -80,13 +80,18 @@ fn utf8DecodeUntilControlSeqScalar( var valid: usize = 1; // lead byte is valid for (0..seq.len - 1) |ci| { if (decode_offset + valid >= decode.len) { - // Truncated at end of buffer: treat as incomplete - // input that may be completed later. Stop decoding - // without consuming these bytes. - return .{ + // The sequence is cut off by the end of the decode + // region. If the region ends at the true end of the + // input then it may be completed by future input, so + // stop without consuming these bytes. If the region + // was bounded by an ESC then the sequence can never + // be completed; the valid-so-far prefix is a maximal + // subpart which maps to a single U+FFFD below. + if (decode.len == input.len) return .{ .consumed = decode_offset, .decoded = decode_count, }; + break; } const cb = decode[decode_offset + valid]; if (cb < seq.ranges[ci][0] or cb > seq.ranges[ci][1]) { @@ -148,6 +153,58 @@ fn utf8SeqInfo(lead: u8) Utf8SeqInfo { }; } +// Differential test: the SIMD implementation must agree with the +// scalar implementation on any input. Exercises random mixtures of +// ASCII, escapes, controls, valid and invalid UTF-8, at various +// lengths (including chunk-boundary straddling cases). +test "decode simd matches scalar" { + if (comptime !options.simd) return error.SkipZigTest; + + const testing = std.testing; + var prng = std.Random.DefaultPrng.init(0xf00dface); + const rand = prng.random(); + + var input: [257]u8 = undefined; + var out_simd: [input.len]u32 = undefined; + var out_scalar: [input.len]u32 = undefined; + + for (0..10_000) |_| { + const len = rand.intRangeAtMost(usize, 0, input.len); + const style = rand.intRangeAtMost(u8, 0, 2); + for (input[0..len]) |*b| { + b.* = switch (style) { + // Mostly ASCII with occasional specials. + 0 => switch (rand.intRangeAtMost(u8, 0, 20)) { + 0 => 0x1B, + 1 => rand.intRangeAtMost(u8, 0, 0x1F), + 2 => rand.int(u8), + else => rand.intRangeAtMost(u8, 0x20, 0x7E), + }, + // Heavy multi-byte/invalid UTF-8. + 1 => switch (rand.intRangeAtMost(u8, 0, 3)) { + 0 => rand.intRangeAtMost(u8, 0x80, 0xBF), + 1 => rand.intRangeAtMost(u8, 0xC0, 0xFF), + 2 => 0x1B, + else => rand.intRangeAtMost(u8, 0x20, 0x7E), + }, + // Fully random bytes. + else => rand.int(u8), + }; + } + + const res_simd = utf8DecodeUntilControlSeq(input[0..len], &out_simd); + const res_scalar = utf8DecodeUntilControlSeqScalar(input[0..len], &out_scalar); + errdefer std.debug.print("input={x}\n", .{input[0..len]}); + try testing.expectEqual(res_scalar.consumed, res_simd.consumed); + try testing.expectEqual(res_scalar.decoded, res_simd.decoded); + try testing.expectEqualSlices( + u32, + out_scalar[0..res_scalar.decoded], + out_simd[0..res_simd.decoded], + ); + } +} + test "decode no escape" { const testing = std.testing; @@ -399,6 +456,35 @@ test "decode valid multibyte surrounded by invalid" { } } +test "decode partial UTF-8 before escape" { + const testing = std.testing; + + // A valid-so-far but incomplete sequence cut off by an ESC can + // never be completed, so it is consumed and replaced by a single + // U+FFFD (maximal subpart) rather than left pending. Only + // sequences cut off by the true end of input are left pending. + var output: [64]u32 = undefined; + + // 2-byte lead cut off by ESC. + { + const str = "hi\xc2\x1b[0m"; + const result = utf8DecodeUntilControlSeq(str, &output); + try testing.expectEqual(@as(usize, 3), result.consumed); + try testing.expectEqual(@as(usize, 3), result.decoded); + try testing.expectEqual(@as(u32, 0xFFFD), output[2]); + } + + // 3-byte lead plus one valid continuation cut off by ESC: + // the whole prefix is one maximal subpart, one U+FFFD. + { + const str = "\xe0\xa0\x1bX"; + const result = utf8DecodeUntilControlSeq(str, &output); + try testing.expectEqual(@as(usize, 2), result.consumed); + try testing.expectEqual(@as(usize, 1), result.decoded); + try testing.expectEqual(@as(u32, 0xFFFD), output[0]); + } +} + test "decode invalid byte before escape" { const testing = std.testing;