From eb09bf82918de51f22b805dc705ed67b2968b984 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:21:32 -0700 Subject: [PATCH] terminal/snapshot: interleave software CRC32C streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slicing tables removed the byte-at-a-time dependency chain, but each 16-byte fold still depends serially on the previous one, leaving the software CRC latency-bound at roughly 2.5-3 GB/s in V8 while snapshot payloads run through it once per direction. wasm has no carry-less multiply, so wider tables are the only classic escape — and measuring slicing-by-32 against interleaving showed the extra 16 KB of tables buys nothing once the chain is hidden. Instead, inputs of 4 KiB and up split into thirds processed as three independent fold chains in one loop, then merge with the GF(2) zero-shift operator: crc(A ++ B, s) = crc(B, 0) XOR zeroShift(crc(A, s), |B|). The shift matrices are comptime, storing only even powers of two (an odd power applies the preceding matrix twice), 4 KB total. Software CRC throughput roughly doubles; hardware backends are untouched, so native is unaffected (tables below are noise). Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.12 ms | 1.73 ms | 2.59 ms | 2.17 ms | | styled | 2.23 ms | 1.47 ms | 6.43 ms | 5.38 ms | | truecolor | 3.44 ms | 2.30 ms | 8.35 ms | 7.17 ms | | cjk | 5.99 ms | 3.89 ms | 11.39 ms | 9.50 ms | | grapheme | 8.33 ms | 6.94 ms | 10.89 ms | 9.24 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.7 ms | 24.8 ms | | ascii | decode | 47.5 ms | 49.2 ms | | utf8 | encode | 41.9 ms | 42.4 ms | | utf8 | decode | 58.8 ms | 59.5 ms | --- src/crc32c.zig | 201 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 171 insertions(+), 30 deletions(-) diff --git a/src/crc32c.zig b/src/crc32c.zig index dbfca7ef2..3d8aaca10 100644 --- a/src/crc32c.zig +++ b/src/crc32c.zig @@ -7,8 +7,7 @@ //! compile time. //! //! Targets without a dedicated instruction, such as WebAssembly, use a -//! slicing-by-16 table implementation that processes sixteen bytes per -//! iteration instead of one. +//! custom implementation that is faster than Zig's stdlib. //! //! The resulting value is identical across all backends: this is the //! iSCSI CRC32C parameter set (reflected, initial and final XOR @@ -149,13 +148,21 @@ const Software = struct { /// The reflected CRC32C (Castagnoli) polynomial. const reflected_poly: u32 = 0x82F63B78; - /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed - /// by `i` zero bytes. Table zero is the classic one-byte-per-step table; - /// the higher tables let one iteration fold sixteen input bytes with - /// sixteen independent lookups instead of a sixteen-step dependency chain. - const tables: [16][256]u32 = tables: { - @setEvalBranchQuota(100_000); - var result: [16][256]u32 = undefined; + /// Number of slicing tables, which is also the bytes folded per + /// iteration. + const slices = 16; + + /// Inputs below this length use the single-stream pass: the + /// stream-combine matrix work would not pay for itself. + const multi_stream_threshold = 4096; + + /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed by + /// `i` zero bytes. Table zero is the classic one-byte-per-step table; + /// the higher tables let one iteration fold a whole block with + /// independent lookups instead of a byte-by-byte dependency chain. + const tables: [slices][256]u32 = tables: { + @setEvalBranchQuota(200_000); + var result: [slices][256]u32 = undefined; for (0..256) |n| { var crc: u32 = n; for (0..8) |_| { @@ -163,7 +170,7 @@ const Software = struct { } result[0][n] = crc; } - for (1..16) |i| { + for (1..slices) |i| { for (0..256) |n| { const prev = result[i - 1][n]; result[i][n] = (prev >> 8) ^ result[0][prev & 0xFF]; @@ -172,34 +179,135 @@ const Software = struct { break :tables result; }; - /// One update pass using slicing-by-16: each iteration XORs the running - /// CRC into the first of four little-endian words and folds all sixteen - /// bytes through per-position tables. The remainder finishes one byte per - /// step through table zero. fn update(initial: u32, bytes: []const u8) u32 { + if (bytes.len >= multi_stream_threshold) return updateMulti( + initial, + bytes, + ); + + return updateSingle(initial, bytes); + } + + /// One single-stream update pass using slicing. + fn updateSingle(initial: u32, bytes: []const u8) u32 { const t = &tables; var crc = initial; - var remaining = bytes; - - while (remaining.len >= 16) : (remaining = remaining[16..]) { - const a = std.mem.readInt(u32, remaining[0..4], .little) ^ crc; - const b = std.mem.readInt(u32, remaining[4..8], .little); - const c = std.mem.readInt(u32, remaining[8..12], .little); - const d = std.mem.readInt(u32, remaining[12..16], .little); - crc = t[15][a & 0xFF] ^ t[14][(a >> 8) & 0xFF] ^ - t[13][(a >> 16) & 0xFF] ^ t[12][a >> 24] ^ - t[11][b & 0xFF] ^ t[10][(b >> 8) & 0xFF] ^ - t[9][(b >> 16) & 0xFF] ^ t[8][b >> 24] ^ - t[7][c & 0xFF] ^ t[6][(c >> 8) & 0xFF] ^ - t[5][(c >> 16) & 0xFF] ^ t[4][c >> 24] ^ - t[3][d & 0xFF] ^ t[2][(d >> 8) & 0xFF] ^ - t[1][(d >> 16) & 0xFF] ^ t[0][d >> 24]; + var i: usize = 0; + while (i + slices <= bytes.len) : (i += slices) { + crc = foldChunk(bytes, i, crc); } - for (remaining) |byte| { + for (bytes[i..]) |byte| { crc = (crc >> 8) ^ t[0][(crc ^ byte) & 0xFF]; } return crc; } + + /// One update pass as three independent interleaved streams. Faster + /// for large enough inputs. + fn updateMulti(initial: u32, bytes: []const u8) u32 { + // Both leading parts are block multiples so the interleaved loop + // needs no tail handling; the third part absorbs the remainder. + const part = (bytes.len / 3) & ~@as(usize, slices - 1); + const p0 = bytes[0..part]; + const p1 = bytes[part..][0..part]; + const p2 = bytes[2 * part ..]; + + var s0 = initial; + var s1: u32 = 0; + var s2: u32 = 0; + var i: usize = 0; + while (i + slices <= part) : (i += slices) { + s0 = foldChunk(p0, i, s0); + s1 = foldChunk(p1, i, s1); + s2 = foldChunk(p2, i, s2); + } + s2 = updateSingle(s2, p2[part..]); + + const s01 = s1 ^ zeroShift(s0, p1.len); + return s2 ^ zeroShift(s01, p2.len); + } + + /// Fold one aligned block through the per-position slicing tables. + /// The running CRC must already be XORed into the block's first word. + inline fn foldBlock(comptime len: usize, words: *const [len / 4]u32) u32 { + const t = &tables; + var crc: u32 = 0; + inline for (0..len / 4) |w| { + const word = words[w]; + const base = len - 1 - w * 4; + crc ^= t[base][word & 0xFF] ^ + t[base - 1][(word >> 8) & 0xFF] ^ + t[base - 2][(word >> 16) & 0xFF] ^ + t[base - 3][word >> 24]; + } + return crc; + } + + /// Fold the block starting at `offset`, chaining the running CRC state. + inline fn foldChunk(bytes: []const u8, offset: usize, crc: u32) u32 { + var words: [slices / 4]u32 = undefined; + inline for (&words, 0..) |*word, w| { + word.* = std.mem.readInt( + u32, + bytes[offset + w * 4 ..][0..4], + .little, + ); + } + words[0] ^= crc; + return foldBlock(slices, &words); + } + + /// Advance a CRC state as if `len` zero bytes had been processed. + fn zeroShift(state: u32, len: usize) u32 { + var s = state; + var remaining = len; + var k: usize = 0; + while (remaining != 0) : ({ + remaining >>= 1; + k += 1; + }) { + if (remaining & 1 != 0) { + const mat = &zero_shift_matrices[k / 2]; + s = matTimesVec(mat, s); + if (k % 2 != 0) s = matTimesVec(mat, s); + } + } + return s; + } + + /// Multiply the GF(2) matrix by a CRC state column vector. + inline fn matTimesVec(mat: *const [32]u32, vec: u32) u32 { + var sum: u32 = 0; + var v = vec; + var i: usize = 0; + while (v != 0) : ({ + v >>= 1; + i += 1; + }) { + if (v & 1 != 0) sum ^= mat[i]; + } + return sum; + } + + const zero_shift_matrices: [32][32]u32 = matrices: { + @setEvalBranchQuota(500_000); + var matrices: [32][32]u32 = undefined; + var previous: [32]u32 = undefined; + for (0..32) |i| { + const unit: u32 = 1 << i; + previous[i] = (unit >> 8) ^ tables[0][unit & 0xFF]; + } + matrices[0] = previous; + for (1..64) |k| { + var squared: [32]u32 = undefined; + for (0..32) |i| { + squared[i] = matTimesVec(&previous, previous[i]); + } + previous = squared; + if (k % 2 == 0) matrices[k / 2] = squared; + } + break :matrices matrices; + }; }; /// The standard-library implementation of the same parameter set. This is @@ -238,6 +346,39 @@ test "software slicing matches the standard library" { } } +test "software multi-stream matches the standard library" { + // Lengths around and far above the interleaving threshold, plus odd + // remainders, so all three streams and both combine steps are covered. + var bytes: [96 * 1024]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0x3517_1A3B); + prng.random().bytes(&bytes); + + for ([_]usize{ + Software.multi_stream_threshold - 1, + Software.multi_stream_threshold, + Software.multi_stream_threshold + 1, + Software.multi_stream_threshold + 97, + 12 * 1024, + 64 * 1024 + 31, + bytes.len, + }) |len| { + const input = bytes[0..len]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(input); + try std.testing.expectEqual( + reference.crc, + Software.update(0xFFFF_FFFF, input), + ); + + // Continuation across a split inside the multi-stream range. + const first = Software.update(0xFFFF_FFFF, input[0 .. len / 2]); + try std.testing.expectEqual( + reference.crc, + Software.update(first, input[len / 2 ..]), + ); + } +} + test "matches the check value" { // The catalog check value for CRC-32/ISCSI. try std.testing.expectEqual(