diff --git a/src/crc32c.zig b/src/crc32c.zig index 8b8ebd1b2..3d8aaca10 100644 --- a/src/crc32c.zig +++ b/src/crc32c.zig @@ -4,8 +4,10 @@ //! lookup (as of Zig 0.16), which is more than an order of magnitude slower //! than the dedicated CRC32C instructions available on aarch64 (CRC //! extension) and x86_64 (SSE4.2). This module selects the best backend at -//! compile time and falls back to the standard library elsewhere, including -//! WebAssembly. +//! compile time. +//! +//! Targets without a dedicated instruction, such as WebAssembly, use a +//! custom implementation that is faster than Zig's stdlib. //! //! The resulting value is identical across all backends: this is the //! iSCSI CRC32C parameter set (reflected, initial and final XOR @@ -14,10 +16,6 @@ const std = @import("std"); const builtin = @import("builtin"); -/// The standard-library implementation of the same parameter set. This is -/// both the portable fallback and the reference the tests compare against. -const Software = std.hash.crc.Crc32Iscsi; - const Backend = enum { aarch64_crc, x86_64_sse42, @@ -58,11 +56,7 @@ pub const Crc32c = struct { pub fn update(self: *Crc32c, bytes: []const u8) void { self.crc = switch (comptime backend) { .aarch64_crc, .x86_64_sse42 => updateHardware(self.crc, bytes), - .software => software: { - var crc: Software = .{ .crc = self.crc }; - crc.update(bytes); - break :software crc.crc; - }, + .software => Software.update(self.crc, bytes), }; } @@ -148,6 +142,243 @@ inline fn step(comptime T: type, crc: u32, value: T) u32 { }; } +/// The portable software backend, used by targets without a dedicated +/// CRC32C instruction. +const Software = struct { + /// The reflected CRC32C (Castagnoli) polynomial. + const reflected_poly: u32 = 0x82F63B78; + + /// Number of slicing tables, which is also the bytes folded per + /// iteration. + const slices = 16; + + /// Inputs below this length use the single-stream pass: the + /// stream-combine matrix work would not pay for itself. + const multi_stream_threshold = 4096; + + /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed by + /// `i` zero bytes. Table zero is the classic one-byte-per-step table; + /// the higher tables let one iteration fold a whole block with + /// independent lookups instead of a byte-by-byte dependency chain. + const tables: [slices][256]u32 = tables: { + @setEvalBranchQuota(200_000); + var result: [slices][256]u32 = undefined; + for (0..256) |n| { + var crc: u32 = n; + for (0..8) |_| { + crc = (crc >> 1) ^ (reflected_poly * (crc & 1)); + } + result[0][n] = crc; + } + for (1..slices) |i| { + for (0..256) |n| { + const prev = result[i - 1][n]; + result[i][n] = (prev >> 8) ^ result[0][prev & 0xFF]; + } + } + break :tables result; + }; + + fn update(initial: u32, bytes: []const u8) u32 { + if (bytes.len >= multi_stream_threshold) return updateMulti( + initial, + bytes, + ); + + return updateSingle(initial, bytes); + } + + /// One single-stream update pass using slicing. + fn updateSingle(initial: u32, bytes: []const u8) u32 { + const t = &tables; + var crc = initial; + var i: usize = 0; + while (i + slices <= bytes.len) : (i += slices) { + crc = foldChunk(bytes, i, crc); + } + for (bytes[i..]) |byte| { + crc = (crc >> 8) ^ t[0][(crc ^ byte) & 0xFF]; + } + return crc; + } + + /// One update pass as three independent interleaved streams. Faster + /// for large enough inputs. + fn updateMulti(initial: u32, bytes: []const u8) u32 { + // Both leading parts are block multiples so the interleaved loop + // needs no tail handling; the third part absorbs the remainder. + const part = (bytes.len / 3) & ~@as(usize, slices - 1); + const p0 = bytes[0..part]; + const p1 = bytes[part..][0..part]; + const p2 = bytes[2 * part ..]; + + var s0 = initial; + var s1: u32 = 0; + var s2: u32 = 0; + var i: usize = 0; + while (i + slices <= part) : (i += slices) { + s0 = foldChunk(p0, i, s0); + s1 = foldChunk(p1, i, s1); + s2 = foldChunk(p2, i, s2); + } + s2 = updateSingle(s2, p2[part..]); + + const s01 = s1 ^ zeroShift(s0, p1.len); + return s2 ^ zeroShift(s01, p2.len); + } + + /// Fold one aligned block through the per-position slicing tables. + /// The running CRC must already be XORed into the block's first word. + inline fn foldBlock(comptime len: usize, words: *const [len / 4]u32) u32 { + const t = &tables; + var crc: u32 = 0; + inline for (0..len / 4) |w| { + const word = words[w]; + const base = len - 1 - w * 4; + crc ^= t[base][word & 0xFF] ^ + t[base - 1][(word >> 8) & 0xFF] ^ + t[base - 2][(word >> 16) & 0xFF] ^ + t[base - 3][word >> 24]; + } + return crc; + } + + /// Fold the block starting at `offset`, chaining the running CRC state. + inline fn foldChunk(bytes: []const u8, offset: usize, crc: u32) u32 { + var words: [slices / 4]u32 = undefined; + inline for (&words, 0..) |*word, w| { + word.* = std.mem.readInt( + u32, + bytes[offset + w * 4 ..][0..4], + .little, + ); + } + words[0] ^= crc; + return foldBlock(slices, &words); + } + + /// Advance a CRC state as if `len` zero bytes had been processed. + fn zeroShift(state: u32, len: usize) u32 { + var s = state; + var remaining = len; + var k: usize = 0; + while (remaining != 0) : ({ + remaining >>= 1; + k += 1; + }) { + if (remaining & 1 != 0) { + const mat = &zero_shift_matrices[k / 2]; + s = matTimesVec(mat, s); + if (k % 2 != 0) s = matTimesVec(mat, s); + } + } + return s; + } + + /// Multiply the GF(2) matrix by a CRC state column vector. + inline fn matTimesVec(mat: *const [32]u32, vec: u32) u32 { + var sum: u32 = 0; + var v = vec; + var i: usize = 0; + while (v != 0) : ({ + v >>= 1; + i += 1; + }) { + if (v & 1 != 0) sum ^= mat[i]; + } + return sum; + } + + const zero_shift_matrices: [32][32]u32 = matrices: { + @setEvalBranchQuota(500_000); + var matrices: [32][32]u32 = undefined; + var previous: [32]u32 = undefined; + for (0..32) |i| { + const unit: u32 = 1 << i; + previous[i] = (unit >> 8) ^ tables[0][unit & 0xFF]; + } + matrices[0] = previous; + for (1..64) |k| { + var squared: [32]u32 = undefined; + for (0..32) |i| { + squared[i] = matTimesVec(&previous, previous[i]); + } + previous = squared; + if (k % 2 == 0) matrices[k / 2] = squared; + } + break :matrices matrices; + }; +}; + +/// The standard-library implementation of the same parameter set. This is +/// the reference the tests compare against. +const Reference = std.hash.crc.Crc32Iscsi; + +test "software slicing matches the standard library" { + // The selected backend may be hardware, so cover the sliced software + // path directly: every length around the sixteen-byte boundary, several + // alignments, and continuation across arbitrary split points. + var bytes: [512 + 19]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0x511C); + prng.random().bytes(&bytes); + + for (0..64 + 1) |len| { + for (0..4) |offset| { + const input = bytes[offset..][0..len]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(input); + try std.testing.expectEqual( + reference.crc, + Software.update(0xFFFF_FFFF, input), + ); + } + } + + const long = bytes[0..512]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(long); + for ([_]usize{ 0, 1, 15, 16, 17, 100, 511, 512 }) |split| { + const first = Software.update(0xFFFF_FFFF, long[0..split]); + try std.testing.expectEqual( + reference.crc, + Software.update(first, long[split..]), + ); + } +} + +test "software multi-stream matches the standard library" { + // Lengths around and far above the interleaving threshold, plus odd + // remainders, so all three streams and both combine steps are covered. + var bytes: [96 * 1024]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0x3517_1A3B); + prng.random().bytes(&bytes); + + for ([_]usize{ + Software.multi_stream_threshold - 1, + Software.multi_stream_threshold, + Software.multi_stream_threshold + 1, + Software.multi_stream_threshold + 97, + 12 * 1024, + 64 * 1024 + 31, + bytes.len, + }) |len| { + const input = bytes[0..len]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(input); + try std.testing.expectEqual( + reference.crc, + Software.update(0xFFFF_FFFF, input), + ); + + // Continuation across a split inside the multi-stream range. + const first = Software.update(0xFFFF_FFFF, input[0 .. len / 2]); + try std.testing.expectEqual( + reference.crc, + Software.update(first, input[len / 2 ..]), + ); + } +} + test "matches the check value" { // The catalog check value for CRC-32/ISCSI. try std.testing.expectEqual( @@ -164,7 +395,7 @@ test "matches the standard library at every length and split" { for (0..bytes.len + 1) |len| { const input = bytes[0..len]; try std.testing.expectEqual( - Software.hash(input), + Reference.hash(input), Crc32c.hash(input), ); @@ -174,7 +405,7 @@ test "matches the standard library at every length and split" { split.update(input[0 .. len / 3]); split.update(input[len / 3 .. len - len / 3]); split.update(input[len - len / 3 ..]); - try std.testing.expectEqual(Software.hash(input), split.final()); + try std.testing.expectEqual(Reference.hash(input), split.final()); } } @@ -186,7 +417,7 @@ test "matches the standard library at every alignment" { for (0..16) |offset| { const input = bytes[offset..][0..64]; try std.testing.expectEqual( - Software.hash(input), + Reference.hash(input), Crc32c.hash(input), ); } diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index fe5d05528..3a890913b 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -468,25 +468,21 @@ pub fn encode( const row = page.getRow(y); const cells = page.getCells(row); - // Trailing default cells decode implicitly. Wide/spacer pairs and + // Trailing default cells decode implicitly: wide/spacer pairs and // hyperlinked or styled cells are always nonzero, so eliding the - // zero suffix never drops encoded state. - const count: usize = count: { - var i: usize = cells.len; - while (i > 0) : (i -= 1) { - if (!cells[i - 1].isZero()) break :count i; - } - break :count 0; - }; + // zero suffix never drops encoded state. The scan also accumulates + // the OR of the row's cell words to select its encoded cell width. + const count: usize, const word_or: u64 = scanRow(cells); // Validate the wide state of every encoded cell so we don't encode - // corrupt data, and accumulate the OR of the row's cell words to - // select its encoded cell width. Trailing default cells are narrow, - // so checking the encoded prefix against the full row width covers - // every pair. - var word_or: u64 = 0; - for (cells[0..count], 0..) |*cell, x| { - switch (cell.wide) { + // corrupt data. The width bits of the OR word witness whether any + // encoded cell is non-narrow at all; rows without them, the common + // case, satisfy every pair rule vacuously. Trailing default cells + // are narrow, so checking the encoded prefix against the full row + // width covers every pair. + const wide_mask: u64 = comptime @bitCast(Cell{ .width = 3 }); + if (word_or & wide_mask != 0) { + for (cells[0..count], 0..) |*cell, x| switch (cell.wide) { .narrow => {}, .wide => if (x + 1 == cells.len or cells[x + 1].wide != .spacer_tail) @@ -501,21 +497,66 @@ pub fn encode( .spacer_head => if (x + 1 != cells.len or !row.wrap) { return error.InvalidWideCell; }, - } - word_or |= classifyWord(cell); + }; } // Canonical rows use the smallest admissible width. const cell_width: Cell.EncodedWidth = .select(word_or); + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + .cell_width = cell_width, + }; + + // The bulk codec emits the row header and its encoded cells directly + // into the destination's spare buffer capacity, so a row costs no + // writer call at all instead of one for the header and one per cell + // chunk. + if (comptime bulk_codec) emit: { + const words: [*]const u64 = @ptrCast(cells.ptr); + switch (cell_width) { + inline .one, .two, .four => |width| { + const size = comptime width.size(); + const needed = 3 + count * size; + if (writer.unusedCapacityLen() < needed) break :emit; + const out = writer.unusedCapacitySlice()[0..needed]; + out[0] = @bitCast(row_header); + std.mem.writeInt(u16, out[1..3], @intCast(count), .little); + encodeNarrowInto(width, words, count, out[3..]); + writer.advance(needed); + continue; + }, + .eight => { + // Hyperlink IDs live in a native side table, but we + // embed them in ours, so if we have any hyperlinks we + // need to fall back to the loop below. + const witness: Cell = @bitCast(word_or); + if (!witness.hyperlink and witness.hyperlink_id == 0) { + const needed = 3 + count * 8; + if (writer.unusedCapacityLen() < needed) break :emit; + const out = writer.unusedCapacitySlice()[0..needed]; + out[0] = @bitCast(row_header); + std.mem.writeInt( + u16, + out[1..3], + @intCast(count), + .little, + ); + @memcpy( + out[3..], + std.mem.sliceAsBytes(cells[0..count]), + ); + writer.advance(needed); + continue; + } + }, + } + } + // Row header: flags then the encoded cell count. { - const row_header: Row = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = @intFromEnum(row.semantic_prompt), - .cell_width = cell_width, - }; var header_bytes: [3]u8 = undefined; header_bytes[0] = @bitCast(row_header); std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); @@ -559,6 +600,47 @@ pub fn encode( try encodeGraphemes(page, writer); } +/// The encoded cell count (through the last nonzero cell) and the bitwise +/// OR of every encoded cell word for one row. +fn scanRow(cells: []const TerminalCell) struct { usize, u64 } { + if (comptime bulk_codec) { + const words: [*]const u64 = @ptrCast(cells.ptr); + const V = @Vector(4, u64); + const VPtr = *align(@alignOf(u64)) const V; + + // Count the zero cells using vectorized instructions + var count = cells.len; + while (count >= 4) { + const tail = @as(VPtr, @ptrCast(words + count - 4)).*; + if (@reduce(.Or, tail) != 0) break; + count -= 4; + } + while (count > 0 and words[count - 1] == 0) count -= 1; + + // Accumulate the OR of the classifaction vectorized, with the + // scalar tail continuing from where the vector loop stopped. + const word_or: u64 = word_or: { + var acc: V = @splat(0); + var i: usize = 0; + while (i + 4 <= count) : (i += 4) { + acc |= @as(VPtr, @ptrCast(words + i)).*; + } + var word_or: u64 = @reduce(.Or, acc); + while (i < count) : (i += 1) word_or |= words[i]; + break :word_or word_or; + }; + + return .{ count, word_or }; + } + + // Scalar path, count backwards + var count: usize = cells.len; + while (count > 0 and cells[count - 1].isZero()) count -= 1; + var word_or: u64 = 0; + for (cells[0..count]) |*cell| word_or |= classifyWord(cell); + return .{ count, word_or }; +} + /// The word used to select a row's encoded cell width. This is the cell's /// wire word with the hyperlink flag reflecting the native cell, so linked /// cells and nonzero native padding disqualify every narrow width. @@ -584,70 +666,146 @@ fn encodeNarrowCells( var i: usize = 0; while (i < cells.len) { const n = @min(cells.len - i, chunk.len / size); + for (cells[i..][0..n], 0..) |*cell, j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(classifyWord(cell)), + .little, + ); + } + try writer.writeAll(chunk[0 .. n * size]); + i += n; + } +} - if (comptime native_matches_wire) { - // A pure truncating loop over integers that the compiler can - // vectorize. - const words: [*]const u64 = @ptrCast(cells.ptr); - for (0..n) |j| { - std.mem.writeInt( - width.Int(), - chunk[j * size ..][0..size], - width.truncate(words[i + j]), - .little, - ); +/// Truncate one row's cell words into `out` at the given encoded width. +fn encodeNarrowInto( + comptime width: Cell.EncodedWidth, + words: [*]const u64, + count: usize, + out: []u8, +) void { + comptime assert(bulk_codec); + const size = comptime width.size(); + const V = @Vector(2, u64); + const shift: V = @splat(comptime switch (width) { + .one, .two => @bitOffsetOf(Cell, "content"), + .four => 0, + .eight => unreachable, + }); + const mask: @Vector(size * 4, i32) = comptime mask: { + var mask: [size * 4]i32 = undefined; + for (0..2) |lane| { + for (0..size) |byte| { + mask[lane * size + byte] = @intCast(lane * 8 + byte); + mask[(lane + 2) * size + byte] = + ~@as(i32, @intCast(lane * 8 + byte)); } + } + break :mask mask; + }; + + var j: usize = 0; + while (j + 4 <= count) : (j += 4) encodeNarrowStep( + width, + words, + j, + out, + shift, + mask, + ); + if (j < count) { + if (count >= 4) { + encodeNarrowStep(width, words, count - 4, out, shift, mask); } else { - for (cells[i..][0..n], 0..) |*cell, j| { + while (j < count) : (j += 1) { std.mem.writeInt( width.Int(), - chunk[j * size ..][0..size], - width.truncate(classifyWord(cell)), + out[j * size ..][0..size], + width.truncate(words[j]), .little, ); } } - - try writer.writeAll(chunk[0 .. n * size]); - i += n; } } +/// Emit four truncated cell words starting at cell index `j`. +inline fn encodeNarrowStep( + comptime width: Cell.EncodedWidth, + words: [*]const u64, + j: usize, + out: []u8, + shift: @Vector(2, u64), + mask: @Vector(width.size() * 4, i32), +) void { + const size = comptime width.size(); + const VPtr = *align(@alignOf(u64)) const @Vector(2, u64); + const lo: @Vector(16, u8) = @bitCast(@as(VPtr, @ptrCast(words + j)).* >> shift); + const hi: @Vector(16, u8) = @bitCast(@as(VPtr, @ptrCast(words + j + 2)).* >> shift); + @as( + *align(1) @Vector(size * 4, u8), + @ptrCast(out[j * size ..].ptr), + ).* = @shuffle(u8, lo, hi, mask); +} + /// Encode the grapheme suffix section for every kind 1 cell in the grid. fn encodeGraphemes( page: *const TerminalPage, writer: *std.Io.Writer, ) EncodeError!void { - // Count and validate entries before the section header so the count is - // always exact. Rows without the native grapheme hint contain no - // grapheme cells in any intact page. - var entries: u32 = 0; - for (0..page.size.rows) |y| { - const row = page.getRow(y); - if (!row.grapheme) continue; - for (page.getCells(row)) |*cell| { - if (!cell.hasGrapheme()) continue; - const cps = page.lookupGrapheme(cell) orelse unreachable; - if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; - entries += 1; - } - } - - try io.writeInt(writer, u32, entries); + // Every grapheme cell owns exactly one entry in the page's grapheme + // map, so the section header comes straight from the page without a + // counting pass over the grid. Rows without the native grapheme hint + // contain no grapheme cells in any intact page. + const entries = page.graphemeCount(); + try io.writeInt(writer, u32, @intCast(entries)); if (entries == 0) return; + // Entries are batched into a local buffer so hot pages perform one + // writer call per flush instead of several per entry. + var buffer: [4096]u8 = undefined; + var used: usize = 0; + var emitted: usize = 0; for (0..page.size.rows) |y| { const row = page.getRow(y); if (!row.grapheme) continue; for (page.getCells(row), 0..) |*cell, x| { if (!cell.hasGrapheme()) continue; const cps = page.lookupGrapheme(cell) orelse unreachable; - try io.writeInt(writer, u16, @intCast(y)); - try io.writeInt(writer, u16, @intCast(x)); - try io.writeInt(writer, u16, @intCast(cps.len)); - for (cps) |cp| try io.writeInt(writer, u32, cp); + if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; + emitted += 1; + + const needed = 6 + cps.len * 4; + if (buffer.len - used < needed) { + try writer.writeAll(buffer[0..used]); + used = 0; + } + if (needed > buffer.len) { + // An entry larger than the whole buffer streams directly. + try io.writeInt(writer, u16, @intCast(y)); + try io.writeInt(writer, u16, @intCast(x)); + try io.writeInt(writer, u16, @intCast(cps.len)); + for (cps) |cp| try io.writeInt(writer, u32, cp); + continue; + } + + std.mem.writeInt(u16, buffer[used..][0..2], @intCast(y), .little); + std.mem.writeInt(u16, buffer[used + 2 ..][0..2], @intCast(x), .little); + std.mem.writeInt(u16, buffer[used + 4 ..][0..2], @intCast(cps.len), .little); + used += 6; + for (cps) |cp| { + std.mem.writeInt(u32, buffer[used..][0..4], cp, .little); + used += 4; + } } } + try writer.writeAll(buffer[0..used]); + + // The declared count is trusted by decoders for framing, so the grid + // must have produced exactly that many entries. + assert(emitted == entries); } pub const DecodeError = std.Io.Reader.Error || error{ @@ -707,13 +865,21 @@ pub fn decode( break :header .{ row_header, count }; }; + // A fully default row needs no work at all: decoded pages start + // zeroed, which is exactly the default row and cell state. + if (@as(u8, @bitCast(row_header)) == 0 and count == 0) continue; + + // Update the row fields through one load and store instead of a + // read-modify-write per packed field. const row = page.getRow(y); - row.wrap = row_header.wrap; - row.wrap_continuation = row_header.wrap_continuation; - row.semantic_prompt = std.enums.fromInt( + var row_value = row.*; + row_value.wrap = row_header.wrap; + row_value.wrap_continuation = row_header.wrap_continuation; + row_value.semantic_prompt = std.enums.fromInt( TerminalRow.SemanticPrompt, row_header.semantic_prompt, ) orelse .none; + row.* = row_value; const cells = page.getCells(row); if (count > cells.len) return error.InvalidRowCellCount; @@ -727,7 +893,7 @@ pub fn decode( reader, cells[0..count], ), - .four => try decodeWordCells( + .four => _ = try decodeWordCells( .four, page, row, @@ -747,7 +913,9 @@ pub fn decode( try reader.readSliceAll( std.mem.sliceAsBytes(cells[0..count]), ); + var row_or: u64 = 0; for (0..count) |x| { + row_or |= words[x]; applyCell( page, row, @@ -758,8 +926,9 @@ pub fn decode( hyperlink_remap, ); } + normalizeWideRow(.eight, row, cells, count, row_or); } else { - try decodeWordCells( + _ = try decodeWordCells( .eight, page, row, @@ -825,8 +994,34 @@ fn widenCells( ) void { const size = comptime width.size(); + // With the bulk codec layout this is a pure widening store: sixteen + // transported bytes per step, shuffling each pair of encoded values + // into u64 lane position against a zero vector and shifting them into + // the content field. Zig 0.16 disables loop auto-vectorization, so the + // scalar loop would issue one widening store per cell. + if (comptime bulk_codec) { + const words: [*]u64 = @ptrCast(cells.ptr); + const step = 16 / size; + var i: usize = 0; + while (i + step <= cells.len) : (i += step) { + widenStep(width, bytes, words, i); + } + if (i < cells.len) { + if (cells.len >= step) { + // Reprocess the final full window with overlapping stores, + // which rewrite the same widened values. + widenStep(width, bytes, words, cells.len - step); + } else { + while (i < cells.len) : (i += 1) { + words[i] = width.extend(widenValue(width, bytes[i * size ..])); + } + } + } + return; + } + // When the native cell matches the wire word, this is a pure widening - // loop over integers that the compiler can vectorize. + // loop over integers. if (comptime native_matches_wire) { const words: [*]u64 = @ptrCast(cells.ptr); for (0..cells.len) |i| { @@ -840,6 +1035,58 @@ fn widenCells( } } +/// Widen sixteen transported bytes into their cell words at cell index `i`: +/// shuffle each pair of encoded values into u64 lane position against a +/// zero vector, then shift the value into the content field. Width two +/// first degrades surrogate lanes to U+FFFD, matching `widenValue`. +inline fn widenStep( + comptime width: Cell.EncodedWidth, + bytes: []const u8, + words: [*]u64, + i: usize, +) void { + const size = comptime width.size(); + const step = 16 / size; + + var in: @Vector(16, u8) = @as( + *align(1) const @Vector(16, u8), + @ptrCast(bytes[i * size ..].ptr), + ).*; + + // Width two admits surrogates, which degrade to U+FFFD exactly + // like `widenValue`. Width one cannot encode an invalid scalar. + if (comptime width == .two) { + const values: @Vector(8, u16) = @bitCast(in); + const invalid = (values & @as(@Vector(8, u16), @splat(0xF800))) == + @as(@Vector(8, u16), @splat(0xD800)); + in = @bitCast(@select( + u16, + invalid, + @as(@Vector(8, u16), @splat(0xFFFD)), + values, + )); + } + + const zero: @Vector(16, u8) = @splat(0); + inline for (0..step / 2) |pair| { + const mask: @Vector(16, i32) = comptime mask: { + var mask: [16]i32 = @splat(~@as(i32, 0)); + for (0..size) |byte| { + mask[byte] = @intCast((2 * pair) * size + byte); + mask[8 + byte] = @intCast((2 * pair + 1) * size + byte); + } + break :mask mask; + }; + const lanes: @Vector(2, u64) = @bitCast( + @shuffle(u8, in, zero, mask), + ); + @as( + *align(@alignOf(u64)) @Vector(2, u64), + @ptrCast(words + i + 2 * pair), + ).* = lanes << @splat(@bitOffsetOf(Cell, "content")); + } +} + /// Read and validate one narrow transported value. inline fn widenValue( comptime width: Cell.EncodedWidth, @@ -858,6 +1105,9 @@ inline fn widenValue( /// Decode one row of width-four or fallback full-width cells through the /// complete per-cell normalization path. +/// +/// Returns the bitwise OR of every decoded wire word so callers can gate +/// the trailing wide-pair resolution pass without a second scan. fn decodeWordCells( comptime width: Cell.EncodedWidth, page: *TerminalPage, @@ -867,8 +1117,9 @@ fn decodeWordCells( reader: *std.Io.Reader, style_remap: *const StyleRemap, hyperlink_remap: *const HyperlinkRemap, -) DecodeError!void { +) DecodeError!u64 { const size = comptime width.size(); + var row_or: u64 = 0; // The staged payload path has the complete row buffered. const total = count * size; @@ -880,6 +1131,7 @@ fn decodeWordCells( bytes[x * size ..][0..size], .little, )); + row_or |= bits; applyCell( page, row, @@ -891,12 +1143,14 @@ fn decodeWordCells( ); } reader.toss(total); - return; + normalizeWideRow(width, row, cells, count, row_or); + return row_or; } // Streaming sources fall back to per-cell reads. for (0..count) |x| { const bits = width.extend(try io.readInt(reader, width.Int())); + row_or |= bits; applyCell( page, row, @@ -907,6 +1161,28 @@ fn decodeWordCells( hyperlink_remap, ); } + normalizeWideRow(width, row, cells, count, row_or); + return row_or; +} + +/// Resolve wide-pair relationships for one decoded row. +/// +/// Cell decoding stores every cell unresolved, so this pass applies +/// `normalizeWide` in order, which is equivalent to interleaving it with +/// the stores. Rows whose word OR carries no wide bits are already +/// normalized: every cell is narrow. Width four and narrower transports +/// cannot encode wide bits at all, so those rows skip the check entirely. +inline fn normalizeWideRow( + comptime width: Cell.EncodedWidth, + row: *const TerminalRow, + cells: []TerminalCell, + count: usize, + row_or: u64, +) void { + if (comptime width != .eight) return; + const wide_mask: u64 = comptime @bitCast(Cell{ .width = 3 }); + if (row_or & wide_mask == 0) return; + for (0..count) |x| normalizeWide(row, cells, x); } /// Whether the value is a valid Unicode scalar value. @@ -916,10 +1192,10 @@ inline fn validScalar(cp: u32) bool { /// Normalize one encoded cell word and store it at `cells[x]`. /// -/// This owns every per-cell decode rule except grapheme suffixes: content -/// validation, reserved-value degradation, style and hyperlink remapping -/// with reference counting, and wide-pair normalization against already -/// decoded neighbors. +/// This owns every per-cell decode rule except grapheme suffixes and +/// wide-pair resolution: content validation, reserved-value degradation, +/// and style and hyperlink remapping with reference counting. Callers run +/// `normalizeWideRow` over the stored row afterward. fn applyCell( page: *TerminalPage, row: *TerminalRow, @@ -935,7 +1211,6 @@ fn applyCell( // reference counting, or table lookups. if (bits_wire == 0) { storeCell(cell, 0); - normalizeWide(row, cells, x); return; } @@ -1003,8 +1278,6 @@ fn applyCell( page.hyperlink_set.release(page.memory, link_native); }; } - - normalizeWide(row, cells, x); } /// Resolve wide-pair relationships for the cell at `x` against its already @@ -1046,9 +1319,23 @@ fn decodeGraphemes( ) DecodeError!void { const entries = try io.readInt(reader, u32); for (0..entries) |_| { - const y = try io.readInt(reader, u16); - const x = try io.readInt(reader, u16); - const cp_count = try io.readInt(reader, u16); + // The staged and borrowed payload paths have every header buffered. + const y: u16, const x: u16, const cp_count: u16 = header: { + if (reader.bufferedLen() >= 6) { + const bytes = reader.buffered()[0..6]; + defer reader.toss(6); + break :header .{ + std.mem.readInt(u16, bytes[0..2], .little), + std.mem.readInt(u16, bytes[2..4], .little), + std.mem.readInt(u16, bytes[4..6], .little), + }; + } + break :header .{ + try io.readInt(reader, u16), + try io.readInt(reader, u16), + try io.readInt(reader, u16), + }; + }; // Resolve the target cell. Entries whose target cannot carry a // suffix are optional detail: their codepoints are consumed to @@ -1070,31 +1357,67 @@ fn decodeGraphemes( break :target .{ .row = row, .cell = cell }; }; - // Always consume every declared codepoint. Invalid scalars and NUL are - // not meaningful grapheme suffix components and are ignored. If native - // capacity is exhausted, remove any prefix already attached so the - // cell never exposes a truncated cluster. - var accept = target != null; - for (0..cp_count) |_| { - const cp = try io.readInt(reader, u32); - if (!accept) continue; - if (cp == 0 or !validScalar(cp)) continue; + // Always consume every declared codepoint. Invalid scalars and NUL + // are not meaningful grapheme suffix components and are ignored. A + // dropped entry's codepoints are discarded in bulk. + const resolved = target orelse { + try reader.discardAll(@as(usize, cp_count) * 4); + continue; + }; - page.appendGrapheme( - target.?.row, - target.?.cell, - @intCast(cp), - ) catch { - if (target.?.cell.hasGrapheme()) { - page.clearGrapheme(target.?.cell); - page.updateRowGraphemeFlag(target.?.row); - } - accept = false; - }; + var accept = true; + var index: usize = 0; + while (index < cp_count) { + const buffered = reader.buffered(); + if (buffered.len >= 4) { + const n = @min(cp_count - index, buffered.len / 4); + for (0..n) |i| applyGraphemeSuffix( + page, + resolved.row, + resolved.cell, + &accept, + std.mem.readInt(u32, buffered[i * 4 ..][0..4], .little), + ); + reader.toss(n * 4); + index += n; + } else { + applyGraphemeSuffix( + page, + resolved.row, + resolved.cell, + &accept, + try io.readInt(reader, u32), + ); + index += 1; + } } } } +/// Attach one decoded suffix codepoint to its resolved target cell. +/// +/// If native capacity is exhausted, any prefix already attached is removed +/// so the cell never exposes a truncated cluster, and `accept` latches +/// false so the entry's remaining codepoints are consumed but dropped. +fn applyGraphemeSuffix( + page: *TerminalPage, + row: *TerminalRow, + cell: *TerminalCell, + accept: *bool, + cp: u32, +) void { + if (!accept.*) return; + if (cp == 0 or !validScalar(cp)) return; + + page.appendGrapheme(row, cell, @intCast(cp)) catch { + if (cell.hasGrapheme()) { + page.clearGrapheme(cell); + page.updateRowGraphemeFlag(row); + } + accept.* = false; + }; +} + /// The encoded word for one native cell and its hyperlink ID. fn cellBits(cell: TerminalCell, link_id: TerminalHyperlinkId) u64 { if (comptime native_matches_wire) { @@ -1192,6 +1515,11 @@ fn Remap(comptime Id: type) type { /// semantics. seen: std.DynamicBitSetUnmanaged, + /// A remap with no entries at all: every lookup is unmapped. Use + /// this instead of `init` when the encoded table is empty so pages + /// without styles or hyperlinks allocate nothing. + pub const empty: Self = .{ .entries = &.{}, .seen = .{} }; + pub fn init(alloc: Allocator) Allocator.Error!Self { const entries = try alloc.alloc(Id, capacity); errdefer alloc.free(entries); @@ -1204,25 +1532,29 @@ fn Remap(comptime Id: type) type { } pub fn deinit(self: *Self, alloc: Allocator) void { - alloc.free(self.entries); - self.seen.deinit(alloc); + if (self.entries.len != 0) { + alloc.free(self.entries); + self.seen.deinit(alloc); + } self.* = undefined; } - /// Record one encoded-to-native mapping. + /// Record one encoded-to-native mapping. Illegal on `empty`. pub fn put(self: *Self, encoded: Id, native: Id) void { assert(!self.seen.isSet(encoded)); self.entries[encoded] = native; self.seen.set(encoded); } - /// Whether the encoded ID already has an entry, even a default one. + /// Whether the encoded ID already has an entry, even a default + /// one. Illegal on `empty`. pub fn contains(self: *const Self, encoded: Id) bool { return self.seen.isSet(encoded); } /// The native ID for an encoded ID, or zero when unmapped. pub inline fn get(self: *const Self, encoded: Id) Id { + if (self.entries.len == 0) return 0; return self.entries[encoded]; } }; diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index 0d166e389..6c8bb5ce8 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -233,7 +233,17 @@ pub const Decoder = struct { // payload, so this is the byte count of the tables and grid. const remaining = self.record_reader.header.payload_len - Header.len; - if (remaining <= max_staged_payload) { + if (self.record_reader.payloadReader().bufferedLen() >= remaining) { + // The complete payload is already buffered, e.g. borrowed from + // an in-memory snapshot. Parse it in place with no staging + // copy; `finish` still enforces the CRC and exact exhaustion. + try decodePayloadBody( + self.record_reader.payloadReader(), + alloc, + destination, + self.header, + ); + } else if (remaining <= max_staged_payload) { // Stage the payload with one bulk read. The whole payload passes // through the checksum hasher as one update and the payload // decoders then parse a flat buffer, which keeps per-row work free @@ -357,31 +367,51 @@ fn decodePayloadBody( page.pauseIntegrityChecks(true); defer page.pauseIntegrityChecks(false); - var style_remap = grid.StyleRemap.init(alloc) catch - return error.OutOfMemory; + // Pages without styles or hyperlinks, the common case for plain + // scrollback, skip the remap tables entirely: every encoded cell ID + // resolves to the default through the empty remap. + var style_remap: grid.StyleRemap = if (header.style_count > 0) + grid.StyleRemap.init(alloc) catch return error.OutOfMemory + else + .empty; defer style_remap.deinit(alloc); - var hyperlink_remap = grid.HyperlinkRemap.init(alloc) catch - return error.OutOfMemory; + var hyperlink_remap: grid.HyperlinkRemap = if (header.hyperlink_count > 0) + grid.HyperlinkRemap.init(alloc) catch return error.OutOfMemory + else + .empty; defer hyperlink_remap.deinit(alloc); - // Styles + // Styles. The complete fixed-size entry is parsed from the buffered + // payload when possible so each entry costs no reader calls. + const style_entry_len = @sizeOf(TerminalStyleId) + style.len; for (0..header.style_count) |_| { - const native_id = try io.readInt(reader, TerminalStyleId); - const value = try style.decodeOrNull(reader); + const native_id: TerminalStyleId, const value = entry: { + if (reader.bufferedLen() >= style_entry_len) { + const bytes = reader.buffered()[0..style_entry_len]; + defer reader.toss(style_entry_len); + break :entry .{ + std.mem.readInt(TerminalStyleId, bytes[0..2], .little), + style.parseOrNull(bytes[2..][0..style.len]), + }; + } + break :entry .{ + try io.readInt(reader, TerminalStyleId), + try style.decodeOrNull(reader), + }; + }; // Zero is reserved for the implicit default. For a duplicate encoded // ID, the first entry wins and this complete entry is simply ignored. if (native_id == 0 or style_remap.contains(native_id)) continue; - // Invalid/default styles map to the native default. Repeated concrete - // values share the existing native entry, while capacity failure also - // degrades only this style. + // Invalid/default styles map to the native default. `add` returns + // the existing entry for a repeated concrete value, taking one + // reference either way, while capacity failure degrades only this + // style. The references are surrendered through the remap below + // once every cell reference is installed. const decoded_id: TerminalStyleId = if (value) |valid| decoded: { if (valid.default()) break :decoded 0; - if (page.styles.lookup(page.memory, valid)) |existing| { - break :decoded existing; - } break :decoded page.styles.add( page.memory, valid, @@ -417,16 +447,17 @@ fn decodePayloadBody( &hyperlink_remap, ); - // A newly inserted table value starts with one reference so grid decoding - // can safely attach it to any number of cells. Unlike organically built - // pages, that initial reference does not itself represent a cell. Release - // it once per distinct live style after every cell reference is installed; - // unused entries then become dead and disappear from canonical re-encoding. - for (1..@as(usize, page.styles.next_id)) |raw_id| { - const id: TerminalStyleId = @intCast(raw_id); - if (page.styles.refCount(page.memory, id) > 0) { - page.styles.release(page.memory, id); - } + // Every accepted table entry took one reference through `add` so grid + // decoding can safely attach its style to any number of cells. Unlike + // organically built pages, those references do not themselves represent + // cells. Release through the encoded-ID remap, so duplicate values + // which deduplicated to the same native ID each surrender their own + // reference; unused entries then become dead and disappear from + // canonical re-encoding. + var style_it = style_remap.seen.iterator(.{}); + while (style_it.next()) |encoded_id| { + const id = style_remap.entries[encoded_id]; + if (id != 0) page.styles.release(page.memory, id); } // Hyperlink insertion likewise creates one temporary reference for every diff --git a/src/terminal/snapshot/record.zig b/src/terminal/snapshot/record.zig index e8dada911..397d00cbe 100644 --- a/src/terminal/snapshot/record.zig +++ b/src/terminal/snapshot/record.zig @@ -257,6 +257,18 @@ pub const Reader = struct { limited: std.Io.Reader.Limited, hashing: std.Io.Reader.Hashed(Crc32c), + /// When the source already has the complete payload buffered, for + /// example an in-memory snapshot, the payload is borrowed straight + /// from the source buffer instead of streaming through the limited + /// and hashing adapters. The checksum is then verified with one bulk + /// update in `finish`, and the source is not advanced until `finish`. + borrowed: ?Borrowed, + + const Borrowed = struct { + source: *std.Io.Reader, + payload: std.Io.Reader, + }; + pub const InitError = Header.DecodeError; /// Errors detected after a payload decoder returns. @@ -278,6 +290,21 @@ pub const Reader = struct { ) InitError!void { self.* = undefined; self.header = try Header.decode(source); + + // The complete payload is already sitting in the source buffer: + // borrow it in place. Payload decoders read from a fixed reader + // over the borrowed bytes and `finish` checksums them in one pass. + if (source.bufferedLen() >= self.header.payload_len) { + self.borrowed = .{ + .source = source, + .payload = .fixed( + source.buffered()[0..self.header.payload_len], + ), + }; + return; + } + + self.borrowed = null; self.limited = .init( source, .limited(self.header.payload_len), @@ -297,11 +324,34 @@ pub const Reader = struct { /// Return the length-limited, checksum-updating payload reader. pub fn payloadReader(self: *Reader) *std.Io.Reader { + if (self.borrowed) |*borrowed| return &borrowed.payload; return &self.hashing.reader; } /// Require exact payload exhaustion and validate its CRC32C. pub fn finish(self: *Reader) FinishError!void { + if (self.borrowed) |*borrowed| { + if (borrowed.payload.bufferedLen() != 0) { + return error.PayloadNotExhausted; + } + + var checksum: Checksum = .init( + self.header.tag, + self.header.payload_len, + ); + checksum.writer().writeAll( + borrowed.payload.buffer[0..borrowed.payload.end], + ) catch unreachable; + if (checksum.final() != self.header.crc32c) { + return error.InvalidChecksum; + } + + // The borrowed bytes validated, so consume them from the + // source only now, leaving it positioned at the next record. + borrowed.source.toss(self.header.payload_len); + return; + } + if (self.hashing.reader.bufferedLen() != 0 or self.limited.remaining != .nothing) { diff --git a/src/terminal/snapshot/style.zig b/src/terminal/snapshot/style.zig index 176190b75..bd0d445ca 100644 --- a/src/terminal/snapshot/style.zig +++ b/src/terminal/snapshot/style.zig @@ -47,19 +47,14 @@ const std = @import("std"); const test_fixture = @import("fixture.zig"); -const io = @import("io.zig"); const sgr = @import("../sgr.zig"); const terminal_style = @import("../style.zig"); -/// Number of bytes written by `encode`, calculated using the encoder itself -/// so this remains synchronized with the field-by-field wire format. -pub const len = computeLen(); - -comptime { - // This size is part of the wire format. If it changes, the snapshot - // version and golden fixtures must also change. - std.debug.assert(len == 16); -} +/// Number of bytes in one encoded style entry. This size is part of the +/// wire format: the codec reads and writes fixed offsets within an entry +/// of exactly this size, so if the layout changes, the snapshot version +/// and golden fixtures must also change. +pub const len = 16; const Flags = packed struct(u16) { bold: bool = false, @@ -80,8 +75,8 @@ const ColorKind = enum(u8) { rgb = 2, }; -/// Errors possible while decoding one style entry. -pub const DecodeError = std.Io.Reader.Error || error{ +/// Semantic validation errors for one complete style entry buffer. +const ParseError = error{ /// A color kind is not defined by snapshot version 1. InvalidColorKind, @@ -98,14 +93,21 @@ pub const DecodeError = std.Io.Reader.Error || error{ InvalidReserved, }; +/// Errors possible while decoding one style entry. +pub const DecodeError = std.Io.Reader.Error || ParseError; + /// Encode one terminal style as a fixed-size snapshot style entry. +/// +/// The entry is assembled in a fixed buffer and written once, so hot +/// encoders perform a single writer call per style. pub fn encode( value: terminal_style.Style, writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - try encodeColor(value.fg_color, writer); - try encodeColor(value.bg_color, writer); - try encodeColor(value.underline_color, writer); + var encoded: [len]u8 = @splat(0); + encodeColorBuf(encoded[0..4], value.fg_color); + encodeColorBuf(encoded[4..8], value.bg_color); + encodeColorBuf(encoded[8..12], value.underline_color); const flags: Flags = .{ .bold = value.flags.bold, @@ -118,17 +120,26 @@ pub fn encode( .overline = value.flags.overline, .underline = @intFromEnum(value.flags.underline), }; - try io.writeInt(writer, u16, @bitCast(flags)); - try io.writeInt(writer, u16, 0); + std.mem.writeInt(u16, encoded[12..14], @bitCast(flags), .little); + try writer.writeAll(&encoded); } /// Decode and validate one fixed-size snapshot style entry. pub fn decode(reader: *std.Io.Reader) DecodeError!terminal_style.Style { - const fg_color = try decodeColor(reader); - const bg_color = try decodeColor(reader); - const underline_color = try decodeColor(reader); + var encoded: [len]u8 = undefined; + try reader.readSliceAll(&encoded); + return parse(&encoded); +} - const flags: Flags = @bitCast(try io.readInt(reader, u16)); +/// Decode and validate one complete fixed-size style entry buffer. +fn parse(encoded: *const [len]u8) ParseError!terminal_style.Style { + const fg_color = try parseColor(encoded[0..4]); + const bg_color = try parseColor(encoded[4..8]); + const underline_color = try parseColor(encoded[8..12]); + + const flags: Flags = @bitCast( + std.mem.readInt(u16, encoded[12..14], .little), + ); if (flags.reserved != 0) return error.InvalidFlags; const underline = std.enums.fromInt( @@ -136,7 +147,7 @@ pub fn decode(reader: *std.Io.Reader) DecodeError!terminal_style.Style { flags.underline, ) orelse return error.InvalidUnderline; - const reserved = try io.readInt(reader, u16); + const reserved = std.mem.readInt(u16, encoded[14..16], .little); if (reserved != 0) return error.InvalidReserved; return .{ @@ -167,9 +178,7 @@ pub fn decodeOrDiscard( ) DecodeError!terminal_style.Style { var encoded: [len]u8 = undefined; try reader.readSliceAll(&encoded); - - var source: std.Io.Reader = .fixed(&encoded); - return decode(&source); + return parse(&encoded); } /// Decode one complete entry, returning null for invalid semantic contents. @@ -193,11 +202,16 @@ pub fn decodeOrNull( }; } -fn encodeColor( +/// `decodeOrNull` over one already-buffered entry, for enclosing codecs +/// that parse many entries from a flat payload without reader calls. +pub fn parseOrNull(encoded: *const [len]u8) ?terminal_style.Style { + return parse(encoded) catch null; +} + +fn encodeColorBuf( + encoded: *[4]u8, value: terminal_style.Style.Color, - writer: *std.Io.Writer, -) std.Io.Writer.Error!void { - var encoded: [4]u8 = @splat(0); +) void { switch (value) { .none => encoded[0] = @intFromEnum(ColorKind.none), .palette => |index| { @@ -211,23 +225,18 @@ fn encodeColor( encoded[3] = rgb.b; }, } - try writer.writeAll(&encoded); } -fn decodeColor( - reader: *std.Io.Reader, -) DecodeError!terminal_style.Style.Color { - // Colors are always 4 bytes - var encoded: [4]u8 = undefined; - try reader.readSliceAll(&encoded); - +fn parseColor( + encoded: *const [4]u8, +) ParseError!terminal_style.Style.Color { // Kind must be something we know about. const kind = std.enums.fromInt(ColorKind, encoded[0]) orelse { return error.InvalidColorKind; }; return switch (kind) { - .none => if (std.mem.eql(u8, encoded[1..], &.{ 0, 0, 0 })) + .none => if (encoded[1] == 0 and encoded[2] == 0 and encoded[3] == 0) .none else error.InvalidColor, @@ -243,15 +252,6 @@ fn decodeColor( }; } -fn computeLen() usize { - comptime { - var buf: [128]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - encode(.{}, &writer) catch unreachable; - return writer.end; - } -} - const test_golden_fixture = test_fixture.parse(@embedFile("testdata/style-v1.hex")); test "golden encoding and decoding" {