From 36d8e3f77779939a4413ddcd72c05ab08aeae57d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 08:50:19 -0700 Subject: [PATCH 1/9] terminal/snapshot: slicing-by-16 software CRC32C The software CRC32C fallback (WebAssembly and any other target without a dedicated instruction) was the std byte-at-a-time table walk, which profiled at ~65% of snapshot encode and ~70% of decode self-time in V8. Replace it with slicing-by-16: sixteen bytes fold per iteration through comptime per-position tables, so the serial dependency advances one block at a time instead of one byte. The hardware backends (aarch64 CRC, x86_64 SSE4.2) are unchanged, so native is expected to be unaffected; its deltas below are run-to-run noise. Benchmarks: wasm is V8 (node 25), ReleaseFast + wasm-opt -O3, 80x24 terminal, 2 MiB VT corpus per workload, complete snapshot including scrollback, best-of-5. Native is aarch64 macOS, hyperfine mean, ghostty-bench +terminal-snapshot --loops=20. "base" is the parent commit. | wasm | encode base | encode | decode base | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 7.07 ms | 3.43 ms | 6.45 ms | 2.81 ms | | styled | 10.54 ms | 3.14 ms | 14.37 ms | 7.12 ms | | truecolor | 15.23 ms | 5.33 ms | 21.76 ms | 12.11 ms | | cjk | 25.41 ms | 5.95 ms | 30.83 ms | 12.04 ms | | grapheme | 27.67 ms | 14.18 ms | 27.00 ms | 13.16 ms | | native | mode | base | this | |--------|--------|--------:|--------:| | ascii | encode | 40.6 ms | 41.7 ms | | ascii | decode | 51.2 ms | 53.2 ms | | utf8 | encode | 45.2 ms | 47.2 ms | | utf8 | decode | 59.8 ms | 61.6 ms | --- src/crc32c.zig | 118 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 104 insertions(+), 14 deletions(-) diff --git a/src/crc32c.zig b/src/crc32c.zig index 8b8ebd1b2..dbfca7ef2 100644 --- a/src/crc32c.zig +++ b/src/crc32c.zig @@ -4,8 +4,11 @@ //! lookup (as of Zig 0.16), which is more than an order of magnitude slower //! than the dedicated CRC32C instructions available on aarch64 (CRC //! extension) and x86_64 (SSE4.2). This module selects the best backend at -//! compile time and falls back to the standard library elsewhere, including -//! WebAssembly. +//! compile time. +//! +//! Targets without a dedicated instruction, such as WebAssembly, use a +//! slicing-by-16 table implementation that processes sixteen bytes per +//! iteration instead of one. //! //! The resulting value is identical across all backends: this is the //! iSCSI CRC32C parameter set (reflected, initial and final XOR @@ -14,10 +17,6 @@ const std = @import("std"); const builtin = @import("builtin"); -/// The standard-library implementation of the same parameter set. This is -/// both the portable fallback and the reference the tests compare against. -const Software = std.hash.crc.Crc32Iscsi; - const Backend = enum { aarch64_crc, x86_64_sse42, @@ -58,11 +57,7 @@ pub const Crc32c = struct { pub fn update(self: *Crc32c, bytes: []const u8) void { self.crc = switch (comptime backend) { .aarch64_crc, .x86_64_sse42 => updateHardware(self.crc, bytes), - .software => software: { - var crc: Software = .{ .crc = self.crc }; - crc.update(bytes); - break :software crc.crc; - }, + .software => Software.update(self.crc, bytes), }; } @@ -148,6 +143,101 @@ inline fn step(comptime T: type, crc: u32, value: T) u32 { }; } +/// The portable software backend, used by targets without a dedicated +/// CRC32C instruction. +const Software = struct { + /// The reflected CRC32C (Castagnoli) polynomial. + const reflected_poly: u32 = 0x82F63B78; + + /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed + /// by `i` zero bytes. Table zero is the classic one-byte-per-step table; + /// the higher tables let one iteration fold sixteen input bytes with + /// sixteen independent lookups instead of a sixteen-step dependency chain. + const tables: [16][256]u32 = tables: { + @setEvalBranchQuota(100_000); + var result: [16][256]u32 = undefined; + for (0..256) |n| { + var crc: u32 = n; + for (0..8) |_| { + crc = (crc >> 1) ^ (reflected_poly * (crc & 1)); + } + result[0][n] = crc; + } + for (1..16) |i| { + for (0..256) |n| { + const prev = result[i - 1][n]; + result[i][n] = (prev >> 8) ^ result[0][prev & 0xFF]; + } + } + break :tables result; + }; + + /// One update pass using slicing-by-16: each iteration XORs the running + /// CRC into the first of four little-endian words and folds all sixteen + /// bytes through per-position tables. The remainder finishes one byte per + /// step through table zero. + fn update(initial: u32, bytes: []const u8) u32 { + const t = &tables; + var crc = initial; + var remaining = bytes; + + while (remaining.len >= 16) : (remaining = remaining[16..]) { + const a = std.mem.readInt(u32, remaining[0..4], .little) ^ crc; + const b = std.mem.readInt(u32, remaining[4..8], .little); + const c = std.mem.readInt(u32, remaining[8..12], .little); + const d = std.mem.readInt(u32, remaining[12..16], .little); + crc = t[15][a & 0xFF] ^ t[14][(a >> 8) & 0xFF] ^ + t[13][(a >> 16) & 0xFF] ^ t[12][a >> 24] ^ + t[11][b & 0xFF] ^ t[10][(b >> 8) & 0xFF] ^ + t[9][(b >> 16) & 0xFF] ^ t[8][b >> 24] ^ + t[7][c & 0xFF] ^ t[6][(c >> 8) & 0xFF] ^ + t[5][(c >> 16) & 0xFF] ^ t[4][c >> 24] ^ + t[3][d & 0xFF] ^ t[2][(d >> 8) & 0xFF] ^ + t[1][(d >> 16) & 0xFF] ^ t[0][d >> 24]; + } + for (remaining) |byte| { + crc = (crc >> 8) ^ t[0][(crc ^ byte) & 0xFF]; + } + return crc; + } +}; + +/// The standard-library implementation of the same parameter set. This is +/// the reference the tests compare against. +const Reference = std.hash.crc.Crc32Iscsi; + +test "software slicing matches the standard library" { + // The selected backend may be hardware, so cover the sliced software + // path directly: every length around the sixteen-byte boundary, several + // alignments, and continuation across arbitrary split points. + var bytes: [512 + 19]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0x511C); + prng.random().bytes(&bytes); + + for (0..64 + 1) |len| { + for (0..4) |offset| { + const input = bytes[offset..][0..len]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(input); + try std.testing.expectEqual( + reference.crc, + Software.update(0xFFFF_FFFF, input), + ); + } + } + + const long = bytes[0..512]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(long); + for ([_]usize{ 0, 1, 15, 16, 17, 100, 511, 512 }) |split| { + const first = Software.update(0xFFFF_FFFF, long[0..split]); + try std.testing.expectEqual( + reference.crc, + Software.update(first, long[split..]), + ); + } +} + test "matches the check value" { // The catalog check value for CRC-32/ISCSI. try std.testing.expectEqual( @@ -164,7 +254,7 @@ test "matches the standard library at every length and split" { for (0..bytes.len + 1) |len| { const input = bytes[0..len]; try std.testing.expectEqual( - Software.hash(input), + Reference.hash(input), Crc32c.hash(input), ); @@ -174,7 +264,7 @@ test "matches the standard library at every length and split" { split.update(input[0 .. len / 3]); split.update(input[len / 3 .. len - len / 3]); split.update(input[len - len / 3 ..]); - try std.testing.expectEqual(Software.hash(input), split.final()); + try std.testing.expectEqual(Reference.hash(input), split.final()); } } @@ -186,7 +276,7 @@ test "matches the standard library at every alignment" { for (0..16) |offset| { const input = bytes[offset..][0..64]; try std.testing.expectEqual( - Software.hash(input), + Reference.hash(input), Crc32c.hash(input), ); } From c1a61fddda00e907c7e66bd3609d6c558cccd26d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 08:54:25 -0700 Subject: [PATCH 2/9] terminal/snapshot: borrow fully buffered record payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the record source already has the complete payload buffered — always the case for in-memory snapshots such as ghostty_snapshot_decoder_new_buf — the record reader now borrows the payload straight out of the source buffer instead of streaming it through the limited and hashing reader adapters. Payload decoders parse a fixed reader over the borrowed bytes, `finish` validates the CRC with a single bulk update, and the source advances only after validation. The page decoder takes a matching fast path: a fully buffered payload is parsed in place, skipping the staging allocation and copy it previously made per PAGE record. Streaming sources are unchanged. This is a modest win on its own; it is also the foundation for later commits whose buffered fast paths rely on the payload being contiguous. Benchmarks (see the first commit in this series for methodology; "prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 3.43 ms | 3.50 ms | 2.81 ms | 2.68 ms | | styled | 3.14 ms | 3.15 ms | 7.12 ms | 7.04 ms | | truecolor | 5.33 ms | 5.27 ms | 12.11 ms | 12.04 ms | | cjk | 5.95 ms | 5.92 ms | 12.04 ms | 11.89 ms | | grapheme | 14.18 ms | 14.33 ms | 13.16 ms | 13.14 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 41.7 ms | 41.8 ms | | ascii | decode | 53.2 ms | 52.4 ms | | utf8 | encode | 47.2 ms | 47.4 ms | | utf8 | decode | 61.6 ms | 61.2 ms | --- src/terminal/snapshot/page.zig | 12 +++++++- src/terminal/snapshot/record.zig | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index 0d166e389..bd64e483e 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -233,7 +233,17 @@ pub const Decoder = struct { // payload, so this is the byte count of the tables and grid. const remaining = self.record_reader.header.payload_len - Header.len; - if (remaining <= max_staged_payload) { + if (self.record_reader.payloadReader().bufferedLen() >= remaining) { + // The complete payload is already buffered, e.g. borrowed from + // an in-memory snapshot. Parse it in place with no staging + // copy; `finish` still enforces the CRC and exact exhaustion. + try decodePayloadBody( + self.record_reader.payloadReader(), + alloc, + destination, + self.header, + ); + } else if (remaining <= max_staged_payload) { // Stage the payload with one bulk read. The whole payload passes // through the checksum hasher as one update and the payload // decoders then parse a flat buffer, which keeps per-row work free diff --git a/src/terminal/snapshot/record.zig b/src/terminal/snapshot/record.zig index e8dada911..397d00cbe 100644 --- a/src/terminal/snapshot/record.zig +++ b/src/terminal/snapshot/record.zig @@ -257,6 +257,18 @@ pub const Reader = struct { limited: std.Io.Reader.Limited, hashing: std.Io.Reader.Hashed(Crc32c), + /// When the source already has the complete payload buffered, for + /// example an in-memory snapshot, the payload is borrowed straight + /// from the source buffer instead of streaming through the limited + /// and hashing adapters. The checksum is then verified with one bulk + /// update in `finish`, and the source is not advanced until `finish`. + borrowed: ?Borrowed, + + const Borrowed = struct { + source: *std.Io.Reader, + payload: std.Io.Reader, + }; + pub const InitError = Header.DecodeError; /// Errors detected after a payload decoder returns. @@ -278,6 +290,21 @@ pub const Reader = struct { ) InitError!void { self.* = undefined; self.header = try Header.decode(source); + + // The complete payload is already sitting in the source buffer: + // borrow it in place. Payload decoders read from a fixed reader + // over the borrowed bytes and `finish` checksums them in one pass. + if (source.bufferedLen() >= self.header.payload_len) { + self.borrowed = .{ + .source = source, + .payload = .fixed( + source.buffered()[0..self.header.payload_len], + ), + }; + return; + } + + self.borrowed = null; self.limited = .init( source, .limited(self.header.payload_len), @@ -297,11 +324,34 @@ pub const Reader = struct { /// Return the length-limited, checksum-updating payload reader. pub fn payloadReader(self: *Reader) *std.Io.Reader { + if (self.borrowed) |*borrowed| return &borrowed.payload; return &self.hashing.reader; } /// Require exact payload exhaustion and validate its CRC32C. pub fn finish(self: *Reader) FinishError!void { + if (self.borrowed) |*borrowed| { + if (borrowed.payload.bufferedLen() != 0) { + return error.PayloadNotExhausted; + } + + var checksum: Checksum = .init( + self.header.tag, + self.header.payload_len, + ); + checksum.writer().writeAll( + borrowed.payload.buffer[0..borrowed.payload.end], + ) catch unreachable; + if (checksum.final() != self.header.crc32c) { + return error.InvalidChecksum; + } + + // The borrowed bytes validated, so consume them from the + // source only now, leaving it positioned at the next record. + borrowed.source.toss(self.header.payload_len); + return; + } + if (self.hashing.reader.bufferedLen() != 0 or self.limited.remaining != .nothing) { From 973f619a2371c50fffb2062bd59a0c5134c783b1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 08:57:29 -0700 Subject: [PATCH 3/9] terminal/snapshot: vectorize grid row encoding Row encoding previously made two scalar passes over every row (a backward scan for the encoded cell count and a validation pass accumulating the width-selection OR), then wrote a 3-byte header and per-width chunked cells through separate writer calls. Three changes, all bulk-codec only with the portable path unchanged: - scanRow computes the count and word-OR in @Vector(4, u64) strides. Trailing default cells are all-zero words, so the OR over the whole row equals the OR over the encoded prefix. - The per-cell wide-pair validation loop is skipped entirely when the OR carries no wide bits, which is every row of plain text. - Rows are emitted with a single reservation in the destination's spare buffer capacity (header plus cells, no writer calls), using explicit i8x16.shuffle truncation for the 1/2/4-byte cell widths. Zig 0.16 disables loop auto-vectorization, so the previous "vectorizable" truncating loop was actually scalar. Destinations without buffered capacity (counting writers, a still-growing scratch) fall through to the streaming path. Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 3.50 ms | 2.09 ms | 2.68 ms | 2.67 ms | | styled | 3.15 ms | 2.23 ms | 7.04 ms | 7.00 ms | | truecolor | 5.27 ms | 4.95 ms | 12.04 ms | 11.85 ms | | cjk | 5.92 ms | 6.03 ms | 11.89 ms | 11.67 ms | | grapheme | 14.33 ms | 13.08 ms | 13.14 ms | 13.28 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 41.8 ms | 24.5 ms | | ascii | decode | 52.4 ms | 53.4 ms | | utf8 | encode | 47.4 ms | 47.6 ms | | utf8 | decode | 61.2 ms | 60.9 ms | --- src/terminal/snapshot/grid.zig | 219 +++++++++++++++++++++++++++------ 1 file changed, 178 insertions(+), 41 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index fe5d05528..0b5402330 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -468,25 +468,21 @@ pub fn encode( const row = page.getRow(y); const cells = page.getCells(row); - // Trailing default cells decode implicitly. Wide/spacer pairs and + // Trailing default cells decode implicitly: wide/spacer pairs and // hyperlinked or styled cells are always nonzero, so eliding the - // zero suffix never drops encoded state. - const count: usize = count: { - var i: usize = cells.len; - while (i > 0) : (i -= 1) { - if (!cells[i - 1].isZero()) break :count i; - } - break :count 0; - }; + // zero suffix never drops encoded state. The scan also accumulates + // the OR of the row's cell words to select its encoded cell width. + const count: usize, const word_or: u64 = scanRow(cells); // Validate the wide state of every encoded cell so we don't encode - // corrupt data, and accumulate the OR of the row's cell words to - // select its encoded cell width. Trailing default cells are narrow, - // so checking the encoded prefix against the full row width covers - // every pair. - var word_or: u64 = 0; - for (cells[0..count], 0..) |*cell, x| { - switch (cell.wide) { + // corrupt data. The width bits of the OR word witness whether any + // encoded cell is non-narrow at all; rows without them, the common + // case, satisfy every pair rule vacuously. Trailing default cells + // are narrow, so checking the encoded prefix against the full row + // width covers every pair. + const wide_mask: u64 = comptime @bitCast(Cell{ .width = 3 }); + if (word_or & wide_mask != 0) { + for (cells[0..count], 0..) |*cell, x| switch (cell.wide) { .narrow => {}, .wide => if (x + 1 == cells.len or cells[x + 1].wide != .spacer_tail) @@ -501,21 +497,66 @@ pub fn encode( .spacer_head => if (x + 1 != cells.len or !row.wrap) { return error.InvalidWideCell; }, - } - word_or |= classifyWord(cell); + }; } // Canonical rows use the smallest admissible width. const cell_width: Cell.EncodedWidth = .select(word_or); + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + .cell_width = cell_width, + }; + + // The bulk codec emits the row header and its encoded cells directly + // into the destination's spare buffer capacity, so a row costs no + // writer call at all instead of one for the header and one per cell + // chunk. + if (comptime bulk_codec) emit: { + const words: [*]const u64 = @ptrCast(cells.ptr); + switch (cell_width) { + inline .one, .two, .four => |width| { + const size = comptime width.size(); + const needed = 3 + count * size; + if (writer.unusedCapacityLen() < needed) break :emit; + const out = writer.unusedCapacitySlice()[0..needed]; + out[0] = @bitCast(row_header); + std.mem.writeInt(u16, out[1..3], @intCast(count), .little); + encodeNarrowInto(width, words, count, out[3..]); + writer.advance(needed); + continue; + }, + .eight => { + // Hyperlink IDs live in a native side table, but we + // embed them in ours, so if we have any hyperlinks we + // need to fall back to the loop below. + const witness: Cell = @bitCast(word_or); + if (!witness.hyperlink and witness.hyperlink_id == 0) { + const needed = 3 + count * 8; + if (writer.unusedCapacityLen() < needed) break :emit; + const out = writer.unusedCapacitySlice()[0..needed]; + out[0] = @bitCast(row_header); + std.mem.writeInt( + u16, + out[1..3], + @intCast(count), + .little, + ); + @memcpy( + out[3..], + std.mem.sliceAsBytes(cells[0..count]), + ); + writer.advance(needed); + continue; + } + }, + } + } + // Row header: flags then the encoded cell count. { - const row_header: Row = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = @intFromEnum(row.semantic_prompt), - .cell_width = cell_width, - }; var header_bytes: [3]u8 = undefined; header_bytes[0] = @bitCast(row_header); std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); @@ -559,6 +600,47 @@ pub fn encode( try encodeGraphemes(page, writer); } +/// The encoded cell count (through the last nonzero cell) and the bitwise +/// OR of every encoded cell word for one row. +fn scanRow(cells: []const TerminalCell) struct { usize, u64 } { + if (comptime bulk_codec) { + const words: [*]const u64 = @ptrCast(cells.ptr); + const V = @Vector(4, u64); + const VPtr = *align(@alignOf(u64)) const V; + + // Count the zero cells using vectorized instructions + var count = cells.len; + while (count >= 4) { + const tail = @as(VPtr, @ptrCast(words + count - 4)).*; + if (@reduce(.Or, tail) != 0) break; + count -= 4; + } + while (count > 0 and words[count - 1] == 0) count -= 1; + + // Accumulate the OR of the classifaction vectorized, with the + // scalar tail continuing from where the vector loop stopped. + const word_or: u64 = word_or: { + var acc: V = @splat(0); + var i: usize = 0; + while (i + 4 <= count) : (i += 4) { + acc |= @as(VPtr, @ptrCast(words + i)).*; + } + var word_or: u64 = @reduce(.Or, acc); + while (i < count) : (i += 1) word_or |= words[i]; + break :word_or word_or; + }; + + return .{ count, word_or }; + } + + // Scalar path, count backwards + var count: usize = cells.len; + while (count > 0 and cells[count - 1].isZero()) count -= 1; + var word_or: u64 = 0; + for (cells[0..count]) |*cell| word_or |= classifyWord(cell); + return .{ count, word_or }; +} + /// The word used to select a row's encoded cell width. This is the cell's /// wire word with the hyperlink flag reflecting the native cell, so linked /// cells and nonzero native padding disqualify every narrow width. @@ -584,35 +666,90 @@ fn encodeNarrowCells( var i: usize = 0; while (i < cells.len) { const n = @min(cells.len - i, chunk.len / size); + for (cells[i..][0..n], 0..) |*cell, j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(classifyWord(cell)), + .little, + ); + } + try writer.writeAll(chunk[0 .. n * size]); + i += n; + } +} - if (comptime native_matches_wire) { - // A pure truncating loop over integers that the compiler can - // vectorize. - const words: [*]const u64 = @ptrCast(cells.ptr); - for (0..n) |j| { - std.mem.writeInt( - width.Int(), - chunk[j * size ..][0..size], - width.truncate(words[i + j]), - .little, - ); +/// Truncate one row's cell words into `out` at the given encoded width. +fn encodeNarrowInto( + comptime width: Cell.EncodedWidth, + words: [*]const u64, + count: usize, + out: []u8, +) void { + comptime assert(bulk_codec); + const size = comptime width.size(); + const V = @Vector(2, u64); + const shift: V = @splat(comptime switch (width) { + .one, .two => @bitOffsetOf(Cell, "content"), + .four => 0, + .eight => unreachable, + }); + const mask: @Vector(size * 4, i32) = comptime mask: { + var mask: [size * 4]i32 = undefined; + for (0..2) |lane| { + for (0..size) |byte| { + mask[lane * size + byte] = @intCast(lane * 8 + byte); + mask[(lane + 2) * size + byte] = + ~@as(i32, @intCast(lane * 8 + byte)); } + } + break :mask mask; + }; + + var j: usize = 0; + while (j + 4 <= count) : (j += 4) encodeNarrowStep( + width, + words, + j, + out, + shift, + mask, + ); + if (j < count) { + if (count >= 4) { + encodeNarrowStep(width, words, count - 4, out, shift, mask); } else { - for (cells[i..][0..n], 0..) |*cell, j| { + while (j < count) : (j += 1) { std.mem.writeInt( width.Int(), - chunk[j * size ..][0..size], - width.truncate(classifyWord(cell)), + out[j * size ..][0..size], + width.truncate(words[j]), .little, ); } } - - try writer.writeAll(chunk[0 .. n * size]); - i += n; } } +/// Emit four truncated cell words starting at cell index `j`. +inline fn encodeNarrowStep( + comptime width: Cell.EncodedWidth, + words: [*]const u64, + j: usize, + out: []u8, + shift: @Vector(2, u64), + mask: @Vector(width.size() * 4, i32), +) void { + const size = comptime width.size(); + const VPtr = *align(@alignOf(u64)) const @Vector(2, u64); + const lo: @Vector(16, u8) = @bitCast(@as(VPtr, @ptrCast(words + j)).* >> shift); + const hi: @Vector(16, u8) = @bitCast(@as(VPtr, @ptrCast(words + j + 2)).* >> shift); + @as( + *align(1) @Vector(size * 4, u8), + @ptrCast(out[j * size ..].ptr), + ).* = @shuffle(u8, lo, hi, mask); +} + /// Encode the grapheme suffix section for every kind 1 cell in the grid. fn encodeGraphemes( page: *const TerminalPage, From 593762cfa11afdd3eaefc9e0bdd9d979d85ab39f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 08:59:48 -0700 Subject: [PATCH 4/9] terminal/snapshot: batch grapheme suffix codec Grapheme suffix encoding made two full passes over the grid (a counting pass, then an emit pass) and issued three writer calls per entry plus one per codepoint. Every grapheme cell owns exactly one entry in the page's grapheme map, so the section count now comes straight from page.graphemeCount() with no counting pass, and entries are batched through a 4 KB buffer with one writer call per flush. The per-entry size check moved into the emit loop; an error still cancels the whole record before any of it is emitted, so error behavior is unchanged. Decoding similarly parsed entry headers and codepoints with one reader call per integer. Fully buffered payloads (the common case after the borrowed-payload commit) now parse entry headers and codepoint runs directly from the buffered bytes, and entries whose target cell cannot carry a suffix discard their codepoints in bulk. Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.09 ms | 2.07 ms | 2.67 ms | 2.69 ms | | styled | 2.23 ms | 2.29 ms | 7.00 ms | 7.06 ms | | truecolor | 4.95 ms | 4.91 ms | 11.85 ms | 11.99 ms | | cjk | 6.03 ms | 6.14 ms | 11.67 ms | 11.83 ms | | grapheme | 13.08 ms | 8.26 ms | 13.28 ms | 11.18 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.5 ms | 24.3 ms | | ascii | decode | 53.4 ms | 50.9 ms | | utf8 | encode | 47.6 ms | 41.2 ms | | utf8 | decode | 60.9 ms | 59.5 ms | --- src/terminal/snapshot/grid.zig | 157 ++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 43 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 0b5402330..2f383ba93 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -755,36 +755,57 @@ fn encodeGraphemes( page: *const TerminalPage, writer: *std.Io.Writer, ) EncodeError!void { - // Count and validate entries before the section header so the count is - // always exact. Rows without the native grapheme hint contain no - // grapheme cells in any intact page. - var entries: u32 = 0; - for (0..page.size.rows) |y| { - const row = page.getRow(y); - if (!row.grapheme) continue; - for (page.getCells(row)) |*cell| { - if (!cell.hasGrapheme()) continue; - const cps = page.lookupGrapheme(cell) orelse unreachable; - if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; - entries += 1; - } - } - - try io.writeInt(writer, u32, entries); + // Every grapheme cell owns exactly one entry in the page's grapheme + // map, so the section header comes straight from the page without a + // counting pass over the grid. Rows without the native grapheme hint + // contain no grapheme cells in any intact page. + const entries = page.graphemeCount(); + try io.writeInt(writer, u32, @intCast(entries)); if (entries == 0) return; + // Entries are batched into a local buffer so hot pages perform one + // writer call per flush instead of several per entry. + var buffer: [4096]u8 = undefined; + var used: usize = 0; + var emitted: usize = 0; for (0..page.size.rows) |y| { const row = page.getRow(y); if (!row.grapheme) continue; for (page.getCells(row), 0..) |*cell, x| { if (!cell.hasGrapheme()) continue; const cps = page.lookupGrapheme(cell) orelse unreachable; - try io.writeInt(writer, u16, @intCast(y)); - try io.writeInt(writer, u16, @intCast(x)); - try io.writeInt(writer, u16, @intCast(cps.len)); - for (cps) |cp| try io.writeInt(writer, u32, cp); + if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; + emitted += 1; + + const needed = 6 + cps.len * 4; + if (buffer.len - used < needed) { + try writer.writeAll(buffer[0..used]); + used = 0; + } + if (needed > buffer.len) { + // An entry larger than the whole buffer streams directly. + try io.writeInt(writer, u16, @intCast(y)); + try io.writeInt(writer, u16, @intCast(x)); + try io.writeInt(writer, u16, @intCast(cps.len)); + for (cps) |cp| try io.writeInt(writer, u32, cp); + continue; + } + + std.mem.writeInt(u16, buffer[used..][0..2], @intCast(y), .little); + std.mem.writeInt(u16, buffer[used + 2 ..][0..2], @intCast(x), .little); + std.mem.writeInt(u16, buffer[used + 4 ..][0..2], @intCast(cps.len), .little); + used += 6; + for (cps) |cp| { + std.mem.writeInt(u32, buffer[used..][0..4], cp, .little); + used += 4; + } } } + try writer.writeAll(buffer[0..used]); + + // The declared count is trusted by decoders for framing, so the grid + // must have produced exactly that many entries. + assert(emitted == entries); } pub const DecodeError = std.Io.Reader.Error || error{ @@ -1183,9 +1204,23 @@ fn decodeGraphemes( ) DecodeError!void { const entries = try io.readInt(reader, u32); for (0..entries) |_| { - const y = try io.readInt(reader, u16); - const x = try io.readInt(reader, u16); - const cp_count = try io.readInt(reader, u16); + // The staged and borrowed payload paths have every header buffered. + const y: u16, const x: u16, const cp_count: u16 = header: { + if (reader.bufferedLen() >= 6) { + const bytes = reader.buffered()[0..6]; + defer reader.toss(6); + break :header .{ + std.mem.readInt(u16, bytes[0..2], .little), + std.mem.readInt(u16, bytes[2..4], .little), + std.mem.readInt(u16, bytes[4..6], .little), + }; + } + break :header .{ + try io.readInt(reader, u16), + try io.readInt(reader, u16), + try io.readInt(reader, u16), + }; + }; // Resolve the target cell. Entries whose target cannot carry a // suffix are optional detail: their codepoints are consumed to @@ -1207,31 +1242,67 @@ fn decodeGraphemes( break :target .{ .row = row, .cell = cell }; }; - // Always consume every declared codepoint. Invalid scalars and NUL are - // not meaningful grapheme suffix components and are ignored. If native - // capacity is exhausted, remove any prefix already attached so the - // cell never exposes a truncated cluster. - var accept = target != null; - for (0..cp_count) |_| { - const cp = try io.readInt(reader, u32); - if (!accept) continue; - if (cp == 0 or !validScalar(cp)) continue; + // Always consume every declared codepoint. Invalid scalars and NUL + // are not meaningful grapheme suffix components and are ignored. A + // dropped entry's codepoints are discarded in bulk. + const resolved = target orelse { + try reader.discardAll(@as(usize, cp_count) * 4); + continue; + }; - page.appendGrapheme( - target.?.row, - target.?.cell, - @intCast(cp), - ) catch { - if (target.?.cell.hasGrapheme()) { - page.clearGrapheme(target.?.cell); - page.updateRowGraphemeFlag(target.?.row); - } - accept = false; - }; + var accept = true; + var index: usize = 0; + while (index < cp_count) { + const buffered = reader.buffered(); + if (buffered.len >= 4) { + const n = @min(cp_count - index, buffered.len / 4); + for (0..n) |i| applyGraphemeSuffix( + page, + resolved.row, + resolved.cell, + &accept, + std.mem.readInt(u32, buffered[i * 4 ..][0..4], .little), + ); + reader.toss(n * 4); + index += n; + } else { + applyGraphemeSuffix( + page, + resolved.row, + resolved.cell, + &accept, + try io.readInt(reader, u32), + ); + index += 1; + } } } } +/// Attach one decoded suffix codepoint to its resolved target cell. +/// +/// If native capacity is exhausted, any prefix already attached is removed +/// so the cell never exposes a truncated cluster, and `accept` latches +/// false so the entry's remaining codepoints are consumed but dropped. +fn applyGraphemeSuffix( + page: *TerminalPage, + row: *TerminalRow, + cell: *TerminalCell, + accept: *bool, + cp: u32, +) void { + if (!accept.*) return; + if (cp == 0 or !validScalar(cp)) return; + + page.appendGrapheme(row, cell, @intCast(cp)) catch { + if (cell.hasGrapheme()) { + page.clearGrapheme(cell); + page.updateRowGraphemeFlag(row); + } + accept.* = false; + }; +} + /// The encoded word for one native cell and its hyperlink ID. fn cellBits(cell: TerminalCell, link_id: TerminalHyperlinkId) u64 { if (comptime native_matches_wire) { From 2aaad3ca99a27d36eff57d6e59e2044005ef2ba4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:01:59 -0700 Subject: [PATCH 5/9] terminal/snapshot: vectorize grid cell decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decoding one- and two-byte cells widened them to their 8-byte words one scalar store at a time. The bulk codec now widens sixteen transported bytes per step with byte shuffles against a zero vector, degrading width-two surrogate lanes to U+FFFD with a vector select, exactly matching the scalar path. Short row tails reprocess the final full window with overlapping stores that rewrite identical bytes. This is a native win: the shuffles lower to NEON and take ascii decode from 38.9 ms to 34.0 ms (measured by toggling this path at the tip of this series). On wasm, V8 runs the scalar fallback at the same speed as the shuffle version — the loop is store-bound either way — so the wasm deltas below are flat. Row decoding also drops per-field packed-struct read-modify-writes in favor of one load and one store per row header, and rows that are fully default (zero header byte, zero encoded cells) skip all work: decoded pages start zeroed, which is exactly the default row and cell state. Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.07 ms | 2.18 ms | 2.69 ms | 2.68 ms | | styled | 2.29 ms | 2.28 ms | 7.06 ms | 6.93 ms | | truecolor | 4.91 ms | 4.93 ms | 11.99 ms | 11.91 ms | | cjk | 6.14 ms | 6.11 ms | 11.83 ms | 11.72 ms | | grapheme | 8.26 ms | 8.28 ms | 11.18 ms | 11.38 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.3 ms | 24.3 ms | | ascii | decode | 50.9 ms | 46.7 ms | | utf8 | encode | 41.2 ms | 41.0 ms | | utf8 | decode | 59.5 ms | 59.5 ms | --- src/terminal/snapshot/grid.zig | 94 ++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 2f383ba93..6ec084e61 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -865,13 +865,21 @@ pub fn decode( break :header .{ row_header, count }; }; + // A fully default row needs no work at all: decoded pages start + // zeroed, which is exactly the default row and cell state. + if (@as(u8, @bitCast(row_header)) == 0 and count == 0) continue; + + // Update the row fields through one load and store instead of a + // read-modify-write per packed field. const row = page.getRow(y); - row.wrap = row_header.wrap; - row.wrap_continuation = row_header.wrap_continuation; - row.semantic_prompt = std.enums.fromInt( + var row_value = row.*; + row_value.wrap = row_header.wrap; + row_value.wrap_continuation = row_header.wrap_continuation; + row_value.semantic_prompt = std.enums.fromInt( TerminalRow.SemanticPrompt, row_header.semantic_prompt, ) orelse .none; + row.* = row_value; const cells = page.getCells(row); if (count > cells.len) return error.InvalidRowCellCount; @@ -983,8 +991,34 @@ fn widenCells( ) void { const size = comptime width.size(); + // With the bulk codec layout this is a pure widening store: sixteen + // transported bytes per step, shuffling each pair of encoded values + // into u64 lane position against a zero vector and shifting them into + // the content field. Zig 0.16 disables loop auto-vectorization, so the + // scalar loop would issue one widening store per cell. + if (comptime bulk_codec) { + const words: [*]u64 = @ptrCast(cells.ptr); + const step = 16 / size; + var i: usize = 0; + while (i + step <= cells.len) : (i += step) { + widenStep(width, bytes, words, i); + } + if (i < cells.len) { + if (cells.len >= step) { + // Reprocess the final full window with overlapping stores, + // which rewrite the same widened values. + widenStep(width, bytes, words, cells.len - step); + } else { + while (i < cells.len) : (i += 1) { + words[i] = width.extend(widenValue(width, bytes[i * size ..])); + } + } + } + return; + } + // When the native cell matches the wire word, this is a pure widening - // loop over integers that the compiler can vectorize. + // loop over integers. if (comptime native_matches_wire) { const words: [*]u64 = @ptrCast(cells.ptr); for (0..cells.len) |i| { @@ -998,6 +1032,58 @@ fn widenCells( } } +/// Widen sixteen transported bytes into their cell words at cell index `i`: +/// shuffle each pair of encoded values into u64 lane position against a +/// zero vector, then shift the value into the content field. Width two +/// first degrades surrogate lanes to U+FFFD, matching `widenValue`. +inline fn widenStep( + comptime width: Cell.EncodedWidth, + bytes: []const u8, + words: [*]u64, + i: usize, +) void { + const size = comptime width.size(); + const step = 16 / size; + + var in: @Vector(16, u8) = @as( + *align(1) const @Vector(16, u8), + @ptrCast(bytes[i * size ..].ptr), + ).*; + + // Width two admits surrogates, which degrade to U+FFFD exactly + // like `widenValue`. Width one cannot encode an invalid scalar. + if (comptime width == .two) { + const values: @Vector(8, u16) = @bitCast(in); + const invalid = (values & @as(@Vector(8, u16), @splat(0xF800))) == + @as(@Vector(8, u16), @splat(0xD800)); + in = @bitCast(@select( + u16, + invalid, + @as(@Vector(8, u16), @splat(0xFFFD)), + values, + )); + } + + const zero: @Vector(16, u8) = @splat(0); + inline for (0..step / 2) |pair| { + const mask: @Vector(16, i32) = comptime mask: { + var mask: [16]i32 = @splat(~@as(i32, 0)); + for (0..size) |byte| { + mask[byte] = @intCast((2 * pair) * size + byte); + mask[8 + byte] = @intCast((2 * pair + 1) * size + byte); + } + break :mask mask; + }; + const lanes: @Vector(2, u64) = @bitCast( + @shuffle(u8, in, zero, mask), + ); + @as( + *align(@alignOf(u64)) @Vector(2, u64), + @ptrCast(words + i + 2 * pair), + ).* = lanes << @splat(@bitOffsetOf(Cell, "content")); + } +} + /// Read and validate one narrow transported value. inline fn widenValue( comptime width: Cell.EncodedWidth, From 7c1014ef662cd4e9b968b5bac6ca062fa17a3b56 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:10:48 -0700 Subject: [PATCH 6/9] terminal/snapshot: resolve wide pairs in a per-row pass Cell decoding ran wide-pair normalization inline for every decoded cell: two neighbor loads and a switch per cell, even though the overwhelming majority of rows contain no wide cells at all. Normalization is defined against already-stored predecessors, so running it as an ordered pass over the stored row afterward is exactly equivalent to interleaving it. The word-cell decoders now accumulate the bitwise OR of the row's wire words as they apply cells, and the pass is gated on it: rows without wide bits are already normalized (every cell narrow), and width-four and narrower transports cannot encode wide bits at all, so their rows skip the check at comptime. That removes the per-cell neighbor traffic from all styled text, which decodes through the four-byte width. Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.18 ms | 2.06 ms | 2.68 ms | 2.70 ms | | styled | 2.28 ms | 2.41 ms | 6.93 ms | 6.56 ms | | truecolor | 4.93 ms | 5.02 ms | 11.91 ms | 11.86 ms | | cjk | 6.11 ms | 6.03 ms | 11.72 ms | 11.60 ms | | grapheme | 8.28 ms | 8.36 ms | 11.38 ms | 11.39 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.3 ms | 25.6 ms | | ascii | decode | 46.7 ms | 48.1 ms | | utf8 | encode | 41.0 ms | 41.7 ms | | utf8 | decode | 59.5 ms | 58.5 ms | --- src/terminal/snapshot/grid.zig | 51 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 6ec084e61..1fc946e31 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -893,7 +893,7 @@ pub fn decode( reader, cells[0..count], ), - .four => try decodeWordCells( + .four => _ = try decodeWordCells( .four, page, row, @@ -913,7 +913,9 @@ pub fn decode( try reader.readSliceAll( std.mem.sliceAsBytes(cells[0..count]), ); + var row_or: u64 = 0; for (0..count) |x| { + row_or |= words[x]; applyCell( page, row, @@ -924,8 +926,9 @@ pub fn decode( hyperlink_remap, ); } + normalizeWideRow(.eight, row, cells, count, row_or); } else { - try decodeWordCells( + _ = try decodeWordCells( .eight, page, row, @@ -1102,6 +1105,9 @@ inline fn widenValue( /// Decode one row of width-four or fallback full-width cells through the /// complete per-cell normalization path. +/// +/// Returns the bitwise OR of every decoded wire word so callers can gate +/// the trailing wide-pair resolution pass without a second scan. fn decodeWordCells( comptime width: Cell.EncodedWidth, page: *TerminalPage, @@ -1111,8 +1117,9 @@ fn decodeWordCells( reader: *std.Io.Reader, style_remap: *const StyleRemap, hyperlink_remap: *const HyperlinkRemap, -) DecodeError!void { +) DecodeError!u64 { const size = comptime width.size(); + var row_or: u64 = 0; // The staged payload path has the complete row buffered. const total = count * size; @@ -1124,6 +1131,7 @@ fn decodeWordCells( bytes[x * size ..][0..size], .little, )); + row_or |= bits; applyCell( page, row, @@ -1135,12 +1143,14 @@ fn decodeWordCells( ); } reader.toss(total); - return; + normalizeWideRow(width, row, cells, count, row_or); + return row_or; } // Streaming sources fall back to per-cell reads. for (0..count) |x| { const bits = width.extend(try io.readInt(reader, width.Int())); + row_or |= bits; applyCell( page, row, @@ -1151,6 +1161,28 @@ fn decodeWordCells( hyperlink_remap, ); } + normalizeWideRow(width, row, cells, count, row_or); + return row_or; +} + +/// Resolve wide-pair relationships for one decoded row. +/// +/// Cell decoding stores every cell unresolved, so this pass applies +/// `normalizeWide` in order, which is equivalent to interleaving it with +/// the stores. Rows whose word OR carries no wide bits are already +/// normalized: every cell is narrow. Width four and narrower transports +/// cannot encode wide bits at all, so those rows skip the check entirely. +inline fn normalizeWideRow( + comptime width: Cell.EncodedWidth, + row: *const TerminalRow, + cells: []TerminalCell, + count: usize, + row_or: u64, +) void { + if (comptime width != .eight) return; + const wide_mask: u64 = comptime @bitCast(Cell{ .width = 3 }); + if (row_or & wide_mask == 0) return; + for (0..count) |x| normalizeWide(row, cells, x); } /// Whether the value is a valid Unicode scalar value. @@ -1160,10 +1192,10 @@ inline fn validScalar(cp: u32) bool { /// Normalize one encoded cell word and store it at `cells[x]`. /// -/// This owns every per-cell decode rule except grapheme suffixes: content -/// validation, reserved-value degradation, style and hyperlink remapping -/// with reference counting, and wide-pair normalization against already -/// decoded neighbors. +/// This owns every per-cell decode rule except grapheme suffixes and +/// wide-pair resolution: content validation, reserved-value degradation, +/// and style and hyperlink remapping with reference counting. Callers run +/// `normalizeWideRow` over the stored row afterward. fn applyCell( page: *TerminalPage, row: *TerminalRow, @@ -1179,7 +1211,6 @@ fn applyCell( // reference counting, or table lookups. if (bits_wire == 0) { storeCell(cell, 0); - normalizeWide(row, cells, x); return; } @@ -1247,8 +1278,6 @@ fn applyCell( page.hyperlink_set.release(page.memory, link_native); }; } - - normalizeWide(row, cells, x); } /// Resolve wide-pair relationships for the cell at `x` against its already From 47a5182621e324f3351683ca2cc422562b3ff792 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:14:31 -0700 Subject: [PATCH 7/9] terminal/snapshot: skip remap tables for pages without styles Decoding a page allocated and zeroed two full remap tables (a 128 KB entries array plus an 8 KB seen bitmap each for styles and hyperlinks) even when the page declared no table entries at all, which is every page of plain scrollback. Empty tables now use a shared `.empty` remap that allocates nothing; `get` reads it as all-unmapped through a length check. Pages that do declare entries are unchanged. (Leaving the entries array unzeroed behind a seen-bitmap-gated `get` was also tried and measured no better than the plain memset, so the table keeps its simple zero-means-unmapped representation.) Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.06 ms | 2.13 ms | 2.70 ms | 2.54 ms | | styled | 2.41 ms | 2.29 ms | 6.56 ms | 6.75 ms | | truecolor | 5.02 ms | 4.95 ms | 11.86 ms | 12.05 ms | | cjk | 6.03 ms | 6.23 ms | 11.60 ms | 11.39 ms | | grapheme | 8.36 ms | 8.72 ms | 11.39 ms | 11.27 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 25.6 ms | 24.4 ms | | ascii | decode | 48.1 ms | 45.1 ms | | utf8 | encode | 41.7 ms | 41.9 ms | | utf8 | decode | 58.5 ms | 58.6 ms | --- src/terminal/snapshot/grid.zig | 17 +++++++++++++---- src/terminal/snapshot/page.zig | 15 +++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 1fc946e31..3a890913b 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -1515,6 +1515,11 @@ fn Remap(comptime Id: type) type { /// semantics. seen: std.DynamicBitSetUnmanaged, + /// A remap with no entries at all: every lookup is unmapped. Use + /// this instead of `init` when the encoded table is empty so pages + /// without styles or hyperlinks allocate nothing. + pub const empty: Self = .{ .entries = &.{}, .seen = .{} }; + pub fn init(alloc: Allocator) Allocator.Error!Self { const entries = try alloc.alloc(Id, capacity); errdefer alloc.free(entries); @@ -1527,25 +1532,29 @@ fn Remap(comptime Id: type) type { } pub fn deinit(self: *Self, alloc: Allocator) void { - alloc.free(self.entries); - self.seen.deinit(alloc); + if (self.entries.len != 0) { + alloc.free(self.entries); + self.seen.deinit(alloc); + } self.* = undefined; } - /// Record one encoded-to-native mapping. + /// Record one encoded-to-native mapping. Illegal on `empty`. pub fn put(self: *Self, encoded: Id, native: Id) void { assert(!self.seen.isSet(encoded)); self.entries[encoded] = native; self.seen.set(encoded); } - /// Whether the encoded ID already has an entry, even a default one. + /// Whether the encoded ID already has an entry, even a default + /// one. Illegal on `empty`. pub fn contains(self: *const Self, encoded: Id) bool { return self.seen.isSet(encoded); } /// The native ID for an encoded ID, or zero when unmapped. pub inline fn get(self: *const Self, encoded: Id) Id { + if (self.entries.len == 0) return 0; return self.entries[encoded]; } }; diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index bd64e483e..0acd51718 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -367,12 +367,19 @@ fn decodePayloadBody( page.pauseIntegrityChecks(true); defer page.pauseIntegrityChecks(false); - var style_remap = grid.StyleRemap.init(alloc) catch - return error.OutOfMemory; + // Pages without styles or hyperlinks, the common case for plain + // scrollback, skip the remap tables entirely: every encoded cell ID + // resolves to the default through the empty remap. + var style_remap: grid.StyleRemap = if (header.style_count > 0) + grid.StyleRemap.init(alloc) catch return error.OutOfMemory + else + .empty; defer style_remap.deinit(alloc); - var hyperlink_remap = grid.HyperlinkRemap.init(alloc) catch - return error.OutOfMemory; + var hyperlink_remap: grid.HyperlinkRemap = if (header.hyperlink_count > 0) + grid.HyperlinkRemap.init(alloc) catch return error.OutOfMemory + else + .empty; defer hyperlink_remap.deinit(alloc); // Styles From 1359973aefb37a9beaa2ec3e8f79df78290ea6f5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:18:20 -0700 Subject: [PATCH 8/9] terminal/snapshot: single-pass style entry codec Style entries went through roughly five writer or reader vtable calls each: encode wrote three 4-byte colors and two u16s separately, and decode read 16 bytes into a stack buffer only to re-parse it through a nested fixed reader, one small read per field. Style-heavy pages carry hundreds of entries per page, so encode now assembles each entry in a 16-byte buffer with a single write, and decode parses the fixed-size entry directly from a byte array. Entries additionally parse straight from the buffered payload (ID and value together) when it is contiguous. Inserting a decoded style also hashed twice: an explicit `lookup` before `add`, even though `add` already returns the existing entry for repeated values. Insert with `add` alone, taking one reference per accepted table entry, and surrender those references through the encoded-ID remap after grid decoding, the same scheme hyperlink entries already use. Refcount outcomes are identical: each distinct style nets its cell references. Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.13 ms | 2.12 ms | 2.54 ms | 2.59 ms | | styled | 2.29 ms | 2.23 ms | 6.75 ms | 6.43 ms | | truecolor | 4.95 ms | 3.44 ms | 12.05 ms | 8.35 ms | | cjk | 6.23 ms | 5.99 ms | 11.39 ms | 11.39 ms | | grapheme | 8.72 ms | 8.33 ms | 11.27 ms | 10.89 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.4 ms | 24.7 ms | | ascii | decode | 45.1 ms | 47.5 ms | | utf8 | encode | 41.9 ms | 41.9 ms | | utf8 | decode | 58.6 ms | 58.8 ms | --- src/terminal/snapshot/page.zig | 52 +++++++++++------- src/terminal/snapshot/style.zig | 94 ++++++++++++++++----------------- 2 files changed, 80 insertions(+), 66 deletions(-) diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index 0acd51718..6c8bb5ce8 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -382,23 +382,36 @@ fn decodePayloadBody( .empty; defer hyperlink_remap.deinit(alloc); - // Styles + // Styles. The complete fixed-size entry is parsed from the buffered + // payload when possible so each entry costs no reader calls. + const style_entry_len = @sizeOf(TerminalStyleId) + style.len; for (0..header.style_count) |_| { - const native_id = try io.readInt(reader, TerminalStyleId); - const value = try style.decodeOrNull(reader); + const native_id: TerminalStyleId, const value = entry: { + if (reader.bufferedLen() >= style_entry_len) { + const bytes = reader.buffered()[0..style_entry_len]; + defer reader.toss(style_entry_len); + break :entry .{ + std.mem.readInt(TerminalStyleId, bytes[0..2], .little), + style.parseOrNull(bytes[2..][0..style.len]), + }; + } + break :entry .{ + try io.readInt(reader, TerminalStyleId), + try style.decodeOrNull(reader), + }; + }; // Zero is reserved for the implicit default. For a duplicate encoded // ID, the first entry wins and this complete entry is simply ignored. if (native_id == 0 or style_remap.contains(native_id)) continue; - // Invalid/default styles map to the native default. Repeated concrete - // values share the existing native entry, while capacity failure also - // degrades only this style. + // Invalid/default styles map to the native default. `add` returns + // the existing entry for a repeated concrete value, taking one + // reference either way, while capacity failure degrades only this + // style. The references are surrendered through the remap below + // once every cell reference is installed. const decoded_id: TerminalStyleId = if (value) |valid| decoded: { if (valid.default()) break :decoded 0; - if (page.styles.lookup(page.memory, valid)) |existing| { - break :decoded existing; - } break :decoded page.styles.add( page.memory, valid, @@ -434,16 +447,17 @@ fn decodePayloadBody( &hyperlink_remap, ); - // A newly inserted table value starts with one reference so grid decoding - // can safely attach it to any number of cells. Unlike organically built - // pages, that initial reference does not itself represent a cell. Release - // it once per distinct live style after every cell reference is installed; - // unused entries then become dead and disappear from canonical re-encoding. - for (1..@as(usize, page.styles.next_id)) |raw_id| { - const id: TerminalStyleId = @intCast(raw_id); - if (page.styles.refCount(page.memory, id) > 0) { - page.styles.release(page.memory, id); - } + // Every accepted table entry took one reference through `add` so grid + // decoding can safely attach its style to any number of cells. Unlike + // organically built pages, those references do not themselves represent + // cells. Release through the encoded-ID remap, so duplicate values + // which deduplicated to the same native ID each surrender their own + // reference; unused entries then become dead and disappear from + // canonical re-encoding. + var style_it = style_remap.seen.iterator(.{}); + while (style_it.next()) |encoded_id| { + const id = style_remap.entries[encoded_id]; + if (id != 0) page.styles.release(page.memory, id); } // Hyperlink insertion likewise creates one temporary reference for every diff --git a/src/terminal/snapshot/style.zig b/src/terminal/snapshot/style.zig index 176190b75..bd0d445ca 100644 --- a/src/terminal/snapshot/style.zig +++ b/src/terminal/snapshot/style.zig @@ -47,19 +47,14 @@ const std = @import("std"); const test_fixture = @import("fixture.zig"); -const io = @import("io.zig"); const sgr = @import("../sgr.zig"); const terminal_style = @import("../style.zig"); -/// Number of bytes written by `encode`, calculated using the encoder itself -/// so this remains synchronized with the field-by-field wire format. -pub const len = computeLen(); - -comptime { - // This size is part of the wire format. If it changes, the snapshot - // version and golden fixtures must also change. - std.debug.assert(len == 16); -} +/// Number of bytes in one encoded style entry. This size is part of the +/// wire format: the codec reads and writes fixed offsets within an entry +/// of exactly this size, so if the layout changes, the snapshot version +/// and golden fixtures must also change. +pub const len = 16; const Flags = packed struct(u16) { bold: bool = false, @@ -80,8 +75,8 @@ const ColorKind = enum(u8) { rgb = 2, }; -/// Errors possible while decoding one style entry. -pub const DecodeError = std.Io.Reader.Error || error{ +/// Semantic validation errors for one complete style entry buffer. +const ParseError = error{ /// A color kind is not defined by snapshot version 1. InvalidColorKind, @@ -98,14 +93,21 @@ pub const DecodeError = std.Io.Reader.Error || error{ InvalidReserved, }; +/// Errors possible while decoding one style entry. +pub const DecodeError = std.Io.Reader.Error || ParseError; + /// Encode one terminal style as a fixed-size snapshot style entry. +/// +/// The entry is assembled in a fixed buffer and written once, so hot +/// encoders perform a single writer call per style. pub fn encode( value: terminal_style.Style, writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - try encodeColor(value.fg_color, writer); - try encodeColor(value.bg_color, writer); - try encodeColor(value.underline_color, writer); + var encoded: [len]u8 = @splat(0); + encodeColorBuf(encoded[0..4], value.fg_color); + encodeColorBuf(encoded[4..8], value.bg_color); + encodeColorBuf(encoded[8..12], value.underline_color); const flags: Flags = .{ .bold = value.flags.bold, @@ -118,17 +120,26 @@ pub fn encode( .overline = value.flags.overline, .underline = @intFromEnum(value.flags.underline), }; - try io.writeInt(writer, u16, @bitCast(flags)); - try io.writeInt(writer, u16, 0); + std.mem.writeInt(u16, encoded[12..14], @bitCast(flags), .little); + try writer.writeAll(&encoded); } /// Decode and validate one fixed-size snapshot style entry. pub fn decode(reader: *std.Io.Reader) DecodeError!terminal_style.Style { - const fg_color = try decodeColor(reader); - const bg_color = try decodeColor(reader); - const underline_color = try decodeColor(reader); + var encoded: [len]u8 = undefined; + try reader.readSliceAll(&encoded); + return parse(&encoded); +} - const flags: Flags = @bitCast(try io.readInt(reader, u16)); +/// Decode and validate one complete fixed-size style entry buffer. +fn parse(encoded: *const [len]u8) ParseError!terminal_style.Style { + const fg_color = try parseColor(encoded[0..4]); + const bg_color = try parseColor(encoded[4..8]); + const underline_color = try parseColor(encoded[8..12]); + + const flags: Flags = @bitCast( + std.mem.readInt(u16, encoded[12..14], .little), + ); if (flags.reserved != 0) return error.InvalidFlags; const underline = std.enums.fromInt( @@ -136,7 +147,7 @@ pub fn decode(reader: *std.Io.Reader) DecodeError!terminal_style.Style { flags.underline, ) orelse return error.InvalidUnderline; - const reserved = try io.readInt(reader, u16); + const reserved = std.mem.readInt(u16, encoded[14..16], .little); if (reserved != 0) return error.InvalidReserved; return .{ @@ -167,9 +178,7 @@ pub fn decodeOrDiscard( ) DecodeError!terminal_style.Style { var encoded: [len]u8 = undefined; try reader.readSliceAll(&encoded); - - var source: std.Io.Reader = .fixed(&encoded); - return decode(&source); + return parse(&encoded); } /// Decode one complete entry, returning null for invalid semantic contents. @@ -193,11 +202,16 @@ pub fn decodeOrNull( }; } -fn encodeColor( +/// `decodeOrNull` over one already-buffered entry, for enclosing codecs +/// that parse many entries from a flat payload without reader calls. +pub fn parseOrNull(encoded: *const [len]u8) ?terminal_style.Style { + return parse(encoded) catch null; +} + +fn encodeColorBuf( + encoded: *[4]u8, value: terminal_style.Style.Color, - writer: *std.Io.Writer, -) std.Io.Writer.Error!void { - var encoded: [4]u8 = @splat(0); +) void { switch (value) { .none => encoded[0] = @intFromEnum(ColorKind.none), .palette => |index| { @@ -211,23 +225,18 @@ fn encodeColor( encoded[3] = rgb.b; }, } - try writer.writeAll(&encoded); } -fn decodeColor( - reader: *std.Io.Reader, -) DecodeError!terminal_style.Style.Color { - // Colors are always 4 bytes - var encoded: [4]u8 = undefined; - try reader.readSliceAll(&encoded); - +fn parseColor( + encoded: *const [4]u8, +) ParseError!terminal_style.Style.Color { // Kind must be something we know about. const kind = std.enums.fromInt(ColorKind, encoded[0]) orelse { return error.InvalidColorKind; }; return switch (kind) { - .none => if (std.mem.eql(u8, encoded[1..], &.{ 0, 0, 0 })) + .none => if (encoded[1] == 0 and encoded[2] == 0 and encoded[3] == 0) .none else error.InvalidColor, @@ -243,15 +252,6 @@ fn decodeColor( }; } -fn computeLen() usize { - comptime { - var buf: [128]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - encode(.{}, &writer) catch unreachable; - return writer.end; - } -} - const test_golden_fixture = test_fixture.parse(@embedFile("testdata/style-v1.hex")); test "golden encoding and decoding" { From eb09bf82918de51f22b805dc705ed67b2968b984 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 15 Aug 2026 09:21:32 -0700 Subject: [PATCH 9/9] terminal/snapshot: interleave software CRC32C streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slicing tables removed the byte-at-a-time dependency chain, but each 16-byte fold still depends serially on the previous one, leaving the software CRC latency-bound at roughly 2.5-3 GB/s in V8 while snapshot payloads run through it once per direction. wasm has no carry-less multiply, so wider tables are the only classic escape — and measuring slicing-by-32 against interleaving showed the extra 16 KB of tables buys nothing once the chain is hidden. Instead, inputs of 4 KiB and up split into thirds processed as three independent fold chains in one loop, then merge with the GF(2) zero-shift operator: crc(A ++ B, s) = crc(B, 0) XOR zeroShift(crc(A, s), |B|). The shift matrices are comptime, storing only even powers of two (an odd power applies the preceding matrix twice), 4 KB total. Software CRC throughput roughly doubles; hardware backends are untouched, so native is unaffected (tables below are noise). Benchmarks ("prev" is the parent commit): | wasm | encode prev | encode | decode prev | decode | |-----------|------------:|---------:|------------:|---------:| | ascii | 2.12 ms | 1.73 ms | 2.59 ms | 2.17 ms | | styled | 2.23 ms | 1.47 ms | 6.43 ms | 5.38 ms | | truecolor | 3.44 ms | 2.30 ms | 8.35 ms | 7.17 ms | | cjk | 5.99 ms | 3.89 ms | 11.39 ms | 9.50 ms | | grapheme | 8.33 ms | 6.94 ms | 10.89 ms | 9.24 ms | | native | mode | prev | this | |--------|--------|--------:|--------:| | ascii | encode | 24.7 ms | 24.8 ms | | ascii | decode | 47.5 ms | 49.2 ms | | utf8 | encode | 41.9 ms | 42.4 ms | | utf8 | decode | 58.8 ms | 59.5 ms | --- src/crc32c.zig | 201 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 171 insertions(+), 30 deletions(-) diff --git a/src/crc32c.zig b/src/crc32c.zig index dbfca7ef2..3d8aaca10 100644 --- a/src/crc32c.zig +++ b/src/crc32c.zig @@ -7,8 +7,7 @@ //! compile time. //! //! Targets without a dedicated instruction, such as WebAssembly, use a -//! slicing-by-16 table implementation that processes sixteen bytes per -//! iteration instead of one. +//! custom implementation that is faster than Zig's stdlib. //! //! The resulting value is identical across all backends: this is the //! iSCSI CRC32C parameter set (reflected, initial and final XOR @@ -149,13 +148,21 @@ const Software = struct { /// The reflected CRC32C (Castagnoli) polynomial. const reflected_poly: u32 = 0x82F63B78; - /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed - /// by `i` zero bytes. Table zero is the classic one-byte-per-step table; - /// the higher tables let one iteration fold sixteen input bytes with - /// sixteen independent lookups instead of a sixteen-step dependency chain. - const tables: [16][256]u32 = tables: { - @setEvalBranchQuota(100_000); - var result: [16][256]u32 = undefined; + /// Number of slicing tables, which is also the bytes folded per + /// iteration. + const slices = 16; + + /// Inputs below this length use the single-stream pass: the + /// stream-combine matrix work would not pay for itself. + const multi_stream_threshold = 4096; + + /// Slicing tables: `tables[i][b]` is the CRC of byte `b` followed by + /// `i` zero bytes. Table zero is the classic one-byte-per-step table; + /// the higher tables let one iteration fold a whole block with + /// independent lookups instead of a byte-by-byte dependency chain. + const tables: [slices][256]u32 = tables: { + @setEvalBranchQuota(200_000); + var result: [slices][256]u32 = undefined; for (0..256) |n| { var crc: u32 = n; for (0..8) |_| { @@ -163,7 +170,7 @@ const Software = struct { } result[0][n] = crc; } - for (1..16) |i| { + for (1..slices) |i| { for (0..256) |n| { const prev = result[i - 1][n]; result[i][n] = (prev >> 8) ^ result[0][prev & 0xFF]; @@ -172,34 +179,135 @@ const Software = struct { break :tables result; }; - /// One update pass using slicing-by-16: each iteration XORs the running - /// CRC into the first of four little-endian words and folds all sixteen - /// bytes through per-position tables. The remainder finishes one byte per - /// step through table zero. fn update(initial: u32, bytes: []const u8) u32 { + if (bytes.len >= multi_stream_threshold) return updateMulti( + initial, + bytes, + ); + + return updateSingle(initial, bytes); + } + + /// One single-stream update pass using slicing. + fn updateSingle(initial: u32, bytes: []const u8) u32 { const t = &tables; var crc = initial; - var remaining = bytes; - - while (remaining.len >= 16) : (remaining = remaining[16..]) { - const a = std.mem.readInt(u32, remaining[0..4], .little) ^ crc; - const b = std.mem.readInt(u32, remaining[4..8], .little); - const c = std.mem.readInt(u32, remaining[8..12], .little); - const d = std.mem.readInt(u32, remaining[12..16], .little); - crc = t[15][a & 0xFF] ^ t[14][(a >> 8) & 0xFF] ^ - t[13][(a >> 16) & 0xFF] ^ t[12][a >> 24] ^ - t[11][b & 0xFF] ^ t[10][(b >> 8) & 0xFF] ^ - t[9][(b >> 16) & 0xFF] ^ t[8][b >> 24] ^ - t[7][c & 0xFF] ^ t[6][(c >> 8) & 0xFF] ^ - t[5][(c >> 16) & 0xFF] ^ t[4][c >> 24] ^ - t[3][d & 0xFF] ^ t[2][(d >> 8) & 0xFF] ^ - t[1][(d >> 16) & 0xFF] ^ t[0][d >> 24]; + var i: usize = 0; + while (i + slices <= bytes.len) : (i += slices) { + crc = foldChunk(bytes, i, crc); } - for (remaining) |byte| { + for (bytes[i..]) |byte| { crc = (crc >> 8) ^ t[0][(crc ^ byte) & 0xFF]; } return crc; } + + /// One update pass as three independent interleaved streams. Faster + /// for large enough inputs. + fn updateMulti(initial: u32, bytes: []const u8) u32 { + // Both leading parts are block multiples so the interleaved loop + // needs no tail handling; the third part absorbs the remainder. + const part = (bytes.len / 3) & ~@as(usize, slices - 1); + const p0 = bytes[0..part]; + const p1 = bytes[part..][0..part]; + const p2 = bytes[2 * part ..]; + + var s0 = initial; + var s1: u32 = 0; + var s2: u32 = 0; + var i: usize = 0; + while (i + slices <= part) : (i += slices) { + s0 = foldChunk(p0, i, s0); + s1 = foldChunk(p1, i, s1); + s2 = foldChunk(p2, i, s2); + } + s2 = updateSingle(s2, p2[part..]); + + const s01 = s1 ^ zeroShift(s0, p1.len); + return s2 ^ zeroShift(s01, p2.len); + } + + /// Fold one aligned block through the per-position slicing tables. + /// The running CRC must already be XORed into the block's first word. + inline fn foldBlock(comptime len: usize, words: *const [len / 4]u32) u32 { + const t = &tables; + var crc: u32 = 0; + inline for (0..len / 4) |w| { + const word = words[w]; + const base = len - 1 - w * 4; + crc ^= t[base][word & 0xFF] ^ + t[base - 1][(word >> 8) & 0xFF] ^ + t[base - 2][(word >> 16) & 0xFF] ^ + t[base - 3][word >> 24]; + } + return crc; + } + + /// Fold the block starting at `offset`, chaining the running CRC state. + inline fn foldChunk(bytes: []const u8, offset: usize, crc: u32) u32 { + var words: [slices / 4]u32 = undefined; + inline for (&words, 0..) |*word, w| { + word.* = std.mem.readInt( + u32, + bytes[offset + w * 4 ..][0..4], + .little, + ); + } + words[0] ^= crc; + return foldBlock(slices, &words); + } + + /// Advance a CRC state as if `len` zero bytes had been processed. + fn zeroShift(state: u32, len: usize) u32 { + var s = state; + var remaining = len; + var k: usize = 0; + while (remaining != 0) : ({ + remaining >>= 1; + k += 1; + }) { + if (remaining & 1 != 0) { + const mat = &zero_shift_matrices[k / 2]; + s = matTimesVec(mat, s); + if (k % 2 != 0) s = matTimesVec(mat, s); + } + } + return s; + } + + /// Multiply the GF(2) matrix by a CRC state column vector. + inline fn matTimesVec(mat: *const [32]u32, vec: u32) u32 { + var sum: u32 = 0; + var v = vec; + var i: usize = 0; + while (v != 0) : ({ + v >>= 1; + i += 1; + }) { + if (v & 1 != 0) sum ^= mat[i]; + } + return sum; + } + + const zero_shift_matrices: [32][32]u32 = matrices: { + @setEvalBranchQuota(500_000); + var matrices: [32][32]u32 = undefined; + var previous: [32]u32 = undefined; + for (0..32) |i| { + const unit: u32 = 1 << i; + previous[i] = (unit >> 8) ^ tables[0][unit & 0xFF]; + } + matrices[0] = previous; + for (1..64) |k| { + var squared: [32]u32 = undefined; + for (0..32) |i| { + squared[i] = matTimesVec(&previous, previous[i]); + } + previous = squared; + if (k % 2 == 0) matrices[k / 2] = squared; + } + break :matrices matrices; + }; }; /// The standard-library implementation of the same parameter set. This is @@ -238,6 +346,39 @@ test "software slicing matches the standard library" { } } +test "software multi-stream matches the standard library" { + // Lengths around and far above the interleaving threshold, plus odd + // remainders, so all three streams and both combine steps are covered. + var bytes: [96 * 1024]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0x3517_1A3B); + prng.random().bytes(&bytes); + + for ([_]usize{ + Software.multi_stream_threshold - 1, + Software.multi_stream_threshold, + Software.multi_stream_threshold + 1, + Software.multi_stream_threshold + 97, + 12 * 1024, + 64 * 1024 + 31, + bytes.len, + }) |len| { + const input = bytes[0..len]; + var reference: Reference = .{ .crc = 0xFFFF_FFFF }; + reference.update(input); + try std.testing.expectEqual( + reference.crc, + Software.update(0xFFFF_FFFF, input), + ); + + // Continuation across a split inside the multi-stream range. + const first = Software.update(0xFFFF_FFFF, input[0 .. len / 2]); + try std.testing.expectEqual( + reference.crc, + Software.update(first, input[len / 2 ..]), + ); + } +} + test "matches the check value" { // The catalog check value for CRC-32/ISCSI. try std.testing.expectEqual(