From 7031c892b2d1309d4463a56654b90bd0501f3970 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:32:29 -0700 Subject: [PATCH 1/7] synthetic: line length options for the ascii generator The ascii generator emits an unbroken stream of printable bytes, which exercises terminal wrapping but produces only full-width rows. Add line-min and line-max options that emit CR LF-terminated lines with a uniformly distributed printable length so generated corpora can also model shell-like output where most rows end well before the last column. The default behavior is unchanged. --- src/synthetic/cli/Ascii.zig | 60 +++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/src/synthetic/cli/Ascii.zig b/src/synthetic/cli/Ascii.zig index 81a4e6073..b9ce2cb15 100644 --- a/src/synthetic/cli/Ascii.zig +++ b/src/synthetic/cli/Ascii.zig @@ -7,7 +7,14 @@ const Bytes = @import("../Bytes.zig"); const log = std.log.scoped(.@"terminal-stream-bench"); -pub const Options = struct {}; +pub const Options = struct { + /// When nonzero, emit lines whose printable length is uniformly + /// distributed in `[line-min, line-max]`, each terminated by CR LF. + /// When zero (the default), emit an unbroken stream of printable + /// bytes that relies on terminal wrapping. + @"line-min": usize = 0, + @"line-max": usize = 0, +}; fn checkAsciiAlphabet(c: u8) bool { return switch (c) { @@ -18,13 +25,16 @@ fn checkAsciiAlphabet(c: u8) bool { pub const ascii = Bytes.generateAlphabet(checkAsciiAlphabet); +opts: Options, + /// Create a new terminal stream handler for the given arguments. pub fn create( alloc: Allocator, - _: Options, + opts: Options, ) !*Ascii { const ptr = try alloc.create(Ascii); errdefer alloc.destroy(ptr); + ptr.* = .{ .opts = opts }; return ptr; } @@ -32,16 +42,18 @@ pub fn destroy(self: *Ascii, alloc: Allocator) void { alloc.destroy(self); } -pub fn run(_: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { +pub fn run(self: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { + const line_max = @max(self.opts.@"line-min", self.opts.@"line-max"); + const lines = line_max > 0; var gen: Bytes = .{ .rand = rand, .alphabet = ascii, - .min_len = 1024, - .max_len = 1024, + .min_len = if (lines) @max(self.opts.@"line-min", 1) else 1024, + .max_len = if (lines) line_max else 1024, }; while (true) { - _ = gen.write(writer) catch |err| { + writeChunk(&gen, writer, lines) catch |err| { const Error = error{ WriteFailed, BrokenPipe } || @TypeOf(err); switch (@as(Error, err)) { error.BrokenPipe => return, // stdout closed @@ -51,6 +63,15 @@ pub fn run(_: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { } } +fn writeChunk( + gen: *const Bytes, + writer: *std.Io.Writer, + lines: bool, +) std.Io.Writer.Error!void { + _ = try gen.write(writer); + if (lines) try writer.writeAll("\r\n"); +} + test Ascii { const testing = std.testing; const alloc = testing.allocator; @@ -65,3 +86,30 @@ test Ascii { var writer: std.Io.Writer = .fixed(&buf); try impl.run(&writer, rand); } + +test "Ascii lines" { + const testing = std.testing; + const alloc = testing.allocator; + + const impl: *Ascii = try .create(alloc, .{ + .@"line-min" = 5, + .@"line-max" = 20, + }); + defer impl.destroy(alloc); + + var prng = std.Random.DefaultPrng.init(1); + const rand = prng.random(); + + var buf: [1024]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try impl.run(&writer, rand); + + // Every emitted line respects the configured bounds. + var it = std.mem.splitSequence(u8, writer.buffered(), "\r\n"); + while (it.next()) |line| { + // The fixed buffer may end mid-line. + if (it.rest().len == 0) break; + try testing.expect(line.len >= 5); + try testing.expect(line.len <= 20); + } +} From 0b5e12453b639af3f904ad57123cb075a97a82f8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:33:02 -0700 Subject: [PATCH 2/7] benchmark: add terminal-snapshot benchmark Measures the terminal binary snapshot codecs in both directions against the same terminal state. Setup feeds a pre-generated VT stream (for example from ghostty-gen ascii) to a terminal outside the timed region. Baseline measurements at this commit (M-series, ReleaseFast, 80x24, unlimited scrollback, 1 MB corpora, per-loop time with setup subtracted): ascii lines 1-70: 34.16 MB encode 92.8 ms decode 119.8 ms ascii full-wrap: 16.01 MB encode 43.6 ms decode 56.2 ms utf8: 4.33 MB encode 12.4 ms decode 19.0 ms --- src/benchmark/TerminalSnapshot.zig | 307 +++++++++++++++++++++++++++++ src/benchmark/cli.zig | 2 + src/benchmark/main.zig | 1 + 3 files changed, 310 insertions(+) create mode 100644 src/benchmark/TerminalSnapshot.zig diff --git a/src/benchmark/TerminalSnapshot.zig b/src/benchmark/TerminalSnapshot.zig new file mode 100644 index 000000000..dd191f1ea --- /dev/null +++ b/src/benchmark/TerminalSnapshot.zig @@ -0,0 +1,307 @@ +//! Benchmarks the terminal binary snapshot codecs (`terminal/snapshot`). +//! +//! Encoding and decoding a snapshot are the hot paths for terminal +//! persistence and handoff, so both directions are measured against the +//! same terminal state. +//! +//! ## Input +//! +//! `--data` names a pre-generated VT byte stream (for example from +//! `ghostty-gen ascii`). The stream is fed to a terminal of the requested +//! dimensions during setup, outside the timed region. The resulting screen +//! and scrollback contents are what each step encodes or decodes. +//! +//! ## Modes +//! +//! * `noop` performs no codec work and establishes loop overhead. +//! * `encode` encodes one complete snapshot per loop into a reusable +//! buffer. Buffer growth happens on the first iteration only. +//! * `decode` restores one complete snapshot per loop from bytes prepared +//! during setup. The restored terminal is destroyed inside the step, so +//! this measures the complete restore lifecycle. +//! * `report` encodes once and prints total and per-record-tag sizes. It +//! is for inspecting the wire shape, not timing comparisons. +//! +//! ## Examples +//! +//! Build benchmarks in ReleaseFast mode: +//! +//! zig build -Demit-bench -Doptimize=ReleaseFast -Demit-macos-app=false +//! +//! Generate a deterministic corpus, then measure both directions: +//! +//! ghostty-gen ascii | head -c 1000000 > /tmp/ascii.vt +//! hyperfine --warmup 3 \ +//! 'ghostty-bench +terminal-snapshot --mode=encode --loops=20 --data=/tmp/ascii.vt' \ +//! 'ghostty-bench +terminal-snapshot --mode=decode --loops=20 --data=/tmp/ascii.vt' +const TerminalSnapshot = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const terminalpkg = @import("../terminal/main.zig"); +const snapshot = terminalpkg.snapshot; +const Benchmark = @import("Benchmark.zig"); +const options = @import("options.zig"); +const Terminal = terminalpkg.Terminal; +const global = @import("../global.zig"); + +const log = std.log.scoped(.@"terminal-snapshot-bench"); + +alloc: Allocator, +opts: Options, +terminal: ?Terminal = null, + +/// Reused across encode steps so buffer growth is a one-time setup cost. +encoded: std.Io.Writer.Allocating, + +pub const Options = struct { + /// Set by the shared CLI parser for string option ownership. + _arena: ?std.heap.ArenaAllocator = null, + + /// Select the operation performed inside the timed benchmark step. + mode: Mode = .encode, + + /// Number of codec operations per benchmark step. Increase this when + /// the state is too small for stable `hyperfine` measurements. + loops: u32 = 1, + + /// The size of the terminal. This affects wrapping, page sizes, and + /// the amount of state per encoded page. + @"terminal-rows": u16 = 24, + @"terminal-cols": u16 = 80, + + /// Pre-generated VT stream fed to the terminal during setup. `-` reads + /// stdin, although a regular file is recommended so identical state can + /// be reused across runs. When unset, the terminal is empty. + data: ?[]const u8 = null, + + pub fn deinit(self: *Options) void { + if (self._arena) |arena| arena.deinit(); + self.* = undefined; + } +}; + +pub const Mode = enum { + /// Establish the benchmark loop overhead. + noop, + + /// Encode one complete snapshot per loop. + encode, + + /// Restore one complete snapshot per loop, including its teardown. + decode, + + /// Print encoded sizes by record tag. Not a timing benchmark. + report, +}; + +pub fn create( + alloc: Allocator, + opts: Options, +) !*TerminalSnapshot { + const ptr = try alloc.create(TerminalSnapshot); + errdefer alloc.destroy(ptr); + ptr.* = .{ + .alloc = alloc, + .opts = opts, + .encoded = .init(alloc), + }; + return ptr; +} + +pub fn destroy(self: *TerminalSnapshot, alloc: Allocator) void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.encoded.deinit(); + alloc.destroy(self); +} + +pub fn benchmark(self: *TerminalSnapshot) Benchmark { + return .init(self, .{ + .stepFn = switch (self.opts.mode) { + .noop => stepNoop, + .encode => stepEncode, + .decode => stepDecode, + .report => stepReport, + }, + .setupFn = setup, + .teardownFn = teardown, + }); +} + +/// Build the terminal state every mode shares and, for decode mode, the +/// encoded snapshot it restores. All of this is outside the timed region. +fn setup(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + self.setupImpl() catch |err| { + log.warn("failed to prepare snapshot benchmark err={}", .{err}); + return error.BenchmarkFailed; + }; +} + +fn setupImpl(self: *TerminalSnapshot) !void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.terminal = try Terminal.init(global.io(), self.alloc, .{ + .cols = self.opts.@"terminal-cols", + .rows = self.opts.@"terminal-rows", + .max_scrollback_bytes = null, + .max_scrollback_lines = null, + }); + const terminal = &self.terminal.?; + + // Feed the input corpus through the standard VT stream. + if (try options.dataFile(self.opts.data)) |data_f| { + defer data_f.close(global.io()); + + var stream = terminal.vtStream(); + defer stream.deinit(); + + var read_buf: [4096]u8 align(std.atomic.cache_line) = undefined; + var f_reader = data_f.reader(global.io(), &read_buf); + const r = &f_reader.interface; + + var buf: [4096]u8 = undefined; + while (true) { + const n = try r.readSliceShort(&buf); + if (n == 0) break; // EOF reached + stream.nextSlice(buf[0..n]); + } + } + + // Decode restores the same bytes every step. Encode mode also uses + // this buffer, in which case this is its one-time growth. + self.encoded.shrinkRetainingCapacity(0); + try snapshot.encode( + self.alloc, + &self.encoded.writer, + terminal, + .{ .continuation = .ground }, + ); +} + +fn teardown(ptr: *anyopaque) void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.encoded.shrinkRetainingCapacity(0); +} + +fn stepNoop(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + for (0..self.opts.loops) |_| { + std.mem.doNotOptimizeAway(self.encoded.written()); + } +} + +fn stepEncode(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const terminal = &self.terminal.?; + for (0..self.opts.loops) |_| { + self.encoded.shrinkRetainingCapacity(0); + snapshot.encode( + self.alloc, + &self.encoded.writer, + terminal, + .{ .continuation = .ground }, + ) catch |err| { + log.warn("snapshot encoding failed err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(self.encoded.written()); + } +} + +fn stepDecode(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const bytes = self.encoded.written(); + for (0..self.opts.loops) |_| { + var reader: std.Io.Reader = .fixed(bytes); + var decoded = snapshot.decode( + self.alloc, + global.io(), + &reader, + .{ .max_continuation_bytes = 1024 * 1024 }, + ) catch |err| { + log.warn("snapshot decoding failed err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(&decoded); + decoded.deinit(self.alloc); + } +} + +/// Print the encoded size grouped by record tag. This shares the encoder +/// with encode mode but deliberately makes no timing claims. +fn stepReport(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const bytes = self.encoded.written(); + + var payload_totals = std.enums.EnumArray( + snapshot.record.Tag, + u64, + ).initFill(0); + var record_counts = std.enums.EnumArray( + snapshot.record.Tag, + u64, + ).initFill(0); + var record_total: u64 = 0; + + var offset: usize = snapshot.envelope.encoded_len; + while (offset + snapshot.record.Header.len <= bytes.len) { + const tag_raw = std.mem.readInt(u16, bytes[offset..][0..2], .little); + const payload_len = std.mem.readInt( + u32, + bytes[offset + 2 ..][0..4], + .little, + ); + const tag = std.enums.fromInt(snapshot.record.Tag, tag_raw) orelse { + log.warn("unknown record tag {}", .{tag_raw}); + return error.BenchmarkFailed; + }; + payload_totals.getPtr(tag).* += payload_len; + record_counts.getPtr(tag).* += 1; + record_total += 1; + offset += snapshot.record.Header.len + payload_len; + } + + var it = payload_totals.iterator(); + while (it.next()) |entry| { + std.debug.print("terminal-snapshot tag={s} records={d} payload={d}\n", .{ + @tagName(entry.key), + record_counts.get(entry.key), + entry.value.*, + }); + } + std.debug.print( + "terminal-snapshot total records={d} framing={d} encoded={d}\n", + .{ + record_total, + record_total * snapshot.record.Header.len + + snapshot.envelope.encoded_len, + bytes.len, + }, + ); +} + +test TerminalSnapshot { + const testing = std.testing; + const impl: *TerminalSnapshot = try .create(testing.allocator, .{}); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} + +test "TerminalSnapshot decode round trip" { + const testing = std.testing; + const impl: *TerminalSnapshot = try .create(testing.allocator, .{ + .mode = .decode, + .@"terminal-rows" = 4, + .@"terminal-cols" = 8, + }); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} diff --git a/src/benchmark/cli.zig b/src/benchmark/cli.zig index 1dc089b71..f6a31f2c5 100644 --- a/src/benchmark/cli.zig +++ b/src/benchmark/cli.zig @@ -16,6 +16,7 @@ pub const Action = enum { @"screen-clone", @"terminal-parser", @"terminal-resize", + @"terminal-snapshot", @"terminal-stream", @"is-symbol", @"osc-parser", @@ -40,6 +41,7 @@ pub const Action = enum { .@"grapheme-break" => @import("GraphemeBreak.zig"), .@"terminal-parser" => @import("TerminalParser.zig"), .@"terminal-resize" => @import("TerminalResize.zig"), + .@"terminal-snapshot" => @import("TerminalSnapshot.zig"), .@"is-symbol" => @import("IsSymbol.zig"), .@"osc-parser" => @import("OscParser.zig"), }; diff --git a/src/benchmark/main.zig b/src/benchmark/main.zig index fb038c0a3..666ffb0f6 100644 --- a/src/benchmark/main.zig +++ b/src/benchmark/main.zig @@ -8,6 +8,7 @@ pub const HyperlinkMap = @import("HyperlinkMap.zig"); pub const ScreenClone = @import("ScreenClone.zig"); pub const TerminalParser = @import("TerminalParser.zig"); pub const TerminalResize = @import("TerminalResize.zig"); +pub const TerminalSnapshot = @import("TerminalSnapshot.zig"); pub const IsSymbol = @import("IsSymbol.zig"); pub const PageCompression = @import("PageCompression.zig"); pub const ScrollbackCompression = @import("ScrollbackCompression.zig"); From ed0f54fb8ccf45fc502de25f2c0f1e60967fc2a4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:33:43 -0700 Subject: [PATCH 3/7] terminal/snapshot: 8-byte grid cells with blank elision Rework the PAGE grid encoding for codec speed and size. This is a breaking change to the work-in-progress version 1 wire format. Cells were previously a fixed 16-byte header plus inline grapheme suffixes: one byte each for content kind, width, and flags, a reserved byte, 16-bit style and hyperlink IDs, a 32-bit value, and an always-present 32-bit suffix count that was almost always zero. Cells are now one 64-bit little-endian word with a documented bit registry that carries the hyperlink ID in its high bits. The layout deliberately coincides with the native cell so clean rows encode as a straight copy of page memory and decode as one bulk read plus an in-place normalization pass; a comptime check falls back to a portable field-by-field codec if the native layout ever diverges. Each row header also gains an encoded cell count so trailing default cells are elided instead of spending 16 bytes apiece encoding nothing: on typical shell output most of every row is blank, and measurement showed 97% of encoded snapshot bytes were zero. Grapheme suffixes move out of the cell stream into a per-grid section of (row, column, codepoints) entries, which keeps row decoding fixed-stride and bulk-copyable. Decode ID remapping switches from hash maps to direct-indexed tables sized by the 16-bit encoded ID space, removing per-styled-cell hash lookups. Benchmark deltas at this commit (terminal-snapshot, M-series, ReleaseFast, 1 MB corpora): ascii lines 1-70: 34.16 MB -> 7.66 MB (4.5x) encode 92.8 -> 18.2 ms, decode 119.8 -> 28.0 ms ascii full-wrap: 16.01 MB -> 8.04 MB (2.0x) encode 43.6 -> 18.4 ms, decode 56.2 -> 25.6 ms utf8: 4.33 MB -> 1.90 MB (2.3x) encode 12.4 -> 4.9 ms, decode 19.0 -> 9.6 ms --- src/terminal/snapshot/grid.zig | 1430 ++++++++++++----- src/terminal/snapshot/page.zig | 133 +- src/terminal/snapshot/screen.zig | 23 +- src/terminal/snapshot/snapshot.ksy | 121 +- .../snapshot/testdata/complete-v1.hex | 93 +- src/terminal/snapshot/testdata/grid-v1.hex | 30 +- .../testdata/page-empty-record-v1.hex | 4 +- src/terminal/snapshot/testdata/page-v1.hex | 12 +- 8 files changed, 1197 insertions(+), 649 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 18e7955ba..163f12bef 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -1,25 +1,37 @@ -//! Grid (rows and cells) encoding. +//! Grid (rows, cells, and grapheme suffixes) encoding. //! //! A grid contains the rows and cells of a terminal page. Its dimensions are //! supplied by the containing record rather than repeated here. The encoder -//! writes exactly `rows` row records, and every row contains exactly `columns` -//! cell records. +//! writes exactly `rows` row records followed by one grapheme suffix section. //! //! All records are tightly packed with no padding between them. All integers //! are unsigned and little-endian. //! +//! The layout is designed so the common case decodes with bulk copies: cells +//! are fixed-size words, trailing default cells are elided per row, and the +//! variable-length grapheme suffixes live outside the row data. +//! +//! ```text +//! +---------------------------+ +//! | Row 0 | +//! +---------------------------+ +//! | ... | +//! +---------------------------+ +//! | Row (rows - 1) | +//! +---------------------------+ +//! | Grapheme suffix section | +//! +---------------------------+ +//! ``` +//! //! ## Row //! //! Each row has the following format: //! -//! | Offset | Size | Field | -//! | -----: | -------: | :-------------------------- | -//! | 0 | 1 | Row flags | -//! | 1 | variable | Exactly `columns` cells | -//! -//! Cells are encoded consecutively. Since cells may contain a variable number -//! of grapheme codepoints, the next row begins immediately after the final -//! cell and its grapheme codepoints. +//! | Offset | Size | Field | +//! | -----: | ----------: | :------------------------ | +//! | 0 | 1 | Row flags | +//! | 1 | 2 | Encoded cell count (`u16`)| +//! | 3 | 8 * `count` | Encoded cells | //! //! The row flag byte has the following format: //! @@ -40,37 +52,100 @@ //! //! Value 3 is not emitted in snapshot version 1. Decoders treat it as none. //! +//! The encoded cell count must not exceed the grid's column count; it is a +//! structural field and decoders reject larger values. Cells at and beyond +//! the count are the default cell: all sixty-four bits zero, meaning an +//! empty narrow codepoint cell with the default style and no hyperlink. +//! Canonical encoders emit exactly through the row's last non-default cell, +//! so a fully default row has a zero count and no cell words. +//! //! Native row cache flags are not encoded. In particular, the Kitty virtual //! placeholder hint is derived while decoding cells containing U+10EEEE. //! //! ## Cell //! -//! Each cell has the following format: +//! Each cell is one 64-bit little-endian word: //! -//! | Offset | Size | Field | -//! | -----: | --------: | :---------------------------- | -//! | 0 | 1 | Content kind | -//! | 1 | 1 | Width kind | -//! | 2 | 1 | Protected and semantic flags | -//! | 3 | 1 | Reserved, zero | -//! | 4 | 2 | Style ID | -//! | 6 | 2 | Hyperlink ID | -//! | 8 | 4 | Codepoint or packed color | -//! | 12 | 4 | Grapheme suffix count | -//! | 16 | 4 * count | Grapheme suffix codepoints | +//! ```text +//! bit 0 +-------------------------------+ +//! | Content kind | +//! | 2 bits | +//! bit 2 +-------------------------------+ +//! | Content | +//! | 24 bits | +//! bit 26 +-------------------------------+ +//! | Style ID | +//! | 16 bits | +//! bit 42 +-------------------------------+ +//! | Width kind | +//! | 2 bits | +//! bit 44 +-------------------------------+ +//! | Protected | +//! bit 45 +-------------------------------+ +//! | Hyperlink flag | +//! bit 46 +-------------------------------+ +//! | Semantic content | +//! | 2 bits | +//! bit 48 +-------------------------------+ +//! | Hyperlink ID | +//! | 16 bits | +//! bit 64 +-------------------------------+ +//! ``` //! //! Content kinds are: //! -//! | Value | Meaning | -//! | ----: | :----------------- | -//! | 0 | Codepoint | -//! | 1 | Palette background | -//! | 2 | RGB background | +//! | Value | Meaning | +//! | ----: | :------------------------------- | +//! | 0 | Codepoint | +//! | 1 | Codepoint with grapheme suffix | +//! | 2 | Palette background | +//! | 3 | RGB background | //! -//! For codepoint content, the value is a Unicode scalar encoded as a `u32`. -//! For a palette background, the low byte is the palette index and the other -//! three bytes are zero. For an RGB background, the low three bytes are red, -//! green, and blue, and the high byte is zero. +//! The content kind selects the layout of the 24-bit content field. All +//! content bit positions below are relative to the start of the content +//! field, so content bit 0 is cell word bit 2. +//! +//! For codepoint content (kinds 0 and 1), the content field is a Unicode +//! scalar value. Values above U+10FFFF and surrogates decode as U+FFFD. +//! Kind 1 additionally declares that the grapheme suffix section contains +//! one entry for this cell; the cell is otherwise identical to kind 0. +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Unicode scalar value | +//! | 24 bits | +//! bit 24 +-------------------------------+ +//! ``` +//! +//! For a palette background (kind 2), the low byte is the palette index. +//! The remaining content bits are reserved, canonically zero, and ignored +//! by decoders. +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Palette index | +//! | 8 bits | +//! bit 8 +-------------------------------+ +//! | Reserved, zero | +//! | 16 bits | +//! bit 24 +-------------------------------+ +//! ``` +//! +//! For an RGB background (kind 3), the content field is one byte per +//! channel: +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Red | +//! | 8 bits | +//! bit 8 +-------------------------------+ +//! | Green | +//! | 8 bits | +//! bit 16 +-------------------------------+ +//! | Blue | +//! | 8 bits | +//! bit 24 +-------------------------------+ +//! ``` //! //! Width kinds are: //! @@ -81,14 +156,6 @@ //! | 2 | Spacer tail | //! | 3 | Spacer head | //! -//! The cell flag byte has the following format: -//! -//! | Bits | Field | -//! | ---: | :---------------- | -//! | 0 | Protected | -//! | 1-2 | Semantic content | -//! | 3-7 | Reserved, zero | -//! //! Semantic content values are: //! //! | Value | Meaning | @@ -101,13 +168,40 @@ //! //! Style and hyperlink ID zero mean no style and no hyperlink. Other IDs //! refer to entries in the containing record's separate style and hyperlink -//! tables. +//! tables. The hyperlink flag is set exactly when the hyperlink ID is +//! nonzero; decoders derive cell linkage from the remapped ID and ignore the +//! flag itself. //! -//! Grapheme suffixes are valid only for codepoint content. Each suffix is a -//! Unicode scalar encoded as a `u32`. A nonzero suffix count requires a -//! nonzero base codepoint. +//! ## Grapheme suffix section +//! +//! The section begins with an entry count followed by that many entries: +//! +//! | Offset | Size | Field | +//! | -----: | ------: | :------------------- | +//! | 0 | 4 | Entry count (`u32`) | +//! | 4 | varies | Entries | +//! +//! Each entry: +//! +//! | Offset | Size | Field | +//! | -----: | ----------: | :-------------------------- | +//! | 0 | 2 | Row (`u16`) | +//! | 2 | 2 | Column (`u16`) | +//! | 4 | 2 | Codepoint count (`u16`) | +//! | 6 | 4 * `count` | Codepoints (`u32` each) | +//! +//! Each codepoint is a Unicode scalar; invalid scalars are individually +//! ignored by decoders. Canonical encoders emit exactly one entry, with at +//! least one codepoint, for every kind 1 cell, in ascending row-then-column +//! order. Decoders consume every declared entry and ignore entries whose +//! target is out of range, is not a codepoint cell, has a zero codepoint, or +//! already received an entry. A kind 1 cell that receives no suffix +//! codepoints decodes as a plain codepoint cell. const std = @import("std"); +const assert = std.debug.assert; +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; const test_fixture = @import("fixture.zig"); const io = @import("io.zig"); const kitty = @import("../kitty.zig"); @@ -121,26 +215,147 @@ const TerminalPage = terminal_page.Page; const TerminalRow = terminal_page.Row; const TerminalStyleId = terminal_style.Id; -/// Maps encoded style table IDs to IDs assigned by the destination page. +/// The header before every row's encoded cells. /// -/// Build this by inserting each decoded style into the page, then recording -/// the encoded ID and the ID returned by the page's style set. Style ID zero is -/// implicit and does not need an entry. -pub const StyleRemap = std.AutoHashMap(TerminalStyleId, TerminalStyleId); +/// The semantic prompt is a raw integer for the same reason as the wire +/// cell fields: decoders must accept its reserved value without +/// instantiating an invalid native enum, so every header byte bit-casts to +/// a valid value. +pub const Row = packed struct(u8) { + wrap: bool = false, + wrap_continuation: bool = false, + semantic_prompt: u2 = 0, + _padding: u4 = 0, +}; -/// Maps encoded hyperlink table IDs to IDs assigned by the destination page. +/// The wire layout of one encoded cell. This is its own registry: the bit +/// positions and field meanings are part of the snapshot format and are +/// documented above independently of the native cell. /// -/// Build this by inserting each decoded hyperlink into the page, then -/// recording the encoded ID and the ID returned by the page's hyperlink set. -/// Hyperlink ID zero is implicit and does not need an entry. -pub const HyperlinkRemap = std.AutoHashMap(TerminalHyperlinkId, TerminalHyperlinkId); +/// Enum-like fields are raw integers because decoders must accept reserved +/// values (for example semantic content 3) without instantiating an invalid +/// native enum. +pub const Cell = packed struct(u64) { + kind: u2 = @intFromEnum(Kind.codepoint), + content: u24 = 0, + style_id: u16 = 0, + width: u2 = 0, + protected: bool = false, + hyperlink: bool = false, + semantic_content: u2 = 0, + hyperlink_id: u16 = 0, + + /// Determines how `content` is interpreted. + pub const Kind = enum(u2) { + codepoint = 0, + codepoint_grapheme = 1, + bg_color_palette = 2, + bg_color_rgb = 3, + }; +}; + +/// Whether the native cell's in-memory layout matches the wire cell layout +/// bit for bit, with the wire hyperlink ID occupying the native padding. +/// +/// The wire format does not require this: it is an optimization. When it +/// holds, whole rows encode and decode as bulk copies. If the native layout +/// ever diverges, the portable field-by-field codec below remains correct +/// and this constant simply becomes false. +const native_matches_wire = native: { + if (@bitSizeOf(TerminalCell) != 64) break :native false; + + // The bulk copies above are only sound if we can prove the two layouts + // agree, and we want native cell changes to demote us to the portable + // codec automatically rather than corrupt snapshots. + // + // Instead of pinning every native field offset and enum value w/ assertions + // that must be maintained by hand, each probe below pairs a native + // cell with the wire cell that must share its exact bit pattern; both + // sides bit cast to a word and any difference means some field moved + // or some enum member was renumbered. + // + // Three probes, one per content payload, cover the whole cell. + const probes = [_]struct { + native: TerminalCell, + wire: Cell, + }{ + .{ + .native = .{ + .content_tag = .codepoint_grapheme, + .content = .{ .codepoint = .{ .data = 0x10FFFF } }, + .style_id = 0xBEEF, + .wide = .spacer_tail, + .protected = true, + ._padding = 0x1D2C, + }, + .wire = .{ + .kind = 1, + .content = 0x10FFFF, + .style_id = 0xBEEF, + .width = 2, + .protected = true, + // The native padding position, canonical wire or not. + .hyperlink_id = 0x1D2C, + }, + }, + .{ + .native = .{ + .content_tag = .bg_color_palette, + .content = .{ .color_palette = .{ .data = 0xAB } }, + .wide = .spacer_head, + .hyperlink = true, + .semantic_content = .input, + }, + .wire = .{ + .kind = 2, + .content = 0xAB, + .width = 3, + .hyperlink = true, + .semantic_content = 1, + }, + }, + .{ + .native = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ + .r = 0x12, + .g = 0x34, + .b = 0x56, + } }, + .wide = .wide, + .semantic_content = .prompt, + }, + .wire = .{ + .kind = 3, + .content = 0x563412, + .width = 1, + .semantic_content = 2, + }, + }, + }; + for (probes) |probe| { + const native_bits: u64 = @bitCast(probe.native); + const wire_bits: u64 = @bitCast(probe.wire); + if (native_bits != wire_bits) break :native false; + } + + break :native true; +}; + +/// Whether rows of cells can be copied between native and wire storage +/// without per-cell transformation. +const bulk_codec = native_matches_wire and + builtin.cpu.arch.endian() == .little; pub const EncodeError = std.Io.Writer.Error || error{ /// Wide and spacer cells do not form a valid row. InvalidWideCell, + + /// One cell's grapheme suffix exceeds the entry's u16 codepoint count. + TooManyGraphemes, }; -/// Encode every row and cell directly from a page. +/// Encode every row, cell, and grapheme suffix directly from a page. /// /// If an error is returned, partial data may have been written. If you /// want transactional writing, the caller is responsible for using something @@ -151,20 +366,39 @@ pub fn encode( ) EncodeError!void { defer page.assertIntegrity(); for (0..page.size.rows) |y| { - // Row header const row = page.getRow(y); - const row_header: RowHeader = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = row.semantic_prompt, - }; - try writer.writeByte(@bitCast(row_header)); - - // Cells const cells = page.getCells(row); - for (cells, 0..) |*cell, x| { - // Validate the wide state of this cell, we don't want - // to encode corrupt data. + + // Trailing default cells decode implicitly. Wide/spacer pairs and + // hyperlinked or styled cells are always nonzero, so eliding the + // zero suffix never drops encoded state. + const count: usize = count: { + var i: usize = cells.len; + while (i > 0) : (i -= 1) { + if (!cells[i - 1].isZero()) break :count i; + } + break :count 0; + }; + + // Row header: flags then the encoded cell count. + { + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + }; + var header_bytes: [3]u8 = undefined; + header_bytes[0] = @bitCast(row_header); + std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); + try writer.writeAll(&header_bytes); + } + + // Validate the wide state of every encoded cell so we don't encode + // corrupt data, and detect the cells that keep this row off the + // direct-copy path. Trailing default cells are narrow, so checking + // the encoded prefix against the full row width covers every pair. + var direct = true; + for (cells[0..count], 0..) |*cell, x| { switch (cell.wide) { .narrow => {}, .wide => if (x + 1 == cells.len or @@ -182,413 +416,506 @@ pub fn encode( }, } - const graphemes: []const u21 = if (cell.hasGrapheme()) - page.lookupGrapheme(cell) orelse unreachable - else - &.{}; + // Hyperlink IDs live in a native side table, and nonzero native + // padding would leak into the wire hyperlink ID field. + if (cell.hyperlink or cell._padding != 0) direct = false; + } - // The page has two codepoint tags depending on whether suffixes - // exist, but the wire represents both with one content kind. - const kind: CellHeader.Kind = switch (cell.content_tag) { - .codepoint, .codepoint_grapheme => .codepoint, - .bg_color_palette => .bg_color_palette, - .bg_color_rgb => .bg_color_rgb, - }; - const value: CellHeader.Value = switch (kind) { - .codepoint => .{ - .codepoint = cell.content.codepoint.data, - }, - .bg_color_palette => .{ .bg_color_palette = .{ - .index = cell.content.color_palette.data, - } }, - .bg_color_rgb => .{ .bg_color_rgb = .{ - .r = cell.content.color_rgb.r, - .g = cell.content.color_rgb.g, - .b = cell.content.color_rgb.b, - } }, - }; + if (comptime bulk_codec) { + if (direct) { + try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); + continue; + } + } - const style_id = cell.style_id; - const hyperlink_id: TerminalHyperlinkId = if (cell.hyperlink) + for (cells[0..count]) |*cell| { + const link_id: TerminalHyperlinkId = if (cell.hyperlink) page.lookupHyperlink(cell) orelse unreachable else 0; + try io.writeInt(writer, u64, cellBits(cell.*, link_id)); + } + } - const header: CellHeader = .{ - .content_kind = kind, - .width = cell.wide, - .protected = cell.protected, - .semantic_content = cell.semantic_content, - .style_id = style_id, - .hyperlink_id = hyperlink_id, - .value = value, - .grapheme_count = @intCast(graphemes.len), - }; - try header.encode(writer); - for (graphemes) |suffix| try io.writeInt( - writer, - u32, - suffix, - ); + try encodeGraphemes(page, writer); +} + +/// Encode the grapheme suffix section for every kind 1 cell in the grid. +fn encodeGraphemes( + page: *const TerminalPage, + writer: *std.Io.Writer, +) EncodeError!void { + // Count and validate entries before the section header so the count is + // always exact. Rows without the native grapheme hint contain no + // grapheme cells in any intact page. + var entries: u32 = 0; + for (0..page.size.rows) |y| { + const row = page.getRow(y); + if (!row.grapheme) continue; + for (page.getCells(row)) |*cell| { + if (!cell.hasGrapheme()) continue; + const cps = page.lookupGrapheme(cell) orelse unreachable; + if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; + entries += 1; + } + } + + try io.writeInt(writer, u32, entries); + if (entries == 0) return; + + for (0..page.size.rows) |y| { + const row = page.getRow(y); + if (!row.grapheme) continue; + for (page.getCells(row), 0..) |*cell, x| { + if (!cell.hasGrapheme()) continue; + const cps = page.lookupGrapheme(cell) orelse unreachable; + try io.writeInt(writer, u16, @intCast(y)); + try io.writeInt(writer, u16, @intCast(x)); + try io.writeInt(writer, u16, @intCast(cps.len)); + for (cps) |cp| try io.writeInt(writer, u32, cp); } } } -/// Decode every row and cell directly into an initialized, empty page. +pub const DecodeError = std.Io.Reader.Error || error{ + /// A row declares more encoded cells than the grid has columns. + InvalidRowCellCount, +}; + +/// Maps encoded style table IDs to page-assigned style IDs. +pub const StyleRemap = Remap(TerminalStyleId); + +/// Maps encoded hyperlink table IDs to page-assigned hyperlink IDs. +pub const HyperlinkRemap = Remap(TerminalHyperlinkId); + +/// Decode every row, cell, and grapheme suffix directly into an +/// initialized, empty page. /// -/// The grid does not encode dimensions, so `page` must already have the exact -/// row and column count expected by the containing record. This function reads -/// exactly `page.size.rows` rows with `page.size.cols` cells each. Capacity -/// hints are advisory. Graphemes and cell hyperlink references that do not fit -/// are discarded without affecting the rest of the grid. +/// The grid does not encode dimensions, so `page` must already have the +/// exact row and column count expected by the containing record. Capacity +/// hints are advisory. Graphemes and cell hyperlink references that do not +/// fit are discarded without affecting the rest of the grid. /// /// Style and hyperlink table entries must be inserted into `page` before -/// calling this function. As each table entry is inserted, the caller records -/// its encoded ID and page-assigned ID in `style_remap` or `hyperlink_remap`. -/// ID zero always means the default style or no hyperlink. A nonzero ID missing -/// from its remap is also treated as zero so unknown table references do not -/// prevent the rest of the grid from decoding. +/// calling this function, with their encoded and page-assigned IDs recorded +/// in `style_remap` and `hyperlink_remap`. ID zero always means the default +/// style or no hyperlink. A nonzero ID missing from its remap is also +/// treated as zero so unknown table references do not prevent the rest of +/// the grid from decoding. /// -/// Invalid semantic data is normalized into a degraded form while preserving -/// the declared byte boundaries. Unknown semantic values use their neutral -/// variants, invalid Unicode becomes U+FFFD, invalid optional data is ignored, -/// and malformed wide-cell relationships become narrow cells. +/// Invalid semantic data is normalized into a degraded form while +/// preserving the declared byte boundaries. Unknown semantic values use +/// their neutral variants, invalid Unicode becomes U+FFFD, invalid optional +/// data is ignored, and malformed wide-cell relationships become narrow +/// cells. The encoded cell count is the only structural field. pub fn decode( page: *TerminalPage, reader: *std.Io.Reader, style_remap: *const StyleRemap, hyperlink_remap: *const HyperlinkRemap, -) std.Io.Reader.Error!void { +) DecodeError!void { for (0..page.size.rows) |y| { - const row_raw = try reader.takeByte(); - const row_header: RowHeader = @bitCast(row_raw); - const semantic_prompt_raw: u2 = @truncate(row_raw >> @bitOffsetOf(RowHeader, "semantic_prompt")); + // Every bit pattern is a valid header: booleans decode directly + // and the raw semantic value gets a default below. Reserved bits + // do not change the known fields. + const row_header: Row = @bitCast(try reader.takeByte()); + const count = try io.readInt(reader, u16); - // Reserved bits do not change the known fields. The semantic enum is - // the only non-exhaustive field, so give its unknown value a default. const row = page.getRow(y); row.wrap = row_header.wrap; row.wrap_continuation = row_header.wrap_continuation; row.semantic_prompt = std.enums.fromInt( TerminalRow.SemanticPrompt, - semantic_prompt_raw, + row_header.semantic_prompt, ) orelse .none; const cells = page.getCells(row); - for (cells, 0..) |*cell, x| { - const decoded_header = try CellHeader.decode(reader); + if (count > cells.len) return error.InvalidRowCellCount; + if (count == 0) continue; - cell.* = .init(0); - var accept_graphemes = false; - const grapheme_count: u32 = switch (decoded_header) { - // The fixed header was fully consumed, but its content kind - // cannot be interpreted. Keep the cell empty and consume only - // the suffix bytes needed to reach the next cell. - .invalid => |count| count, + if (comptime bulk_codec) { + // Read the encoded words directly into page storage, then + // normalize them in place. The raw words are only ever touched + // as integers until normalization makes them valid cells. + const words: [*]u64 = @ptrCast(cells.ptr); + try reader.readSliceAll( + std.mem.sliceAsBytes(cells[0..count]), + ); + for (0..count) |x| { + applyCell( + page, + row, + cells, + x, + words[x], + style_remap, + hyperlink_remap, + ); + } + } else { + for (0..count) |x| { + const bits = try io.readInt(reader, u64); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } + } - .valid => |header| valid: { - // IDs belong to the encoded page. Translate them to IDs - // assigned by the destination page before storing them on - // cells. - const encoded_style_id = header.style_id; - const style_id = if (encoded_style_id == 0) - 0 - else - style_remap.get(encoded_style_id) orelse 0; + // The implicit cell after a short row is narrow, which resolves a + // trailing wide marker exactly like an explicit narrow neighbor. + if (count < cells.len and cells[count - 1].wide == .wide) { + cells[count - 1].wide = .narrow; + } + } - const encoded_hyperlink_id = header.hyperlink_id; - const hyperlink_id = if (encoded_hyperlink_id == 0) - 0 - else - hyperlink_remap.get(encoded_hyperlink_id) orelse 0; + try decodeGraphemes(page, reader); +} - switch (header.content_kind) { - .codepoint => { - var cp = std.math.cast( - u21, - header.value.codepoint, - ) orelse 0xFFFD; - if (cp > 0x10FFFF or - (cp >= 0xD800 and cp <= 0xDFFF)) - { - cp = 0xFFFD; - } - cell.content = .{ .codepoint = .{ .data = cp } }; - accept_graphemes = cp != 0; +/// Normalize one encoded cell word and store it at `cells[x]`. +/// +/// This owns every per-cell decode rule except grapheme suffixes: content +/// validation, reserved-value degradation, style and hyperlink remapping +/// with reference counting, and wide-pair normalization against already +/// decoded neighbors. +fn applyCell( + page: *TerminalPage, + row: *TerminalRow, + cells: []TerminalCell, + x: usize, + bits_wire: u64, + style_remap: *const StyleRemap, + hyperlink_remap: *const HyperlinkRemap, +) void { + const cell = &cells[x]; - // Kitty image and placement state is not part of this - // snapshot version, but the placeholder is still a - // valid Unicode scalar. Preserve it and derive the - // native row hint so later row operations remain - // correct. - if (cp == kitty.graphics.unicode.placeholder) { - row.kitty_virtual_placeholder = true; - } - }, - .bg_color_palette => { - const palette = header.value.bg_color_palette; - cell.content_tag = .bg_color_palette; - cell.content = .{ - .color_palette = .{ .data = palette.index }, - }; - }, - .bg_color_rgb => { - const rgb = header.value.bg_color_rgb; - cell.content_tag = .bg_color_rgb; - cell.content = .{ .color_rgb = .{ - .r = rgb.r, - .g = rgb.g, - .b = rgb.b, - } }; - }, - } + // The default cell is the common case and needs no normalization, + // reference counting, or table lookups. + if (bits_wire == 0) { + storeCell(cell, 0); + normalizeWide(row, cells, x); + return; + } - cell.wide = header.width; - cell.protected = header.protected; - cell.semantic_content = header.semantic_content; + // Any word is a valid wire cell because its fields are raw integers, + // so all normalization below is plain field access. + var wire: Cell = @bitCast(bits_wire); - if (style_id != 0) { - // The table owns one reference and each decoded cell owns - // one additional reference. - page.styles.use(page.memory, style_id); - cell.style_id = style_id; - row.styled = true; - } + // Hyperlink linkage is derived from the remapped ID below. The stored + // native cell starts unlinked either way. + const link_encoded = wire.hyperlink_id; + wire.hyperlink_id = 0; + wire.hyperlink = false; - if (hyperlink_id != 0) { - // setHyperlink records the cell mapping but intentionally - // does not increment the set's reference count. If its map - // is full, undo our reference and leave this cell unlinked. - page.hyperlink_set.use(page.memory, hyperlink_id); - page.setHyperlink(row, cell, hyperlink_id) catch { - page.hyperlink_set.release(page.memory, hyperlink_id); - }; - } + switch (@as(Cell.Kind, @enumFromInt(wire.kind))) { + // Kind 1 differs from 0 only by declaring a grapheme suffix section + // entry, which reattaches through the native grapheme APIs later. + .codepoint, .codepoint_grapheme => { + wire.kind = @intFromEnum(Cell.Kind.codepoint); + if (!validScalar(wire.content)) wire.content = 0xFFFD; - break :valid header.grapheme_count; - }, + // Kitty image and placement state is not part of this snapshot + // version, but the placeholder is still a valid Unicode scalar. + // Preserve it and derive the native row hint so later row + // operations remain correct. + if (wire.content == kitty.graphics.unicode.placeholder) { + row.kitty_virtual_placeholder = true; + } + }, + + // Only the palette index is meaningful; the remaining content bits + // are reserved and must not obscure it. + .bg_color_palette => wire.content = @as(u8, @truncate(wire.content)), + + .bg_color_rgb => {}, + } + + // Reserved semantic content degrades to plain output. + wire.semantic_content = @intFromEnum(std.enums.fromInt( + TerminalCell.SemanticContent, + wire.semantic_content, + ) orelse .output); + + // IDs belong to the encoded page. Translate them to IDs assigned by the + // destination page before storing them on cells. The table owns one + // reference and each decoded cell owns one additional reference. + if (wire.style_id != 0) { + wire.style_id = style_remap.get(wire.style_id); + if (wire.style_id != 0) { + page.styles.use(page.memory, wire.style_id); + row.styled = true; + } + } + + storeCell(cell, @bitCast(wire)); + + if (link_encoded != 0) link: { + const link_native = hyperlink_remap.get(link_encoded); + if (link_native == 0) break :link; + + // setHyperlink records the cell mapping but intentionally does not + // increment the set's reference count. If its map is full, undo our + // reference and leave this cell unlinked. + page.hyperlink_set.use(page.memory, link_native); + page.setHyperlink(row, cell, link_native) catch { + page.hyperlink_set.release(page.memory, link_native); + }; + } + + normalizeWide(row, cells, x); +} + +/// Resolve wide-pair relationships for the cell at `x` against its already +/// normalized predecessors. +fn normalizeWide(row: *const TerminalRow, cells: []TerminalCell, x: usize) void { + // A following cell resolves whether the previous wide marker owns a + // tail. Normalize the current marker immediately when possible, + // including a wide marker at the row end. + if (x > 0 and + cells[x - 1].wide == .wide and + cells[x].wide != .spacer_tail) + { + cells[x - 1].wide = .narrow; + } + switch (cells[x].wide) { + .narrow => {}, + + // A non-final wide marker remains pending until the next cell. + .wide => if (x + 1 == cells.len) { + cells[x].wide = .narrow; + }, + + .spacer_tail => if (x == 0 or + cells[x - 1].wide != .wide) + { + cells[x].wide = .narrow; + }, + + .spacer_head => if (x + 1 != cells.len or !row.wrap) { + cells[x].wide = .narrow; + }, + } +} + +/// Whether the value is a valid Unicode scalar value. +inline fn validScalar(cp: u32) bool { + return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); +} + +/// Decode the grapheme suffix section into already decoded cells. +fn decodeGraphemes( + page: *TerminalPage, + reader: *std.Io.Reader, +) DecodeError!void { + const entries = try io.readInt(reader, u32); + for (0..entries) |_| { + const y = try io.readInt(reader, u16); + const x = try io.readInt(reader, u16); + const cp_count = try io.readInt(reader, u16); + + // Resolve the target cell. Entries whose target cannot carry a + // suffix are optional detail: their codepoints are consumed to + // preserve framing and then dropped. + const target: ?struct { + row: *TerminalRow, + cell: *TerminalCell, + } = target: { + if (y >= page.size.rows or x >= page.size.cols) { + break :target null; + } + const row = page.getRow(y); + const cell = &page.getCells(row)[x]; + + // Cell decoding stores every kind 1 cell as a plain codepoint, + // so this also gives duplicate entries first-wins semantics. + if (cell.content_tag != .codepoint) break :target null; + if (cell.content.codepoint.data == 0) break :target null; + break :target .{ .row = row, .cell = cell }; + }; + + // Always consume every declared codepoint. Invalid scalars and + // suffixes that exceed the native capacity are dropped + // independently without affecting the rest of the grid. + var accept = target != null; + for (0..cp_count) |_| { + const cp = try io.readInt(reader, u32); + if (!accept) continue; + if (!validScalar(cp)) continue; + + page.appendGrapheme( + target.?.row, + target.?.cell, + @intCast(cp), + ) catch { + accept = false; }; - - // Always consume every declared suffix. Invalid scalars, suffixes - // on non-text cells, and suffixes that exceed the native capacity - // are optional detail and can be dropped independently. - for (0..grapheme_count) |_| { - const suffix_raw = try io.readInt(reader, u32); - if (!accept_graphemes) continue; - - const suffix = std.math.cast(u21, suffix_raw) orelse continue; - if (suffix > 0x10FFFF or - (suffix >= 0xD800 and suffix <= 0xDFFF)) - { - continue; - } - page.appendGrapheme(row, cell, suffix) catch { - accept_graphemes = false; - }; - } - - // A following cell resolves whether the previous wide marker owns - // a tail. Normalize the current marker immediately when possible, - // including a wide marker at the row end. - if (x > 0 and - cells[x - 1].wide == .wide and - cell.wide != .spacer_tail) - { - cells[x - 1].wide = .narrow; - } - switch (cell.wide) { - .narrow => {}, - - // A non-final wide marker remains pending until the next cell. - .wide => if (x + 1 == cells.len) { - cell.wide = .narrow; - }, - - .spacer_tail => if (x == 0 or - cells[x - 1].wide != .wide) - { - cell.wide = .narrow; - }, - - .spacer_head => if (x + 1 != cells.len or !row.wrap) { - cell.wide = .narrow; - }, - } } } } -/// The header before every row. -const RowHeader = packed struct(u8) { - wrap: bool = false, - wrap_continuation: bool = false, - semantic_prompt: TerminalRow.SemanticPrompt = .none, - _padding: u4 = 0, -}; - -/// The fixed fields that precede a cell's grapheme suffix codepoints. -const CellHeader = struct { - /// Number of bytes written by `encode`, calculated using the encoder itself - /// so this remains synchronized with the field-by-field wire format. - pub const len = computeLen(); - - comptime { - // This size is part of the wire format. If it changes, the snapshot - // version and golden fixtures must also change. - std.debug.assert(len == 16); +/// The encoded word for one native cell and its hyperlink ID. +fn cellBits(cell: TerminalCell, link_id: TerminalHyperlinkId) u64 { + if (comptime native_matches_wire) { + var wire: Cell = @bitCast(cell); + wire.hyperlink = link_id != 0; + wire.hyperlink_id = link_id; + return @bitCast(wire); } - /// Interpretation of `value`. - content_kind: Kind = .codepoint, - - /// Display width and spacer role of the cell. - width: TerminalCell.Wide = .narrow, - - /// Whether selective erase operations protect the cell. - protected: bool = false, - - /// Semantic role assigned by shell integration. - semantic_content: TerminalCell.SemanticContent = .output, - - /// ID in the encoded page's style table, or zero for the default style. - style_id: TerminalStyleId = 0, - - /// ID in the encoded page's hyperlink table, or zero for no hyperlink. - hyperlink_id: TerminalHyperlinkId = 0, - - /// Codepoint or packed background color selected by `content_kind`. - value: Value = .{ .codepoint = 0 }, - - /// Number of grapheme suffix codepoints immediately following the header. - grapheme_count: u32 = 0, - - /// Encode the fixed cell header. - pub fn encode( - self: CellHeader, - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - const flags: Flags = .{ - .protected = self.protected, - .semantic_content = self.semantic_content, - }; - - // 0 1 2 3 4 6 8 12 16 - // +-------+-------+-------+-------+--------+--------+--------+--------+ - // | kind | width | flags | zero | style | link | value | count | - // | u8 | u8 | u8 | u8 | u16 LE | u16 LE | u32 LE | u32 LE | - // +-------+-------+-------+-------+--------+--------+--------+--------+ - try writer.writeByte(@intFromEnum(self.content_kind)); - try writer.writeByte(@intFromEnum(self.width)); - try writer.writeByte(@bitCast(flags)); - try writer.writeByte(0); - try io.writeInt(writer, TerminalStyleId, self.style_id); - try io.writeInt(writer, TerminalHyperlinkId, self.hyperlink_id); - try io.writeInt(writer, u32, @bitCast(self.value)); - try io.writeInt(writer, u32, self.grapheme_count); - } - - /// A valid header, or the suffix count from an uninterpretable header. - const DecodeResult = union(enum) { - valid: CellHeader, - invalid: u32, + const wire: Cell = .{ + .kind = @intFromEnum(cell.content_tag), + .content = switch (cell.content_tag) { + .codepoint, + .codepoint_grapheme, + => cell.content.codepoint.data, + .bg_color_palette => cell.content.color_palette.data, + .bg_color_rgb => @as(u24, cell.content.color_rgb.r) | + (@as(u24, cell.content.color_rgb.g) << 8) | + (@as(u24, cell.content.color_rgb.b) << 16), + }, + .style_id = cell.style_id, + .width = switch (cell.wide) { + .narrow => 0, + .wide => 1, + .spacer_tail => 2, + .spacer_head => 3, + }, + .protected = cell.protected, + .hyperlink = link_id != 0, + .semantic_content = switch (cell.semantic_content) { + .output => 0, + .input => 1, + .prompt => 2, + }, + .hyperlink_id = link_id, }; + return @bitCast(wire); +} - /// Decode a fixed cell header, normalizing unknown semantic values. - pub fn decode(reader: *std.Io.Reader) std.Io.Reader.Error!DecodeResult { - // Decode various fields. We need to be very defensive here - // because it can come from an untrusted source and may be invalid. - const content_kind_optional = kind: { - const raw = try reader.takeByte(); - break :kind std.enums.fromInt(Kind, raw); - }; - const width = width: { - const raw = try reader.takeByte(); - break :width std.enums.fromInt(TerminalCell.Wide, raw) orelse .narrow; - }; - - const flags: Flags = @bitCast(try reader.takeByte()); - - // We can't trust `flags` to have a valid semantic content so - // we need extract the bits and do a safe conversion. - const semantic_content = semantic: { - const flags_raw: u8 = @bitCast(flags); - const raw: u2 = @truncate(flags_raw >> @bitOffsetOf(Flags, "semantic_content")); - break :semantic std.enums.fromInt( - TerminalCell.SemanticContent, - raw, - ) orelse .output; - }; - - // This byte is reserved so the IDs and content value remain naturally - // aligned within the fixed cell header. Ignore it so future versions - // can give it meaning without making old decoders reject the cell. - _ = try reader.takeByte(); - - const style_id = try io.readInt(reader, TerminalStyleId); - const hyperlink_id = try io.readInt(reader, TerminalHyperlinkId); - const value: Value = @bitCast(try io.readInt(reader, u32)); - const grapheme_count = try io.readInt(reader, u32); - - // If we don't have a valid content kind, we return invalid and - // note the suffix bytes so that the reader can discard - const content_kind = content_kind_optional orelse - return .{ .invalid = grapheme_count }; - - return .{ .valid = .{ - .content_kind = content_kind, - .width = width, - .protected = flags.protected, - .semantic_content = semantic_content, - .style_id = style_id, - .hyperlink_id = hyperlink_id, - .value = value, - .grapheme_count = grapheme_count, - } }; +/// Store one normalized, hyperlink-free wire word as a native cell. +fn storeCell(cell: *TerminalCell, bits: u64) void { + if (comptime native_matches_wire) { + const words: [*]u64 = @ptrCast(cell); + words[0] = bits; + return; } - /// Computes the fixed header size using the encoder itself. - fn computeLen() usize { - comptime { - var buf: [128]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - CellHeader.encode(.{}, &writer) catch unreachable; - return writer.end; + const wire: Cell = @bitCast(bits); + var native: TerminalCell = .init(0); + switch (@as(Cell.Kind, @enumFromInt(wire.kind))) { + .codepoint, .codepoint_grapheme => native.content = .{ + .codepoint = .{ .data = @intCast(wire.content) }, + }, + .bg_color_palette => { + native.content_tag = .bg_color_palette; + native.content = .{ + .color_palette = .{ .data = @truncate(wire.content) }, + }; + }, + .bg_color_rgb => { + native.content_tag = .bg_color_rgb; + native.content = .{ .color_rgb = .{ + .r = @truncate(wire.content), + .g = @truncate(wire.content >> 8), + .b = @truncate(wire.content >> 16), + } }; + }, + } + native.style_id = wire.style_id; + native.wide = @enumFromInt(wire.width); + native.protected = wire.protected; + native.semantic_content = @enumFromInt(wire.semantic_content); + cell.* = native; +} + +/// Maps encoded table IDs to IDs assigned by the destination page. +/// +/// Build this by inserting each decoded table entry into the page, then +/// recording the encoded ID and the ID returned by the page's set. ID zero +/// is implicit and does not need an entry. Lookup must be cheap because the +/// grid decoder consults it for every styled or linked cell, so this is a +/// direct-indexed table rather than a hash map. +fn Remap(comptime Id: type) type { + return struct { + const Self = @This(); + + /// One slot for every possible encoded ID. + pub const capacity = std.math.maxInt(Id) + 1; + + /// Indexed by encoded ID; zero means unmapped or default. + entries: []Id, + + /// Tracks IDs that received an entry, including ones mapped to the + /// default, so callers can give duplicate table entries first-wins + /// semantics. + seen: std.DynamicBitSetUnmanaged, + + pub fn init(alloc: Allocator) Allocator.Error!Self { + const entries = try alloc.alloc(Id, capacity); + errdefer alloc.free(entries); + @memset(entries, 0); + const seen = try std.DynamicBitSetUnmanaged.initEmpty( + alloc, + capacity, + ); + return .{ .entries = entries, .seen = seen }; } - } - /// Determines how `value` is interpreted. - pub const Kind = enum(u8) { - codepoint = 0, - bg_color_palette = 1, - bg_color_rgb = 2, - }; + pub fn deinit(self: *Self, alloc: Allocator) void { + alloc.free(self.entries); + self.seen.deinit(alloc); + self.* = undefined; + } - /// The alternate interpretations of the four-byte content field. - pub const Value = packed union(u32) { - codepoint: u32, - bg_color_palette: packed struct(u32) { - index: u8, - _padding: u24 = 0, - }, - bg_color_rgb: packed struct(u32) { - r: u8, - g: u8, - b: u8, - _padding: u8 = 0, - }, - }; + /// Record one encoded-to-native mapping. + pub fn put(self: *Self, encoded: Id, native: Id) void { + assert(!self.seen.isSet(encoded)); + self.entries[encoded] = native; + self.seen.set(encoded); + } - const Flags = packed struct(u8) { - protected: bool = false, - semantic_content: TerminalCell.SemanticContent = .output, - _padding: u5 = 0, + /// Whether the encoded ID already has an entry, even a default one. + pub fn contains(self: *const Self, encoded: Id) bool { + return self.seen.isSet(encoded); + } + + /// The native ID for an encoded ID, or zero when unmapped. + pub inline fn get(self: *const Self, encoded: Id) Id { + return self.entries[encoded]; + } }; -}; +} const test_golden_fixture = test_fixture.parse( @embedFile("testdata/grid-v1.hex"), ); +test "cell wire layout registry" { + const testing = std.testing; + + // The wire bit positions are format constants. The shifts and masks + // derive from the packed struct, so pin the struct itself to the + // documented registry so an edit cannot silently move wire bits. + try testing.expectEqual(0, @bitOffsetOf(Cell, "kind")); + try testing.expectEqual(2, @bitOffsetOf(Cell, "content")); + try testing.expectEqual(26, @bitOffsetOf(Cell, "style_id")); + try testing.expectEqual(42, @bitOffsetOf(Cell, "width")); + try testing.expectEqual(44, @bitOffsetOf(Cell, "protected")); + try testing.expectEqual(45, @bitOffsetOf(Cell, "hyperlink")); + try testing.expectEqual(46, @bitOffsetOf(Cell, "semantic_content")); + try testing.expectEqual(48, @bitOffsetOf(Cell, "hyperlink_id")); + + // The native cell currently matches the wire registry, which enables + // the bulk row codec. If this fails, the native layout diverged: either + // restore it or accept the portable codec and update this expectation. + try testing.expect(native_matches_wire); +} + test "grid golden encoding and decoding" { const capacity: terminal_page.Capacity = .{ .cols = 3, @@ -656,10 +983,10 @@ test "grid golden encoding and decoding" { var destination = try TerminalPage.init(capacity); defer destination.deinit(); - var style_remap = StyleRemap.init(std.testing.allocator); - defer style_remap.deinit(); - var hyperlink_remap = HyperlinkRemap.init(std.testing.allocator); - defer hyperlink_remap.deinit(); + var style_remap = try StyleRemap.init(std.testing.allocator); + defer style_remap.deinit(std.testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(std.testing.allocator); + defer hyperlink_remap.deinit(std.testing.allocator); // Decode the checked-in reference through a one-byte reader buffer. var fixture_reader: std.Io.Reader = .fixed(&test_golden_fixture); @@ -746,10 +1073,80 @@ test "grid golden encoding and decoding" { ); } +test "grid elides trailing default cells" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 80, .rows = 3 }); + defer page.deinit(); + + // Row 0 is fully default. Row 1 has content in columns zero and two. + // Row 2 has one protected-only cell at column four. + const middle = page.getRowAndCell(2, 1); + middle.cell.* = .init('b'); + page.getRowAndCell(0, 1).cell.* = .init('a'); + page.getRowAndCell(4, 2).cell.protected = true; + + var encoded: [256]u8 = undefined; + var writer: std.Io.Writer = .fixed(&encoded); + try encode(&page, &writer); + + // 3 bytes per row header, cells only through the last non-default + // cell, and an empty grapheme section. + try testing.expectEqual( + @as(usize, 3 + (3 + 3 * 8) + (3 + 5 * 8) + 4), + writer.buffered().len, + ); + + var destination = try TerminalPage.init(.{ .cols = 80, .rows = 3 }); + defer destination.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&destination, &reader, &style_remap, &hyperlink_remap); + try destination.verifyIntegrity(testing.allocator); + + try testing.expectEqual( + @as(u21, 'a'), + destination.getRowAndCell(0, 1).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 'b'), + destination.getRowAndCell(2, 1).cell.codepoint(), + ); + try testing.expect(destination.getRowAndCell(4, 2).cell.protected); + try testing.expect(destination.getRowAndCell(79, 1).cell.isZero()); +} + +test "grid rejects a row cell count above the column count" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{})); + try io.writeInt(&writer, u16, 3); + for (0..3) |_| try io.writeInt(&writer, u64, 0); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try testing.expectError( + error.InvalidRowCellCount, + decode(&page, &reader, &style_remap, &hyperlink_remap), + ); +} + test "grid normalizes incomplete wide cells" { const testing = std.testing; // One column puts the wide cell at the row end. Two columns put an // ordinary narrow cell after it. Neither case supplies a spacer tail. + // Both rely on the implicit narrow cell rule when the tail is elided. for ([_]u16{ 1, 2 }) |columns| { const capacity: terminal_page.Capacity = .{ .cols = columns, @@ -760,21 +1157,23 @@ test "grid normalizes incomplete wide cells" { // content but makes the incomplete wide cell narrow. var payload: [64]u8 = undefined; var payload_writer: std.Io.Writer = .fixed(&payload); - try payload_writer.writeByte(@bitCast(RowHeader{})); - try (CellHeader{ - .width = .wide, - .value = .{ .codepoint = 'A' }, - }).encode(&payload_writer); - if (capacity.cols > 1) try (CellHeader{ - .value = .{ .codepoint = 'B' }, - }).encode(&payload_writer); + try payload_writer.writeByte(@bitCast(Row{})); + try io.writeInt(&payload_writer, u16, columns); + try io.writeInt(&payload_writer, u64, @bitCast(Cell{ + .width = 1, // wide + .content = 'A', + })); + if (columns > 1) try io.writeInt(&payload_writer, u64, @bitCast(Cell{ + .content = 'B', + })); + try io.writeInt(&payload_writer, u32, 0); var destination = try TerminalPage.init(capacity); defer destination.deinit(); - var style_remap = StyleRemap.init(testing.allocator); - defer style_remap.deinit(); - var hyperlink_remap = HyperlinkRemap.init(testing.allocator); - defer hyperlink_remap.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); var payload_reader: std.Io.Reader = .fixed( payload_writer.buffered(), @@ -801,4 +1200,173 @@ test "grid normalizes incomplete wide cells" { try testing.expectEqual(TerminalCell.Wide.narrow, second.wide); } } + + // A wide marker whose spacer tail was elided by a short cell count is + // also normalized, and the trailing cells stay default. + var page = try TerminalPage.init(.{ .cols = 4, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{})); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u64, @bitCast(Cell{ + .width = 1, // wide + .content = 'W', + })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + TerminalCell.Wide.narrow, + page.getRowAndCell(0, 0).cell.wide, + ); + try testing.expect(page.getRowAndCell(1, 0).cell.isZero()); +} + +test "grid normalizes reserved cell values" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 3, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + + // Reserved row flag bits are ignored while the unknown semantic-prompt + // value degrades to none and wrap survives. + try writer.writeByte(0xFD); + try io.writeInt(&writer, u16, 3); + + // A surrogate codepoint with reserved semantic content 3. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .content = 0xD800, + .semantic_content = 3, + })); + + // Reserved palette content bits do not obscure the palette index. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .kind = 2, + .content = 0xFFFF07, + })); + + // An unknown style reference degrades to the default style, and an + // unknown hyperlink reference leaves the cell unlinked. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .content = 'x', + .style_id = 5, + .hyperlink = true, + .hyperlink_id = 9, + })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expect(first.row.wrap); + try testing.expectEqual( + TerminalRow.SemanticPrompt.none, + first.row.semantic_prompt, + ); + try testing.expectEqual(@as(u21, 0xFFFD), first.cell.codepoint()); + try testing.expectEqual( + TerminalCell.SemanticContent.output, + first.cell.semantic_content, + ); + + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual( + TerminalCell.ContentTag.bg_color_palette, + second.content_tag, + ); + try testing.expectEqual(@as(u8, 7), second.content.color_palette.data); + + const third = page.getRowAndCell(2, 0).cell; + try testing.expectEqual(@as(u21, 'x'), third.codepoint()); + try testing.expectEqual(@as(TerminalStyleId, 0), third.style_id); + try testing.expect(!third.hyperlink); + try testing.expectEqual(null, page.lookupHyperlink(third)); +} + +test "grid drops undeliverable grapheme entries" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 4, + .rows = 1, + .grapheme_bytes = 64, + }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [128]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{})); + try io.writeInt(&writer, u16, 4); + // A kind 1 cell that receives a valid entry. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'x' })); + // A kind 1 cell without an entry decodes as a plain codepoint. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'y' })); + // A background cell cannot carry a suffix. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 2, .content = 7 })); + // An empty codepoint cannot carry a suffix. + try io.writeInt(&writer, u64, @bitCast(Cell{})); + + try io.writeInt(&writer, u32, 5); + // Valid entry with one invalid scalar dropped from within it. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 3); + try io.writeInt(&writer, u32, 0x0301); + try io.writeInt(&writer, u32, 0xD800); + try io.writeInt(&writer, u32, 0x0302); + // Duplicate entry for the same cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0303); + // Entry for the background cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + // Entry for the empty cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 3); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + // Entry outside the grid is consumed and dropped. + try io.writeInt(&writer, u16, 7); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expectEqualSlices( + u21, + &.{ 0x0301, 0x0302 }, + page.lookupGrapheme(first.cell).?, + ); + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual(@as(u21, 'y'), second.codepoint()); + try testing.expect(!second.hasGrapheme()); + try testing.expect(!page.getRowAndCell(2, 0).cell.hasGrapheme()); + try testing.expect(!page.getRowAndCell(3, 0).cell.hasGrapheme()); } diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index e71bc53c4..d96e118f5 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -63,19 +63,19 @@ //! | hyperlink_count entries | //! +---------------------------+ //! | Grid | -//! | one record per row | +//! | rows and grapheme section | //! +---------------------------+ //! //! Style entry = encoded ID + style record //! Hyperlink entry = encoded ID + hyperlink record -//! Grid = row 0 ... row (rows - 1) -//! Row = row header + cell 0 ... cell (columns - 1) -//! Cell = cell header + grapheme suffix codepoints +//! Grid = row 0 ... row (rows - 1) + grapheme suffix section +//! Row = row header + encoded cells //! ``` //! //! Following the header, the payload contains exactly `style_count` style //! records and `hyperlink_count` hyperlink records, followed by exactly -//! `rows` row records. There is no padding between records. +//! `rows` row records and one grapheme suffix section. There is no padding +//! between records. //! //! Each style begins with an ID (`u16`) followed by the typical style //! binary representation (in style.zig). The ID is what cells will use @@ -119,6 +119,7 @@ const PayloadEncodeError = hyperlink.EncodeError || grid.EncodeError; const PayloadDecodeError = std.Io.Reader.Error || Header.CapacityError || + grid.DecodeError || error{ /// The hyperlink kind is not defined by snapshot version 1. InvalidKind, @@ -284,15 +285,13 @@ fn decodePayloadBody( page.pauseIntegrityChecks(true); defer page.pauseIntegrityChecks(false); - var style_remap = grid.StyleRemap.init(alloc); - defer style_remap.deinit(); - style_remap.ensureTotalCapacity(header.style_count) catch + var style_remap = grid.StyleRemap.init(alloc) catch return error.OutOfMemory; + defer style_remap.deinit(alloc); - var hyperlink_remap = grid.HyperlinkRemap.init(alloc); - defer hyperlink_remap.deinit(); - hyperlink_remap.ensureTotalCapacity(header.hyperlink_count) catch + var hyperlink_remap = grid.HyperlinkRemap.init(alloc) catch return error.OutOfMemory; + defer hyperlink_remap.deinit(alloc); // Styles for (0..header.style_count) |_| { @@ -316,10 +315,7 @@ fn decodePayloadBody( valid, ) catch 0; } else 0; - style_remap.putAssumeCapacityNoClobber( - native_id, - decoded_id, - ); + style_remap.put(native_id, decoded_id); } // Hyperlinks @@ -338,10 +334,7 @@ fn decodePayloadBody( // Zero records an ignored table entry. Grid decoding treats every // reference to it as no hyperlink. - hyperlink_remap.putAssumeCapacityNoClobber( - native_id, - decoded_id, - ); + hyperlink_remap.put(native_id, decoded_id); } // Rows and cells @@ -860,8 +853,7 @@ test "decode accepts unordered sparse style IDs and ignores zero" { var descending: [ Header.len + 2 * (2 + style.len) + - 1 + - 16 + 7 ]u8 = undefined; var descending_writer: std.Io.Writer = .fixed(&descending); try header.encode(&descending_writer); @@ -869,8 +861,9 @@ test "decode accepts unordered sparse style IDs and ignores zero" { try style.encode(.{ .flags = .{ .bold = true } }, &descending_writer); try io.writeInt(&descending_writer, TerminalStyleId, 2); try style.encode(.{ .flags = .{ .italic = true } }, &descending_writer); - try descending_writer.writeByte(0); - try descending_writer.splatByteAll(0, 16); + try descending_writer.writeByte(0); // row flags + try io.writeInt(&descending_writer, u16, 0); // cell count + try io.writeInt(&descending_writer, u32, 0); // grapheme section var descending_reader: std.Io.Reader = .fixed( descending_writer.buffered(), @@ -893,7 +886,7 @@ test "decode accepts unordered sparse style IDs and ignores zero" { .string_capacity_bytes = 0, }; - var zero: [Header.len + 2 + style.len + 17]u8 = undefined; + var zero: [Header.len + 2 + style.len + 7]u8 = undefined; var zero_writer: std.Io.Writer = .fixed(&zero); try one_header.encode(&zero_writer); try io.writeInt(&zero_writer, TerminalStyleId, 0); @@ -936,8 +929,7 @@ test "decode accepts unordered sparse hyperlink IDs" { var encoded: [ Header.len + 2 * 14 + - 1 + - 16 + 7 ]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try header.encode(&writer); @@ -945,8 +937,9 @@ test "decode accepts unordered sparse hyperlink IDs" { try hyperlink.encode(first, &writer); try io.writeInt(&writer, TerminalHyperlinkId, 2); try hyperlink.encode(second, &writer); - try writer.writeByte(0); - try writer.splatByteAll(0, 16); + try writer.writeByte(0); // row flags + try io.writeInt(&writer, u16, 0); // cell count + try io.writeInt(&writer, u32, 0); // grapheme section var reader: std.Io.Reader = .fixed(writer.buffered()); var decoded = try decodePayload( @@ -973,15 +966,15 @@ test "decode defaults missing sparse cell references" { .string_capacity_bytes = 0, }; - var style_encoded: [Header.len + 1 + 16]u8 = undefined; + var style_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var style_writer: std.Io.Writer = .fixed(&style_encoded); try style_header.encode(&style_writer); - try style_writer.writeByte(0); - try style_writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&style_writer, TerminalStyleId, 1); - try io.writeInt(&style_writer, TerminalHyperlinkId, 0); - try io.writeInt(&style_writer, u32, 0); - try io.writeInt(&style_writer, u32, 0); + try style_writer.writeByte(0); // row flags + try io.writeInt(&style_writer, u16, 1); // cell count + try io.writeInt(&style_writer, u64, @bitCast(grid.Cell{ + .style_id = 1, + })); + try io.writeInt(&style_writer, u32, 0); // grapheme section var style_reader: std.Io.Reader = .fixed(style_writer.buffered()); var style_page = try decodePayload( @@ -1006,15 +999,16 @@ test "decode defaults missing sparse cell references" { .string_capacity_bytes = 0, }; - var hyperlink_encoded: [Header.len + 1 + 16]u8 = undefined; + var hyperlink_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var hyperlink_writer: std.Io.Writer = .fixed(&hyperlink_encoded); try hyperlink_header.encode(&hyperlink_writer); - try hyperlink_writer.writeByte(0); - try hyperlink_writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&hyperlink_writer, TerminalStyleId, 0); - try io.writeInt(&hyperlink_writer, TerminalHyperlinkId, 1); - try io.writeInt(&hyperlink_writer, u32, 0); - try io.writeInt(&hyperlink_writer, u32, 0); + try hyperlink_writer.writeByte(0); // row flags + try io.writeInt(&hyperlink_writer, u16, 1); // cell count + try io.writeInt(&hyperlink_writer, u64, @bitCast(grid.Cell{ + .hyperlink = true, + .hyperlink_id = 1, + })); + try io.writeInt(&hyperlink_writer, u32, 0); // grapheme section var hyperlink_reader: std.Io.Reader = .fixed( hyperlink_writer.buffered(), @@ -1044,41 +1038,42 @@ test "decode normalizes invalid grid semantics" { .string_capacity_bytes = 0, }; - var encoded: [Header.len + 1 + 3 * 16 + 12]u8 = undefined; + var encoded: [Header.len + 3 + 3 * 8 + 4 + 10]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try header.encode(&writer); // Preserve wrap while degrading the unknown semantic-prompt value to none // and ignoring all reserved row bits. try writer.writeByte(0xFD); + try io.writeInt(&writer, u16, 3); - // An unknown content and width kind, invalid semantic content, reserved - // bytes, and suffix data all degrade to a blank narrow output cell. - try writer.writeAll(&.{ 3, 4, 0xFF, 0xFF }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xD800); - try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 0x110000); + // A text cell replaces an invalid Unicode scalar with U+FFFD while its + // reserved semantic content degrades to output. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .content = 0xD800, + .semantic_content = 3, + })); - // A recognized text cell replaces an invalid Unicode scalar with U+FFFD. - // Its valid suffix cannot fit the advertised zero capacity, so decoding - // preserves the base character and drops the optional suffix. - try writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xD800); - try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 'x'); + // A declared grapheme suffix cannot fit the advertised zero capacity, + // so decoding preserves the base character and drops the suffix. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .kind = 1, + .content = 'x', + })); - // Reserved palette bits and a nonsensical suffix do not obscure the valid - // low palette byte; the suffix is consumed and ignored. - try writer.writeAll(&.{ 1, 0, 0, 0xFF }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xFFFFFF07); + // Reserved palette content bits do not obscure the valid low palette + // byte. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .kind = 2, + .content = 0xFFFF07, + })); + + // The grapheme section carries the suffix for the second cell. try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 'x'); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); var reader: std.Io.Reader = .fixed(writer.buffered()); var page = try decodePayload(&reader, std.testing.allocator); @@ -1091,7 +1086,7 @@ test "decode normalizes invalid grid semantics" { terminal_page.Row.SemanticPrompt.none, first.row.semantic_prompt, ); - try std.testing.expectEqual(@as(u21, 0), first.cell.codepoint()); + try std.testing.expectEqual(@as(u21, 0xFFFD), first.cell.codepoint()); try std.testing.expectEqual( terminal_page.Cell.Wide.narrow, first.cell.wide, @@ -1104,7 +1099,7 @@ test "decode normalizes invalid grid semantics" { try std.testing.expect(!first.cell.hasGrapheme()); const second = page.getRowAndCell(1, 0).cell; - try std.testing.expectEqual(@as(u21, 0xFFFD), second.codepoint()); + try std.testing.expectEqual(@as(u21, 'x'), second.codepoint()); try std.testing.expect(!second.hasGrapheme()); const third = page.getRowAndCell(2, 0).cell; diff --git a/src/terminal/snapshot/screen.zig b/src/terminal/snapshot/screen.zig index 875d2bb0e..06dc8bce6 100644 --- a/src/terminal/snapshot/screen.zig +++ b/src/terminal/snapshot/screen.zig @@ -205,6 +205,7 @@ const Allocator = std.mem.Allocator; const hyperlink = @import("hyperlink.zig"); const test_fixture = @import("fixture.zig"); const io = @import("io.zig"); +const grid = @import("grid.zig"); const page = @import("page.zig"); const record = @import("record.zig"); const style = @import("style.zig"); @@ -2261,20 +2262,14 @@ test "SCREEN decode ignores a PAGE with an empty hyperlink URI" { // One narrow codepoint cell refers to the hyperlink table entry above. // Since that entry is ignored, the cell must restore without a hyperlink. - try page_payload.writeByte(0); - try page_payload.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt( - page_payload, - terminal_style.Id, - 0, - ); - try io.writeInt( - page_payload, - terminal_hyperlink.Id, - 1, - ); - try io.writeInt(page_payload, u32, 'A'); - try io.writeInt(page_payload, u32, 0); + try page_payload.writeByte(0); // row flags + try io.writeInt(page_payload, u16, 1); // cell count + try io.writeInt(page_payload, u64, @bitCast(grid.Cell{ + .content = 'A', + .hyperlink = true, + .hyperlink_id = 1, + })); + try io.writeInt(page_payload, u32, 0); // grapheme section try stream.finish(); var source: std.Io.Reader = .fixed(destination.written()); diff --git a/src/terminal/snapshot/snapshot.ksy b/src/terminal/snapshot/snapshot.ksy index 6a31fccd6..c503fe2f7 100644 --- a/src/terminal/snapshot/snapshot.ksy +++ b/src/terminal/snapshot/snapshot.ksy @@ -914,18 +914,28 @@ types: type: grid_row(columns) repeat: expr repeat-expr: num_rows + - id: num_grapheme_entries + type: u4 + - id: grapheme_entries + type: grapheme_entry(num_rows, columns) + repeat: expr + repeat-expr: num_grapheme_entries grid_row: params: - - id: num_cells + - id: columns type: u2 seq: - id: flags type: grid_row_flags + - id: cell_count + type: u2 + valid: + expr: _ <= columns - id: cells - type: grid_cell(_index, num_cells, flags.wrap) + type: grid_cell(_index, columns, flags.wrap) repeat: expr - repeat-expr: num_cells + repeat-expr: cell_count grid_row_flags: seq: @@ -942,6 +952,17 @@ types: value: (raw >> 2) & 0x3 grid_cell: + doc: | + One 64-bit little-endian cell word. The word is parsed as two 32-bit + halves so every derived field stays within JavaScript's safe integer + range, following the same approach as mode_set. + + Canonical rules validated here: reserved semantic content is not + emitted, the hyperlink flag matches a nonzero hyperlink ID, codepoint + content is a Unicode scalar value, palette content uses only its low + eight bits, and spacer relationships match the preceding cell. Wide + markers cannot be validated forward because their spacer tail may be + the next cell or the implicit narrow cell after a short row. params: - id: index type: u2 @@ -950,59 +971,61 @@ types: - id: row_wrap type: bool seq: - - id: content_kind - type: u1 - valid: - max: 2 - - id: width - type: u1 + - id: lo + type: u4 + - id: hi + type: u4 valid: expr: | - _ <= 3 and - (_ != 2 or + semantic_content <= 2 and + hyperlink == (hyperlink_id != 0) and + (content_kind >= 2 or + (content <= 0x10ffff and + not (content >= 0xd800 and content <= 0xdfff))) and + (content_kind != 2 or content <= 0xff) and + (width != 2 or (index > 0 and _parent.cells[index - 1].width == 1)) and - (_ != 3 or - (index + 1 == columns and row_wrap)) - - id: flags - type: grid_cell_flags - - id: reserved - type: u1 - valid: 0 - - id: style_id + (width != 3 or (index + 1 == columns and row_wrap)) + instances: + content_kind: + value: lo % 4 + content: + value: (lo / 4) % 16777216 + style_id: + value: (lo / 67108864) + (hi % 1024) * 64 + width: + value: (hi / 1024) % 4 + protected: + value: (hi / 4096) % 2 != 0 + hyperlink: + value: (hi / 8192) % 2 != 0 + semantic_content: + value: (hi / 16384) % 4 + hyperlink_id: + value: hi / 65536 + + grapheme_entry: + seq: + - id: row type: u2 - - id: hyperlink_id + valid: + expr: _ < num_rows + - id: col type: u2 - - id: value - type: u4 valid: - expr: | - content_kind == 0 ? - (_ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) and _ != 0x10eeee) : - content_kind == 1 ? - _ <= 0xff : - _ <= 0xffffff - - id: num_graphemes - type: u4 + expr: _ < columns + - id: num_codepoints + type: u2 valid: - expr: | - content_kind == 0 ? - (_ == 0 or value != 0) : - _ == 0 - - id: graphemes + min: 1 + - id: codepoints type: u4 repeat: expr - repeat-expr: num_graphemes + repeat-expr: num_codepoints valid: - expr: _ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) and _ != 0x10eeee - - grid_cell_flags: - seq: - - id: raw - type: u1 - valid: - expr: (_ & 0xf8) == 0 and ((_ >> 1) & 0x3) <= 2 - instances: - protected: - value: (raw & (1 << 0)) != 0 - semantic_content: - value: (raw >> 1) & 0x3 + expr: _ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) + params: + - id: num_rows + type: u2 + - id: columns + type: u2 diff --git a/src/terminal/snapshot/testdata/complete-v1.hex b/src/terminal/snapshot/testdata/complete-v1.hex index 83e3e174c..e88db31c0 100644 --- a/src/terminal/snapshot/testdata/complete-v1.hex +++ b/src/terminal/snapshot/testdata/complete-v1.hex @@ -78,65 +78,52 @@ ee ee 80 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000037a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003ec 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003fc -# offset 0x0000040c: page record, payload 119 bytes -03 00 77 00 00 00 48 b7 91 cb 02 00 03 00 00 00 # 0x0000040c -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x0000041c -00 00 00 00 00 00 00 43 00 00 00 00 00 00 00 00 # 0x0000042c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000043c -00 00 00 00 00 00 00 00 44 00 00 00 00 00 00 00 # 0x0000044c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000045c -00 00 00 00 00 00 00 00 00 45 00 00 00 00 00 00 # 0x0000046c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000047c -00 # 0x0000048c +# offset 0x0000040c: page record, payload 57 bytes +03 00 39 00 00 00 26 5a 94 0b 02 00 03 00 00 00 # 0x0000040c +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x0000041c +00 0c 01 00 00 00 00 00 00 00 01 00 10 01 00 00 # 0x0000042c +00 00 00 00 00 01 00 14 01 00 00 00 00 00 00 00 # 0x0000043c +00 00 00 # 0x0000044c -# offset 0x0000048d: screen record, payload 54 bytes -02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000048d -00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000049d -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000004ad -00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000004bd +# offset 0x0000044f: screen record, payload 54 bytes +02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000044f +00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000045f +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046f +00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000047f -# offset 0x000004cd: page record, payload 119 bytes -03 00 77 00 00 00 84 a6 e9 08 02 00 03 00 00 00 # 0x000004cd -00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 00 # 0x000004dd -00 00 00 00 00 00 00 72 00 00 00 00 00 00 00 00 # 0x000004ed -00 00 00 00 00 00 00 6e 00 00 00 00 00 00 00 03 # 0x000004fd -00 00 00 00 00 00 00 00 61 00 00 00 00 00 00 00 # 0x0000050d -00 00 00 00 00 00 00 00 74 00 00 00 00 00 00 00 # 0x0000051d -03 00 00 00 00 00 00 00 00 65 00 00 00 00 00 00 # 0x0000052d -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000053d -00 # 0x0000054d +# offset 0x0000048f: page record, payload 73 bytes +03 00 49 00 00 00 37 af 30 0e 02 00 03 00 00 00 # 0x0000048f +00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000049f +00 c8 01 00 00 00 00 00 00 b8 01 00 00 00 00 00 # 0x000004af +00 03 02 00 84 01 00 00 00 00 00 00 d0 01 00 00 # 0x000004bf +00 00 00 00 03 01 00 94 01 00 00 00 00 00 00 00 # 0x000004cf +00 00 00 # 0x000004df -# offset 0x0000054e: continuation record, payload 0 bytes -07 00 00 00 00 00 27 80 63 d1 # 0x0000054e +# offset 0x000004e2: continuation record, payload 0 bytes +07 00 00 00 00 00 27 80 63 d1 # 0x000004e2 -# offset 0x00000558: ready record, payload 32 bytes -05 00 20 00 00 00 d4 d1 fc d0 8d 69 76 c1 6e c6 # 0x00000558 -88 21 bd 10 97 41 97 35 63 2d ce 9e 3c b8 fa cf # 0x00000568 -57 04 f5 4b e0 70 66 5e 7e b6 # 0x00000578 +# offset 0x000004ec: ready record, payload 32 bytes +05 00 20 00 00 00 5d f3 97 80 41 a2 18 18 cb 82 # 0x000004ec +8c 97 b6 58 14 60 26 9a 88 2e be d4 c3 4b 83 c9 # 0x000004fc +9d 32 9d de e6 70 17 4a 5d bc # 0x0000050c -# offset 0x00000582: history record, payload 6 bytes -04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x00000582 +# offset 0x00000516: history record, payload 6 bytes +04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x00000516 -# offset 0x00000592: page record, payload 86 bytes -03 00 56 00 00 00 52 3b 6e 8d 02 00 02 00 00 00 # 0x00000592 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x000005a2 -00 00 00 00 00 00 00 42 00 00 00 00 00 00 00 00 # 0x000005b2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005c2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005d2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005e2 +# offset 0x00000526: page record, payload 38 bytes +03 00 26 00 00 00 9e 10 a2 93 02 00 02 00 00 00 # 0x00000526 +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000536 +00 08 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000546 -# offset 0x000005f2: page record, payload 86 bytes -03 00 56 00 00 00 fb bc 15 06 02 00 02 00 00 00 # 0x000005f2 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x00000602 -00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 00 # 0x00000612 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000622 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000632 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000642 +# offset 0x00000556: page record, payload 38 bytes +03 00 26 00 00 00 70 e1 36 3a 02 00 02 00 00 00 # 0x00000556 +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000566 +00 04 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000576 -# offset 0x00000652: history record, payload 6 bytes -04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000652 +# offset 0x00000586: history record, payload 6 bytes +04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000586 -# offset 0x00000662: finish record, payload 32 bytes -06 00 20 00 00 00 f3 a4 cf b6 f2 2b 17 88 1a 28 # 0x00000662 -a7 27 6e f7 de bc 7a 52 f9 a4 f9 c3 32 5b e0 09 # 0x00000672 -8a 1a 7d a0 9f 8b 32 31 67 41 # 0x00000682 +# offset 0x00000596: finish record, payload 32 bytes +06 00 20 00 00 00 20 0f e0 7e 10 4f 2c da 90 fa # 0x00000596 +65 19 a9 73 26 75 2a 5b 33 cd 9d fa 73 81 86 07 # 0x000005a6 +9e 57 d4 83 41 9f a9 81 64 e7 # 0x000005b6 diff --git a/src/terminal/snapshot/testdata/grid-v1.hex b/src/terminal/snapshot/testdata/grid-v1.hex index 3389a876f..b266f5438 100644 --- a/src/terminal/snapshot/testdata/grid-v1.hex +++ b/src/terminal/snapshot/testdata/grid-v1.hex @@ -6,27 +6,9 @@ # Generated by its snapshot test; review before replacing. # On mismatch, the candidate is copied to the repository root. -# row 0: prompt -04 - -# row 0, cell 0: protected prompt, wide "A" -00 01 05 00 00 00 00 00 41 00 00 00 00 00 00 00 - -# row 0, cell 1: input spacer tail -00 02 02 00 00 00 00 00 00 00 00 00 00 00 00 00 - -# row 0, cell 2: "x" with two grapheme suffixes -00 00 00 00 00 00 00 00 78 00 00 00 02 00 00 00 -01 03 00 00 02 03 00 00 - -# row 1: wrapped prompt continuation -0b - -# row 1, cell 0: palette background 7 -01 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 - -# row 1, cell 1: protected RGB background -02 00 01 00 00 00 00 00 aa bb cc 00 00 00 00 00 - -# row 1, cell 2: spacer head -00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# offset 0x00000000: encoded bytes +04 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 +48 00 00 e1 01 00 00 00 00 00 00 0b 03 00 1e 00 # 0x00000010 +00 00 00 00 00 00 ab ee 32 03 00 10 00 00 00 00 # 0x00000020 +00 00 00 0c 00 00 01 00 00 00 00 00 02 00 02 00 # 0x00000030 +01 03 00 00 02 03 00 00 # 0x00000040 diff --git a/src/terminal/snapshot/testdata/page-empty-record-v1.hex b/src/terminal/snapshot/testdata/page-empty-record-v1.hex index 4b759b704..cda4f1d91 100644 --- a/src/terminal/snapshot/testdata/page-empty-record-v1.hex +++ b/src/terminal/snapshot/testdata/page-empty-record-v1.hex @@ -7,6 +7,6 @@ # On mismatch, the candidate is copied to the repository root. # offset 0x00000000: encoded bytes -03 00 25 00 00 00 8c 05 d6 d3 01 00 01 00 00 00 # 0x00000000 +03 00 1b 00 00 00 5e 0d 0a d4 01 00 01 00 00 00 # 0x00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000010 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000020 +00 00 00 00 00 # 0x00000020 diff --git a/src/terminal/snapshot/testdata/page-v1.hex b/src/terminal/snapshot/testdata/page-v1.hex index dfc53eeaa..c572decc8 100644 --- a/src/terminal/snapshot/testdata/page-v1.hex +++ b/src/terminal/snapshot/testdata/page-v1.hex @@ -15,10 +15,8 @@ 00 00 03 00 00 00 00 00 01 2a 00 00 00 00 00 00 # 0x00000024 00 00 00 00 01 00 02 01 00 00 00 61 05 00 00 00 # 0x00000034 61 6c 70 68 61 03 00 01 04 03 02 01 04 00 00 00 # 0x00000044 -62 65 74 61 04 00 01 05 00 01 00 01 00 41 00 00 # 0x00000054 -00 00 00 00 00 00 02 02 00 03 00 03 00 00 00 00 # 0x00000064 -00 00 00 00 00 01 00 00 00 01 00 01 00 07 00 00 # 0x00000074 -00 00 00 00 00 0b 00 00 00 00 00 00 00 00 78 00 # 0x00000084 -00 00 02 00 00 00 01 03 00 00 02 03 00 00 02 00 # 0x00000094 -01 00 00 00 00 00 aa bb cc 00 00 00 00 00 00 03 # 0x000000a4 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000000b4 +62 65 74 61 04 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 +00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 0b # 0x00000064 +03 00 e1 01 00 00 00 00 00 00 ab ee 32 03 00 10 # 0x00000074 +00 00 00 00 00 00 00 0c 00 00 01 00 00 00 01 00 # 0x00000084 +00 00 02 00 01 03 00 00 02 03 00 00 # 0x00000094 From 9cc061c28cc79f5c11eae1299fe06fe486ada157 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:35:25 -0700 Subject: [PATCH 4/7] terminal/snapshot: hardware-accelerated CRC32C --- src/crc32c.zig | 193 +++++++++++++++++++++++++++++++ src/terminal/snapshot/record.zig | 7 +- 2 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 src/crc32c.zig diff --git a/src/crc32c.zig b/src/crc32c.zig new file mode 100644 index 000000000..8b8ebd1b2 --- /dev/null +++ b/src/crc32c.zig @@ -0,0 +1,193 @@ +//! CRC32C with hardware acceleration. +//! +//! The Zig standard library implementation processes one byte per table +//! lookup (as of Zig 0.16), which is more than an order of magnitude slower +//! than the dedicated CRC32C instructions available on aarch64 (CRC +//! extension) and x86_64 (SSE4.2). This module selects the best backend at +//! compile time and falls back to the standard library elsewhere, including +//! WebAssembly. +//! +//! The resulting value is identical across all backends: this is the +//! iSCSI CRC32C parameter set (reflected, initial and final XOR +//! `0xFFFFFFFF`), matching `std.hash.crc.Crc32Iscsi`. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// The standard-library implementation of the same parameter set. This is +/// both the portable fallback and the reference the tests compare against. +const Software = std.hash.crc.Crc32Iscsi; + +const Backend = enum { + aarch64_crc, + x86_64_sse42, + software, +}; + +const backend: Backend = backend: { + switch (builtin.cpu.arch) { + .aarch64, + .aarch64_be, + => if (std.Target.aarch64.featureSetHas( + builtin.cpu.features, + .crc, + )) break :backend .aarch64_crc, + + // The self-hosted x86_64 backend cannot encode the CRC32 + // instruction forms used below, so that combination falls back to + // the portable implementation. + .x86_64 => if (builtin.zig_backend == .stage2_llvm and + std.Target.x86.featureSetHas( + builtin.cpu.features, + .sse4_2, + )) break :backend .x86_64_sse42, + + else => {}, + } + break :backend .software; +}; + +/// Streaming CRC32C with the same interface shape as `std.hash.crc` types. +pub const Crc32c = struct { + crc: u32, + + pub fn init() Crc32c { + return .{ .crc = 0xFFFF_FFFF }; + } + + pub fn update(self: *Crc32c, bytes: []const u8) void { + self.crc = switch (comptime backend) { + .aarch64_crc, .x86_64_sse42 => updateHardware(self.crc, bytes), + .software => software: { + var crc: Software = .{ .crc = self.crc }; + crc.update(bytes); + break :software crc.crc; + }, + }; + } + + pub fn final(self: Crc32c) u32 { + return self.crc ^ 0xFFFF_FFFF; + } + + pub fn hash(bytes: []const u8) u32 { + var c: Crc32c = .init(); + c.update(bytes); + return c.final(); + } +}; + +/// One update pass using the dedicated CRC32C instructions. Both supported +/// architectures handle unaligned loads efficiently, so the loop reads +/// little-endian words directly from the input. +fn updateHardware(initial: u32, bytes: []const u8) u32 { + var crc = initial; + var remaining = bytes; + + while (remaining.len >= 8) : (remaining = remaining[8..]) { + crc = step(u64, crc, std.mem.readInt( + u64, + remaining[0..8], + .little, + )); + } + if (remaining.len >= 4) { + crc = step(u32, crc, std.mem.readInt( + u32, + remaining[0..4], + .little, + )); + remaining = remaining[4..]; + } + for (remaining) |byte| crc = step(u8, crc, byte); + return crc; +} + +/// One CRC32C instruction folding `value` into the running CRC. +inline fn step(comptime T: type, crc: u32, value: T) u32 { + return switch (comptime backend) { + .aarch64_crc => switch (T) { + u8 => asm ("crc32cb %[out:w], %[crc:w], %[value:w]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + u32 => asm ("crc32cw %[out:w], %[crc:w], %[value:w]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + u64 => asm ("crc32cx %[out:w], %[crc:w], %[value:x]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + else => comptime unreachable, + }, + + .x86_64_sse42 => switch (T) { + u8 => asm ("crc32b %[value], %[out]" + : [out] "=r" (-> u32), + : [value] "r" (value), + [crc_in] "0" (crc), + ), + u32 => asm ("crc32l %[value], %[out]" + : [out] "=r" (-> u32), + : [value] "r" (value), + [crc_in] "0" (crc), + ), + u64 => @truncate(asm ("crc32q %[value], %[out]" + : [out] "=r" (-> u64), + : [value] "r" (value), + [crc_in] "0" (@as(u64, crc)), + )), + else => comptime unreachable, + }, + + .software => comptime unreachable, + }; +} + +test "matches the check value" { + // The catalog check value for CRC-32/ISCSI. + try std.testing.expectEqual( + @as(u32, 0xE3069283), + Crc32c.hash("123456789"), + ); +} + +test "matches the standard library at every length and split" { + var bytes: [259]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0xC5C32C); + prng.random().bytes(&bytes); + + for (0..bytes.len + 1) |len| { + const input = bytes[0..len]; + try std.testing.expectEqual( + Software.hash(input), + Crc32c.hash(input), + ); + + // Streaming across arbitrary split points must not change the + // result: word batching may not leak state between updates. + var split: Crc32c = .init(); + split.update(input[0 .. len / 3]); + split.update(input[len / 3 .. len - len / 3]); + split.update(input[len - len / 3 ..]); + try std.testing.expectEqual(Software.hash(input), split.final()); + } +} + +test "matches the standard library at every alignment" { + var bytes: [64 + 16]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0xA11C); + prng.random().bytes(&bytes); + + for (0..16) |offset| { + const input = bytes[offset..][0..64]; + try std.testing.expectEqual( + Software.hash(input), + Crc32c.hash(input), + ); + } +} diff --git a/src/terminal/snapshot/record.zig b/src/terminal/snapshot/record.zig index fd69d0b41..c50987f18 100644 --- a/src/terminal/snapshot/record.zig +++ b/src/terminal/snapshot/record.zig @@ -29,9 +29,10 @@ const Blake3 = std.crypto.hash.Blake3; /// The running digest shared by snapshot stream codecs and checkpoints. pub const PrefixDigest = [Blake3.digest_length]u8; -/// CRC32C as specified by the snapshot format. Zig names this standard -/// parameter set after its iSCSI use. -pub const Crc32c = std.hash.crc.Crc32Iscsi; +/// CRC32C as specified by the snapshot format. This is the parameter set +/// Zig's standard library names after its iSCSI use, backed by dedicated +/// CRC32C instructions where the target has them. +pub const Crc32c = @import("../../crc32c.zig").Crc32c; /// Identifies the layout and meaning of a record payload. /// The current snapshot version rejects every value not listed here. From 9f66563479df3b12e08d28fca2bb7bbe4ce65e16 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:37:17 -0700 Subject: [PATCH 5/7] terminal/snapshot: gate page verification on slow runtime safety PAGE decoding verified the complete native integrity of every decoded page unconditionally, building per-cell reference maps that accounted for roughly a fifth of decode time. The decoder normalizes every semantic value while decoding, so a completed decode upholds page invariants by construction and the verification only defends against decoder bugs. Follow the native page policy instead: assertIntegrity and friends run full verification only when slow runtime safety is enabled, which keeps the check in debug and test builds where those bugs are caught. Benchmark deltas at this commit (terminal-snapshot, 1 MB corpora): ascii lines 1-70: decode 15.5 -> 12.2 ms (encode unchanged) --- src/terminal/snapshot/page.zig | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index d96e118f5..fb6359455 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -93,6 +93,7 @@ //! Rows and cells use the grid encoding documented in `grid.zig`. const std = @import("std"); +const build_options = @import("terminal_options"); const Allocator = std.mem.Allocator; const test_fixture = @import("fixture.zig"); const grid = @import("grid.zig"); @@ -227,7 +228,15 @@ pub const Decoder = struct { self.header, ); try self.record_reader.finish(); - try destination.verifyIntegrity(alloc); + + // The decoder normalizes every semantic value, so a complete decode + // upholds native page invariants by construction. Verifying them + // again is a defense against decoder bugs and follows the native + // page policy: full integrity verification only when slow runtime + // safety is enabled. + if (comptime build_options.slow_runtime_safety) { + try destination.verifyIntegrity(alloc); + } } }; From 3e5d128353171df595a9535e595f18a4406db0c2 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 09:37:25 -0700 Subject: [PATCH 6/7] terminal/snapshot: stage PAGE payloads while decoding PAGE payloads were decoded through a stack of stream adapters: a CRC32C-hashing reader over a length-limited reader over the BLAKE3-hashing snapshot reader. Every row paid several adapter crossings and both hashes were fed row-sized chunks, which kept BLAKE3 out of its efficient many-block path and made adapter overhead about a quarter of decode time. Decode now reads the remaining payload into a scratch buffer with one bulk read, so each hash sees the payload as a single update, and then parses the tables and grid from a flat in-memory reader. Row headers are also read as one three-byte read instead of two calls. Staging is capped at 8 MiB, far above any standard-capacity page payload, so a hostile declared length cannot force a large allocation; larger payloads fall back to the streaming path. CRC validation and exact-exhaustion checks are unchanged, with the staged reader checked for leftover bytes to preserve PayloadNotExhausted semantics. Benchmark deltas at this commit (terminal-snapshot, 1 MB corpora): ascii lines 1-70: decode 12.2 -> 8.1 ms (encode unchanged) ascii full-wrap: decode 11.1 -> 7.2 ms utf8: decode 3.1 -> 2.1 ms Relative to the previous wire format and codecs, the series is a 16.0x encode and 14.8x decode improvement on line-shaped scrollback at 4.5x smaller wire size. --- src/terminal/snapshot/grid.zig | 23 ++++++++++---- src/terminal/snapshot/page.zig | 51 +++++++++++++++++++++++++++----- src/terminal/snapshot/record.zig | 5 ++-- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 163f12bef..a3c5678b2 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -515,11 +515,24 @@ pub fn decode( hyperlink_remap: *const HyperlinkRemap, ) DecodeError!void { for (0..page.size.rows) |y| { - // Every bit pattern is a valid header: booleans decode directly - // and the raw semantic value gets a default below. Reserved bits - // do not change the known fields. - const row_header: Row = @bitCast(try reader.takeByte()); - const count = try io.readInt(reader, u16); + // Read the row header and cell count. + const row_header: Row, const count: u16 = header: { + // The staged payload path has every header buffered. + var row_header_bytes: [3]u8 = undefined; + if (reader.bufferedLen() >= 3) { + row_header_bytes = reader.buffered()[0..3].*; + reader.toss(3); + } else { + try reader.readSliceAll(&row_header_bytes); + } + + // Every bit pattern is a valid header: booleans decode directly + // and the raw semantic value gets a default below. Reserved + // bits do not change the known fields. + const row_header: Row = @bitCast(row_header_bytes[0]); + const count = std.mem.readInt(u16, row_header_bytes[1..3], .little); + break :header .{ row_header, count }; + }; const row = page.getRow(y); row.wrap = row_header.wrap; diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index fb6359455..e9d115026 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -204,10 +204,17 @@ pub const Decoder = struct { return self.header.pageCapacity() catch unreachable; } + /// Largest remaining PAGE payload staged into one contiguous buffer. + /// This comfortably covers every payload a standard-capacity page can + /// produce while keeping the allocation an untrusted length can force + /// far below the record framing's four-byte length limit. + const max_staged_payload = 8 * 1024 * 1024; + /// Decode the remaining payload into caller-owned native page storage. /// /// The destination must be freshly initialized with `capacity`. `alloc` - /// is used only for temporary ID remaps and integrity-check storage. + /// is used only for temporary staging, ID remaps, and integrity-check + /// storage. pub fn decode( self: *Decoder, destination: *TerminalPage, @@ -221,12 +228,42 @@ pub const Decoder = struct { .cols = self.header.columns, .rows = self.header.rows, }; - try decodePayloadBody( - self.record_reader.payloadReader(), - alloc, - destination, - self.header, - ); + + // `init` consumed exactly the fixed header from the declared + // payload, so this is the byte count of the tables and grid. + const remaining = self.record_reader.header.payload_len - Header.len; + + if (remaining <= max_staged_payload) { + // Stage the payload with one bulk read. The whole payload + // passes through both hashes as one update and the payload + // decoders then parse a flat buffer, which keeps per-row work + // free of stream adapters. The CRC and exact-length checks in + // `finish` are unaffected. + const staged = try alloc.alloc(u8, remaining); + defer alloc.free(staged); + try self.record_reader.payloadReader().readSliceAll(staged); + + var staged_reader: std.Io.Reader = .fixed(staged); + try decodePayloadBody( + &staged_reader, + alloc, + destination, + self.header, + ); + if (staged_reader.bufferedLen() != 0) { + return error.PayloadNotExhausted; + } + } else { + // A payload this large is either hostile or a page far beyond + // native capacities. Decode it through the streaming payload + // reader so its declared length cannot force an allocation. + try decodePayloadBody( + self.record_reader.payloadReader(), + alloc, + destination, + self.header, + ); + } try self.record_reader.finish(); // The decoder normalizes every semantic value, so a complete decode diff --git a/src/terminal/snapshot/record.zig b/src/terminal/snapshot/record.zig index c50987f18..1327ca23d 100644 --- a/src/terminal/snapshot/record.zig +++ b/src/terminal/snapshot/record.zig @@ -302,8 +302,9 @@ pub const Reader = struct { // such as peek and discard. limited_buffer: [1]u8, - // PAGE decoding performs many small reads. 256 bytes batches several cells - // while CRC32C is calculated without making Reader large on the stack. + // Fixed-header decoding performs several small reads. 256 bytes batches + // them while CRC32C is calculated without making Reader large on the + // stack. Bulk payload reads bypass this buffer entirely. hashing_buffer: [256]u8, limited: std.Io.Reader.Limited, From 9e3019f1905496970e90eb9169fac4ebc0804321 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 2 Aug 2026 10:12:30 -0700 Subject: [PATCH 7/7] terminal/snapshot: variable-width grid cell encoding Add a per-row encoded cell width to the PAGE grid format. Rows previously always spent eight bytes per cell, but a plain text cell carries only a codepoint: on line-shaped scrollback most encoded bytes were predictable zeros that still had to pass through CRC32C, BLAKE3, both codecs, and any transport compression the caller applies. Each row now declares one of four widths in previously reserved row flag bits, chosen canonically as the smallest width admitted by the bitwise OR of the row cell words: one or two bytes transport a bare codepoint, four bytes transport the low word half (any content kind, style IDs up to sixty-three, no wide or flag or hyperlink bits), and eight bytes remain the full word. Every width is a truncation on encode and a zero-extension on decode, so narrow rows encode and decode as vectorizable integer loops, one and two byte rows need at most surrogate replacement and skip cell normalization entirely, and full-width rows keep the existing bulk copy. Decoders use the declared width for framing and accept rows encoded wider than necessary. Rows containing wide characters, hyperlinks, semantic content, or large style IDs still use the full width, which leaves CJK-heavy content unchanged. Benchmark deltas at this commit (terminal-snapshot, M-series, ReleaseFast, 1 MB corpora): ascii lines 1-70: 7.66 MB -> 1.03 MB (7.4x) encode 5.8 -> 2.0 ms, decode 8.1 -> 2.7 ms ascii full-wrap: 8.04 MB -> 1.04 MB (7.7x) encode 5.4 -> 1.3 ms, decode 7.2 -> 1.8 ms utf8: unchanged (wide cells keep rows at full width) For a caller compressing the stream, the lines snapshot end to end with zstd -1: encode plus compress 18.5 -> 2.8 ms, decompress plus decode 15.8 -> 3.6 ms, and the compressed size itself drops from 1.35 MB to 0.86 MB because the packed stream is denser for the entropy coder. --- src/terminal/snapshot/grid.zig | 825 +++++++++++++++--- src/terminal/snapshot/page.zig | 4 +- src/terminal/snapshot/screen.zig | 2 +- src/terminal/snapshot/snapshot.ksy | 53 +- .../snapshot/testdata/complete-v1.hex | 73 +- src/terminal/snapshot/testdata/grid-v1.hex | 11 +- src/terminal/snapshot/testdata/page-v1.hex | 4 +- 7 files changed, 821 insertions(+), 151 deletions(-) diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index a3c5678b2..221d545f8 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -27,11 +27,11 @@ //! //! Each row has the following format: //! -//! | Offset | Size | Field | -//! | -----: | ----------: | :------------------------ | -//! | 0 | 1 | Row flags | -//! | 1 | 2 | Encoded cell count (`u16`)| -//! | 3 | 8 * `count` | Encoded cells | +//! | Offset | Size | Field | +//! | -----: | ----------------: | :------------------------- | +//! | 0 | 1 | Row flags | +//! | 1 | 2 | Encoded cell count (`u16`) | +//! | 3 | `width` * `count` | Encoded cells | //! //! The row flag byte has the following format: //! @@ -40,7 +40,8 @@ //! | 0 | Wrap | //! | 1 | Wrap continuation | //! | 2-3 | Semantic prompt | -//! | 4-7 | Reserved, zero | +//! | 4-5 | Encoded cell width | +//! | 6-7 | Reserved, zero | //! //! Semantic prompt values are: //! @@ -59,12 +60,40 @@ //! Canonical encoders emit exactly through the row's last non-default cell, //! so a fully default row has a zero count and no cell words. //! +//! ## Encoded cell width +//! +//! The two width bits select how many bytes encode each of the row's +//! cells: `1 << width` is the size, so zero through three select one, two, +//! four, or eight bytes. Narrower widths are truncated transports of the +//! same cell word: a cell qualifies for a width when all of its higher +//! word bits are zero. +//! +//! | Width | Bytes | Encoded value and admitted cells | +//! | ----: | ----: | :--------------------------------------------------- | +//! | 0 | 1 | Codepoint at or below U+00FF; all other bits zero | +//! | 1 | 2 | Codepoint at or below U+FFFF; all other bits zero | +//! | 2 | 4 | Word bits 0-31: any content kind and codepoint, style | +//! | | | IDs 1-63, narrow, no flags, no hyperlink | +//! | 3 | 8 | The complete word | +//! +//! Widths zero and one store the codepoint value itself, which is word +//! bits 2-25 shifted down; the reconstructed word is the codepoint shifted +//! left by two. Width two stores the word's low half unshifted. This makes +//! every width a zero-extension on decode and a truncation on encode. +//! +//! Canonical encoders choose each row's smallest admissible width, which +//! follows directly from the bitwise OR of the row's cell words. Decoders +//! use the declared width for framing and accept rows encoded wider than +//! necessary. The width of a row with a zero cell count is canonically +//! zero and carries no meaning. +//! //! Native row cache flags are not encoded. In particular, the Kitty virtual //! placeholder hint is derived while decoding cells containing U+10EEEE. //! //! ## Cell //! -//! Each cell is one 64-bit little-endian word: +//! Each cell is one 64-bit little-endian word, transported at the row's +//! encoded cell width as described above: //! //! ```text //! bit 0 +-------------------------------+ @@ -219,13 +248,14 @@ const TerminalStyleId = terminal_style.Id; /// /// The semantic prompt is a raw integer for the same reason as the wire /// cell fields: decoders must accept its reserved value without -/// instantiating an invalid native enum, so every header byte bit-casts to -/// a valid value. +/// instantiating an invalid native enum. The width enum is exhaustive over +/// its two bits, so every header byte bit-casts to a valid value. pub const Row = packed struct(u8) { wrap: bool = false, wrap_continuation: bool = false, semantic_prompt: u2 = 0, - _padding: u4 = 0, + cell_width: Cell.EncodedWidth = .one, + _padding: u2 = 0, }; /// The wire layout of one encoded cell. This is its own registry: the bit @@ -252,6 +282,72 @@ pub const Cell = packed struct(u64) { bg_color_palette = 2, bg_color_rgb = 3, }; + + /// The encoded cell width declared by a row header: how many bytes + /// transport each of the row's cell words. See the format + /// documentation above for the value each width transports and the + /// cells it admits. + /// + /// `truncate` and `extend` are the transport transform itself, so the + /// admission masks derive from them rather than being maintained by + /// hand: a cell word is admitted exactly when it round-trips. + pub const EncodedWidth = enum(u2) { + one = 0, + two = 1, + four = 2, + eight = 3, + + /// The number of bytes transporting one cell word. + pub fn size(self: EncodedWidth) usize { + return @as(usize, 1) << @intFromEnum(self); + } + + /// The integer type transporting one cell word. + pub fn Int(comptime self: EncodedWidth) type { + return switch (self) { + .one => u8, + .two => u16, + .four => u32, + .eight => u64, + }; + } + + /// Truncate one cell word to its transported value. Only words + /// admitted by this width round-trip; `select` proves that for + /// every cell in a row before an encoder may use it. + pub fn truncate(comptime self: EncodedWidth, word: u64) self.Int() { + return @truncate(switch (self) { + // The bare codepoint, shifted down from the content field. + .one, .two => @as(Cell, @bitCast(word)).content, + + // The word itself. + .four, .eight => word, + }); + } + + /// Widen one transported value back to its cell word. + pub fn extend(comptime self: EncodedWidth, value: self.Int()) u64 { + return switch (self) { + .one, .two => @bitCast(Cell{ .content = value }), + .four, .eight => value, + }; + } + + /// The word bits a cell may use and still round-trip through this + /// width. + pub fn mask(comptime self: EncodedWidth) u64 { + return comptime self.extend(std.math.maxInt(self.Int())); + } + + /// The smallest width admitting the word, typically the bitwise + /// OR of every cell word in a row. + pub fn select(word: u64) EncodedWidth { + inline for ([_]EncodedWidth{ .one, .two, .four }) |width| { + if (word & ~width.mask() == 0) return width; + } + return .eight; + } + }; }; /// Whether the native cell's in-memory layout matches the wire cell layout @@ -380,24 +476,12 @@ pub fn encode( break :count 0; }; - // Row header: flags then the encoded cell count. - { - const row_header: Row = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = @intFromEnum(row.semantic_prompt), - }; - var header_bytes: [3]u8 = undefined; - header_bytes[0] = @bitCast(row_header); - std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); - try writer.writeAll(&header_bytes); - } - // Validate the wide state of every encoded cell so we don't encode - // corrupt data, and detect the cells that keep this row off the - // direct-copy path. Trailing default cells are narrow, so checking - // the encoded prefix against the full row width covers every pair. - var direct = true; + // corrupt data, and accumulate the OR of the row's cell words to + // select its encoded cell width. Trailing default cells are narrow, + // so checking the encoded prefix against the full row width covers + // every pair. + var word_or: u64 = 0; for (cells[0..count], 0..) |*cell, x| { switch (cell.wide) { .narrow => {}, @@ -415,31 +499,117 @@ pub fn encode( return error.InvalidWideCell; }, } - - // Hyperlink IDs live in a native side table, and nonzero native - // padding would leak into the wire hyperlink ID field. - if (cell.hyperlink or cell._padding != 0) direct = false; + word_or |= classifyWord(cell); } - if (comptime bulk_codec) { - if (direct) { - try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); - continue; - } + // Canonical rows use the smallest admissible width. + const cell_width: Cell.EncodedWidth = .select(word_or); + + // Row header: flags then the encoded cell count. + { + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + .cell_width = cell_width, + }; + var header_bytes: [3]u8 = undefined; + header_bytes[0] = @bitCast(row_header); + std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); + try writer.writeAll(&header_bytes); } - for (cells[0..count]) |*cell| { - const link_id: TerminalHyperlinkId = if (cell.hyperlink) - page.lookupHyperlink(cell) orelse unreachable - else - 0; - try io.writeInt(writer, u64, cellBits(cell.*, link_id)); + switch (cell_width) { + inline .one, .two, .four => |width| try encodeNarrowCells( + width, + cells[0..count], + writer, + ), + + .eight => { + // Hyperlink IDs live in a native side table, but we embed + // them in ours, so if we have any hyperlinks we need + // to fallback to the loop below. + if (comptime bulk_codec) { + const witness: Cell = @bitCast(word_or); + if (!witness.hyperlink and witness.hyperlink_id == 0) { + try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); + continue; + } + } + + for (cells[0..count]) |*cell| { + const link_id: TerminalHyperlinkId = if (cell.hyperlink) + page.lookupHyperlink(cell) orelse unreachable + else + 0; + try io.writeInt( + writer, + u64, + cellBits(cell.*, link_id), + ); + } + }, } } try encodeGraphemes(page, writer); } +/// The word used to select a row's encoded cell width. This is the cell's +/// wire word with the hyperlink flag reflecting the native cell, so linked +/// cells and nonzero native padding disqualify every narrow width. +inline fn classifyWord(cell: *const TerminalCell) u64 { + if (comptime native_matches_wire) return @bitCast(cell.*); + + var wire: Cell = @bitCast(cellBits(cell.*, 0)); + wire.hyperlink = cell.hyperlink; + return @bitCast(wire); +} + +/// Write one row's cells truncated to the given encoded width. +/// +/// Every admitted cell round-trips exactly because the row's width +/// selection proved every cell word survives `truncate` then `extend`. +fn encodeNarrowCells( + comptime width: Cell.EncodedWidth, + cells: []const TerminalCell, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + const size = comptime width.size(); + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + + if (comptime native_matches_wire) { + // A pure truncating loop over integers that the compiler can + // vectorize. + const words: [*]const u64 = @ptrCast(cells.ptr); + for (0..n) |j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(words[i + j]), + .little, + ); + } + } else { + for (cells[i..][0..n], 0..) |*cell, j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(classifyWord(cell)), + .little, + ); + } + } + + try writer.writeAll(chunk[0 .. n * size]); + i += n; + } +} + /// Encode the grapheme suffix section for every kind 1 cell in the grid. fn encodeGraphemes( page: *const TerminalPage, @@ -526,9 +696,9 @@ pub fn decode( try reader.readSliceAll(&row_header_bytes); } - // Every bit pattern is a valid header: booleans decode directly - // and the raw semantic value gets a default below. Reserved - // bits do not change the known fields. + // Every bit pattern is a valid header: booleans and the exhaustive + // width enum decode directly, and the raw semantic value gets a + // default below. Reserved bits do not change the known fields. const row_header: Row = @bitCast(row_header_bytes[0]); const count = std.mem.readInt(u16, row_header_bytes[1..3], .little); break :header .{ row_header, count }; @@ -546,50 +716,201 @@ pub fn decode( if (count > cells.len) return error.InvalidRowCellCount; if (count == 0) continue; - if (comptime bulk_codec) { - // Read the encoded words directly into page storage, then - // normalize them in place. The raw words are only ever touched - // as integers until normalization makes them valid cells. - const words: [*]u64 = @ptrCast(cells.ptr); - try reader.readSliceAll( - std.mem.sliceAsBytes(cells[0..count]), - ); - for (0..count) |x| { - applyCell( - page, - row, - cells, - x, - words[x], - style_remap, - hyperlink_remap, - ); - } - } else { - for (0..count) |x| { - const bits = try io.readInt(reader, u64); - applyCell( - page, - row, - cells, - x, - bits, - style_remap, - hyperlink_remap, - ); - } - } + // The encoded cell width is framing: it determines exactly how many + // bytes this row occupies. + switch (row_header.cell_width) { + inline .one, .two => |width| try decodeNarrowCells( + width, + reader, + cells[0..count], + ), + .four => try decodeWordCells( + .four, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ), + .eight => { + if (comptime bulk_codec) { + // Read the encoded words directly into page storage, + // then normalize them in place. The raw words are only + // ever touched as integers until normalization makes + // them valid cells. + const words: [*]u64 = @ptrCast(cells.ptr); + try reader.readSliceAll( + std.mem.sliceAsBytes(cells[0..count]), + ); + for (0..count) |x| { + applyCell( + page, + row, + cells, + x, + words[x], + style_remap, + hyperlink_remap, + ); + } + } else { + try decodeWordCells( + .eight, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ); + } - // The implicit cell after a short row is narrow, which resolves a - // trailing wide marker exactly like an explicit narrow neighbor. - if (count < cells.len and cells[count - 1].wide == .wide) { - cells[count - 1].wide = .narrow; + // The implicit cell after a short row is narrow, which + // resolves a trailing wide marker exactly like an explicit + // narrow neighbor. Only full-width rows can encode wide + // markers. + if (count < cells.len and cells[count - 1].wide == .wide) { + cells[count - 1].wide = .narrow; + } + }, } } try decodeGraphemes(page, reader); } +/// Decode one row of width-one or width-two cells. +/// +/// These widths admit only bare codepoints, so cells store directly with at +/// most Unicode scalar validation: no styles, hyperlinks, wide pairs, or +/// row hints are reachable and no normalization state is needed. +fn decodeNarrowCells( + comptime width: Cell.EncodedWidth, + reader: *std.Io.Reader, + cells: []TerminalCell, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered, making this + // one bounds check followed by a vectorizable widening loop. + const total = cells.len * size; + if (reader.bufferedLen() >= total) { + widenCells(width, reader.buffered()[0..total], cells); + reader.toss(total); + return; + } + + // Streaming sources fall back to bounded chunks. + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + try reader.readSliceAll(chunk[0 .. n * size]); + widenCells(width, chunk[0 .. n * size], cells[i..][0..n]); + i += n; + } +} + +/// Store codepoint-valued encoded cells of the given width. +fn widenCells( + comptime width: Cell.EncodedWidth, + bytes: []const u8, + cells: []TerminalCell, +) void { + const size = comptime width.size(); + + // When the native cell matches the wire word, this is a pure widening + // loop over integers that the compiler can vectorize. + if (comptime native_matches_wire) { + const words: [*]u64 = @ptrCast(cells.ptr); + for (0..cells.len) |i| { + words[i] = width.extend(widenValue(width, bytes[i * size ..])); + } + return; + } + + for (cells, 0..) |*cell, i| { + storeCell(cell, width.extend(widenValue(width, bytes[i * size ..]))); + } +} + +/// Read and validate one narrow transported value. +inline fn widenValue( + comptime width: Cell.EncodedWidth, + bytes: []const u8, +) width.Int() { + const size = comptime width.size(); + const value = std.mem.readInt(width.Int(), bytes[0..size], .little); + + // Width one cannot encode an invalid scalar. Width two admits + // surrogates, which degrade exactly like their full-width form. + if (comptime width != .one) { + if (!validScalar(value)) return 0xFFFD; + } + return value; +} + +/// Decode one row of width-four or fallback full-width cells through the +/// complete per-cell normalization path. +fn decodeWordCells( + comptime width: Cell.EncodedWidth, + page: *TerminalPage, + row: *TerminalRow, + cells: []TerminalCell, + count: usize, + reader: *std.Io.Reader, + style_remap: *const StyleRemap, + hyperlink_remap: *const HyperlinkRemap, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered. + const total = count * size; + if (reader.bufferedLen() >= total) { + const bytes = reader.buffered()[0..total]; + for (0..count) |x| { + const bits = width.extend(std.mem.readInt( + width.Int(), + bytes[x * size ..][0..size], + .little, + )); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } + reader.toss(total); + return; + } + + // Streaming sources fall back to per-cell reads. + for (0..count) |x| { + const bits = width.extend(try io.readInt(reader, width.Int())); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } +} + +/// Whether the value is a valid Unicode scalar value. +inline fn validScalar(cp: u32) bool { + return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); +} + /// Normalize one encoded cell word and store it at `cells[x]`. /// /// This owns every per-cell decode rule except grapheme suffixes: content @@ -715,11 +1036,6 @@ fn normalizeWide(row: *const TerminalRow, cells: []TerminalCell, x: usize) void } } -/// Whether the value is a valid Unicode scalar value. -inline fn validScalar(cp: u32) bool { - return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); -} - /// Decode the grapheme suffix section into already decoded cells. fn decodeGraphemes( page: *TerminalPage, @@ -903,7 +1219,6 @@ fn Remap(comptime Id: type) type { } }; } - const test_golden_fixture = test_fixture.parse( @embedFile("testdata/grid-v1.hex"), ); @@ -929,10 +1244,76 @@ test "cell wire layout registry" { try testing.expect(native_matches_wire); } +test "encoded cell width transport registry" { + const testing = std.testing; + + // Pin the transported value and admission mask of every width against + // the documented format: widths one and two carry the bare codepoint, + // width four the low word half, width eight the complete word. + const word: u64 = @bitCast(Cell{ + .kind = 1, + .content = 0xABCDEF, + .style_id = 0x1234, + .hyperlink_id = 0x5678, + }); + try testing.expectEqual(@as(u8, 0xEF), Cell.EncodedWidth.one.truncate(word)); + try testing.expectEqual(@as(u16, 0xCDEF), Cell.EncodedWidth.two.truncate(word)); + try testing.expectEqual( + @as(u32, @truncate(word)), + Cell.EncodedWidth.four.truncate(word), + ); + try testing.expectEqual(word, Cell.EncodedWidth.eight.truncate(word)); + + try testing.expectEqual( + @as(u64, 0x0000_0000_0000_03FC), + Cell.EncodedWidth.one.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_0003_FFFC), + Cell.EncodedWidth.two.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_FFFF_FFFF), + Cell.EncodedWidth.four.mask(), + ); + try testing.expectEqual( + @as(u64, 0xFFFF_FFFF_FFFF_FFFF), + Cell.EncodedWidth.eight.mask(), + ); + + // Selection returns the smallest admissible width, and every admitted + // word round-trips through its transport. + const cases = [_]struct { cell: Cell, width: Cell.EncodedWidth }{ + .{ .cell = .{}, .width = .one }, + .{ .cell = .{ .content = 0xFF }, .width = .one }, + .{ .cell = .{ .content = 0x100 }, .width = .two }, + .{ .cell = .{ .content = 0xFFFF }, .width = .two }, + .{ .cell = .{ .content = 0x10000 }, .width = .four }, + .{ .cell = .{ .kind = 2, .content = 7 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 63 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 64 }, .width = .eight }, + .{ .cell = .{ .width = 1, .content = 'a' }, .width = .eight }, + .{ .cell = .{ .protected = true }, .width = .eight }, + .{ .cell = .{ .semantic_content = 1 }, .width = .eight }, + .{ .cell = .{ .hyperlink = true, .hyperlink_id = 1 }, .width = .eight }, + }; + inline for (cases) |case| { + const case_word: u64 = @bitCast(case.cell); + try testing.expectEqual( + case.width, + Cell.EncodedWidth.select(case_word), + ); + try testing.expectEqual( + case_word, + case.width.extend(case.width.truncate(case_word)), + ); + } +} + test "grid golden encoding and decoding" { const capacity: terminal_page.Capacity = .{ .cols = 3, - .rows = 2, + .rows = 4, .styles = 0, .hyperlink_bytes = 0, .grapheme_bytes = 64, @@ -983,7 +1364,16 @@ test "grid golden encoding and decoding" { head.row.wrap_continuation = true; head.row.semantic_prompt = .prompt_continuation; - var encoded: [128]u8 = undefined; + // The third row is bare ASCII text with an interior blank, which uses + // the one-byte encoded cell width. + source.getRowAndCell(0, 2).cell.* = .init('h'); + source.getRowAndCell(2, 2).cell.* = .init('i'); + + // The fourth row needs the two-byte width for a codepoint above U+00FF. + source.getRowAndCell(0, 3).cell.* = .init(0x0416); // Ж + source.getRowAndCell(1, 3).cell.* = .init('!'); + + var encoded: [160]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try encode(&source, &writer); try test_fixture.expectEqual( @@ -1076,8 +1466,26 @@ test "grid golden encoding and decoding" { decoded_head.row.semantic_prompt, ); + try std.testing.expectEqual( + @as(u21, 'h'), + destination.getRowAndCell(0, 2).cell.codepoint(), + ); + try std.testing.expect(destination.getRowAndCell(1, 2).cell.isZero()); + try std.testing.expectEqual( + @as(u21, 'i'), + destination.getRowAndCell(2, 2).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, 0x0416), + destination.getRowAndCell(0, 3).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, '!'), + destination.getRowAndCell(1, 3).cell.codepoint(), + ); + // A re-encode proves the decoded native page retains every wire field. - var reencoded: [128]u8 = undefined; + var reencoded: [160]u8 = undefined; var rewriter: std.Io.Writer = .fixed(&reencoded); try encode(&destination, &rewriter); try std.testing.expectEqualStrings( @@ -1102,10 +1510,11 @@ test "grid elides trailing default cells" { var writer: std.Io.Writer = .fixed(&encoded); try encode(&page, &writer); - // 3 bytes per row header, cells only through the last non-default - // cell, and an empty grapheme section. + // 3 bytes per row header and cells only through the last non-default + // cell, followed by an empty grapheme section. The text row uses the + // one-byte width while the protected flag forces the full width. try testing.expectEqual( - @as(usize, 3 + (3 + 3 * 8) + (3 + 5 * 8) + 4), + @as(usize, 3 + (3 + 3 * 1) + (3 + 5 * 8) + 4), writer.buffered().len, ); @@ -1170,7 +1579,7 @@ test "grid normalizes incomplete wide cells" { // content but makes the incomplete wide cell narrow. var payload: [64]u8 = undefined; var payload_writer: std.Io.Writer = .fixed(&payload); - try payload_writer.writeByte(@bitCast(Row{})); + try payload_writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&payload_writer, u16, columns); try io.writeInt(&payload_writer, u64, @bitCast(Cell{ .width = 1, // wide @@ -1225,7 +1634,7 @@ test "grid normalizes incomplete wide cells" { var payload: [64]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - try writer.writeByte(@bitCast(Row{})); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&writer, u16, 1); try io.writeInt(&writer, u64, @bitCast(Cell{ .width = 1, // wide @@ -1255,8 +1664,9 @@ test "grid normalizes reserved cell values" { var payload: [64]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - // Reserved row flag bits are ignored while the unknown semantic-prompt - // value degrades to none and wrap survives. + // Reserved row flag bits six and seven are ignored while the unknown + // semantic-prompt value degrades to none and wrap survives. Bits four + // and five select the full encoded cell width. try writer.writeByte(0xFD); try io.writeInt(&writer, u16, 3); @@ -1327,7 +1737,7 @@ test "grid drops undeliverable grapheme entries" { var payload: [128]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - try writer.writeByte(@bitCast(Row{})); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&writer, u16, 4); // A kind 1 cell that receives a valid entry. try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'x' })); @@ -1383,3 +1793,220 @@ test "grid drops undeliverable grapheme entries" { try testing.expect(!page.getRowAndCell(2, 0).cell.hasGrapheme()); try testing.expect(!page.getRowAndCell(3, 0).cell.hasGrapheme()); } + +test "grid encodes rows at their narrowest width" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer page.deinit(); + + // Width zero: bare Latin-1 text. + page.getRowAndCell(0, 0).cell.* = .init('A'); + page.getRowAndCell(1, 0).cell.* = .init(0xFF); + + // Width one: any BMP codepoint. + page.getRowAndCell(0, 1).cell.* = .init(0x0100); + + // Width two: a small style ID and a background color. + const style_id = try page.styles.add(page.memory, .{ + .flags = .{ .bold = true }, + }); + try testing.expect(style_id <= 63); + const styled = page.getRowAndCell(0, 2); + styled.cell.* = .init('s'); + styled.cell.style_id = style_id; + styled.row.styled = true; + const bg = page.getRowAndCell(1, 2); + bg.cell.content_tag = .bg_color_palette; + bg.cell.content = .{ .color_palette = .{ .data = 7 } }; + + // Width three: a protected cell. + page.getRowAndCell(0, 3).cell.protected = true; + + var encoded: [128]u8 = undefined; + var writer: std.Io.Writer = .fixed(&encoded); + try encode(&page, &writer); + + const bytes = writer.buffered(); + try testing.expectEqual( + @as(usize, (3 + 2 * 1) + (3 + 1 * 2) + (3 + 2 * 4) + (3 + 1 * 8) + 4), + bytes.len, + ); + + // Each row header carries the expected width bits. + try testing.expectEqual(@as(u8, 0 << 4), bytes[0] & 0x30); + try testing.expectEqual(@as(u8, 1 << 4), bytes[5] & 0x30); + try testing.expectEqual(@as(u8, 2 << 4), bytes[10] & 0x30); + try testing.expectEqual(@as(u8, 3 << 4), bytes[21] & 0x30); + + var destination = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer destination.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + const native_style = try destination.styles.add(destination.memory, .{ + .flags = .{ .bold = true }, + }); + style_remap.put(style_id, native_style); + + var reader: std.Io.Reader = .fixed(bytes); + try decode(&destination, &reader, &style_remap, &hyperlink_remap); + try destination.verifyIntegrity(testing.allocator); + + try testing.expectEqual( + @as(u21, 'A'), + destination.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0xFF), + destination.getRowAndCell(1, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0100), + destination.getRowAndCell(0, 1).cell.codepoint(), + ); + const decoded_styled = destination.getRowAndCell(0, 2); + try testing.expectEqual(@as(u21, 's'), decoded_styled.cell.codepoint()); + try testing.expectEqual(native_style, decoded_styled.cell.style_id); + try testing.expect(decoded_styled.row.styled); + try testing.expectEqual( + @as(u8, 7), + destination.getRowAndCell(1, 2).cell.content.color_palette.data, + ); + try testing.expect(destination.getRowAndCell(0, 3).cell.protected); +} + +test "grid decodes non-canonical cell widths" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + // A bare ASCII row encoded at the full width is wasteful but valid. + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'o' })); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'k' })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 'o'), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 'k'), + page.getRowAndCell(1, 0).cell.codepoint(), + ); + + // Re-encoding canonicalizes the row back to the one-byte width. + var reencoded: [16]u8 = undefined; + var rewriter: std.Io.Writer = .fixed(&reencoded); + try encode(&page, &rewriter); + try testing.expectEqual(@as(usize, 3 + 2 + 4), rewriter.buffered().len); +} + +test "grid normalizes surrogates in two-byte cells" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [16]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .two })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u16, 0xD800); + try io.writeInt(&writer, u16, 0x0416); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 0xFFFD), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0416), + page.getRowAndCell(1, 0).cell.codepoint(), + ); +} + +test "grid four-byte cells run full normalization" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 3, + .rows = 1, + .styles = 8, + }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [32]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .four })); + try io.writeInt(&writer, u16, 3); + + // The Kitty placeholder does not fit two-byte cells but fits here + // and must still derive the native row hint. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = kitty.graphics.unicode.placeholder, + })))); + + // An unknown small style reference degrades to the default style. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = 'x', + .style_id = 63, + })))); + + // Reserved palette content bits are cleared at this width too. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .kind = 2, + .content = 0xFFFF07, + })))); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expectEqual( + @as(u21, kitty.graphics.unicode.placeholder), + first.cell.codepoint(), + ); + try testing.expect(first.row.kitty_virtual_placeholder); + + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual(@as(u21, 'x'), second.codepoint()); + try testing.expectEqual(@as(TerminalStyleId, 0), second.style_id); + + const third = page.getRowAndCell(2, 0).cell; + try testing.expectEqual( + TerminalCell.ContentTag.bg_color_palette, + third.content_tag, + ); + try testing.expectEqual(@as(u8, 7), third.content.color_palette.data); +} diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index e9d115026..c09fd9dad 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -1015,7 +1015,7 @@ test "decode defaults missing sparse cell references" { var style_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var style_writer: std.Io.Writer = .fixed(&style_encoded); try style_header.encode(&style_writer); - try style_writer.writeByte(0); // row flags + try style_writer.writeByte(0x30); // row flags, full cell width try io.writeInt(&style_writer, u16, 1); // cell count try io.writeInt(&style_writer, u64, @bitCast(grid.Cell{ .style_id = 1, @@ -1048,7 +1048,7 @@ test "decode defaults missing sparse cell references" { var hyperlink_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var hyperlink_writer: std.Io.Writer = .fixed(&hyperlink_encoded); try hyperlink_header.encode(&hyperlink_writer); - try hyperlink_writer.writeByte(0); // row flags + try hyperlink_writer.writeByte(0x30); // row flags, full cell width try io.writeInt(&hyperlink_writer, u16, 1); // cell count try io.writeInt(&hyperlink_writer, u64, @bitCast(grid.Cell{ .hyperlink = true, diff --git a/src/terminal/snapshot/screen.zig b/src/terminal/snapshot/screen.zig index 06dc8bce6..d05ff00ba 100644 --- a/src/terminal/snapshot/screen.zig +++ b/src/terminal/snapshot/screen.zig @@ -2262,7 +2262,7 @@ test "SCREEN decode ignores a PAGE with an empty hyperlink URI" { // One narrow codepoint cell refers to the hyperlink table entry above. // Since that entry is ignored, the cell must restore without a hyperlink. - try page_payload.writeByte(0); // row flags + try page_payload.writeByte(0x30); // row flags, full cell width try io.writeInt(page_payload, u16, 1); // cell count try io.writeInt(page_payload, u64, @bitCast(grid.Cell{ .content = 'A', diff --git a/src/terminal/snapshot/snapshot.ksy b/src/terminal/snapshot/snapshot.ksy index c503fe2f7..6ff925ef3 100644 --- a/src/terminal/snapshot/snapshot.ksy +++ b/src/terminal/snapshot/snapshot.ksy @@ -933,7 +933,13 @@ types: valid: expr: _ <= columns - id: cells - type: grid_cell(_index, columns, flags.wrap) + type: + switch-on: flags.width_log2 + cases: + 0: grid_cell_1 + 1: grid_cell_2 + 2: grid_cell_4 + 3: grid_cell(_index, columns, flags.wrap) repeat: expr repeat-expr: cell_count @@ -942,7 +948,7 @@ types: - id: raw type: u1 valid: - expr: (_ & 0xf0) == 0 and ((_ >> 2) & 0x3) <= 2 + expr: (_ & 0xc0) == 0 and ((_ >> 2) & 0x3) <= 2 instances: wrap: value: (raw & 1) != 0 @@ -950,6 +956,46 @@ types: value: (raw & 2) != 0 semantic_prompt: value: (raw >> 2) & 0x3 + width_log2: + value: (raw >> 4) & 0x3 + + grid_cell_1: + doc: One-byte encoded cell; the value is a codepoint at or below U+00FF. + seq: + - id: codepoint + type: u1 + + grid_cell_2: + doc: | + Two-byte encoded cell; the value is a codepoint at or below U+FFFF. + Canonical encoders never emit surrogates. + seq: + - id: codepoint + type: u2 + valid: + expr: not (_ >= 0xd800 and _ <= 0xdfff) + + grid_cell_4: + doc: | + Four-byte encoded cell holding the low half of the cell word: any + content kind and codepoint, style IDs one through sixty-three, and no + width, flag, or hyperlink bits. + seq: + - id: raw + type: u4 + valid: + expr: | + (content_kind >= 2 or + (content <= 0x10ffff and + not (content >= 0xd800 and content <= 0xdfff))) and + (content_kind != 2 or content <= 0xff) + instances: + content_kind: + value: raw % 4 + content: + value: (raw / 4) % 16777216 + style_id: + value: raw / 67108864 grid_cell: doc: | @@ -984,7 +1030,8 @@ types: not (content >= 0xd800 and content <= 0xdfff))) and (content_kind != 2 or content <= 0xff) and (width != 2 or - (index > 0 and _parent.cells[index - 1].width == 1)) and + (index > 0 and + _parent.cells[index - 1].as.width == 1)) and (width != 3 or (index + 1 == columns and row_wrap)) instances: content_kind: diff --git a/src/terminal/snapshot/testdata/complete-v1.hex b/src/terminal/snapshot/testdata/complete-v1.hex index e88db31c0..8e3df8287 100644 --- a/src/terminal/snapshot/testdata/complete-v1.hex +++ b/src/terminal/snapshot/testdata/complete-v1.hex @@ -78,52 +78,47 @@ ee ee 80 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000037a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003ec 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003fc -# offset 0x0000040c: page record, payload 57 bytes -03 00 39 00 00 00 26 5a 94 0b 02 00 03 00 00 00 # 0x0000040c +# offset 0x0000040c: page record, payload 36 bytes +03 00 24 00 00 00 a2 4f 26 d1 02 00 03 00 00 00 # 0x0000040c 00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x0000041c -00 0c 01 00 00 00 00 00 00 00 01 00 10 01 00 00 # 0x0000042c -00 00 00 00 00 01 00 14 01 00 00 00 00 00 00 00 # 0x0000043c -00 00 00 # 0x0000044c +00 43 00 01 00 44 00 01 00 45 00 00 00 00 # 0x0000042c -# offset 0x0000044f: screen record, payload 54 bytes -02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000044f -00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000045f -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046f -00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000047f +# offset 0x0000043a: screen record, payload 54 bytes +02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000043a +00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000044a +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000045a +00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046a -# offset 0x0000048f: page record, payload 73 bytes -03 00 49 00 00 00 37 af 30 0e 02 00 03 00 00 00 # 0x0000048f -00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000049f -00 c8 01 00 00 00 00 00 00 b8 01 00 00 00 00 00 # 0x000004af -00 03 02 00 84 01 00 00 00 00 00 00 d0 01 00 00 # 0x000004bf -00 00 00 00 03 01 00 94 01 00 00 00 00 00 00 00 # 0x000004cf -00 00 00 # 0x000004df +# offset 0x0000047a: page record, payload 38 bytes +03 00 26 00 00 00 45 61 97 32 02 00 03 00 00 00 # 0x0000047a +00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000048a +00 72 6e 03 02 00 61 74 03 01 00 65 00 00 00 00 # 0x0000049a -# offset 0x000004e2: continuation record, payload 0 bytes -07 00 00 00 00 00 27 80 63 d1 # 0x000004e2 +# offset 0x000004aa: continuation record, payload 0 bytes +07 00 00 00 00 00 27 80 63 d1 # 0x000004aa -# offset 0x000004ec: ready record, payload 32 bytes -05 00 20 00 00 00 5d f3 97 80 41 a2 18 18 cb 82 # 0x000004ec -8c 97 b6 58 14 60 26 9a 88 2e be d4 c3 4b 83 c9 # 0x000004fc -9d 32 9d de e6 70 17 4a 5d bc # 0x0000050c +# offset 0x000004b4: ready record, payload 32 bytes +05 00 20 00 00 00 4d 17 72 ed dd 87 26 75 cf 8e # 0x000004b4 +e5 1f 37 e3 05 92 0a e2 f8 ef c1 16 54 be 49 e7 # 0x000004c4 +b2 6c df 40 d2 77 fe 05 82 ea # 0x000004d4 -# offset 0x00000516: history record, payload 6 bytes -04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x00000516 +# offset 0x000004de: history record, payload 6 bytes +04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x000004de -# offset 0x00000526: page record, payload 38 bytes -03 00 26 00 00 00 9e 10 a2 93 02 00 02 00 00 00 # 0x00000526 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000536 -00 08 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000546 +# offset 0x000004ee: page record, payload 31 bytes +03 00 1f 00 00 00 4a ed 2f c2 02 00 02 00 00 00 # 0x000004ee +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x000004fe +00 42 00 00 00 00 00 00 00 # 0x0000050e -# offset 0x00000556: page record, payload 38 bytes -03 00 26 00 00 00 70 e1 36 3a 02 00 02 00 00 00 # 0x00000556 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000566 -00 04 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000576 +# offset 0x00000517: page record, payload 31 bytes +03 00 1f 00 00 00 23 6a 6b 19 02 00 02 00 00 00 # 0x00000517 +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000527 +00 41 00 00 00 00 00 00 00 # 0x00000537 -# offset 0x00000586: history record, payload 6 bytes -04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000586 +# offset 0x00000540: history record, payload 6 bytes +04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000540 -# offset 0x00000596: finish record, payload 32 bytes -06 00 20 00 00 00 20 0f e0 7e 10 4f 2c da 90 fa # 0x00000596 -65 19 a9 73 26 75 2a 5b 33 cd 9d fa 73 81 86 07 # 0x000005a6 -9e 57 d4 83 41 9f a9 81 64 e7 # 0x000005b6 +# offset 0x00000550: finish record, payload 32 bytes +06 00 20 00 00 00 5a 2d a8 f4 25 b2 49 b4 3e 64 # 0x00000550 +5a 2e d6 7d 7f 3d 60 5b db 9b 4c 3a 29 00 97 ec # 0x00000560 +d6 c7 1d ea da d3 a9 fd 3d 68 # 0x00000570 diff --git a/src/terminal/snapshot/testdata/grid-v1.hex b/src/terminal/snapshot/testdata/grid-v1.hex index b266f5438..b9a74af32 100644 --- a/src/terminal/snapshot/testdata/grid-v1.hex +++ b/src/terminal/snapshot/testdata/grid-v1.hex @@ -1,14 +1,15 @@ # Ghostty snapshot fixture # Kaitai type: grid -# Kaitai params: 2 3 +# Kaitai params: 4 3 # Kaitai offset: 0 # Wire version: 1 # Generated by its snapshot test; review before replacing. # On mismatch, the candidate is copied to the repository root. # offset 0x00000000: encoded bytes -04 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 -48 00 00 e1 01 00 00 00 00 00 00 0b 03 00 1e 00 # 0x00000010 +34 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 +48 00 00 e1 01 00 00 00 00 00 00 3b 03 00 1e 00 # 0x00000010 00 00 00 00 00 00 ab ee 32 03 00 10 00 00 00 00 # 0x00000020 -00 00 00 0c 00 00 01 00 00 00 00 00 02 00 02 00 # 0x00000030 -01 03 00 00 02 03 00 00 # 0x00000040 +00 00 00 0c 00 00 00 03 00 68 00 69 10 02 00 16 # 0x00000030 +04 21 00 01 00 00 00 00 00 02 00 02 00 01 03 00 # 0x00000040 +00 02 03 00 00 # 0x00000050 diff --git a/src/terminal/snapshot/testdata/page-v1.hex b/src/terminal/snapshot/testdata/page-v1.hex index c572decc8..78018ebb6 100644 --- a/src/terminal/snapshot/testdata/page-v1.hex +++ b/src/terminal/snapshot/testdata/page-v1.hex @@ -15,8 +15,8 @@ 00 00 03 00 00 00 00 00 01 2a 00 00 00 00 00 00 # 0x00000024 00 00 00 00 01 00 02 01 00 00 00 61 05 00 00 00 # 0x00000034 61 6c 70 68 61 03 00 01 04 03 02 01 04 00 00 00 # 0x00000044 -62 65 74 61 04 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 -00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 0b # 0x00000064 +62 65 74 61 34 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 +00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 3b # 0x00000064 03 00 e1 01 00 00 00 00 00 00 ab ee 32 03 00 10 # 0x00000074 00 00 00 00 00 00 00 0c 00 00 01 00 00 00 01 00 # 0x00000084 00 00 02 00 01 03 00 00 02 03 00 00 # 0x00000094