diff --git a/src/benchmark/TerminalSnapshot.zig b/src/benchmark/TerminalSnapshot.zig new file mode 100644 index 000000000..dd191f1ea --- /dev/null +++ b/src/benchmark/TerminalSnapshot.zig @@ -0,0 +1,307 @@ +//! Benchmarks the terminal binary snapshot codecs (`terminal/snapshot`). +//! +//! Encoding and decoding a snapshot are the hot paths for terminal +//! persistence and handoff, so both directions are measured against the +//! same terminal state. +//! +//! ## Input +//! +//! `--data` names a pre-generated VT byte stream (for example from +//! `ghostty-gen ascii`). The stream is fed to a terminal of the requested +//! dimensions during setup, outside the timed region. The resulting screen +//! and scrollback contents are what each step encodes or decodes. +//! +//! ## Modes +//! +//! * `noop` performs no codec work and establishes loop overhead. +//! * `encode` encodes one complete snapshot per loop into a reusable +//! buffer. Buffer growth happens on the first iteration only. +//! * `decode` restores one complete snapshot per loop from bytes prepared +//! during setup. The restored terminal is destroyed inside the step, so +//! this measures the complete restore lifecycle. +//! * `report` encodes once and prints total and per-record-tag sizes. It +//! is for inspecting the wire shape, not timing comparisons. +//! +//! ## Examples +//! +//! Build benchmarks in ReleaseFast mode: +//! +//! zig build -Demit-bench -Doptimize=ReleaseFast -Demit-macos-app=false +//! +//! Generate a deterministic corpus, then measure both directions: +//! +//! ghostty-gen ascii | head -c 1000000 > /tmp/ascii.vt +//! hyperfine --warmup 3 \ +//! 'ghostty-bench +terminal-snapshot --mode=encode --loops=20 --data=/tmp/ascii.vt' \ +//! 'ghostty-bench +terminal-snapshot --mode=decode --loops=20 --data=/tmp/ascii.vt' +const TerminalSnapshot = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const terminalpkg = @import("../terminal/main.zig"); +const snapshot = terminalpkg.snapshot; +const Benchmark = @import("Benchmark.zig"); +const options = @import("options.zig"); +const Terminal = terminalpkg.Terminal; +const global = @import("../global.zig"); + +const log = std.log.scoped(.@"terminal-snapshot-bench"); + +alloc: Allocator, +opts: Options, +terminal: ?Terminal = null, + +/// Reused across encode steps so buffer growth is a one-time setup cost. +encoded: std.Io.Writer.Allocating, + +pub const Options = struct { + /// Set by the shared CLI parser for string option ownership. + _arena: ?std.heap.ArenaAllocator = null, + + /// Select the operation performed inside the timed benchmark step. + mode: Mode = .encode, + + /// Number of codec operations per benchmark step. Increase this when + /// the state is too small for stable `hyperfine` measurements. + loops: u32 = 1, + + /// The size of the terminal. This affects wrapping, page sizes, and + /// the amount of state per encoded page. + @"terminal-rows": u16 = 24, + @"terminal-cols": u16 = 80, + + /// Pre-generated VT stream fed to the terminal during setup. `-` reads + /// stdin, although a regular file is recommended so identical state can + /// be reused across runs. When unset, the terminal is empty. + data: ?[]const u8 = null, + + pub fn deinit(self: *Options) void { + if (self._arena) |arena| arena.deinit(); + self.* = undefined; + } +}; + +pub const Mode = enum { + /// Establish the benchmark loop overhead. + noop, + + /// Encode one complete snapshot per loop. + encode, + + /// Restore one complete snapshot per loop, including its teardown. + decode, + + /// Print encoded sizes by record tag. Not a timing benchmark. + report, +}; + +pub fn create( + alloc: Allocator, + opts: Options, +) !*TerminalSnapshot { + const ptr = try alloc.create(TerminalSnapshot); + errdefer alloc.destroy(ptr); + ptr.* = .{ + .alloc = alloc, + .opts = opts, + .encoded = .init(alloc), + }; + return ptr; +} + +pub fn destroy(self: *TerminalSnapshot, alloc: Allocator) void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.encoded.deinit(); + alloc.destroy(self); +} + +pub fn benchmark(self: *TerminalSnapshot) Benchmark { + return .init(self, .{ + .stepFn = switch (self.opts.mode) { + .noop => stepNoop, + .encode => stepEncode, + .decode => stepDecode, + .report => stepReport, + }, + .setupFn = setup, + .teardownFn = teardown, + }); +} + +/// Build the terminal state every mode shares and, for decode mode, the +/// encoded snapshot it restores. All of this is outside the timed region. +fn setup(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + self.setupImpl() catch |err| { + log.warn("failed to prepare snapshot benchmark err={}", .{err}); + return error.BenchmarkFailed; + }; +} + +fn setupImpl(self: *TerminalSnapshot) !void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.terminal = try Terminal.init(global.io(), self.alloc, .{ + .cols = self.opts.@"terminal-cols", + .rows = self.opts.@"terminal-rows", + .max_scrollback_bytes = null, + .max_scrollback_lines = null, + }); + const terminal = &self.terminal.?; + + // Feed the input corpus through the standard VT stream. + if (try options.dataFile(self.opts.data)) |data_f| { + defer data_f.close(global.io()); + + var stream = terminal.vtStream(); + defer stream.deinit(); + + var read_buf: [4096]u8 align(std.atomic.cache_line) = undefined; + var f_reader = data_f.reader(global.io(), &read_buf); + const r = &f_reader.interface; + + var buf: [4096]u8 = undefined; + while (true) { + const n = try r.readSliceShort(&buf); + if (n == 0) break; // EOF reached + stream.nextSlice(buf[0..n]); + } + } + + // Decode restores the same bytes every step. Encode mode also uses + // this buffer, in which case this is its one-time growth. + self.encoded.shrinkRetainingCapacity(0); + try snapshot.encode( + self.alloc, + &self.encoded.writer, + terminal, + .{ .continuation = .ground }, + ); +} + +fn teardown(ptr: *anyopaque) void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.encoded.shrinkRetainingCapacity(0); +} + +fn stepNoop(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + for (0..self.opts.loops) |_| { + std.mem.doNotOptimizeAway(self.encoded.written()); + } +} + +fn stepEncode(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const terminal = &self.terminal.?; + for (0..self.opts.loops) |_| { + self.encoded.shrinkRetainingCapacity(0); + snapshot.encode( + self.alloc, + &self.encoded.writer, + terminal, + .{ .continuation = .ground }, + ) catch |err| { + log.warn("snapshot encoding failed err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(self.encoded.written()); + } +} + +fn stepDecode(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const bytes = self.encoded.written(); + for (0..self.opts.loops) |_| { + var reader: std.Io.Reader = .fixed(bytes); + var decoded = snapshot.decode( + self.alloc, + global.io(), + &reader, + .{ .max_continuation_bytes = 1024 * 1024 }, + ) catch |err| { + log.warn("snapshot decoding failed err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(&decoded); + decoded.deinit(self.alloc); + } +} + +/// Print the encoded size grouped by record tag. This shares the encoder +/// with encode mode but deliberately makes no timing claims. +fn stepReport(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalSnapshot = @ptrCast(@alignCast(ptr)); + const bytes = self.encoded.written(); + + var payload_totals = std.enums.EnumArray( + snapshot.record.Tag, + u64, + ).initFill(0); + var record_counts = std.enums.EnumArray( + snapshot.record.Tag, + u64, + ).initFill(0); + var record_total: u64 = 0; + + var offset: usize = snapshot.envelope.encoded_len; + while (offset + snapshot.record.Header.len <= bytes.len) { + const tag_raw = std.mem.readInt(u16, bytes[offset..][0..2], .little); + const payload_len = std.mem.readInt( + u32, + bytes[offset + 2 ..][0..4], + .little, + ); + const tag = std.enums.fromInt(snapshot.record.Tag, tag_raw) orelse { + log.warn("unknown record tag {}", .{tag_raw}); + return error.BenchmarkFailed; + }; + payload_totals.getPtr(tag).* += payload_len; + record_counts.getPtr(tag).* += 1; + record_total += 1; + offset += snapshot.record.Header.len + payload_len; + } + + var it = payload_totals.iterator(); + while (it.next()) |entry| { + std.debug.print("terminal-snapshot tag={s} records={d} payload={d}\n", .{ + @tagName(entry.key), + record_counts.get(entry.key), + entry.value.*, + }); + } + std.debug.print( + "terminal-snapshot total records={d} framing={d} encoded={d}\n", + .{ + record_total, + record_total * snapshot.record.Header.len + + snapshot.envelope.encoded_len, + bytes.len, + }, + ); +} + +test TerminalSnapshot { + const testing = std.testing; + const impl: *TerminalSnapshot = try .create(testing.allocator, .{}); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} + +test "TerminalSnapshot decode round trip" { + const testing = std.testing; + const impl: *TerminalSnapshot = try .create(testing.allocator, .{ + .mode = .decode, + .@"terminal-rows" = 4, + .@"terminal-cols" = 8, + }); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} diff --git a/src/benchmark/cli.zig b/src/benchmark/cli.zig index 1dc089b71..f6a31f2c5 100644 --- a/src/benchmark/cli.zig +++ b/src/benchmark/cli.zig @@ -16,6 +16,7 @@ pub const Action = enum { @"screen-clone", @"terminal-parser", @"terminal-resize", + @"terminal-snapshot", @"terminal-stream", @"is-symbol", @"osc-parser", @@ -40,6 +41,7 @@ pub const Action = enum { .@"grapheme-break" => @import("GraphemeBreak.zig"), .@"terminal-parser" => @import("TerminalParser.zig"), .@"terminal-resize" => @import("TerminalResize.zig"), + .@"terminal-snapshot" => @import("TerminalSnapshot.zig"), .@"is-symbol" => @import("IsSymbol.zig"), .@"osc-parser" => @import("OscParser.zig"), }; diff --git a/src/benchmark/main.zig b/src/benchmark/main.zig index fb038c0a3..666ffb0f6 100644 --- a/src/benchmark/main.zig +++ b/src/benchmark/main.zig @@ -8,6 +8,7 @@ pub const HyperlinkMap = @import("HyperlinkMap.zig"); pub const ScreenClone = @import("ScreenClone.zig"); pub const TerminalParser = @import("TerminalParser.zig"); pub const TerminalResize = @import("TerminalResize.zig"); +pub const TerminalSnapshot = @import("TerminalSnapshot.zig"); pub const IsSymbol = @import("IsSymbol.zig"); pub const PageCompression = @import("PageCompression.zig"); pub const ScrollbackCompression = @import("ScrollbackCompression.zig"); diff --git a/src/crc32c.zig b/src/crc32c.zig new file mode 100644 index 000000000..8b8ebd1b2 --- /dev/null +++ b/src/crc32c.zig @@ -0,0 +1,193 @@ +//! CRC32C with hardware acceleration. +//! +//! The Zig standard library implementation processes one byte per table +//! lookup (as of Zig 0.16), which is more than an order of magnitude slower +//! than the dedicated CRC32C instructions available on aarch64 (CRC +//! extension) and x86_64 (SSE4.2). This module selects the best backend at +//! compile time and falls back to the standard library elsewhere, including +//! WebAssembly. +//! +//! The resulting value is identical across all backends: this is the +//! iSCSI CRC32C parameter set (reflected, initial and final XOR +//! `0xFFFFFFFF`), matching `std.hash.crc.Crc32Iscsi`. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// The standard-library implementation of the same parameter set. This is +/// both the portable fallback and the reference the tests compare against. +const Software = std.hash.crc.Crc32Iscsi; + +const Backend = enum { + aarch64_crc, + x86_64_sse42, + software, +}; + +const backend: Backend = backend: { + switch (builtin.cpu.arch) { + .aarch64, + .aarch64_be, + => if (std.Target.aarch64.featureSetHas( + builtin.cpu.features, + .crc, + )) break :backend .aarch64_crc, + + // The self-hosted x86_64 backend cannot encode the CRC32 + // instruction forms used below, so that combination falls back to + // the portable implementation. + .x86_64 => if (builtin.zig_backend == .stage2_llvm and + std.Target.x86.featureSetHas( + builtin.cpu.features, + .sse4_2, + )) break :backend .x86_64_sse42, + + else => {}, + } + break :backend .software; +}; + +/// Streaming CRC32C with the same interface shape as `std.hash.crc` types. +pub const Crc32c = struct { + crc: u32, + + pub fn init() Crc32c { + return .{ .crc = 0xFFFF_FFFF }; + } + + pub fn update(self: *Crc32c, bytes: []const u8) void { + self.crc = switch (comptime backend) { + .aarch64_crc, .x86_64_sse42 => updateHardware(self.crc, bytes), + .software => software: { + var crc: Software = .{ .crc = self.crc }; + crc.update(bytes); + break :software crc.crc; + }, + }; + } + + pub fn final(self: Crc32c) u32 { + return self.crc ^ 0xFFFF_FFFF; + } + + pub fn hash(bytes: []const u8) u32 { + var c: Crc32c = .init(); + c.update(bytes); + return c.final(); + } +}; + +/// One update pass using the dedicated CRC32C instructions. Both supported +/// architectures handle unaligned loads efficiently, so the loop reads +/// little-endian words directly from the input. +fn updateHardware(initial: u32, bytes: []const u8) u32 { + var crc = initial; + var remaining = bytes; + + while (remaining.len >= 8) : (remaining = remaining[8..]) { + crc = step(u64, crc, std.mem.readInt( + u64, + remaining[0..8], + .little, + )); + } + if (remaining.len >= 4) { + crc = step(u32, crc, std.mem.readInt( + u32, + remaining[0..4], + .little, + )); + remaining = remaining[4..]; + } + for (remaining) |byte| crc = step(u8, crc, byte); + return crc; +} + +/// One CRC32C instruction folding `value` into the running CRC. +inline fn step(comptime T: type, crc: u32, value: T) u32 { + return switch (comptime backend) { + .aarch64_crc => switch (T) { + u8 => asm ("crc32cb %[out:w], %[crc:w], %[value:w]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + u32 => asm ("crc32cw %[out:w], %[crc:w], %[value:w]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + u64 => asm ("crc32cx %[out:w], %[crc:w], %[value:x]" + : [out] "=r" (-> u32), + : [crc] "r" (crc), + [value] "r" (value), + ), + else => comptime unreachable, + }, + + .x86_64_sse42 => switch (T) { + u8 => asm ("crc32b %[value], %[out]" + : [out] "=r" (-> u32), + : [value] "r" (value), + [crc_in] "0" (crc), + ), + u32 => asm ("crc32l %[value], %[out]" + : [out] "=r" (-> u32), + : [value] "r" (value), + [crc_in] "0" (crc), + ), + u64 => @truncate(asm ("crc32q %[value], %[out]" + : [out] "=r" (-> u64), + : [value] "r" (value), + [crc_in] "0" (@as(u64, crc)), + )), + else => comptime unreachable, + }, + + .software => comptime unreachable, + }; +} + +test "matches the check value" { + // The catalog check value for CRC-32/ISCSI. + try std.testing.expectEqual( + @as(u32, 0xE3069283), + Crc32c.hash("123456789"), + ); +} + +test "matches the standard library at every length and split" { + var bytes: [259]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0xC5C32C); + prng.random().bytes(&bytes); + + for (0..bytes.len + 1) |len| { + const input = bytes[0..len]; + try std.testing.expectEqual( + Software.hash(input), + Crc32c.hash(input), + ); + + // Streaming across arbitrary split points must not change the + // result: word batching may not leak state between updates. + var split: Crc32c = .init(); + split.update(input[0 .. len / 3]); + split.update(input[len / 3 .. len - len / 3]); + split.update(input[len - len / 3 ..]); + try std.testing.expectEqual(Software.hash(input), split.final()); + } +} + +test "matches the standard library at every alignment" { + var bytes: [64 + 16]u8 = undefined; + var prng = std.Random.DefaultPrng.init(0xA11C); + prng.random().bytes(&bytes); + + for (0..16) |offset| { + const input = bytes[offset..][0..64]; + try std.testing.expectEqual( + Software.hash(input), + Crc32c.hash(input), + ); + } +} diff --git a/src/synthetic/cli/Ascii.zig b/src/synthetic/cli/Ascii.zig index 81a4e6073..b9ce2cb15 100644 --- a/src/synthetic/cli/Ascii.zig +++ b/src/synthetic/cli/Ascii.zig @@ -7,7 +7,14 @@ const Bytes = @import("../Bytes.zig"); const log = std.log.scoped(.@"terminal-stream-bench"); -pub const Options = struct {}; +pub const Options = struct { + /// When nonzero, emit lines whose printable length is uniformly + /// distributed in `[line-min, line-max]`, each terminated by CR LF. + /// When zero (the default), emit an unbroken stream of printable + /// bytes that relies on terminal wrapping. + @"line-min": usize = 0, + @"line-max": usize = 0, +}; fn checkAsciiAlphabet(c: u8) bool { return switch (c) { @@ -18,13 +25,16 @@ fn checkAsciiAlphabet(c: u8) bool { pub const ascii = Bytes.generateAlphabet(checkAsciiAlphabet); +opts: Options, + /// Create a new terminal stream handler for the given arguments. pub fn create( alloc: Allocator, - _: Options, + opts: Options, ) !*Ascii { const ptr = try alloc.create(Ascii); errdefer alloc.destroy(ptr); + ptr.* = .{ .opts = opts }; return ptr; } @@ -32,16 +42,18 @@ pub fn destroy(self: *Ascii, alloc: Allocator) void { alloc.destroy(self); } -pub fn run(_: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { +pub fn run(self: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { + const line_max = @max(self.opts.@"line-min", self.opts.@"line-max"); + const lines = line_max > 0; var gen: Bytes = .{ .rand = rand, .alphabet = ascii, - .min_len = 1024, - .max_len = 1024, + .min_len = if (lines) @max(self.opts.@"line-min", 1) else 1024, + .max_len = if (lines) line_max else 1024, }; while (true) { - _ = gen.write(writer) catch |err| { + writeChunk(&gen, writer, lines) catch |err| { const Error = error{ WriteFailed, BrokenPipe } || @TypeOf(err); switch (@as(Error, err)) { error.BrokenPipe => return, // stdout closed @@ -51,6 +63,15 @@ pub fn run(_: *Ascii, writer: *std.Io.Writer, rand: std.Random) !void { } } +fn writeChunk( + gen: *const Bytes, + writer: *std.Io.Writer, + lines: bool, +) std.Io.Writer.Error!void { + _ = try gen.write(writer); + if (lines) try writer.writeAll("\r\n"); +} + test Ascii { const testing = std.testing; const alloc = testing.allocator; @@ -65,3 +86,30 @@ test Ascii { var writer: std.Io.Writer = .fixed(&buf); try impl.run(&writer, rand); } + +test "Ascii lines" { + const testing = std.testing; + const alloc = testing.allocator; + + const impl: *Ascii = try .create(alloc, .{ + .@"line-min" = 5, + .@"line-max" = 20, + }); + defer impl.destroy(alloc); + + var prng = std.Random.DefaultPrng.init(1); + const rand = prng.random(); + + var buf: [1024]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try impl.run(&writer, rand); + + // Every emitted line respects the configured bounds. + var it = std.mem.splitSequence(u8, writer.buffered(), "\r\n"); + while (it.next()) |line| { + // The fixed buffer may end mid-line. + if (it.rest().len == 0) break; + try testing.expect(line.len >= 5); + try testing.expect(line.len <= 20); + } +} diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index 18e7955ba..221d545f8 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -1,25 +1,37 @@ -//! Grid (rows and cells) encoding. +//! Grid (rows, cells, and grapheme suffixes) encoding. //! //! A grid contains the rows and cells of a terminal page. Its dimensions are //! supplied by the containing record rather than repeated here. The encoder -//! writes exactly `rows` row records, and every row contains exactly `columns` -//! cell records. +//! writes exactly `rows` row records followed by one grapheme suffix section. //! //! All records are tightly packed with no padding between them. All integers //! are unsigned and little-endian. //! +//! The layout is designed so the common case decodes with bulk copies: cells +//! are fixed-size words, trailing default cells are elided per row, and the +//! variable-length grapheme suffixes live outside the row data. +//! +//! ```text +//! +---------------------------+ +//! | Row 0 | +//! +---------------------------+ +//! | ... | +//! +---------------------------+ +//! | Row (rows - 1) | +//! +---------------------------+ +//! | Grapheme suffix section | +//! +---------------------------+ +//! ``` +//! //! ## Row //! //! Each row has the following format: //! -//! | Offset | Size | Field | -//! | -----: | -------: | :-------------------------- | -//! | 0 | 1 | Row flags | -//! | 1 | variable | Exactly `columns` cells | -//! -//! Cells are encoded consecutively. Since cells may contain a variable number -//! of grapheme codepoints, the next row begins immediately after the final -//! cell and its grapheme codepoints. +//! | Offset | Size | Field | +//! | -----: | ----------------: | :------------------------- | +//! | 0 | 1 | Row flags | +//! | 1 | 2 | Encoded cell count (`u16`) | +//! | 3 | `width` * `count` | Encoded cells | //! //! The row flag byte has the following format: //! @@ -28,7 +40,8 @@ //! | 0 | Wrap | //! | 1 | Wrap continuation | //! | 2-3 | Semantic prompt | -//! | 4-7 | Reserved, zero | +//! | 4-5 | Encoded cell width | +//! | 6-7 | Reserved, zero | //! //! Semantic prompt values are: //! @@ -40,37 +53,128 @@ //! //! Value 3 is not emitted in snapshot version 1. Decoders treat it as none. //! +//! The encoded cell count must not exceed the grid's column count; it is a +//! structural field and decoders reject larger values. Cells at and beyond +//! the count are the default cell: all sixty-four bits zero, meaning an +//! empty narrow codepoint cell with the default style and no hyperlink. +//! Canonical encoders emit exactly through the row's last non-default cell, +//! so a fully default row has a zero count and no cell words. +//! +//! ## Encoded cell width +//! +//! The two width bits select how many bytes encode each of the row's +//! cells: `1 << width` is the size, so zero through three select one, two, +//! four, or eight bytes. Narrower widths are truncated transports of the +//! same cell word: a cell qualifies for a width when all of its higher +//! word bits are zero. +//! +//! | Width | Bytes | Encoded value and admitted cells | +//! | ----: | ----: | :--------------------------------------------------- | +//! | 0 | 1 | Codepoint at or below U+00FF; all other bits zero | +//! | 1 | 2 | Codepoint at or below U+FFFF; all other bits zero | +//! | 2 | 4 | Word bits 0-31: any content kind and codepoint, style | +//! | | | IDs 1-63, narrow, no flags, no hyperlink | +//! | 3 | 8 | The complete word | +//! +//! Widths zero and one store the codepoint value itself, which is word +//! bits 2-25 shifted down; the reconstructed word is the codepoint shifted +//! left by two. Width two stores the word's low half unshifted. This makes +//! every width a zero-extension on decode and a truncation on encode. +//! +//! Canonical encoders choose each row's smallest admissible width, which +//! follows directly from the bitwise OR of the row's cell words. Decoders +//! use the declared width for framing and accept rows encoded wider than +//! necessary. The width of a row with a zero cell count is canonically +//! zero and carries no meaning. +//! //! Native row cache flags are not encoded. In particular, the Kitty virtual //! placeholder hint is derived while decoding cells containing U+10EEEE. //! //! ## Cell //! -//! Each cell has the following format: +//! Each cell is one 64-bit little-endian word, transported at the row's +//! encoded cell width as described above: //! -//! | Offset | Size | Field | -//! | -----: | --------: | :---------------------------- | -//! | 0 | 1 | Content kind | -//! | 1 | 1 | Width kind | -//! | 2 | 1 | Protected and semantic flags | -//! | 3 | 1 | Reserved, zero | -//! | 4 | 2 | Style ID | -//! | 6 | 2 | Hyperlink ID | -//! | 8 | 4 | Codepoint or packed color | -//! | 12 | 4 | Grapheme suffix count | -//! | 16 | 4 * count | Grapheme suffix codepoints | +//! ```text +//! bit 0 +-------------------------------+ +//! | Content kind | +//! | 2 bits | +//! bit 2 +-------------------------------+ +//! | Content | +//! | 24 bits | +//! bit 26 +-------------------------------+ +//! | Style ID | +//! | 16 bits | +//! bit 42 +-------------------------------+ +//! | Width kind | +//! | 2 bits | +//! bit 44 +-------------------------------+ +//! | Protected | +//! bit 45 +-------------------------------+ +//! | Hyperlink flag | +//! bit 46 +-------------------------------+ +//! | Semantic content | +//! | 2 bits | +//! bit 48 +-------------------------------+ +//! | Hyperlink ID | +//! | 16 bits | +//! bit 64 +-------------------------------+ +//! ``` //! //! Content kinds are: //! -//! | Value | Meaning | -//! | ----: | :----------------- | -//! | 0 | Codepoint | -//! | 1 | Palette background | -//! | 2 | RGB background | +//! | Value | Meaning | +//! | ----: | :------------------------------- | +//! | 0 | Codepoint | +//! | 1 | Codepoint with grapheme suffix | +//! | 2 | Palette background | +//! | 3 | RGB background | //! -//! For codepoint content, the value is a Unicode scalar encoded as a `u32`. -//! For a palette background, the low byte is the palette index and the other -//! three bytes are zero. For an RGB background, the low three bytes are red, -//! green, and blue, and the high byte is zero. +//! The content kind selects the layout of the 24-bit content field. All +//! content bit positions below are relative to the start of the content +//! field, so content bit 0 is cell word bit 2. +//! +//! For codepoint content (kinds 0 and 1), the content field is a Unicode +//! scalar value. Values above U+10FFFF and surrogates decode as U+FFFD. +//! Kind 1 additionally declares that the grapheme suffix section contains +//! one entry for this cell; the cell is otherwise identical to kind 0. +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Unicode scalar value | +//! | 24 bits | +//! bit 24 +-------------------------------+ +//! ``` +//! +//! For a palette background (kind 2), the low byte is the palette index. +//! The remaining content bits are reserved, canonically zero, and ignored +//! by decoders. +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Palette index | +//! | 8 bits | +//! bit 8 +-------------------------------+ +//! | Reserved, zero | +//! | 16 bits | +//! bit 24 +-------------------------------+ +//! ``` +//! +//! For an RGB background (kind 3), the content field is one byte per +//! channel: +//! +//! ```text +//! bit 0 +-------------------------------+ +//! | Red | +//! | 8 bits | +//! bit 8 +-------------------------------+ +//! | Green | +//! | 8 bits | +//! bit 16 +-------------------------------+ +//! | Blue | +//! | 8 bits | +//! bit 24 +-------------------------------+ +//! ``` //! //! Width kinds are: //! @@ -81,14 +185,6 @@ //! | 2 | Spacer tail | //! | 3 | Spacer head | //! -//! The cell flag byte has the following format: -//! -//! | Bits | Field | -//! | ---: | :---------------- | -//! | 0 | Protected | -//! | 1-2 | Semantic content | -//! | 3-7 | Reserved, zero | -//! //! Semantic content values are: //! //! | Value | Meaning | @@ -101,13 +197,40 @@ //! //! Style and hyperlink ID zero mean no style and no hyperlink. Other IDs //! refer to entries in the containing record's separate style and hyperlink -//! tables. +//! tables. The hyperlink flag is set exactly when the hyperlink ID is +//! nonzero; decoders derive cell linkage from the remapped ID and ignore the +//! flag itself. //! -//! Grapheme suffixes are valid only for codepoint content. Each suffix is a -//! Unicode scalar encoded as a `u32`. A nonzero suffix count requires a -//! nonzero base codepoint. +//! ## Grapheme suffix section +//! +//! The section begins with an entry count followed by that many entries: +//! +//! | Offset | Size | Field | +//! | -----: | ------: | :------------------- | +//! | 0 | 4 | Entry count (`u32`) | +//! | 4 | varies | Entries | +//! +//! Each entry: +//! +//! | Offset | Size | Field | +//! | -----: | ----------: | :-------------------------- | +//! | 0 | 2 | Row (`u16`) | +//! | 2 | 2 | Column (`u16`) | +//! | 4 | 2 | Codepoint count (`u16`) | +//! | 6 | 4 * `count` | Codepoints (`u32` each) | +//! +//! Each codepoint is a Unicode scalar; invalid scalars are individually +//! ignored by decoders. Canonical encoders emit exactly one entry, with at +//! least one codepoint, for every kind 1 cell, in ascending row-then-column +//! order. Decoders consume every declared entry and ignore entries whose +//! target is out of range, is not a codepoint cell, has a zero codepoint, or +//! already received an entry. A kind 1 cell that receives no suffix +//! codepoints decodes as a plain codepoint cell. const std = @import("std"); +const assert = std.debug.assert; +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; const test_fixture = @import("fixture.zig"); const io = @import("io.zig"); const kitty = @import("../kitty.zig"); @@ -121,26 +244,214 @@ const TerminalPage = terminal_page.Page; const TerminalRow = terminal_page.Row; const TerminalStyleId = terminal_style.Id; -/// Maps encoded style table IDs to IDs assigned by the destination page. +/// The header before every row's encoded cells. /// -/// Build this by inserting each decoded style into the page, then recording -/// the encoded ID and the ID returned by the page's style set. Style ID zero is -/// implicit and does not need an entry. -pub const StyleRemap = std.AutoHashMap(TerminalStyleId, TerminalStyleId); +/// The semantic prompt is a raw integer for the same reason as the wire +/// cell fields: decoders must accept its reserved value without +/// instantiating an invalid native enum. The width enum is exhaustive over +/// its two bits, so every header byte bit-casts to a valid value. +pub const Row = packed struct(u8) { + wrap: bool = false, + wrap_continuation: bool = false, + semantic_prompt: u2 = 0, + cell_width: Cell.EncodedWidth = .one, + _padding: u2 = 0, +}; -/// Maps encoded hyperlink table IDs to IDs assigned by the destination page. +/// The wire layout of one encoded cell. This is its own registry: the bit +/// positions and field meanings are part of the snapshot format and are +/// documented above independently of the native cell. /// -/// Build this by inserting each decoded hyperlink into the page, then -/// recording the encoded ID and the ID returned by the page's hyperlink set. -/// Hyperlink ID zero is implicit and does not need an entry. -pub const HyperlinkRemap = std.AutoHashMap(TerminalHyperlinkId, TerminalHyperlinkId); +/// Enum-like fields are raw integers because decoders must accept reserved +/// values (for example semantic content 3) without instantiating an invalid +/// native enum. +pub const Cell = packed struct(u64) { + kind: u2 = @intFromEnum(Kind.codepoint), + content: u24 = 0, + style_id: u16 = 0, + width: u2 = 0, + protected: bool = false, + hyperlink: bool = false, + semantic_content: u2 = 0, + hyperlink_id: u16 = 0, + + /// Determines how `content` is interpreted. + pub const Kind = enum(u2) { + codepoint = 0, + codepoint_grapheme = 1, + bg_color_palette = 2, + bg_color_rgb = 3, + }; + + /// The encoded cell width declared by a row header: how many bytes + /// transport each of the row's cell words. See the format + /// documentation above for the value each width transports and the + /// cells it admits. + /// + /// `truncate` and `extend` are the transport transform itself, so the + /// admission masks derive from them rather than being maintained by + /// hand: a cell word is admitted exactly when it round-trips. + pub const EncodedWidth = enum(u2) { + one = 0, + two = 1, + four = 2, + eight = 3, + + /// The number of bytes transporting one cell word. + pub fn size(self: EncodedWidth) usize { + return @as(usize, 1) << @intFromEnum(self); + } + + /// The integer type transporting one cell word. + pub fn Int(comptime self: EncodedWidth) type { + return switch (self) { + .one => u8, + .two => u16, + .four => u32, + .eight => u64, + }; + } + + /// Truncate one cell word to its transported value. Only words + /// admitted by this width round-trip; `select` proves that for + /// every cell in a row before an encoder may use it. + pub fn truncate(comptime self: EncodedWidth, word: u64) self.Int() { + return @truncate(switch (self) { + // The bare codepoint, shifted down from the content field. + .one, .two => @as(Cell, @bitCast(word)).content, + + // The word itself. + .four, .eight => word, + }); + } + + /// Widen one transported value back to its cell word. + pub fn extend(comptime self: EncodedWidth, value: self.Int()) u64 { + return switch (self) { + .one, .two => @bitCast(Cell{ .content = value }), + .four, .eight => value, + }; + } + + /// The word bits a cell may use and still round-trip through this + /// width. + pub fn mask(comptime self: EncodedWidth) u64 { + return comptime self.extend(std.math.maxInt(self.Int())); + } + + /// The smallest width admitting the word, typically the bitwise + /// OR of every cell word in a row. + pub fn select(word: u64) EncodedWidth { + inline for ([_]EncodedWidth{ .one, .two, .four }) |width| { + if (word & ~width.mask() == 0) return width; + } + return .eight; + } + }; +}; + +/// Whether the native cell's in-memory layout matches the wire cell layout +/// bit for bit, with the wire hyperlink ID occupying the native padding. +/// +/// The wire format does not require this: it is an optimization. When it +/// holds, whole rows encode and decode as bulk copies. If the native layout +/// ever diverges, the portable field-by-field codec below remains correct +/// and this constant simply becomes false. +const native_matches_wire = native: { + if (@bitSizeOf(TerminalCell) != 64) break :native false; + + // The bulk copies above are only sound if we can prove the two layouts + // agree, and we want native cell changes to demote us to the portable + // codec automatically rather than corrupt snapshots. + // + // Instead of pinning every native field offset and enum value w/ assertions + // that must be maintained by hand, each probe below pairs a native + // cell with the wire cell that must share its exact bit pattern; both + // sides bit cast to a word and any difference means some field moved + // or some enum member was renumbered. + // + // Three probes, one per content payload, cover the whole cell. + const probes = [_]struct { + native: TerminalCell, + wire: Cell, + }{ + .{ + .native = .{ + .content_tag = .codepoint_grapheme, + .content = .{ .codepoint = .{ .data = 0x10FFFF } }, + .style_id = 0xBEEF, + .wide = .spacer_tail, + .protected = true, + ._padding = 0x1D2C, + }, + .wire = .{ + .kind = 1, + .content = 0x10FFFF, + .style_id = 0xBEEF, + .width = 2, + .protected = true, + // The native padding position, canonical wire or not. + .hyperlink_id = 0x1D2C, + }, + }, + .{ + .native = .{ + .content_tag = .bg_color_palette, + .content = .{ .color_palette = .{ .data = 0xAB } }, + .wide = .spacer_head, + .hyperlink = true, + .semantic_content = .input, + }, + .wire = .{ + .kind = 2, + .content = 0xAB, + .width = 3, + .hyperlink = true, + .semantic_content = 1, + }, + }, + .{ + .native = .{ + .content_tag = .bg_color_rgb, + .content = .{ .color_rgb = .{ + .r = 0x12, + .g = 0x34, + .b = 0x56, + } }, + .wide = .wide, + .semantic_content = .prompt, + }, + .wire = .{ + .kind = 3, + .content = 0x563412, + .width = 1, + .semantic_content = 2, + }, + }, + }; + for (probes) |probe| { + const native_bits: u64 = @bitCast(probe.native); + const wire_bits: u64 = @bitCast(probe.wire); + if (native_bits != wire_bits) break :native false; + } + + break :native true; +}; + +/// Whether rows of cells can be copied between native and wire storage +/// without per-cell transformation. +const bulk_codec = native_matches_wire and + builtin.cpu.arch.endian() == .little; pub const EncodeError = std.Io.Writer.Error || error{ /// Wide and spacer cells do not form a valid row. InvalidWideCell, + + /// One cell's grapheme suffix exceeds the entry's u16 codepoint count. + TooManyGraphemes, }; -/// Encode every row and cell directly from a page. +/// Encode every row, cell, and grapheme suffix directly from a page. /// /// If an error is returned, partial data may have been written. If you /// want transactional writing, the caller is responsible for using something @@ -151,20 +462,27 @@ pub fn encode( ) EncodeError!void { defer page.assertIntegrity(); for (0..page.size.rows) |y| { - // Row header const row = page.getRow(y); - const row_header: RowHeader = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = row.semantic_prompt, - }; - try writer.writeByte(@bitCast(row_header)); - - // Cells const cells = page.getCells(row); - for (cells, 0..) |*cell, x| { - // Validate the wide state of this cell, we don't want - // to encode corrupt data. + + // Trailing default cells decode implicitly. Wide/spacer pairs and + // hyperlinked or styled cells are always nonzero, so eliding the + // zero suffix never drops encoded state. + const count: usize = count: { + var i: usize = cells.len; + while (i > 0) : (i -= 1) { + if (!cells[i - 1].isZero()) break :count i; + } + break :count 0; + }; + + // Validate the wide state of every encoded cell so we don't encode + // corrupt data, and accumulate the OR of the row's cell words to + // select its encoded cell width. Trailing default cells are narrow, + // so checking the encoded prefix against the full row width covers + // every pair. + var word_or: u64 = 0; + for (cells[0..count], 0..) |*cell, x| { switch (cell.wide) { .narrow => {}, .wide => if (x + 1 == cells.len or @@ -181,418 +499,821 @@ pub fn encode( return error.InvalidWideCell; }, } + word_or |= classifyWord(cell); + } - const graphemes: []const u21 = if (cell.hasGrapheme()) - page.lookupGrapheme(cell) orelse unreachable - else - &.{}; + // Canonical rows use the smallest admissible width. + const cell_width: Cell.EncodedWidth = .select(word_or); - // The page has two codepoint tags depending on whether suffixes - // exist, but the wire represents both with one content kind. - const kind: CellHeader.Kind = switch (cell.content_tag) { - .codepoint, .codepoint_grapheme => .codepoint, - .bg_color_palette => .bg_color_palette, - .bg_color_rgb => .bg_color_rgb, - }; - const value: CellHeader.Value = switch (kind) { - .codepoint => .{ - .codepoint = cell.content.codepoint.data, - }, - .bg_color_palette => .{ .bg_color_palette = .{ - .index = cell.content.color_palette.data, - } }, - .bg_color_rgb => .{ .bg_color_rgb = .{ - .r = cell.content.color_rgb.r, - .g = cell.content.color_rgb.g, - .b = cell.content.color_rgb.b, - } }, + // Row header: flags then the encoded cell count. + { + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + .cell_width = cell_width, }; + var header_bytes: [3]u8 = undefined; + header_bytes[0] = @bitCast(row_header); + std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); + try writer.writeAll(&header_bytes); + } - const style_id = cell.style_id; - const hyperlink_id: TerminalHyperlinkId = if (cell.hyperlink) - page.lookupHyperlink(cell) orelse unreachable - else - 0; - - const header: CellHeader = .{ - .content_kind = kind, - .width = cell.wide, - .protected = cell.protected, - .semantic_content = cell.semantic_content, - .style_id = style_id, - .hyperlink_id = hyperlink_id, - .value = value, - .grapheme_count = @intCast(graphemes.len), - }; - try header.encode(writer); - for (graphemes) |suffix| try io.writeInt( + switch (cell_width) { + inline .one, .two, .four => |width| try encodeNarrowCells( + width, + cells[0..count], writer, - u32, - suffix, - ); + ), + + .eight => { + // Hyperlink IDs live in a native side table, but we embed + // them in ours, so if we have any hyperlinks we need + // to fallback to the loop below. + if (comptime bulk_codec) { + const witness: Cell = @bitCast(word_or); + if (!witness.hyperlink and witness.hyperlink_id == 0) { + try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); + continue; + } + } + + for (cells[0..count]) |*cell| { + const link_id: TerminalHyperlinkId = if (cell.hyperlink) + page.lookupHyperlink(cell) orelse unreachable + else + 0; + try io.writeInt( + writer, + u64, + cellBits(cell.*, link_id), + ); + } + }, + } + } + + try encodeGraphemes(page, writer); +} + +/// The word used to select a row's encoded cell width. This is the cell's +/// wire word with the hyperlink flag reflecting the native cell, so linked +/// cells and nonzero native padding disqualify every narrow width. +inline fn classifyWord(cell: *const TerminalCell) u64 { + if (comptime native_matches_wire) return @bitCast(cell.*); + + var wire: Cell = @bitCast(cellBits(cell.*, 0)); + wire.hyperlink = cell.hyperlink; + return @bitCast(wire); +} + +/// Write one row's cells truncated to the given encoded width. +/// +/// Every admitted cell round-trips exactly because the row's width +/// selection proved every cell word survives `truncate` then `extend`. +fn encodeNarrowCells( + comptime width: Cell.EncodedWidth, + cells: []const TerminalCell, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + const size = comptime width.size(); + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + + if (comptime native_matches_wire) { + // A pure truncating loop over integers that the compiler can + // vectorize. + const words: [*]const u64 = @ptrCast(cells.ptr); + for (0..n) |j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(words[i + j]), + .little, + ); + } + } else { + for (cells[i..][0..n], 0..) |*cell, j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(classifyWord(cell)), + .little, + ); + } + } + + try writer.writeAll(chunk[0 .. n * size]); + i += n; + } +} + +/// Encode the grapheme suffix section for every kind 1 cell in the grid. +fn encodeGraphemes( + page: *const TerminalPage, + writer: *std.Io.Writer, +) EncodeError!void { + // Count and validate entries before the section header so the count is + // always exact. Rows without the native grapheme hint contain no + // grapheme cells in any intact page. + var entries: u32 = 0; + for (0..page.size.rows) |y| { + const row = page.getRow(y); + if (!row.grapheme) continue; + for (page.getCells(row)) |*cell| { + if (!cell.hasGrapheme()) continue; + const cps = page.lookupGrapheme(cell) orelse unreachable; + if (cps.len > std.math.maxInt(u16)) return error.TooManyGraphemes; + entries += 1; + } + } + + try io.writeInt(writer, u32, entries); + if (entries == 0) return; + + for (0..page.size.rows) |y| { + const row = page.getRow(y); + if (!row.grapheme) continue; + for (page.getCells(row), 0..) |*cell, x| { + if (!cell.hasGrapheme()) continue; + const cps = page.lookupGrapheme(cell) orelse unreachable; + try io.writeInt(writer, u16, @intCast(y)); + try io.writeInt(writer, u16, @intCast(x)); + try io.writeInt(writer, u16, @intCast(cps.len)); + for (cps) |cp| try io.writeInt(writer, u32, cp); } } } -/// Decode every row and cell directly into an initialized, empty page. +pub const DecodeError = std.Io.Reader.Error || error{ + /// A row declares more encoded cells than the grid has columns. + InvalidRowCellCount, +}; + +/// Maps encoded style table IDs to page-assigned style IDs. +pub const StyleRemap = Remap(TerminalStyleId); + +/// Maps encoded hyperlink table IDs to page-assigned hyperlink IDs. +pub const HyperlinkRemap = Remap(TerminalHyperlinkId); + +/// Decode every row, cell, and grapheme suffix directly into an +/// initialized, empty page. /// -/// The grid does not encode dimensions, so `page` must already have the exact -/// row and column count expected by the containing record. This function reads -/// exactly `page.size.rows` rows with `page.size.cols` cells each. Capacity -/// hints are advisory. Graphemes and cell hyperlink references that do not fit -/// are discarded without affecting the rest of the grid. +/// The grid does not encode dimensions, so `page` must already have the +/// exact row and column count expected by the containing record. Capacity +/// hints are advisory. Graphemes and cell hyperlink references that do not +/// fit are discarded without affecting the rest of the grid. /// /// Style and hyperlink table entries must be inserted into `page` before -/// calling this function. As each table entry is inserted, the caller records -/// its encoded ID and page-assigned ID in `style_remap` or `hyperlink_remap`. -/// ID zero always means the default style or no hyperlink. A nonzero ID missing -/// from its remap is also treated as zero so unknown table references do not -/// prevent the rest of the grid from decoding. +/// calling this function, with their encoded and page-assigned IDs recorded +/// in `style_remap` and `hyperlink_remap`. ID zero always means the default +/// style or no hyperlink. A nonzero ID missing from its remap is also +/// treated as zero so unknown table references do not prevent the rest of +/// the grid from decoding. /// -/// Invalid semantic data is normalized into a degraded form while preserving -/// the declared byte boundaries. Unknown semantic values use their neutral -/// variants, invalid Unicode becomes U+FFFD, invalid optional data is ignored, -/// and malformed wide-cell relationships become narrow cells. +/// Invalid semantic data is normalized into a degraded form while +/// preserving the declared byte boundaries. Unknown semantic values use +/// their neutral variants, invalid Unicode becomes U+FFFD, invalid optional +/// data is ignored, and malformed wide-cell relationships become narrow +/// cells. The encoded cell count is the only structural field. pub fn decode( page: *TerminalPage, reader: *std.Io.Reader, style_remap: *const StyleRemap, hyperlink_remap: *const HyperlinkRemap, -) std.Io.Reader.Error!void { +) DecodeError!void { for (0..page.size.rows) |y| { - const row_raw = try reader.takeByte(); - const row_header: RowHeader = @bitCast(row_raw); - const semantic_prompt_raw: u2 = @truncate(row_raw >> @bitOffsetOf(RowHeader, "semantic_prompt")); + // Read the row header and cell count. + const row_header: Row, const count: u16 = header: { + // The staged payload path has every header buffered. + var row_header_bytes: [3]u8 = undefined; + if (reader.bufferedLen() >= 3) { + row_header_bytes = reader.buffered()[0..3].*; + reader.toss(3); + } else { + try reader.readSliceAll(&row_header_bytes); + } + + // Every bit pattern is a valid header: booleans and the exhaustive + // width enum decode directly, and the raw semantic value gets a + // default below. Reserved bits do not change the known fields. + const row_header: Row = @bitCast(row_header_bytes[0]); + const count = std.mem.readInt(u16, row_header_bytes[1..3], .little); + break :header .{ row_header, count }; + }; - // Reserved bits do not change the known fields. The semantic enum is - // the only non-exhaustive field, so give its unknown value a default. const row = page.getRow(y); row.wrap = row_header.wrap; row.wrap_continuation = row_header.wrap_continuation; row.semantic_prompt = std.enums.fromInt( TerminalRow.SemanticPrompt, - semantic_prompt_raw, + row_header.semantic_prompt, ) orelse .none; const cells = page.getCells(row); - for (cells, 0..) |*cell, x| { - const decoded_header = try CellHeader.decode(reader); + if (count > cells.len) return error.InvalidRowCellCount; + if (count == 0) continue; - cell.* = .init(0); - var accept_graphemes = false; - const grapheme_count: u32 = switch (decoded_header) { - // The fixed header was fully consumed, but its content kind - // cannot be interpreted. Keep the cell empty and consume only - // the suffix bytes needed to reach the next cell. - .invalid => |count| count, - - .valid => |header| valid: { - // IDs belong to the encoded page. Translate them to IDs - // assigned by the destination page before storing them on - // cells. - const encoded_style_id = header.style_id; - const style_id = if (encoded_style_id == 0) - 0 - else - style_remap.get(encoded_style_id) orelse 0; - - const encoded_hyperlink_id = header.hyperlink_id; - const hyperlink_id = if (encoded_hyperlink_id == 0) - 0 - else - hyperlink_remap.get(encoded_hyperlink_id) orelse 0; - - switch (header.content_kind) { - .codepoint => { - var cp = std.math.cast( - u21, - header.value.codepoint, - ) orelse 0xFFFD; - if (cp > 0x10FFFF or - (cp >= 0xD800 and cp <= 0xDFFF)) - { - cp = 0xFFFD; - } - cell.content = .{ .codepoint = .{ .data = cp } }; - accept_graphemes = cp != 0; - - // Kitty image and placement state is not part of this - // snapshot version, but the placeholder is still a - // valid Unicode scalar. Preserve it and derive the - // native row hint so later row operations remain - // correct. - if (cp == kitty.graphics.unicode.placeholder) { - row.kitty_virtual_placeholder = true; - } - }, - .bg_color_palette => { - const palette = header.value.bg_color_palette; - cell.content_tag = .bg_color_palette; - cell.content = .{ - .color_palette = .{ .data = palette.index }, - }; - }, - .bg_color_rgb => { - const rgb = header.value.bg_color_rgb; - cell.content_tag = .bg_color_rgb; - cell.content = .{ .color_rgb = .{ - .r = rgb.r, - .g = rgb.g, - .b = rgb.b, - } }; - }, + // The encoded cell width is framing: it determines exactly how many + // bytes this row occupies. + switch (row_header.cell_width) { + inline .one, .two => |width| try decodeNarrowCells( + width, + reader, + cells[0..count], + ), + .four => try decodeWordCells( + .four, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ), + .eight => { + if (comptime bulk_codec) { + // Read the encoded words directly into page storage, + // then normalize them in place. The raw words are only + // ever touched as integers until normalization makes + // them valid cells. + const words: [*]u64 = @ptrCast(cells.ptr); + try reader.readSliceAll( + std.mem.sliceAsBytes(cells[0..count]), + ); + for (0..count) |x| { + applyCell( + page, + row, + cells, + x, + words[x], + style_remap, + hyperlink_remap, + ); } - - cell.wide = header.width; - cell.protected = header.protected; - cell.semantic_content = header.semantic_content; - - if (style_id != 0) { - // The table owns one reference and each decoded cell owns - // one additional reference. - page.styles.use(page.memory, style_id); - cell.style_id = style_id; - row.styled = true; - } - - if (hyperlink_id != 0) { - // setHyperlink records the cell mapping but intentionally - // does not increment the set's reference count. If its map - // is full, undo our reference and leave this cell unlinked. - page.hyperlink_set.use(page.memory, hyperlink_id); - page.setHyperlink(row, cell, hyperlink_id) catch { - page.hyperlink_set.release(page.memory, hyperlink_id); - }; - } - - break :valid header.grapheme_count; - }, - }; - - // Always consume every declared suffix. Invalid scalars, suffixes - // on non-text cells, and suffixes that exceed the native capacity - // are optional detail and can be dropped independently. - for (0..grapheme_count) |_| { - const suffix_raw = try io.readInt(reader, u32); - if (!accept_graphemes) continue; - - const suffix = std.math.cast(u21, suffix_raw) orelse continue; - if (suffix > 0x10FFFF or - (suffix >= 0xD800 and suffix <= 0xDFFF)) - { - continue; + } else { + try decodeWordCells( + .eight, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ); } - page.appendGrapheme(row, cell, suffix) catch { - accept_graphemes = false; - }; + + // The implicit cell after a short row is narrow, which + // resolves a trailing wide marker exactly like an explicit + // narrow neighbor. Only full-width rows can encode wide + // markers. + if (count < cells.len and cells[count - 1].wide == .wide) { + cells[count - 1].wide = .narrow; + } + }, + } + } + + try decodeGraphemes(page, reader); +} + +/// Decode one row of width-one or width-two cells. +/// +/// These widths admit only bare codepoints, so cells store directly with at +/// most Unicode scalar validation: no styles, hyperlinks, wide pairs, or +/// row hints are reachable and no normalization state is needed. +fn decodeNarrowCells( + comptime width: Cell.EncodedWidth, + reader: *std.Io.Reader, + cells: []TerminalCell, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered, making this + // one bounds check followed by a vectorizable widening loop. + const total = cells.len * size; + if (reader.bufferedLen() >= total) { + widenCells(width, reader.buffered()[0..total], cells); + reader.toss(total); + return; + } + + // Streaming sources fall back to bounded chunks. + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + try reader.readSliceAll(chunk[0 .. n * size]); + widenCells(width, chunk[0 .. n * size], cells[i..][0..n]); + i += n; + } +} + +/// Store codepoint-valued encoded cells of the given width. +fn widenCells( + comptime width: Cell.EncodedWidth, + bytes: []const u8, + cells: []TerminalCell, +) void { + const size = comptime width.size(); + + // When the native cell matches the wire word, this is a pure widening + // loop over integers that the compiler can vectorize. + if (comptime native_matches_wire) { + const words: [*]u64 = @ptrCast(cells.ptr); + for (0..cells.len) |i| { + words[i] = width.extend(widenValue(width, bytes[i * size ..])); + } + return; + } + + for (cells, 0..) |*cell, i| { + storeCell(cell, width.extend(widenValue(width, bytes[i * size ..]))); + } +} + +/// Read and validate one narrow transported value. +inline fn widenValue( + comptime width: Cell.EncodedWidth, + bytes: []const u8, +) width.Int() { + const size = comptime width.size(); + const value = std.mem.readInt(width.Int(), bytes[0..size], .little); + + // Width one cannot encode an invalid scalar. Width two admits + // surrogates, which degrade exactly like their full-width form. + if (comptime width != .one) { + if (!validScalar(value)) return 0xFFFD; + } + return value; +} + +/// Decode one row of width-four or fallback full-width cells through the +/// complete per-cell normalization path. +fn decodeWordCells( + comptime width: Cell.EncodedWidth, + page: *TerminalPage, + row: *TerminalRow, + cells: []TerminalCell, + count: usize, + reader: *std.Io.Reader, + style_remap: *const StyleRemap, + hyperlink_remap: *const HyperlinkRemap, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered. + const total = count * size; + if (reader.bufferedLen() >= total) { + const bytes = reader.buffered()[0..total]; + for (0..count) |x| { + const bits = width.extend(std.mem.readInt( + width.Int(), + bytes[x * size ..][0..size], + .little, + )); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } + reader.toss(total); + return; + } + + // Streaming sources fall back to per-cell reads. + for (0..count) |x| { + const bits = width.extend(try io.readInt(reader, width.Int())); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } +} + +/// Whether the value is a valid Unicode scalar value. +inline fn validScalar(cp: u32) bool { + return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); +} + +/// Normalize one encoded cell word and store it at `cells[x]`. +/// +/// This owns every per-cell decode rule except grapheme suffixes: content +/// validation, reserved-value degradation, style and hyperlink remapping +/// with reference counting, and wide-pair normalization against already +/// decoded neighbors. +fn applyCell( + page: *TerminalPage, + row: *TerminalRow, + cells: []TerminalCell, + x: usize, + bits_wire: u64, + style_remap: *const StyleRemap, + hyperlink_remap: *const HyperlinkRemap, +) void { + const cell = &cells[x]; + + // The default cell is the common case and needs no normalization, + // reference counting, or table lookups. + if (bits_wire == 0) { + storeCell(cell, 0); + normalizeWide(row, cells, x); + return; + } + + // Any word is a valid wire cell because its fields are raw integers, + // so all normalization below is plain field access. + var wire: Cell = @bitCast(bits_wire); + + // Hyperlink linkage is derived from the remapped ID below. The stored + // native cell starts unlinked either way. + const link_encoded = wire.hyperlink_id; + wire.hyperlink_id = 0; + wire.hyperlink = false; + + switch (@as(Cell.Kind, @enumFromInt(wire.kind))) { + // Kind 1 differs from 0 only by declaring a grapheme suffix section + // entry, which reattaches through the native grapheme APIs later. + .codepoint, .codepoint_grapheme => { + wire.kind = @intFromEnum(Cell.Kind.codepoint); + if (!validScalar(wire.content)) wire.content = 0xFFFD; + + // Kitty image and placement state is not part of this snapshot + // version, but the placeholder is still a valid Unicode scalar. + // Preserve it and derive the native row hint so later row + // operations remain correct. + if (wire.content == kitty.graphics.unicode.placeholder) { + row.kitty_virtual_placeholder = true; } + }, - // A following cell resolves whether the previous wide marker owns - // a tail. Normalize the current marker immediately when possible, - // including a wide marker at the row end. - if (x > 0 and - cells[x - 1].wide == .wide and - cell.wide != .spacer_tail) - { - cells[x - 1].wide = .narrow; + // Only the palette index is meaningful; the remaining content bits + // are reserved and must not obscure it. + .bg_color_palette => wire.content = @as(u8, @truncate(wire.content)), + + .bg_color_rgb => {}, + } + + // Reserved semantic content degrades to plain output. + wire.semantic_content = @intFromEnum(std.enums.fromInt( + TerminalCell.SemanticContent, + wire.semantic_content, + ) orelse .output); + + // IDs belong to the encoded page. Translate them to IDs assigned by the + // destination page before storing them on cells. The table owns one + // reference and each decoded cell owns one additional reference. + if (wire.style_id != 0) { + wire.style_id = style_remap.get(wire.style_id); + if (wire.style_id != 0) { + page.styles.use(page.memory, wire.style_id); + row.styled = true; + } + } + + storeCell(cell, @bitCast(wire)); + + if (link_encoded != 0) link: { + const link_native = hyperlink_remap.get(link_encoded); + if (link_native == 0) break :link; + + // setHyperlink records the cell mapping but intentionally does not + // increment the set's reference count. If its map is full, undo our + // reference and leave this cell unlinked. + page.hyperlink_set.use(page.memory, link_native); + page.setHyperlink(row, cell, link_native) catch { + page.hyperlink_set.release(page.memory, link_native); + }; + } + + normalizeWide(row, cells, x); +} + +/// Resolve wide-pair relationships for the cell at `x` against its already +/// normalized predecessors. +fn normalizeWide(row: *const TerminalRow, cells: []TerminalCell, x: usize) void { + // A following cell resolves whether the previous wide marker owns a + // tail. Normalize the current marker immediately when possible, + // including a wide marker at the row end. + if (x > 0 and + cells[x - 1].wide == .wide and + cells[x].wide != .spacer_tail) + { + cells[x - 1].wide = .narrow; + } + switch (cells[x].wide) { + .narrow => {}, + + // A non-final wide marker remains pending until the next cell. + .wide => if (x + 1 == cells.len) { + cells[x].wide = .narrow; + }, + + .spacer_tail => if (x == 0 or + cells[x - 1].wide != .wide) + { + cells[x].wide = .narrow; + }, + + .spacer_head => if (x + 1 != cells.len or !row.wrap) { + cells[x].wide = .narrow; + }, + } +} + +/// Decode the grapheme suffix section into already decoded cells. +fn decodeGraphemes( + page: *TerminalPage, + reader: *std.Io.Reader, +) DecodeError!void { + const entries = try io.readInt(reader, u32); + for (0..entries) |_| { + const y = try io.readInt(reader, u16); + const x = try io.readInt(reader, u16); + const cp_count = try io.readInt(reader, u16); + + // Resolve the target cell. Entries whose target cannot carry a + // suffix are optional detail: their codepoints are consumed to + // preserve framing and then dropped. + const target: ?struct { + row: *TerminalRow, + cell: *TerminalCell, + } = target: { + if (y >= page.size.rows or x >= page.size.cols) { + break :target null; } - switch (cell.wide) { - .narrow => {}, + const row = page.getRow(y); + const cell = &page.getCells(row)[x]; - // A non-final wide marker remains pending until the next cell. - .wide => if (x + 1 == cells.len) { - cell.wide = .narrow; - }, + // Cell decoding stores every kind 1 cell as a plain codepoint, + // so this also gives duplicate entries first-wins semantics. + if (cell.content_tag != .codepoint) break :target null; + if (cell.content.codepoint.data == 0) break :target null; + break :target .{ .row = row, .cell = cell }; + }; - .spacer_tail => if (x == 0 or - cells[x - 1].wide != .wide) - { - cell.wide = .narrow; - }, + // Always consume every declared codepoint. Invalid scalars and + // suffixes that exceed the native capacity are dropped + // independently without affecting the rest of the grid. + var accept = target != null; + for (0..cp_count) |_| { + const cp = try io.readInt(reader, u32); + if (!accept) continue; + if (!validScalar(cp)) continue; - .spacer_head => if (x + 1 != cells.len or !row.wrap) { - cell.wide = .narrow; - }, - } + page.appendGrapheme( + target.?.row, + target.?.cell, + @intCast(cp), + ) catch { + accept = false; + }; } } } -/// The header before every row. -const RowHeader = packed struct(u8) { - wrap: bool = false, - wrap_continuation: bool = false, - semantic_prompt: TerminalRow.SemanticPrompt = .none, - _padding: u4 = 0, -}; - -/// The fixed fields that precede a cell's grapheme suffix codepoints. -const CellHeader = struct { - /// Number of bytes written by `encode`, calculated using the encoder itself - /// so this remains synchronized with the field-by-field wire format. - pub const len = computeLen(); - - comptime { - // This size is part of the wire format. If it changes, the snapshot - // version and golden fixtures must also change. - std.debug.assert(len == 16); +/// The encoded word for one native cell and its hyperlink ID. +fn cellBits(cell: TerminalCell, link_id: TerminalHyperlinkId) u64 { + if (comptime native_matches_wire) { + var wire: Cell = @bitCast(cell); + wire.hyperlink = link_id != 0; + wire.hyperlink_id = link_id; + return @bitCast(wire); } - /// Interpretation of `value`. - content_kind: Kind = .codepoint, - - /// Display width and spacer role of the cell. - width: TerminalCell.Wide = .narrow, - - /// Whether selective erase operations protect the cell. - protected: bool = false, - - /// Semantic role assigned by shell integration. - semantic_content: TerminalCell.SemanticContent = .output, - - /// ID in the encoded page's style table, or zero for the default style. - style_id: TerminalStyleId = 0, - - /// ID in the encoded page's hyperlink table, or zero for no hyperlink. - hyperlink_id: TerminalHyperlinkId = 0, - - /// Codepoint or packed background color selected by `content_kind`. - value: Value = .{ .codepoint = 0 }, - - /// Number of grapheme suffix codepoints immediately following the header. - grapheme_count: u32 = 0, - - /// Encode the fixed cell header. - pub fn encode( - self: CellHeader, - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - const flags: Flags = .{ - .protected = self.protected, - .semantic_content = self.semantic_content, - }; - - // 0 1 2 3 4 6 8 12 16 - // +-------+-------+-------+-------+--------+--------+--------+--------+ - // | kind | width | flags | zero | style | link | value | count | - // | u8 | u8 | u8 | u8 | u16 LE | u16 LE | u32 LE | u32 LE | - // +-------+-------+-------+-------+--------+--------+--------+--------+ - try writer.writeByte(@intFromEnum(self.content_kind)); - try writer.writeByte(@intFromEnum(self.width)); - try writer.writeByte(@bitCast(flags)); - try writer.writeByte(0); - try io.writeInt(writer, TerminalStyleId, self.style_id); - try io.writeInt(writer, TerminalHyperlinkId, self.hyperlink_id); - try io.writeInt(writer, u32, @bitCast(self.value)); - try io.writeInt(writer, u32, self.grapheme_count); - } - - /// A valid header, or the suffix count from an uninterpretable header. - const DecodeResult = union(enum) { - valid: CellHeader, - invalid: u32, + const wire: Cell = .{ + .kind = @intFromEnum(cell.content_tag), + .content = switch (cell.content_tag) { + .codepoint, + .codepoint_grapheme, + => cell.content.codepoint.data, + .bg_color_palette => cell.content.color_palette.data, + .bg_color_rgb => @as(u24, cell.content.color_rgb.r) | + (@as(u24, cell.content.color_rgb.g) << 8) | + (@as(u24, cell.content.color_rgb.b) << 16), + }, + .style_id = cell.style_id, + .width = switch (cell.wide) { + .narrow => 0, + .wide => 1, + .spacer_tail => 2, + .spacer_head => 3, + }, + .protected = cell.protected, + .hyperlink = link_id != 0, + .semantic_content = switch (cell.semantic_content) { + .output => 0, + .input => 1, + .prompt => 2, + }, + .hyperlink_id = link_id, }; + return @bitCast(wire); +} - /// Decode a fixed cell header, normalizing unknown semantic values. - pub fn decode(reader: *std.Io.Reader) std.Io.Reader.Error!DecodeResult { - // Decode various fields. We need to be very defensive here - // because it can come from an untrusted source and may be invalid. - const content_kind_optional = kind: { - const raw = try reader.takeByte(); - break :kind std.enums.fromInt(Kind, raw); - }; - const width = width: { - const raw = try reader.takeByte(); - break :width std.enums.fromInt(TerminalCell.Wide, raw) orelse .narrow; - }; - - const flags: Flags = @bitCast(try reader.takeByte()); - - // We can't trust `flags` to have a valid semantic content so - // we need extract the bits and do a safe conversion. - const semantic_content = semantic: { - const flags_raw: u8 = @bitCast(flags); - const raw: u2 = @truncate(flags_raw >> @bitOffsetOf(Flags, "semantic_content")); - break :semantic std.enums.fromInt( - TerminalCell.SemanticContent, - raw, - ) orelse .output; - }; - - // This byte is reserved so the IDs and content value remain naturally - // aligned within the fixed cell header. Ignore it so future versions - // can give it meaning without making old decoders reject the cell. - _ = try reader.takeByte(); - - const style_id = try io.readInt(reader, TerminalStyleId); - const hyperlink_id = try io.readInt(reader, TerminalHyperlinkId); - const value: Value = @bitCast(try io.readInt(reader, u32)); - const grapheme_count = try io.readInt(reader, u32); - - // If we don't have a valid content kind, we return invalid and - // note the suffix bytes so that the reader can discard - const content_kind = content_kind_optional orelse - return .{ .invalid = grapheme_count }; - - return .{ .valid = .{ - .content_kind = content_kind, - .width = width, - .protected = flags.protected, - .semantic_content = semantic_content, - .style_id = style_id, - .hyperlink_id = hyperlink_id, - .value = value, - .grapheme_count = grapheme_count, - } }; +/// Store one normalized, hyperlink-free wire word as a native cell. +fn storeCell(cell: *TerminalCell, bits: u64) void { + if (comptime native_matches_wire) { + const words: [*]u64 = @ptrCast(cell); + words[0] = bits; + return; } - /// Computes the fixed header size using the encoder itself. - fn computeLen() usize { - comptime { - var buf: [128]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - CellHeader.encode(.{}, &writer) catch unreachable; - return writer.end; + const wire: Cell = @bitCast(bits); + var native: TerminalCell = .init(0); + switch (@as(Cell.Kind, @enumFromInt(wire.kind))) { + .codepoint, .codepoint_grapheme => native.content = .{ + .codepoint = .{ .data = @intCast(wire.content) }, + }, + .bg_color_palette => { + native.content_tag = .bg_color_palette; + native.content = .{ + .color_palette = .{ .data = @truncate(wire.content) }, + }; + }, + .bg_color_rgb => { + native.content_tag = .bg_color_rgb; + native.content = .{ .color_rgb = .{ + .r = @truncate(wire.content), + .g = @truncate(wire.content >> 8), + .b = @truncate(wire.content >> 16), + } }; + }, + } + native.style_id = wire.style_id; + native.wide = @enumFromInt(wire.width); + native.protected = wire.protected; + native.semantic_content = @enumFromInt(wire.semantic_content); + cell.* = native; +} + +/// Maps encoded table IDs to IDs assigned by the destination page. +/// +/// Build this by inserting each decoded table entry into the page, then +/// recording the encoded ID and the ID returned by the page's set. ID zero +/// is implicit and does not need an entry. Lookup must be cheap because the +/// grid decoder consults it for every styled or linked cell, so this is a +/// direct-indexed table rather than a hash map. +fn Remap(comptime Id: type) type { + return struct { + const Self = @This(); + + /// One slot for every possible encoded ID. + pub const capacity = std.math.maxInt(Id) + 1; + + /// Indexed by encoded ID; zero means unmapped or default. + entries: []Id, + + /// Tracks IDs that received an entry, including ones mapped to the + /// default, so callers can give duplicate table entries first-wins + /// semantics. + seen: std.DynamicBitSetUnmanaged, + + pub fn init(alloc: Allocator) Allocator.Error!Self { + const entries = try alloc.alloc(Id, capacity); + errdefer alloc.free(entries); + @memset(entries, 0); + const seen = try std.DynamicBitSetUnmanaged.initEmpty( + alloc, + capacity, + ); + return .{ .entries = entries, .seen = seen }; } - } - /// Determines how `value` is interpreted. - pub const Kind = enum(u8) { - codepoint = 0, - bg_color_palette = 1, - bg_color_rgb = 2, + pub fn deinit(self: *Self, alloc: Allocator) void { + alloc.free(self.entries); + self.seen.deinit(alloc); + self.* = undefined; + } + + /// Record one encoded-to-native mapping. + pub fn put(self: *Self, encoded: Id, native: Id) void { + assert(!self.seen.isSet(encoded)); + self.entries[encoded] = native; + self.seen.set(encoded); + } + + /// Whether the encoded ID already has an entry, even a default one. + pub fn contains(self: *const Self, encoded: Id) bool { + return self.seen.isSet(encoded); + } + + /// The native ID for an encoded ID, or zero when unmapped. + pub inline fn get(self: *const Self, encoded: Id) Id { + return self.entries[encoded]; + } }; - - /// The alternate interpretations of the four-byte content field. - pub const Value = packed union(u32) { - codepoint: u32, - bg_color_palette: packed struct(u32) { - index: u8, - _padding: u24 = 0, - }, - bg_color_rgb: packed struct(u32) { - r: u8, - g: u8, - b: u8, - _padding: u8 = 0, - }, - }; - - const Flags = packed struct(u8) { - protected: bool = false, - semantic_content: TerminalCell.SemanticContent = .output, - _padding: u5 = 0, - }; -}; - +} const test_golden_fixture = test_fixture.parse( @embedFile("testdata/grid-v1.hex"), ); +test "cell wire layout registry" { + const testing = std.testing; + + // The wire bit positions are format constants. The shifts and masks + // derive from the packed struct, so pin the struct itself to the + // documented registry so an edit cannot silently move wire bits. + try testing.expectEqual(0, @bitOffsetOf(Cell, "kind")); + try testing.expectEqual(2, @bitOffsetOf(Cell, "content")); + try testing.expectEqual(26, @bitOffsetOf(Cell, "style_id")); + try testing.expectEqual(42, @bitOffsetOf(Cell, "width")); + try testing.expectEqual(44, @bitOffsetOf(Cell, "protected")); + try testing.expectEqual(45, @bitOffsetOf(Cell, "hyperlink")); + try testing.expectEqual(46, @bitOffsetOf(Cell, "semantic_content")); + try testing.expectEqual(48, @bitOffsetOf(Cell, "hyperlink_id")); + + // The native cell currently matches the wire registry, which enables + // the bulk row codec. If this fails, the native layout diverged: either + // restore it or accept the portable codec and update this expectation. + try testing.expect(native_matches_wire); +} + +test "encoded cell width transport registry" { + const testing = std.testing; + + // Pin the transported value and admission mask of every width against + // the documented format: widths one and two carry the bare codepoint, + // width four the low word half, width eight the complete word. + const word: u64 = @bitCast(Cell{ + .kind = 1, + .content = 0xABCDEF, + .style_id = 0x1234, + .hyperlink_id = 0x5678, + }); + try testing.expectEqual(@as(u8, 0xEF), Cell.EncodedWidth.one.truncate(word)); + try testing.expectEqual(@as(u16, 0xCDEF), Cell.EncodedWidth.two.truncate(word)); + try testing.expectEqual( + @as(u32, @truncate(word)), + Cell.EncodedWidth.four.truncate(word), + ); + try testing.expectEqual(word, Cell.EncodedWidth.eight.truncate(word)); + + try testing.expectEqual( + @as(u64, 0x0000_0000_0000_03FC), + Cell.EncodedWidth.one.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_0003_FFFC), + Cell.EncodedWidth.two.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_FFFF_FFFF), + Cell.EncodedWidth.four.mask(), + ); + try testing.expectEqual( + @as(u64, 0xFFFF_FFFF_FFFF_FFFF), + Cell.EncodedWidth.eight.mask(), + ); + + // Selection returns the smallest admissible width, and every admitted + // word round-trips through its transport. + const cases = [_]struct { cell: Cell, width: Cell.EncodedWidth }{ + .{ .cell = .{}, .width = .one }, + .{ .cell = .{ .content = 0xFF }, .width = .one }, + .{ .cell = .{ .content = 0x100 }, .width = .two }, + .{ .cell = .{ .content = 0xFFFF }, .width = .two }, + .{ .cell = .{ .content = 0x10000 }, .width = .four }, + .{ .cell = .{ .kind = 2, .content = 7 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 63 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 64 }, .width = .eight }, + .{ .cell = .{ .width = 1, .content = 'a' }, .width = .eight }, + .{ .cell = .{ .protected = true }, .width = .eight }, + .{ .cell = .{ .semantic_content = 1 }, .width = .eight }, + .{ .cell = .{ .hyperlink = true, .hyperlink_id = 1 }, .width = .eight }, + }; + inline for (cases) |case| { + const case_word: u64 = @bitCast(case.cell); + try testing.expectEqual( + case.width, + Cell.EncodedWidth.select(case_word), + ); + try testing.expectEqual( + case_word, + case.width.extend(case.width.truncate(case_word)), + ); + } +} + test "grid golden encoding and decoding" { const capacity: terminal_page.Capacity = .{ .cols = 3, - .rows = 2, + .rows = 4, .styles = 0, .hyperlink_bytes = 0, .grapheme_bytes = 64, @@ -643,7 +1364,16 @@ test "grid golden encoding and decoding" { head.row.wrap_continuation = true; head.row.semantic_prompt = .prompt_continuation; - var encoded: [128]u8 = undefined; + // The third row is bare ASCII text with an interior blank, which uses + // the one-byte encoded cell width. + source.getRowAndCell(0, 2).cell.* = .init('h'); + source.getRowAndCell(2, 2).cell.* = .init('i'); + + // The fourth row needs the two-byte width for a codepoint above U+00FF. + source.getRowAndCell(0, 3).cell.* = .init(0x0416); // Ж + source.getRowAndCell(1, 3).cell.* = .init('!'); + + var encoded: [160]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try encode(&source, &writer); try test_fixture.expectEqual( @@ -656,10 +1386,10 @@ test "grid golden encoding and decoding" { var destination = try TerminalPage.init(capacity); defer destination.deinit(); - var style_remap = StyleRemap.init(std.testing.allocator); - defer style_remap.deinit(); - var hyperlink_remap = HyperlinkRemap.init(std.testing.allocator); - defer hyperlink_remap.deinit(); + var style_remap = try StyleRemap.init(std.testing.allocator); + defer style_remap.deinit(std.testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(std.testing.allocator); + defer hyperlink_remap.deinit(std.testing.allocator); // Decode the checked-in reference through a one-byte reader buffer. var fixture_reader: std.Io.Reader = .fixed(&test_golden_fixture); @@ -736,8 +1466,26 @@ test "grid golden encoding and decoding" { decoded_head.row.semantic_prompt, ); + try std.testing.expectEqual( + @as(u21, 'h'), + destination.getRowAndCell(0, 2).cell.codepoint(), + ); + try std.testing.expect(destination.getRowAndCell(1, 2).cell.isZero()); + try std.testing.expectEqual( + @as(u21, 'i'), + destination.getRowAndCell(2, 2).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, 0x0416), + destination.getRowAndCell(0, 3).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, '!'), + destination.getRowAndCell(1, 3).cell.codepoint(), + ); + // A re-encode proves the decoded native page retains every wire field. - var reencoded: [128]u8 = undefined; + var reencoded: [160]u8 = undefined; var rewriter: std.Io.Writer = .fixed(&reencoded); try encode(&destination, &rewriter); try std.testing.expectEqualStrings( @@ -746,10 +1494,81 @@ test "grid golden encoding and decoding" { ); } +test "grid elides trailing default cells" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 80, .rows = 3 }); + defer page.deinit(); + + // Row 0 is fully default. Row 1 has content in columns zero and two. + // Row 2 has one protected-only cell at column four. + const middle = page.getRowAndCell(2, 1); + middle.cell.* = .init('b'); + page.getRowAndCell(0, 1).cell.* = .init('a'); + page.getRowAndCell(4, 2).cell.protected = true; + + var encoded: [256]u8 = undefined; + var writer: std.Io.Writer = .fixed(&encoded); + try encode(&page, &writer); + + // 3 bytes per row header and cells only through the last non-default + // cell, followed by an empty grapheme section. The text row uses the + // one-byte width while the protected flag forces the full width. + try testing.expectEqual( + @as(usize, 3 + (3 + 3 * 1) + (3 + 5 * 8) + 4), + writer.buffered().len, + ); + + var destination = try TerminalPage.init(.{ .cols = 80, .rows = 3 }); + defer destination.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&destination, &reader, &style_remap, &hyperlink_remap); + try destination.verifyIntegrity(testing.allocator); + + try testing.expectEqual( + @as(u21, 'a'), + destination.getRowAndCell(0, 1).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 'b'), + destination.getRowAndCell(2, 1).cell.codepoint(), + ); + try testing.expect(destination.getRowAndCell(4, 2).cell.protected); + try testing.expect(destination.getRowAndCell(79, 1).cell.isZero()); +} + +test "grid rejects a row cell count above the column count" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{})); + try io.writeInt(&writer, u16, 3); + for (0..3) |_| try io.writeInt(&writer, u64, 0); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try testing.expectError( + error.InvalidRowCellCount, + decode(&page, &reader, &style_remap, &hyperlink_remap), + ); +} + test "grid normalizes incomplete wide cells" { const testing = std.testing; // One column puts the wide cell at the row end. Two columns put an // ordinary narrow cell after it. Neither case supplies a spacer tail. + // Both rely on the implicit narrow cell rule when the tail is elided. for ([_]u16{ 1, 2 }) |columns| { const capacity: terminal_page.Capacity = .{ .cols = columns, @@ -760,21 +1579,23 @@ test "grid normalizes incomplete wide cells" { // content but makes the incomplete wide cell narrow. var payload: [64]u8 = undefined; var payload_writer: std.Io.Writer = .fixed(&payload); - try payload_writer.writeByte(@bitCast(RowHeader{})); - try (CellHeader{ - .width = .wide, - .value = .{ .codepoint = 'A' }, - }).encode(&payload_writer); - if (capacity.cols > 1) try (CellHeader{ - .value = .{ .codepoint = 'B' }, - }).encode(&payload_writer); + try payload_writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&payload_writer, u16, columns); + try io.writeInt(&payload_writer, u64, @bitCast(Cell{ + .width = 1, // wide + .content = 'A', + })); + if (columns > 1) try io.writeInt(&payload_writer, u64, @bitCast(Cell{ + .content = 'B', + })); + try io.writeInt(&payload_writer, u32, 0); var destination = try TerminalPage.init(capacity); defer destination.deinit(); - var style_remap = StyleRemap.init(testing.allocator); - defer style_remap.deinit(); - var hyperlink_remap = HyperlinkRemap.init(testing.allocator); - defer hyperlink_remap.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); var payload_reader: std.Io.Reader = .fixed( payload_writer.buffered(), @@ -801,4 +1622,391 @@ test "grid normalizes incomplete wide cells" { try testing.expectEqual(TerminalCell.Wide.narrow, second.wide); } } + + // A wide marker whose spacer tail was elided by a short cell count is + // also normalized, and the trailing cells stay default. + var page = try TerminalPage.init(.{ .cols = 4, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u64, @bitCast(Cell{ + .width = 1, // wide + .content = 'W', + })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + TerminalCell.Wide.narrow, + page.getRowAndCell(0, 0).cell.wide, + ); + try testing.expect(page.getRowAndCell(1, 0).cell.isZero()); +} + +test "grid normalizes reserved cell values" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 3, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + + // Reserved row flag bits six and seven are ignored while the unknown + // semantic-prompt value degrades to none and wrap survives. Bits four + // and five select the full encoded cell width. + try writer.writeByte(0xFD); + try io.writeInt(&writer, u16, 3); + + // A surrogate codepoint with reserved semantic content 3. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .content = 0xD800, + .semantic_content = 3, + })); + + // Reserved palette content bits do not obscure the palette index. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .kind = 2, + .content = 0xFFFF07, + })); + + // An unknown style reference degrades to the default style, and an + // unknown hyperlink reference leaves the cell unlinked. + try io.writeInt(&writer, u64, @bitCast(Cell{ + .content = 'x', + .style_id = 5, + .hyperlink = true, + .hyperlink_id = 9, + })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expect(first.row.wrap); + try testing.expectEqual( + TerminalRow.SemanticPrompt.none, + first.row.semantic_prompt, + ); + try testing.expectEqual(@as(u21, 0xFFFD), first.cell.codepoint()); + try testing.expectEqual( + TerminalCell.SemanticContent.output, + first.cell.semantic_content, + ); + + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual( + TerminalCell.ContentTag.bg_color_palette, + second.content_tag, + ); + try testing.expectEqual(@as(u8, 7), second.content.color_palette.data); + + const third = page.getRowAndCell(2, 0).cell; + try testing.expectEqual(@as(u21, 'x'), third.codepoint()); + try testing.expectEqual(@as(TerminalStyleId, 0), third.style_id); + try testing.expect(!third.hyperlink); + try testing.expectEqual(null, page.lookupHyperlink(third)); +} + +test "grid drops undeliverable grapheme entries" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 4, + .rows = 1, + .grapheme_bytes = 64, + }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [128]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&writer, u16, 4); + // A kind 1 cell that receives a valid entry. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'x' })); + // A kind 1 cell without an entry decodes as a plain codepoint. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'y' })); + // A background cell cannot carry a suffix. + try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 2, .content = 7 })); + // An empty codepoint cannot carry a suffix. + try io.writeInt(&writer, u64, @bitCast(Cell{})); + + try io.writeInt(&writer, u32, 5); + // Valid entry with one invalid scalar dropped from within it. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 3); + try io.writeInt(&writer, u32, 0x0301); + try io.writeInt(&writer, u32, 0xD800); + try io.writeInt(&writer, u32, 0x0302); + // Duplicate entry for the same cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0303); + // Entry for the background cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + // Entry for the empty cell is consumed and dropped. + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 3); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + // Entry outside the grid is consumed and dropped. + try io.writeInt(&writer, u16, 7); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expectEqualSlices( + u21, + &.{ 0x0301, 0x0302 }, + page.lookupGrapheme(first.cell).?, + ); + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual(@as(u21, 'y'), second.codepoint()); + try testing.expect(!second.hasGrapheme()); + try testing.expect(!page.getRowAndCell(2, 0).cell.hasGrapheme()); + try testing.expect(!page.getRowAndCell(3, 0).cell.hasGrapheme()); +} + +test "grid encodes rows at their narrowest width" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer page.deinit(); + + // Width zero: bare Latin-1 text. + page.getRowAndCell(0, 0).cell.* = .init('A'); + page.getRowAndCell(1, 0).cell.* = .init(0xFF); + + // Width one: any BMP codepoint. + page.getRowAndCell(0, 1).cell.* = .init(0x0100); + + // Width two: a small style ID and a background color. + const style_id = try page.styles.add(page.memory, .{ + .flags = .{ .bold = true }, + }); + try testing.expect(style_id <= 63); + const styled = page.getRowAndCell(0, 2); + styled.cell.* = .init('s'); + styled.cell.style_id = style_id; + styled.row.styled = true; + const bg = page.getRowAndCell(1, 2); + bg.cell.content_tag = .bg_color_palette; + bg.cell.content = .{ .color_palette = .{ .data = 7 } }; + + // Width three: a protected cell. + page.getRowAndCell(0, 3).cell.protected = true; + + var encoded: [128]u8 = undefined; + var writer: std.Io.Writer = .fixed(&encoded); + try encode(&page, &writer); + + const bytes = writer.buffered(); + try testing.expectEqual( + @as(usize, (3 + 2 * 1) + (3 + 1 * 2) + (3 + 2 * 4) + (3 + 1 * 8) + 4), + bytes.len, + ); + + // Each row header carries the expected width bits. + try testing.expectEqual(@as(u8, 0 << 4), bytes[0] & 0x30); + try testing.expectEqual(@as(u8, 1 << 4), bytes[5] & 0x30); + try testing.expectEqual(@as(u8, 2 << 4), bytes[10] & 0x30); + try testing.expectEqual(@as(u8, 3 << 4), bytes[21] & 0x30); + + var destination = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer destination.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + const native_style = try destination.styles.add(destination.memory, .{ + .flags = .{ .bold = true }, + }); + style_remap.put(style_id, native_style); + + var reader: std.Io.Reader = .fixed(bytes); + try decode(&destination, &reader, &style_remap, &hyperlink_remap); + try destination.verifyIntegrity(testing.allocator); + + try testing.expectEqual( + @as(u21, 'A'), + destination.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0xFF), + destination.getRowAndCell(1, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0100), + destination.getRowAndCell(0, 1).cell.codepoint(), + ); + const decoded_styled = destination.getRowAndCell(0, 2); + try testing.expectEqual(@as(u21, 's'), decoded_styled.cell.codepoint()); + try testing.expectEqual(native_style, decoded_styled.cell.style_id); + try testing.expect(decoded_styled.row.styled); + try testing.expectEqual( + @as(u8, 7), + destination.getRowAndCell(1, 2).cell.content.color_palette.data, + ); + try testing.expect(destination.getRowAndCell(0, 3).cell.protected); +} + +test "grid decodes non-canonical cell widths" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + // A bare ASCII row encoded at the full width is wasteful but valid. + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'o' })); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'k' })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 'o'), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 'k'), + page.getRowAndCell(1, 0).cell.codepoint(), + ); + + // Re-encoding canonicalizes the row back to the one-byte width. + var reencoded: [16]u8 = undefined; + var rewriter: std.Io.Writer = .fixed(&reencoded); + try encode(&page, &rewriter); + try testing.expectEqual(@as(usize, 3 + 2 + 4), rewriter.buffered().len); +} + +test "grid normalizes surrogates in two-byte cells" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [16]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .two })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u16, 0xD800); + try io.writeInt(&writer, u16, 0x0416); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 0xFFFD), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0416), + page.getRowAndCell(1, 0).cell.codepoint(), + ); +} + +test "grid four-byte cells run full normalization" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 3, + .rows = 1, + .styles = 8, + }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [32]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .four })); + try io.writeInt(&writer, u16, 3); + + // The Kitty placeholder does not fit two-byte cells but fits here + // and must still derive the native row hint. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = kitty.graphics.unicode.placeholder, + })))); + + // An unknown small style reference degrades to the default style. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = 'x', + .style_id = 63, + })))); + + // Reserved palette content bits are cleared at this width too. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .kind = 2, + .content = 0xFFFF07, + })))); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expectEqual( + @as(u21, kitty.graphics.unicode.placeholder), + first.cell.codepoint(), + ); + try testing.expect(first.row.kitty_virtual_placeholder); + + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual(@as(u21, 'x'), second.codepoint()); + try testing.expectEqual(@as(TerminalStyleId, 0), second.style_id); + + const third = page.getRowAndCell(2, 0).cell; + try testing.expectEqual( + TerminalCell.ContentTag.bg_color_palette, + third.content_tag, + ); + try testing.expectEqual(@as(u8, 7), third.content.color_palette.data); } diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index e71bc53c4..c09fd9dad 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -63,19 +63,19 @@ //! | hyperlink_count entries | //! +---------------------------+ //! | Grid | -//! | one record per row | +//! | rows and grapheme section | //! +---------------------------+ //! //! Style entry = encoded ID + style record //! Hyperlink entry = encoded ID + hyperlink record -//! Grid = row 0 ... row (rows - 1) -//! Row = row header + cell 0 ... cell (columns - 1) -//! Cell = cell header + grapheme suffix codepoints +//! Grid = row 0 ... row (rows - 1) + grapheme suffix section +//! Row = row header + encoded cells //! ``` //! //! Following the header, the payload contains exactly `style_count` style //! records and `hyperlink_count` hyperlink records, followed by exactly -//! `rows` row records. There is no padding between records. +//! `rows` row records and one grapheme suffix section. There is no padding +//! between records. //! //! Each style begins with an ID (`u16`) followed by the typical style //! binary representation (in style.zig). The ID is what cells will use @@ -93,6 +93,7 @@ //! Rows and cells use the grid encoding documented in `grid.zig`. const std = @import("std"); +const build_options = @import("terminal_options"); const Allocator = std.mem.Allocator; const test_fixture = @import("fixture.zig"); const grid = @import("grid.zig"); @@ -119,6 +120,7 @@ const PayloadEncodeError = hyperlink.EncodeError || grid.EncodeError; const PayloadDecodeError = std.Io.Reader.Error || Header.CapacityError || + grid.DecodeError || error{ /// The hyperlink kind is not defined by snapshot version 1. InvalidKind, @@ -202,10 +204,17 @@ pub const Decoder = struct { return self.header.pageCapacity() catch unreachable; } + /// Largest remaining PAGE payload staged into one contiguous buffer. + /// This comfortably covers every payload a standard-capacity page can + /// produce while keeping the allocation an untrusted length can force + /// far below the record framing's four-byte length limit. + const max_staged_payload = 8 * 1024 * 1024; + /// Decode the remaining payload into caller-owned native page storage. /// /// The destination must be freshly initialized with `capacity`. `alloc` - /// is used only for temporary ID remaps and integrity-check storage. + /// is used only for temporary staging, ID remaps, and integrity-check + /// storage. pub fn decode( self: *Decoder, destination: *TerminalPage, @@ -219,14 +228,52 @@ pub const Decoder = struct { .cols = self.header.columns, .rows = self.header.rows, }; - try decodePayloadBody( - self.record_reader.payloadReader(), - alloc, - destination, - self.header, - ); + + // `init` consumed exactly the fixed header from the declared + // payload, so this is the byte count of the tables and grid. + const remaining = self.record_reader.header.payload_len - Header.len; + + if (remaining <= max_staged_payload) { + // Stage the payload with one bulk read. The whole payload + // passes through both hashes as one update and the payload + // decoders then parse a flat buffer, which keeps per-row work + // free of stream adapters. The CRC and exact-length checks in + // `finish` are unaffected. + const staged = try alloc.alloc(u8, remaining); + defer alloc.free(staged); + try self.record_reader.payloadReader().readSliceAll(staged); + + var staged_reader: std.Io.Reader = .fixed(staged); + try decodePayloadBody( + &staged_reader, + alloc, + destination, + self.header, + ); + if (staged_reader.bufferedLen() != 0) { + return error.PayloadNotExhausted; + } + } else { + // A payload this large is either hostile or a page far beyond + // native capacities. Decode it through the streaming payload + // reader so its declared length cannot force an allocation. + try decodePayloadBody( + self.record_reader.payloadReader(), + alloc, + destination, + self.header, + ); + } try self.record_reader.finish(); - try destination.verifyIntegrity(alloc); + + // The decoder normalizes every semantic value, so a complete decode + // upholds native page invariants by construction. Verifying them + // again is a defense against decoder bugs and follows the native + // page policy: full integrity verification only when slow runtime + // safety is enabled. + if (comptime build_options.slow_runtime_safety) { + try destination.verifyIntegrity(alloc); + } } }; @@ -284,15 +331,13 @@ fn decodePayloadBody( page.pauseIntegrityChecks(true); defer page.pauseIntegrityChecks(false); - var style_remap = grid.StyleRemap.init(alloc); - defer style_remap.deinit(); - style_remap.ensureTotalCapacity(header.style_count) catch + var style_remap = grid.StyleRemap.init(alloc) catch return error.OutOfMemory; + defer style_remap.deinit(alloc); - var hyperlink_remap = grid.HyperlinkRemap.init(alloc); - defer hyperlink_remap.deinit(); - hyperlink_remap.ensureTotalCapacity(header.hyperlink_count) catch + var hyperlink_remap = grid.HyperlinkRemap.init(alloc) catch return error.OutOfMemory; + defer hyperlink_remap.deinit(alloc); // Styles for (0..header.style_count) |_| { @@ -316,10 +361,7 @@ fn decodePayloadBody( valid, ) catch 0; } else 0; - style_remap.putAssumeCapacityNoClobber( - native_id, - decoded_id, - ); + style_remap.put(native_id, decoded_id); } // Hyperlinks @@ -338,10 +380,7 @@ fn decodePayloadBody( // Zero records an ignored table entry. Grid decoding treats every // reference to it as no hyperlink. - hyperlink_remap.putAssumeCapacityNoClobber( - native_id, - decoded_id, - ); + hyperlink_remap.put(native_id, decoded_id); } // Rows and cells @@ -860,8 +899,7 @@ test "decode accepts unordered sparse style IDs and ignores zero" { var descending: [ Header.len + 2 * (2 + style.len) + - 1 + - 16 + 7 ]u8 = undefined; var descending_writer: std.Io.Writer = .fixed(&descending); try header.encode(&descending_writer); @@ -869,8 +907,9 @@ test "decode accepts unordered sparse style IDs and ignores zero" { try style.encode(.{ .flags = .{ .bold = true } }, &descending_writer); try io.writeInt(&descending_writer, TerminalStyleId, 2); try style.encode(.{ .flags = .{ .italic = true } }, &descending_writer); - try descending_writer.writeByte(0); - try descending_writer.splatByteAll(0, 16); + try descending_writer.writeByte(0); // row flags + try io.writeInt(&descending_writer, u16, 0); // cell count + try io.writeInt(&descending_writer, u32, 0); // grapheme section var descending_reader: std.Io.Reader = .fixed( descending_writer.buffered(), @@ -893,7 +932,7 @@ test "decode accepts unordered sparse style IDs and ignores zero" { .string_capacity_bytes = 0, }; - var zero: [Header.len + 2 + style.len + 17]u8 = undefined; + var zero: [Header.len + 2 + style.len + 7]u8 = undefined; var zero_writer: std.Io.Writer = .fixed(&zero); try one_header.encode(&zero_writer); try io.writeInt(&zero_writer, TerminalStyleId, 0); @@ -936,8 +975,7 @@ test "decode accepts unordered sparse hyperlink IDs" { var encoded: [ Header.len + 2 * 14 + - 1 + - 16 + 7 ]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try header.encode(&writer); @@ -945,8 +983,9 @@ test "decode accepts unordered sparse hyperlink IDs" { try hyperlink.encode(first, &writer); try io.writeInt(&writer, TerminalHyperlinkId, 2); try hyperlink.encode(second, &writer); - try writer.writeByte(0); - try writer.splatByteAll(0, 16); + try writer.writeByte(0); // row flags + try io.writeInt(&writer, u16, 0); // cell count + try io.writeInt(&writer, u32, 0); // grapheme section var reader: std.Io.Reader = .fixed(writer.buffered()); var decoded = try decodePayload( @@ -973,15 +1012,15 @@ test "decode defaults missing sparse cell references" { .string_capacity_bytes = 0, }; - var style_encoded: [Header.len + 1 + 16]u8 = undefined; + var style_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var style_writer: std.Io.Writer = .fixed(&style_encoded); try style_header.encode(&style_writer); - try style_writer.writeByte(0); - try style_writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&style_writer, TerminalStyleId, 1); - try io.writeInt(&style_writer, TerminalHyperlinkId, 0); - try io.writeInt(&style_writer, u32, 0); - try io.writeInt(&style_writer, u32, 0); + try style_writer.writeByte(0x30); // row flags, full cell width + try io.writeInt(&style_writer, u16, 1); // cell count + try io.writeInt(&style_writer, u64, @bitCast(grid.Cell{ + .style_id = 1, + })); + try io.writeInt(&style_writer, u32, 0); // grapheme section var style_reader: std.Io.Reader = .fixed(style_writer.buffered()); var style_page = try decodePayload( @@ -1006,15 +1045,16 @@ test "decode defaults missing sparse cell references" { .string_capacity_bytes = 0, }; - var hyperlink_encoded: [Header.len + 1 + 16]u8 = undefined; + var hyperlink_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var hyperlink_writer: std.Io.Writer = .fixed(&hyperlink_encoded); try hyperlink_header.encode(&hyperlink_writer); - try hyperlink_writer.writeByte(0); - try hyperlink_writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&hyperlink_writer, TerminalStyleId, 0); - try io.writeInt(&hyperlink_writer, TerminalHyperlinkId, 1); - try io.writeInt(&hyperlink_writer, u32, 0); - try io.writeInt(&hyperlink_writer, u32, 0); + try hyperlink_writer.writeByte(0x30); // row flags, full cell width + try io.writeInt(&hyperlink_writer, u16, 1); // cell count + try io.writeInt(&hyperlink_writer, u64, @bitCast(grid.Cell{ + .hyperlink = true, + .hyperlink_id = 1, + })); + try io.writeInt(&hyperlink_writer, u32, 0); // grapheme section var hyperlink_reader: std.Io.Reader = .fixed( hyperlink_writer.buffered(), @@ -1044,41 +1084,42 @@ test "decode normalizes invalid grid semantics" { .string_capacity_bytes = 0, }; - var encoded: [Header.len + 1 + 3 * 16 + 12]u8 = undefined; + var encoded: [Header.len + 3 + 3 * 8 + 4 + 10]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try header.encode(&writer); // Preserve wrap while degrading the unknown semantic-prompt value to none // and ignoring all reserved row bits. try writer.writeByte(0xFD); + try io.writeInt(&writer, u16, 3); - // An unknown content and width kind, invalid semantic content, reserved - // bytes, and suffix data all degrade to a blank narrow output cell. - try writer.writeAll(&.{ 3, 4, 0xFF, 0xFF }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xD800); - try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 0x110000); + // A text cell replaces an invalid Unicode scalar with U+FFFD while its + // reserved semantic content degrades to output. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .content = 0xD800, + .semantic_content = 3, + })); - // A recognized text cell replaces an invalid Unicode scalar with U+FFFD. - // Its valid suffix cannot fit the advertised zero capacity, so decoding - // preserves the base character and drops the optional suffix. - try writer.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xD800); - try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 'x'); + // A declared grapheme suffix cannot fit the advertised zero capacity, + // so decoding preserves the base character and drops the suffix. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .kind = 1, + .content = 'x', + })); - // Reserved palette bits and a nonsensical suffix do not obscure the valid - // low palette byte; the suffix is consumed and ignored. - try writer.writeAll(&.{ 1, 0, 0, 0xFF }); - try io.writeInt(&writer, TerminalStyleId, 0); - try io.writeInt(&writer, TerminalHyperlinkId, 0); - try io.writeInt(&writer, u32, 0xFFFFFF07); + // Reserved palette content bits do not obscure the valid low palette + // byte. + try io.writeInt(&writer, u64, @bitCast(grid.Cell{ + .kind = 2, + .content = 0xFFFF07, + })); + + // The grapheme section carries the suffix for the second cell. try io.writeInt(&writer, u32, 1); - try io.writeInt(&writer, u32, 'x'); + try io.writeInt(&writer, u16, 0); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u16, 1); + try io.writeInt(&writer, u32, 0x0301); var reader: std.Io.Reader = .fixed(writer.buffered()); var page = try decodePayload(&reader, std.testing.allocator); @@ -1091,7 +1132,7 @@ test "decode normalizes invalid grid semantics" { terminal_page.Row.SemanticPrompt.none, first.row.semantic_prompt, ); - try std.testing.expectEqual(@as(u21, 0), first.cell.codepoint()); + try std.testing.expectEqual(@as(u21, 0xFFFD), first.cell.codepoint()); try std.testing.expectEqual( terminal_page.Cell.Wide.narrow, first.cell.wide, @@ -1104,7 +1145,7 @@ test "decode normalizes invalid grid semantics" { try std.testing.expect(!first.cell.hasGrapheme()); const second = page.getRowAndCell(1, 0).cell; - try std.testing.expectEqual(@as(u21, 0xFFFD), second.codepoint()); + try std.testing.expectEqual(@as(u21, 'x'), second.codepoint()); try std.testing.expect(!second.hasGrapheme()); const third = page.getRowAndCell(2, 0).cell; diff --git a/src/terminal/snapshot/record.zig b/src/terminal/snapshot/record.zig index fd69d0b41..1327ca23d 100644 --- a/src/terminal/snapshot/record.zig +++ b/src/terminal/snapshot/record.zig @@ -29,9 +29,10 @@ const Blake3 = std.crypto.hash.Blake3; /// The running digest shared by snapshot stream codecs and checkpoints. pub const PrefixDigest = [Blake3.digest_length]u8; -/// CRC32C as specified by the snapshot format. Zig names this standard -/// parameter set after its iSCSI use. -pub const Crc32c = std.hash.crc.Crc32Iscsi; +/// CRC32C as specified by the snapshot format. This is the parameter set +/// Zig's standard library names after its iSCSI use, backed by dedicated +/// CRC32C instructions where the target has them. +pub const Crc32c = @import("../../crc32c.zig").Crc32c; /// Identifies the layout and meaning of a record payload. /// The current snapshot version rejects every value not listed here. @@ -301,8 +302,9 @@ pub const Reader = struct { // such as peek and discard. limited_buffer: [1]u8, - // PAGE decoding performs many small reads. 256 bytes batches several cells - // while CRC32C is calculated without making Reader large on the stack. + // Fixed-header decoding performs several small reads. 256 bytes batches + // them while CRC32C is calculated without making Reader large on the + // stack. Bulk payload reads bypass this buffer entirely. hashing_buffer: [256]u8, limited: std.Io.Reader.Limited, diff --git a/src/terminal/snapshot/screen.zig b/src/terminal/snapshot/screen.zig index 875d2bb0e..d05ff00ba 100644 --- a/src/terminal/snapshot/screen.zig +++ b/src/terminal/snapshot/screen.zig @@ -205,6 +205,7 @@ const Allocator = std.mem.Allocator; const hyperlink = @import("hyperlink.zig"); const test_fixture = @import("fixture.zig"); const io = @import("io.zig"); +const grid = @import("grid.zig"); const page = @import("page.zig"); const record = @import("record.zig"); const style = @import("style.zig"); @@ -2261,20 +2262,14 @@ test "SCREEN decode ignores a PAGE with an empty hyperlink URI" { // One narrow codepoint cell refers to the hyperlink table entry above. // Since that entry is ignored, the cell must restore without a hyperlink. - try page_payload.writeByte(0); - try page_payload.writeAll(&.{ 0, 0, 0, 0 }); - try io.writeInt( - page_payload, - terminal_style.Id, - 0, - ); - try io.writeInt( - page_payload, - terminal_hyperlink.Id, - 1, - ); - try io.writeInt(page_payload, u32, 'A'); - try io.writeInt(page_payload, u32, 0); + try page_payload.writeByte(0x30); // row flags, full cell width + try io.writeInt(page_payload, u16, 1); // cell count + try io.writeInt(page_payload, u64, @bitCast(grid.Cell{ + .content = 'A', + .hyperlink = true, + .hyperlink_id = 1, + })); + try io.writeInt(page_payload, u32, 0); // grapheme section try stream.finish(); var source: std.Io.Reader = .fixed(destination.written()); diff --git a/src/terminal/snapshot/snapshot.ksy b/src/terminal/snapshot/snapshot.ksy index 6a31fccd6..6ff925ef3 100644 --- a/src/terminal/snapshot/snapshot.ksy +++ b/src/terminal/snapshot/snapshot.ksy @@ -914,25 +914,41 @@ types: type: grid_row(columns) repeat: expr repeat-expr: num_rows + - id: num_grapheme_entries + type: u4 + - id: grapheme_entries + type: grapheme_entry(num_rows, columns) + repeat: expr + repeat-expr: num_grapheme_entries grid_row: params: - - id: num_cells + - id: columns type: u2 seq: - id: flags type: grid_row_flags + - id: cell_count + type: u2 + valid: + expr: _ <= columns - id: cells - type: grid_cell(_index, num_cells, flags.wrap) + type: + switch-on: flags.width_log2 + cases: + 0: grid_cell_1 + 1: grid_cell_2 + 2: grid_cell_4 + 3: grid_cell(_index, columns, flags.wrap) repeat: expr - repeat-expr: num_cells + repeat-expr: cell_count grid_row_flags: seq: - id: raw type: u1 valid: - expr: (_ & 0xf0) == 0 and ((_ >> 2) & 0x3) <= 2 + expr: (_ & 0xc0) == 0 and ((_ >> 2) & 0x3) <= 2 instances: wrap: value: (raw & 1) != 0 @@ -940,8 +956,59 @@ types: value: (raw & 2) != 0 semantic_prompt: value: (raw >> 2) & 0x3 + width_log2: + value: (raw >> 4) & 0x3 + + grid_cell_1: + doc: One-byte encoded cell; the value is a codepoint at or below U+00FF. + seq: + - id: codepoint + type: u1 + + grid_cell_2: + doc: | + Two-byte encoded cell; the value is a codepoint at or below U+FFFF. + Canonical encoders never emit surrogates. + seq: + - id: codepoint + type: u2 + valid: + expr: not (_ >= 0xd800 and _ <= 0xdfff) + + grid_cell_4: + doc: | + Four-byte encoded cell holding the low half of the cell word: any + content kind and codepoint, style IDs one through sixty-three, and no + width, flag, or hyperlink bits. + seq: + - id: raw + type: u4 + valid: + expr: | + (content_kind >= 2 or + (content <= 0x10ffff and + not (content >= 0xd800 and content <= 0xdfff))) and + (content_kind != 2 or content <= 0xff) + instances: + content_kind: + value: raw % 4 + content: + value: (raw / 4) % 16777216 + style_id: + value: raw / 67108864 grid_cell: + doc: | + One 64-bit little-endian cell word. The word is parsed as two 32-bit + halves so every derived field stays within JavaScript's safe integer + range, following the same approach as mode_set. + + Canonical rules validated here: reserved semantic content is not + emitted, the hyperlink flag matches a nonzero hyperlink ID, codepoint + content is a Unicode scalar value, palette content uses only its low + eight bits, and spacer relationships match the preceding cell. Wide + markers cannot be validated forward because their spacer tail may be + the next cell or the implicit narrow cell after a short row. params: - id: index type: u2 @@ -950,59 +1017,62 @@ types: - id: row_wrap type: bool seq: - - id: content_kind - type: u1 - valid: - max: 2 - - id: width - type: u1 - valid: - expr: | - _ <= 3 and - (_ != 2 or - (index > 0 and _parent.cells[index - 1].width == 1)) and - (_ != 3 or - (index + 1 == columns and row_wrap)) - - id: flags - type: grid_cell_flags - - id: reserved - type: u1 - valid: 0 - - id: style_id - type: u2 - - id: hyperlink_id - type: u2 - - id: value + - id: lo + type: u4 + - id: hi type: u4 valid: expr: | - content_kind == 0 ? - (_ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) and _ != 0x10eeee) : - content_kind == 1 ? - _ <= 0xff : - _ <= 0xffffff - - id: num_graphemes - type: u4 + semantic_content <= 2 and + hyperlink == (hyperlink_id != 0) and + (content_kind >= 2 or + (content <= 0x10ffff and + not (content >= 0xd800 and content <= 0xdfff))) and + (content_kind != 2 or content <= 0xff) and + (width != 2 or + (index > 0 and + _parent.cells[index - 1].as.width == 1)) and + (width != 3 or (index + 1 == columns and row_wrap)) + instances: + content_kind: + value: lo % 4 + content: + value: (lo / 4) % 16777216 + style_id: + value: (lo / 67108864) + (hi % 1024) * 64 + width: + value: (hi / 1024) % 4 + protected: + value: (hi / 4096) % 2 != 0 + hyperlink: + value: (hi / 8192) % 2 != 0 + semantic_content: + value: (hi / 16384) % 4 + hyperlink_id: + value: hi / 65536 + + grapheme_entry: + seq: + - id: row + type: u2 valid: - expr: | - content_kind == 0 ? - (_ == 0 or value != 0) : - _ == 0 - - id: graphemes + expr: _ < num_rows + - id: col + type: u2 + valid: + expr: _ < columns + - id: num_codepoints + type: u2 + valid: + min: 1 + - id: codepoints type: u4 repeat: expr - repeat-expr: num_graphemes + repeat-expr: num_codepoints valid: - expr: _ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) and _ != 0x10eeee - - grid_cell_flags: - seq: - - id: raw - type: u1 - valid: - expr: (_ & 0xf8) == 0 and ((_ >> 1) & 0x3) <= 2 - instances: - protected: - value: (raw & (1 << 0)) != 0 - semantic_content: - value: (raw >> 1) & 0x3 + expr: _ <= 0x10ffff and not (_ >= 0xd800 and _ <= 0xdfff) + params: + - id: num_rows + type: u2 + - id: columns + type: u2 diff --git a/src/terminal/snapshot/testdata/complete-v1.hex b/src/terminal/snapshot/testdata/complete-v1.hex index 83e3e174c..8e3df8287 100644 --- a/src/terminal/snapshot/testdata/complete-v1.hex +++ b/src/terminal/snapshot/testdata/complete-v1.hex @@ -78,65 +78,47 @@ ee ee 80 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000037a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003ec 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003fc -# offset 0x0000040c: page record, payload 119 bytes -03 00 77 00 00 00 48 b7 91 cb 02 00 03 00 00 00 # 0x0000040c -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x0000041c -00 00 00 00 00 00 00 43 00 00 00 00 00 00 00 00 # 0x0000042c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000043c -00 00 00 00 00 00 00 00 44 00 00 00 00 00 00 00 # 0x0000044c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000045c -00 00 00 00 00 00 00 00 00 45 00 00 00 00 00 00 # 0x0000046c -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000047c -00 # 0x0000048c +# offset 0x0000040c: page record, payload 36 bytes +03 00 24 00 00 00 a2 4f 26 d1 02 00 03 00 00 00 # 0x0000040c +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x0000041c +00 43 00 01 00 44 00 01 00 45 00 00 00 00 # 0x0000042c -# offset 0x0000048d: screen record, payload 54 bytes -02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000048d -00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000049d -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000004ad -00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000004bd +# offset 0x0000043a: screen record, payload 54 bytes +02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000043a +00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000044a +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000045a +00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046a -# offset 0x000004cd: page record, payload 119 bytes -03 00 77 00 00 00 84 a6 e9 08 02 00 03 00 00 00 # 0x000004cd -00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 00 # 0x000004dd -00 00 00 00 00 00 00 72 00 00 00 00 00 00 00 00 # 0x000004ed -00 00 00 00 00 00 00 6e 00 00 00 00 00 00 00 03 # 0x000004fd -00 00 00 00 00 00 00 00 61 00 00 00 00 00 00 00 # 0x0000050d -00 00 00 00 00 00 00 00 74 00 00 00 00 00 00 00 # 0x0000051d -03 00 00 00 00 00 00 00 00 65 00 00 00 00 00 00 # 0x0000052d -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000053d -00 # 0x0000054d +# offset 0x0000047a: page record, payload 38 bytes +03 00 26 00 00 00 45 61 97 32 02 00 03 00 00 00 # 0x0000047a +00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000048a +00 72 6e 03 02 00 61 74 03 01 00 65 00 00 00 00 # 0x0000049a -# offset 0x0000054e: continuation record, payload 0 bytes -07 00 00 00 00 00 27 80 63 d1 # 0x0000054e +# offset 0x000004aa: continuation record, payload 0 bytes +07 00 00 00 00 00 27 80 63 d1 # 0x000004aa -# offset 0x00000558: ready record, payload 32 bytes -05 00 20 00 00 00 d4 d1 fc d0 8d 69 76 c1 6e c6 # 0x00000558 -88 21 bd 10 97 41 97 35 63 2d ce 9e 3c b8 fa cf # 0x00000568 -57 04 f5 4b e0 70 66 5e 7e b6 # 0x00000578 +# offset 0x000004b4: ready record, payload 32 bytes +05 00 20 00 00 00 4d 17 72 ed dd 87 26 75 cf 8e # 0x000004b4 +e5 1f 37 e3 05 92 0a e2 f8 ef c1 16 54 be 49 e7 # 0x000004c4 +b2 6c df 40 d2 77 fe 05 82 ea # 0x000004d4 -# offset 0x00000582: history record, payload 6 bytes -04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x00000582 +# offset 0x000004de: history record, payload 6 bytes +04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x000004de -# offset 0x00000592: page record, payload 86 bytes -03 00 56 00 00 00 52 3b 6e 8d 02 00 02 00 00 00 # 0x00000592 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x000005a2 -00 00 00 00 00 00 00 42 00 00 00 00 00 00 00 00 # 0x000005b2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005c2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005d2 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000005e2 +# offset 0x000004ee: page record, payload 31 bytes +03 00 1f 00 00 00 4a ed 2f c2 02 00 02 00 00 00 # 0x000004ee +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x000004fe +00 42 00 00 00 00 00 00 00 # 0x0000050e -# offset 0x000005f2: page record, payload 86 bytes -03 00 56 00 00 00 fb bc 15 06 02 00 02 00 00 00 # 0x000005f2 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 00 # 0x00000602 -00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 00 # 0x00000612 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000622 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000632 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000642 +# offset 0x00000517: page record, payload 31 bytes +03 00 1f 00 00 00 23 6a 6b 19 02 00 02 00 00 00 # 0x00000517 +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000527 +00 41 00 00 00 00 00 00 00 # 0x00000537 -# offset 0x00000652: history record, payload 6 bytes -04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000652 +# offset 0x00000540: history record, payload 6 bytes +04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000540 -# offset 0x00000662: finish record, payload 32 bytes -06 00 20 00 00 00 f3 a4 cf b6 f2 2b 17 88 1a 28 # 0x00000662 -a7 27 6e f7 de bc 7a 52 f9 a4 f9 c3 32 5b e0 09 # 0x00000672 -8a 1a 7d a0 9f 8b 32 31 67 41 # 0x00000682 +# offset 0x00000550: finish record, payload 32 bytes +06 00 20 00 00 00 5a 2d a8 f4 25 b2 49 b4 3e 64 # 0x00000550 +5a 2e d6 7d 7f 3d 60 5b db 9b 4c 3a 29 00 97 ec # 0x00000560 +d6 c7 1d ea da d3 a9 fd 3d 68 # 0x00000570 diff --git a/src/terminal/snapshot/testdata/grid-v1.hex b/src/terminal/snapshot/testdata/grid-v1.hex index 3389a876f..b9a74af32 100644 --- a/src/terminal/snapshot/testdata/grid-v1.hex +++ b/src/terminal/snapshot/testdata/grid-v1.hex @@ -1,32 +1,15 @@ # Ghostty snapshot fixture # Kaitai type: grid -# Kaitai params: 2 3 +# Kaitai params: 4 3 # Kaitai offset: 0 # Wire version: 1 # Generated by its snapshot test; review before replacing. # On mismatch, the candidate is copied to the repository root. -# row 0: prompt -04 - -# row 0, cell 0: protected prompt, wide "A" -00 01 05 00 00 00 00 00 41 00 00 00 00 00 00 00 - -# row 0, cell 1: input spacer tail -00 02 02 00 00 00 00 00 00 00 00 00 00 00 00 00 - -# row 0, cell 2: "x" with two grapheme suffixes -00 00 00 00 00 00 00 00 78 00 00 00 02 00 00 00 -01 03 00 00 02 03 00 00 - -# row 1: wrapped prompt continuation -0b - -# row 1, cell 0: palette background 7 -01 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 - -# row 1, cell 1: protected RGB background -02 00 01 00 00 00 00 00 aa bb cc 00 00 00 00 00 - -# row 1, cell 2: spacer head -00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +# offset 0x00000000: encoded bytes +34 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 +48 00 00 e1 01 00 00 00 00 00 00 3b 03 00 1e 00 # 0x00000010 +00 00 00 00 00 00 ab ee 32 03 00 10 00 00 00 00 # 0x00000020 +00 00 00 0c 00 00 00 03 00 68 00 69 10 02 00 16 # 0x00000030 +04 21 00 01 00 00 00 00 00 02 00 02 00 01 03 00 # 0x00000040 +00 02 03 00 00 # 0x00000050 diff --git a/src/terminal/snapshot/testdata/page-empty-record-v1.hex b/src/terminal/snapshot/testdata/page-empty-record-v1.hex index 4b759b704..cda4f1d91 100644 --- a/src/terminal/snapshot/testdata/page-empty-record-v1.hex +++ b/src/terminal/snapshot/testdata/page-empty-record-v1.hex @@ -7,6 +7,6 @@ # On mismatch, the candidate is copied to the repository root. # offset 0x00000000: encoded bytes -03 00 25 00 00 00 8c 05 d6 d3 01 00 01 00 00 00 # 0x00000000 +03 00 1b 00 00 00 5e 0d 0a d4 01 00 01 00 00 00 # 0x00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000010 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000020 +00 00 00 00 00 # 0x00000020 diff --git a/src/terminal/snapshot/testdata/page-v1.hex b/src/terminal/snapshot/testdata/page-v1.hex index dfc53eeaa..78018ebb6 100644 --- a/src/terminal/snapshot/testdata/page-v1.hex +++ b/src/terminal/snapshot/testdata/page-v1.hex @@ -15,10 +15,8 @@ 00 00 03 00 00 00 00 00 01 2a 00 00 00 00 00 00 # 0x00000024 00 00 00 00 01 00 02 01 00 00 00 61 05 00 00 00 # 0x00000034 61 6c 70 68 61 03 00 01 04 03 02 01 04 00 00 00 # 0x00000044 -62 65 74 61 04 00 01 05 00 01 00 01 00 41 00 00 # 0x00000054 -00 00 00 00 00 00 02 02 00 03 00 03 00 00 00 00 # 0x00000064 -00 00 00 00 00 01 00 00 00 01 00 01 00 07 00 00 # 0x00000074 -00 00 00 00 00 0b 00 00 00 00 00 00 00 00 78 00 # 0x00000084 -00 00 02 00 00 00 01 03 00 00 02 03 00 00 02 00 # 0x00000094 -01 00 00 00 00 00 aa bb cc 00 00 00 00 00 00 03 # 0x000000a4 -00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000000b4 +62 65 74 61 34 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 +00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 3b # 0x00000064 +03 00 e1 01 00 00 00 00 00 00 ab ee 32 03 00 10 # 0x00000074 +00 00 00 00 00 00 00 0c 00 00 01 00 00 00 01 00 # 0x00000084 +00 00 02 00 01 03 00 00 02 03 00 00 # 0x00000094