diff --git a/src/terminal/snapshot/grid.zig b/src/terminal/snapshot/grid.zig index a3c5678b2..221d545f8 100644 --- a/src/terminal/snapshot/grid.zig +++ b/src/terminal/snapshot/grid.zig @@ -27,11 +27,11 @@ //! //! Each row has the following format: //! -//! | Offset | Size | Field | -//! | -----: | ----------: | :------------------------ | -//! | 0 | 1 | Row flags | -//! | 1 | 2 | Encoded cell count (`u16`)| -//! | 3 | 8 * `count` | Encoded cells | +//! | Offset | Size | Field | +//! | -----: | ----------------: | :------------------------- | +//! | 0 | 1 | Row flags | +//! | 1 | 2 | Encoded cell count (`u16`) | +//! | 3 | `width` * `count` | Encoded cells | //! //! The row flag byte has the following format: //! @@ -40,7 +40,8 @@ //! | 0 | Wrap | //! | 1 | Wrap continuation | //! | 2-3 | Semantic prompt | -//! | 4-7 | Reserved, zero | +//! | 4-5 | Encoded cell width | +//! | 6-7 | Reserved, zero | //! //! Semantic prompt values are: //! @@ -59,12 +60,40 @@ //! Canonical encoders emit exactly through the row's last non-default cell, //! so a fully default row has a zero count and no cell words. //! +//! ## Encoded cell width +//! +//! The two width bits select how many bytes encode each of the row's +//! cells: `1 << width` is the size, so zero through three select one, two, +//! four, or eight bytes. Narrower widths are truncated transports of the +//! same cell word: a cell qualifies for a width when all of its higher +//! word bits are zero. +//! +//! | Width | Bytes | Encoded value and admitted cells | +//! | ----: | ----: | :--------------------------------------------------- | +//! | 0 | 1 | Codepoint at or below U+00FF; all other bits zero | +//! | 1 | 2 | Codepoint at or below U+FFFF; all other bits zero | +//! | 2 | 4 | Word bits 0-31: any content kind and codepoint, style | +//! | | | IDs 1-63, narrow, no flags, no hyperlink | +//! | 3 | 8 | The complete word | +//! +//! Widths zero and one store the codepoint value itself, which is word +//! bits 2-25 shifted down; the reconstructed word is the codepoint shifted +//! left by two. Width two stores the word's low half unshifted. This makes +//! every width a zero-extension on decode and a truncation on encode. +//! +//! Canonical encoders choose each row's smallest admissible width, which +//! follows directly from the bitwise OR of the row's cell words. Decoders +//! use the declared width for framing and accept rows encoded wider than +//! necessary. The width of a row with a zero cell count is canonically +//! zero and carries no meaning. +//! //! Native row cache flags are not encoded. In particular, the Kitty virtual //! placeholder hint is derived while decoding cells containing U+10EEEE. //! //! ## Cell //! -//! Each cell is one 64-bit little-endian word: +//! Each cell is one 64-bit little-endian word, transported at the row's +//! encoded cell width as described above: //! //! ```text //! bit 0 +-------------------------------+ @@ -219,13 +248,14 @@ const TerminalStyleId = terminal_style.Id; /// /// The semantic prompt is a raw integer for the same reason as the wire /// cell fields: decoders must accept its reserved value without -/// instantiating an invalid native enum, so every header byte bit-casts to -/// a valid value. +/// instantiating an invalid native enum. The width enum is exhaustive over +/// its two bits, so every header byte bit-casts to a valid value. pub const Row = packed struct(u8) { wrap: bool = false, wrap_continuation: bool = false, semantic_prompt: u2 = 0, - _padding: u4 = 0, + cell_width: Cell.EncodedWidth = .one, + _padding: u2 = 0, }; /// The wire layout of one encoded cell. This is its own registry: the bit @@ -252,6 +282,72 @@ pub const Cell = packed struct(u64) { bg_color_palette = 2, bg_color_rgb = 3, }; + + /// The encoded cell width declared by a row header: how many bytes + /// transport each of the row's cell words. See the format + /// documentation above for the value each width transports and the + /// cells it admits. + /// + /// `truncate` and `extend` are the transport transform itself, so the + /// admission masks derive from them rather than being maintained by + /// hand: a cell word is admitted exactly when it round-trips. + pub const EncodedWidth = enum(u2) { + one = 0, + two = 1, + four = 2, + eight = 3, + + /// The number of bytes transporting one cell word. + pub fn size(self: EncodedWidth) usize { + return @as(usize, 1) << @intFromEnum(self); + } + + /// The integer type transporting one cell word. + pub fn Int(comptime self: EncodedWidth) type { + return switch (self) { + .one => u8, + .two => u16, + .four => u32, + .eight => u64, + }; + } + + /// Truncate one cell word to its transported value. Only words + /// admitted by this width round-trip; `select` proves that for + /// every cell in a row before an encoder may use it. + pub fn truncate(comptime self: EncodedWidth, word: u64) self.Int() { + return @truncate(switch (self) { + // The bare codepoint, shifted down from the content field. + .one, .two => @as(Cell, @bitCast(word)).content, + + // The word itself. + .four, .eight => word, + }); + } + + /// Widen one transported value back to its cell word. + pub fn extend(comptime self: EncodedWidth, value: self.Int()) u64 { + return switch (self) { + .one, .two => @bitCast(Cell{ .content = value }), + .four, .eight => value, + }; + } + + /// The word bits a cell may use and still round-trip through this + /// width. + pub fn mask(comptime self: EncodedWidth) u64 { + return comptime self.extend(std.math.maxInt(self.Int())); + } + + /// The smallest width admitting the word, typically the bitwise + /// OR of every cell word in a row. + pub fn select(word: u64) EncodedWidth { + inline for ([_]EncodedWidth{ .one, .two, .four }) |width| { + if (word & ~width.mask() == 0) return width; + } + return .eight; + } + }; }; /// Whether the native cell's in-memory layout matches the wire cell layout @@ -380,24 +476,12 @@ pub fn encode( break :count 0; }; - // Row header: flags then the encoded cell count. - { - const row_header: Row = .{ - .wrap = row.wrap, - .wrap_continuation = row.wrap_continuation, - .semantic_prompt = @intFromEnum(row.semantic_prompt), - }; - var header_bytes: [3]u8 = undefined; - header_bytes[0] = @bitCast(row_header); - std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); - try writer.writeAll(&header_bytes); - } - // Validate the wide state of every encoded cell so we don't encode - // corrupt data, and detect the cells that keep this row off the - // direct-copy path. Trailing default cells are narrow, so checking - // the encoded prefix against the full row width covers every pair. - var direct = true; + // corrupt data, and accumulate the OR of the row's cell words to + // select its encoded cell width. Trailing default cells are narrow, + // so checking the encoded prefix against the full row width covers + // every pair. + var word_or: u64 = 0; for (cells[0..count], 0..) |*cell, x| { switch (cell.wide) { .narrow => {}, @@ -415,31 +499,117 @@ pub fn encode( return error.InvalidWideCell; }, } - - // Hyperlink IDs live in a native side table, and nonzero native - // padding would leak into the wire hyperlink ID field. - if (cell.hyperlink or cell._padding != 0) direct = false; + word_or |= classifyWord(cell); } - if (comptime bulk_codec) { - if (direct) { - try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); - continue; - } + // Canonical rows use the smallest admissible width. + const cell_width: Cell.EncodedWidth = .select(word_or); + + // Row header: flags then the encoded cell count. + { + const row_header: Row = .{ + .wrap = row.wrap, + .wrap_continuation = row.wrap_continuation, + .semantic_prompt = @intFromEnum(row.semantic_prompt), + .cell_width = cell_width, + }; + var header_bytes: [3]u8 = undefined; + header_bytes[0] = @bitCast(row_header); + std.mem.writeInt(u16, header_bytes[1..3], @intCast(count), .little); + try writer.writeAll(&header_bytes); } - for (cells[0..count]) |*cell| { - const link_id: TerminalHyperlinkId = if (cell.hyperlink) - page.lookupHyperlink(cell) orelse unreachable - else - 0; - try io.writeInt(writer, u64, cellBits(cell.*, link_id)); + switch (cell_width) { + inline .one, .two, .four => |width| try encodeNarrowCells( + width, + cells[0..count], + writer, + ), + + .eight => { + // Hyperlink IDs live in a native side table, but we embed + // them in ours, so if we have any hyperlinks we need + // to fallback to the loop below. + if (comptime bulk_codec) { + const witness: Cell = @bitCast(word_or); + if (!witness.hyperlink and witness.hyperlink_id == 0) { + try writer.writeAll(std.mem.sliceAsBytes(cells[0..count])); + continue; + } + } + + for (cells[0..count]) |*cell| { + const link_id: TerminalHyperlinkId = if (cell.hyperlink) + page.lookupHyperlink(cell) orelse unreachable + else + 0; + try io.writeInt( + writer, + u64, + cellBits(cell.*, link_id), + ); + } + }, } } try encodeGraphemes(page, writer); } +/// The word used to select a row's encoded cell width. This is the cell's +/// wire word with the hyperlink flag reflecting the native cell, so linked +/// cells and nonzero native padding disqualify every narrow width. +inline fn classifyWord(cell: *const TerminalCell) u64 { + if (comptime native_matches_wire) return @bitCast(cell.*); + + var wire: Cell = @bitCast(cellBits(cell.*, 0)); + wire.hyperlink = cell.hyperlink; + return @bitCast(wire); +} + +/// Write one row's cells truncated to the given encoded width. +/// +/// Every admitted cell round-trips exactly because the row's width +/// selection proved every cell word survives `truncate` then `extend`. +fn encodeNarrowCells( + comptime width: Cell.EncodedWidth, + cells: []const TerminalCell, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + const size = comptime width.size(); + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + + if (comptime native_matches_wire) { + // A pure truncating loop over integers that the compiler can + // vectorize. + const words: [*]const u64 = @ptrCast(cells.ptr); + for (0..n) |j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(words[i + j]), + .little, + ); + } + } else { + for (cells[i..][0..n], 0..) |*cell, j| { + std.mem.writeInt( + width.Int(), + chunk[j * size ..][0..size], + width.truncate(classifyWord(cell)), + .little, + ); + } + } + + try writer.writeAll(chunk[0 .. n * size]); + i += n; + } +} + /// Encode the grapheme suffix section for every kind 1 cell in the grid. fn encodeGraphemes( page: *const TerminalPage, @@ -526,9 +696,9 @@ pub fn decode( try reader.readSliceAll(&row_header_bytes); } - // Every bit pattern is a valid header: booleans decode directly - // and the raw semantic value gets a default below. Reserved - // bits do not change the known fields. + // Every bit pattern is a valid header: booleans and the exhaustive + // width enum decode directly, and the raw semantic value gets a + // default below. Reserved bits do not change the known fields. const row_header: Row = @bitCast(row_header_bytes[0]); const count = std.mem.readInt(u16, row_header_bytes[1..3], .little); break :header .{ row_header, count }; @@ -546,50 +716,201 @@ pub fn decode( if (count > cells.len) return error.InvalidRowCellCount; if (count == 0) continue; - if (comptime bulk_codec) { - // Read the encoded words directly into page storage, then - // normalize them in place. The raw words are only ever touched - // as integers until normalization makes them valid cells. - const words: [*]u64 = @ptrCast(cells.ptr); - try reader.readSliceAll( - std.mem.sliceAsBytes(cells[0..count]), - ); - for (0..count) |x| { - applyCell( - page, - row, - cells, - x, - words[x], - style_remap, - hyperlink_remap, - ); - } - } else { - for (0..count) |x| { - const bits = try io.readInt(reader, u64); - applyCell( - page, - row, - cells, - x, - bits, - style_remap, - hyperlink_remap, - ); - } - } + // The encoded cell width is framing: it determines exactly how many + // bytes this row occupies. + switch (row_header.cell_width) { + inline .one, .two => |width| try decodeNarrowCells( + width, + reader, + cells[0..count], + ), + .four => try decodeWordCells( + .four, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ), + .eight => { + if (comptime bulk_codec) { + // Read the encoded words directly into page storage, + // then normalize them in place. The raw words are only + // ever touched as integers until normalization makes + // them valid cells. + const words: [*]u64 = @ptrCast(cells.ptr); + try reader.readSliceAll( + std.mem.sliceAsBytes(cells[0..count]), + ); + for (0..count) |x| { + applyCell( + page, + row, + cells, + x, + words[x], + style_remap, + hyperlink_remap, + ); + } + } else { + try decodeWordCells( + .eight, + page, + row, + cells, + count, + reader, + style_remap, + hyperlink_remap, + ); + } - // The implicit cell after a short row is narrow, which resolves a - // trailing wide marker exactly like an explicit narrow neighbor. - if (count < cells.len and cells[count - 1].wide == .wide) { - cells[count - 1].wide = .narrow; + // The implicit cell after a short row is narrow, which + // resolves a trailing wide marker exactly like an explicit + // narrow neighbor. Only full-width rows can encode wide + // markers. + if (count < cells.len and cells[count - 1].wide == .wide) { + cells[count - 1].wide = .narrow; + } + }, } } try decodeGraphemes(page, reader); } +/// Decode one row of width-one or width-two cells. +/// +/// These widths admit only bare codepoints, so cells store directly with at +/// most Unicode scalar validation: no styles, hyperlinks, wide pairs, or +/// row hints are reachable and no normalization state is needed. +fn decodeNarrowCells( + comptime width: Cell.EncodedWidth, + reader: *std.Io.Reader, + cells: []TerminalCell, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered, making this + // one bounds check followed by a vectorizable widening loop. + const total = cells.len * size; + if (reader.bufferedLen() >= total) { + widenCells(width, reader.buffered()[0..total], cells); + reader.toss(total); + return; + } + + // Streaming sources fall back to bounded chunks. + var chunk: [1024]u8 = undefined; + var i: usize = 0; + while (i < cells.len) { + const n = @min(cells.len - i, chunk.len / size); + try reader.readSliceAll(chunk[0 .. n * size]); + widenCells(width, chunk[0 .. n * size], cells[i..][0..n]); + i += n; + } +} + +/// Store codepoint-valued encoded cells of the given width. +fn widenCells( + comptime width: Cell.EncodedWidth, + bytes: []const u8, + cells: []TerminalCell, +) void { + const size = comptime width.size(); + + // When the native cell matches the wire word, this is a pure widening + // loop over integers that the compiler can vectorize. + if (comptime native_matches_wire) { + const words: [*]u64 = @ptrCast(cells.ptr); + for (0..cells.len) |i| { + words[i] = width.extend(widenValue(width, bytes[i * size ..])); + } + return; + } + + for (cells, 0..) |*cell, i| { + storeCell(cell, width.extend(widenValue(width, bytes[i * size ..]))); + } +} + +/// Read and validate one narrow transported value. +inline fn widenValue( + comptime width: Cell.EncodedWidth, + bytes: []const u8, +) width.Int() { + const size = comptime width.size(); + const value = std.mem.readInt(width.Int(), bytes[0..size], .little); + + // Width one cannot encode an invalid scalar. Width two admits + // surrogates, which degrade exactly like their full-width form. + if (comptime width != .one) { + if (!validScalar(value)) return 0xFFFD; + } + return value; +} + +/// Decode one row of width-four or fallback full-width cells through the +/// complete per-cell normalization path. +fn decodeWordCells( + comptime width: Cell.EncodedWidth, + page: *TerminalPage, + row: *TerminalRow, + cells: []TerminalCell, + count: usize, + reader: *std.Io.Reader, + style_remap: *const StyleRemap, + hyperlink_remap: *const HyperlinkRemap, +) DecodeError!void { + const size = comptime width.size(); + + // The staged payload path has the complete row buffered. + const total = count * size; + if (reader.bufferedLen() >= total) { + const bytes = reader.buffered()[0..total]; + for (0..count) |x| { + const bits = width.extend(std.mem.readInt( + width.Int(), + bytes[x * size ..][0..size], + .little, + )); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } + reader.toss(total); + return; + } + + // Streaming sources fall back to per-cell reads. + for (0..count) |x| { + const bits = width.extend(try io.readInt(reader, width.Int())); + applyCell( + page, + row, + cells, + x, + bits, + style_remap, + hyperlink_remap, + ); + } +} + +/// Whether the value is a valid Unicode scalar value. +inline fn validScalar(cp: u32) bool { + return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); +} + /// Normalize one encoded cell word and store it at `cells[x]`. /// /// This owns every per-cell decode rule except grapheme suffixes: content @@ -715,11 +1036,6 @@ fn normalizeWide(row: *const TerminalRow, cells: []TerminalCell, x: usize) void } } -/// Whether the value is a valid Unicode scalar value. -inline fn validScalar(cp: u32) bool { - return cp <= 0x10FFFF and (cp < 0xD800 or cp > 0xDFFF); -} - /// Decode the grapheme suffix section into already decoded cells. fn decodeGraphemes( page: *TerminalPage, @@ -903,7 +1219,6 @@ fn Remap(comptime Id: type) type { } }; } - const test_golden_fixture = test_fixture.parse( @embedFile("testdata/grid-v1.hex"), ); @@ -929,10 +1244,76 @@ test "cell wire layout registry" { try testing.expect(native_matches_wire); } +test "encoded cell width transport registry" { + const testing = std.testing; + + // Pin the transported value and admission mask of every width against + // the documented format: widths one and two carry the bare codepoint, + // width four the low word half, width eight the complete word. + const word: u64 = @bitCast(Cell{ + .kind = 1, + .content = 0xABCDEF, + .style_id = 0x1234, + .hyperlink_id = 0x5678, + }); + try testing.expectEqual(@as(u8, 0xEF), Cell.EncodedWidth.one.truncate(word)); + try testing.expectEqual(@as(u16, 0xCDEF), Cell.EncodedWidth.two.truncate(word)); + try testing.expectEqual( + @as(u32, @truncate(word)), + Cell.EncodedWidth.four.truncate(word), + ); + try testing.expectEqual(word, Cell.EncodedWidth.eight.truncate(word)); + + try testing.expectEqual( + @as(u64, 0x0000_0000_0000_03FC), + Cell.EncodedWidth.one.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_0003_FFFC), + Cell.EncodedWidth.two.mask(), + ); + try testing.expectEqual( + @as(u64, 0x0000_0000_FFFF_FFFF), + Cell.EncodedWidth.four.mask(), + ); + try testing.expectEqual( + @as(u64, 0xFFFF_FFFF_FFFF_FFFF), + Cell.EncodedWidth.eight.mask(), + ); + + // Selection returns the smallest admissible width, and every admitted + // word round-trips through its transport. + const cases = [_]struct { cell: Cell, width: Cell.EncodedWidth }{ + .{ .cell = .{}, .width = .one }, + .{ .cell = .{ .content = 0xFF }, .width = .one }, + .{ .cell = .{ .content = 0x100 }, .width = .two }, + .{ .cell = .{ .content = 0xFFFF }, .width = .two }, + .{ .cell = .{ .content = 0x10000 }, .width = .four }, + .{ .cell = .{ .kind = 2, .content = 7 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 63 }, .width = .four }, + .{ .cell = .{ .content = 'a', .style_id = 64 }, .width = .eight }, + .{ .cell = .{ .width = 1, .content = 'a' }, .width = .eight }, + .{ .cell = .{ .protected = true }, .width = .eight }, + .{ .cell = .{ .semantic_content = 1 }, .width = .eight }, + .{ .cell = .{ .hyperlink = true, .hyperlink_id = 1 }, .width = .eight }, + }; + inline for (cases) |case| { + const case_word: u64 = @bitCast(case.cell); + try testing.expectEqual( + case.width, + Cell.EncodedWidth.select(case_word), + ); + try testing.expectEqual( + case_word, + case.width.extend(case.width.truncate(case_word)), + ); + } +} + test "grid golden encoding and decoding" { const capacity: terminal_page.Capacity = .{ .cols = 3, - .rows = 2, + .rows = 4, .styles = 0, .hyperlink_bytes = 0, .grapheme_bytes = 64, @@ -983,7 +1364,16 @@ test "grid golden encoding and decoding" { head.row.wrap_continuation = true; head.row.semantic_prompt = .prompt_continuation; - var encoded: [128]u8 = undefined; + // The third row is bare ASCII text with an interior blank, which uses + // the one-byte encoded cell width. + source.getRowAndCell(0, 2).cell.* = .init('h'); + source.getRowAndCell(2, 2).cell.* = .init('i'); + + // The fourth row needs the two-byte width for a codepoint above U+00FF. + source.getRowAndCell(0, 3).cell.* = .init(0x0416); // Ж + source.getRowAndCell(1, 3).cell.* = .init('!'); + + var encoded: [160]u8 = undefined; var writer: std.Io.Writer = .fixed(&encoded); try encode(&source, &writer); try test_fixture.expectEqual( @@ -1076,8 +1466,26 @@ test "grid golden encoding and decoding" { decoded_head.row.semantic_prompt, ); + try std.testing.expectEqual( + @as(u21, 'h'), + destination.getRowAndCell(0, 2).cell.codepoint(), + ); + try std.testing.expect(destination.getRowAndCell(1, 2).cell.isZero()); + try std.testing.expectEqual( + @as(u21, 'i'), + destination.getRowAndCell(2, 2).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, 0x0416), + destination.getRowAndCell(0, 3).cell.codepoint(), + ); + try std.testing.expectEqual( + @as(u21, '!'), + destination.getRowAndCell(1, 3).cell.codepoint(), + ); + // A re-encode proves the decoded native page retains every wire field. - var reencoded: [128]u8 = undefined; + var reencoded: [160]u8 = undefined; var rewriter: std.Io.Writer = .fixed(&reencoded); try encode(&destination, &rewriter); try std.testing.expectEqualStrings( @@ -1102,10 +1510,11 @@ test "grid elides trailing default cells" { var writer: std.Io.Writer = .fixed(&encoded); try encode(&page, &writer); - // 3 bytes per row header, cells only through the last non-default - // cell, and an empty grapheme section. + // 3 bytes per row header and cells only through the last non-default + // cell, followed by an empty grapheme section. The text row uses the + // one-byte width while the protected flag forces the full width. try testing.expectEqual( - @as(usize, 3 + (3 + 3 * 8) + (3 + 5 * 8) + 4), + @as(usize, 3 + (3 + 3 * 1) + (3 + 5 * 8) + 4), writer.buffered().len, ); @@ -1170,7 +1579,7 @@ test "grid normalizes incomplete wide cells" { // content but makes the incomplete wide cell narrow. var payload: [64]u8 = undefined; var payload_writer: std.Io.Writer = .fixed(&payload); - try payload_writer.writeByte(@bitCast(Row{})); + try payload_writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&payload_writer, u16, columns); try io.writeInt(&payload_writer, u64, @bitCast(Cell{ .width = 1, // wide @@ -1225,7 +1634,7 @@ test "grid normalizes incomplete wide cells" { var payload: [64]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - try writer.writeByte(@bitCast(Row{})); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&writer, u16, 1); try io.writeInt(&writer, u64, @bitCast(Cell{ .width = 1, // wide @@ -1255,8 +1664,9 @@ test "grid normalizes reserved cell values" { var payload: [64]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - // Reserved row flag bits are ignored while the unknown semantic-prompt - // value degrades to none and wrap survives. + // Reserved row flag bits six and seven are ignored while the unknown + // semantic-prompt value degrades to none and wrap survives. Bits four + // and five select the full encoded cell width. try writer.writeByte(0xFD); try io.writeInt(&writer, u16, 3); @@ -1327,7 +1737,7 @@ test "grid drops undeliverable grapheme entries" { var payload: [128]u8 = undefined; var writer: std.Io.Writer = .fixed(&payload); - try writer.writeByte(@bitCast(Row{})); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); try io.writeInt(&writer, u16, 4); // A kind 1 cell that receives a valid entry. try io.writeInt(&writer, u64, @bitCast(Cell{ .kind = 1, .content = 'x' })); @@ -1383,3 +1793,220 @@ test "grid drops undeliverable grapheme entries" { try testing.expect(!page.getRowAndCell(2, 0).cell.hasGrapheme()); try testing.expect(!page.getRowAndCell(3, 0).cell.hasGrapheme()); } + +test "grid encodes rows at their narrowest width" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer page.deinit(); + + // Width zero: bare Latin-1 text. + page.getRowAndCell(0, 0).cell.* = .init('A'); + page.getRowAndCell(1, 0).cell.* = .init(0xFF); + + // Width one: any BMP codepoint. + page.getRowAndCell(0, 1).cell.* = .init(0x0100); + + // Width two: a small style ID and a background color. + const style_id = try page.styles.add(page.memory, .{ + .flags = .{ .bold = true }, + }); + try testing.expect(style_id <= 63); + const styled = page.getRowAndCell(0, 2); + styled.cell.* = .init('s'); + styled.cell.style_id = style_id; + styled.row.styled = true; + const bg = page.getRowAndCell(1, 2); + bg.cell.content_tag = .bg_color_palette; + bg.cell.content = .{ .color_palette = .{ .data = 7 } }; + + // Width three: a protected cell. + page.getRowAndCell(0, 3).cell.protected = true; + + var encoded: [128]u8 = undefined; + var writer: std.Io.Writer = .fixed(&encoded); + try encode(&page, &writer); + + const bytes = writer.buffered(); + try testing.expectEqual( + @as(usize, (3 + 2 * 1) + (3 + 1 * 2) + (3 + 2 * 4) + (3 + 1 * 8) + 4), + bytes.len, + ); + + // Each row header carries the expected width bits. + try testing.expectEqual(@as(u8, 0 << 4), bytes[0] & 0x30); + try testing.expectEqual(@as(u8, 1 << 4), bytes[5] & 0x30); + try testing.expectEqual(@as(u8, 2 << 4), bytes[10] & 0x30); + try testing.expectEqual(@as(u8, 3 << 4), bytes[21] & 0x30); + + var destination = try TerminalPage.init(.{ + .cols = 2, + .rows = 4, + .styles = 8, + }); + defer destination.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + const native_style = try destination.styles.add(destination.memory, .{ + .flags = .{ .bold = true }, + }); + style_remap.put(style_id, native_style); + + var reader: std.Io.Reader = .fixed(bytes); + try decode(&destination, &reader, &style_remap, &hyperlink_remap); + try destination.verifyIntegrity(testing.allocator); + + try testing.expectEqual( + @as(u21, 'A'), + destination.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0xFF), + destination.getRowAndCell(1, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0100), + destination.getRowAndCell(0, 1).cell.codepoint(), + ); + const decoded_styled = destination.getRowAndCell(0, 2); + try testing.expectEqual(@as(u21, 's'), decoded_styled.cell.codepoint()); + try testing.expectEqual(native_style, decoded_styled.cell.style_id); + try testing.expect(decoded_styled.row.styled); + try testing.expectEqual( + @as(u8, 7), + destination.getRowAndCell(1, 2).cell.content.color_palette.data, + ); + try testing.expect(destination.getRowAndCell(0, 3).cell.protected); +} + +test "grid decodes non-canonical cell widths" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + // A bare ASCII row encoded at the full width is wasteful but valid. + var payload: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .eight })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'o' })); + try io.writeInt(&writer, u64, @bitCast(Cell{ .content = 'k' })); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 'o'), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 'k'), + page.getRowAndCell(1, 0).cell.codepoint(), + ); + + // Re-encoding canonicalizes the row back to the one-byte width. + var reencoded: [16]u8 = undefined; + var rewriter: std.Io.Writer = .fixed(&reencoded); + try encode(&page, &rewriter); + try testing.expectEqual(@as(usize, 3 + 2 + 4), rewriter.buffered().len); +} + +test "grid normalizes surrogates in two-byte cells" { + const testing = std.testing; + var page = try TerminalPage.init(.{ .cols = 2, .rows = 1 }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [16]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .two })); + try io.writeInt(&writer, u16, 2); + try io.writeInt(&writer, u16, 0xD800); + try io.writeInt(&writer, u16, 0x0416); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + try testing.expectEqual( + @as(u21, 0xFFFD), + page.getRowAndCell(0, 0).cell.codepoint(), + ); + try testing.expectEqual( + @as(u21, 0x0416), + page.getRowAndCell(1, 0).cell.codepoint(), + ); +} + +test "grid four-byte cells run full normalization" { + const testing = std.testing; + var page = try TerminalPage.init(.{ + .cols = 3, + .rows = 1, + .styles = 8, + }); + defer page.deinit(); + var style_remap = try StyleRemap.init(testing.allocator); + defer style_remap.deinit(testing.allocator); + var hyperlink_remap = try HyperlinkRemap.init(testing.allocator); + defer hyperlink_remap.deinit(testing.allocator); + + var payload: [32]u8 = undefined; + var writer: std.Io.Writer = .fixed(&payload); + try writer.writeByte(@bitCast(Row{ .cell_width = .four })); + try io.writeInt(&writer, u16, 3); + + // The Kitty placeholder does not fit two-byte cells but fits here + // and must still derive the native row hint. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = kitty.graphics.unicode.placeholder, + })))); + + // An unknown small style reference degrades to the default style. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .content = 'x', + .style_id = 63, + })))); + + // Reserved palette content bits are cleared at this width too. + try io.writeInt(&writer, u32, @truncate(@as(u64, @bitCast(Cell{ + .kind = 2, + .content = 0xFFFF07, + })))); + try io.writeInt(&writer, u32, 0); + + var reader: std.Io.Reader = .fixed(writer.buffered()); + try decode(&page, &reader, &style_remap, &hyperlink_remap); + try page.verifyIntegrity(testing.allocator); + + const first = page.getRowAndCell(0, 0); + try testing.expectEqual( + @as(u21, kitty.graphics.unicode.placeholder), + first.cell.codepoint(), + ); + try testing.expect(first.row.kitty_virtual_placeholder); + + const second = page.getRowAndCell(1, 0).cell; + try testing.expectEqual(@as(u21, 'x'), second.codepoint()); + try testing.expectEqual(@as(TerminalStyleId, 0), second.style_id); + + const third = page.getRowAndCell(2, 0).cell; + try testing.expectEqual( + TerminalCell.ContentTag.bg_color_palette, + third.content_tag, + ); + try testing.expectEqual(@as(u8, 7), third.content.color_palette.data); +} diff --git a/src/terminal/snapshot/page.zig b/src/terminal/snapshot/page.zig index e9d115026..c09fd9dad 100644 --- a/src/terminal/snapshot/page.zig +++ b/src/terminal/snapshot/page.zig @@ -1015,7 +1015,7 @@ test "decode defaults missing sparse cell references" { var style_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var style_writer: std.Io.Writer = .fixed(&style_encoded); try style_header.encode(&style_writer); - try style_writer.writeByte(0); // row flags + try style_writer.writeByte(0x30); // row flags, full cell width try io.writeInt(&style_writer, u16, 1); // cell count try io.writeInt(&style_writer, u64, @bitCast(grid.Cell{ .style_id = 1, @@ -1048,7 +1048,7 @@ test "decode defaults missing sparse cell references" { var hyperlink_encoded: [Header.len + 3 + 8 + 4]u8 = undefined; var hyperlink_writer: std.Io.Writer = .fixed(&hyperlink_encoded); try hyperlink_header.encode(&hyperlink_writer); - try hyperlink_writer.writeByte(0); // row flags + try hyperlink_writer.writeByte(0x30); // row flags, full cell width try io.writeInt(&hyperlink_writer, u16, 1); // cell count try io.writeInt(&hyperlink_writer, u64, @bitCast(grid.Cell{ .hyperlink = true, diff --git a/src/terminal/snapshot/screen.zig b/src/terminal/snapshot/screen.zig index 06dc8bce6..d05ff00ba 100644 --- a/src/terminal/snapshot/screen.zig +++ b/src/terminal/snapshot/screen.zig @@ -2262,7 +2262,7 @@ test "SCREEN decode ignores a PAGE with an empty hyperlink URI" { // One narrow codepoint cell refers to the hyperlink table entry above. // Since that entry is ignored, the cell must restore without a hyperlink. - try page_payload.writeByte(0); // row flags + try page_payload.writeByte(0x30); // row flags, full cell width try io.writeInt(page_payload, u16, 1); // cell count try io.writeInt(page_payload, u64, @bitCast(grid.Cell{ .content = 'A', diff --git a/src/terminal/snapshot/snapshot.ksy b/src/terminal/snapshot/snapshot.ksy index c503fe2f7..6ff925ef3 100644 --- a/src/terminal/snapshot/snapshot.ksy +++ b/src/terminal/snapshot/snapshot.ksy @@ -933,7 +933,13 @@ types: valid: expr: _ <= columns - id: cells - type: grid_cell(_index, columns, flags.wrap) + type: + switch-on: flags.width_log2 + cases: + 0: grid_cell_1 + 1: grid_cell_2 + 2: grid_cell_4 + 3: grid_cell(_index, columns, flags.wrap) repeat: expr repeat-expr: cell_count @@ -942,7 +948,7 @@ types: - id: raw type: u1 valid: - expr: (_ & 0xf0) == 0 and ((_ >> 2) & 0x3) <= 2 + expr: (_ & 0xc0) == 0 and ((_ >> 2) & 0x3) <= 2 instances: wrap: value: (raw & 1) != 0 @@ -950,6 +956,46 @@ types: value: (raw & 2) != 0 semantic_prompt: value: (raw >> 2) & 0x3 + width_log2: + value: (raw >> 4) & 0x3 + + grid_cell_1: + doc: One-byte encoded cell; the value is a codepoint at or below U+00FF. + seq: + - id: codepoint + type: u1 + + grid_cell_2: + doc: | + Two-byte encoded cell; the value is a codepoint at or below U+FFFF. + Canonical encoders never emit surrogates. + seq: + - id: codepoint + type: u2 + valid: + expr: not (_ >= 0xd800 and _ <= 0xdfff) + + grid_cell_4: + doc: | + Four-byte encoded cell holding the low half of the cell word: any + content kind and codepoint, style IDs one through sixty-three, and no + width, flag, or hyperlink bits. + seq: + - id: raw + type: u4 + valid: + expr: | + (content_kind >= 2 or + (content <= 0x10ffff and + not (content >= 0xd800 and content <= 0xdfff))) and + (content_kind != 2 or content <= 0xff) + instances: + content_kind: + value: raw % 4 + content: + value: (raw / 4) % 16777216 + style_id: + value: raw / 67108864 grid_cell: doc: | @@ -984,7 +1030,8 @@ types: not (content >= 0xd800 and content <= 0xdfff))) and (content_kind != 2 or content <= 0xff) and (width != 2 or - (index > 0 and _parent.cells[index - 1].width == 1)) and + (index > 0 and + _parent.cells[index - 1].as.width == 1)) and (width != 3 or (index + 1 == columns and row_wrap)) instances: content_kind: diff --git a/src/terminal/snapshot/testdata/complete-v1.hex b/src/terminal/snapshot/testdata/complete-v1.hex index e88db31c0..8e3df8287 100644 --- a/src/terminal/snapshot/testdata/complete-v1.hex +++ b/src/terminal/snapshot/testdata/complete-v1.hex @@ -78,52 +78,47 @@ ee ee 80 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000037a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003ec 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x000003fc -# offset 0x0000040c: page record, payload 57 bytes -03 00 39 00 00 00 26 5a 94 0b 02 00 03 00 00 00 # 0x0000040c +# offset 0x0000040c: page record, payload 36 bytes +03 00 24 00 00 00 a2 4f 26 d1 02 00 03 00 00 00 # 0x0000040c 00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x0000041c -00 0c 01 00 00 00 00 00 00 00 01 00 10 01 00 00 # 0x0000042c -00 00 00 00 00 01 00 14 01 00 00 00 00 00 00 00 # 0x0000043c -00 00 00 # 0x0000044c +00 43 00 01 00 44 00 01 00 45 00 00 00 00 # 0x0000042c -# offset 0x0000044f: screen record, payload 54 bytes -02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000044f -00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000045f -00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046f -00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000047f +# offset 0x0000043a: screen record, payload 54 bytes +02 00 36 00 00 00 ce 1f 9f 08 01 00 01 00 00 00 # 0x0000043a +00 00 00 00 00 00 01 00 02 00 01 00 00 00 00 00 # 0x0000044a +00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000045a +00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x0000046a -# offset 0x0000048f: page record, payload 73 bytes -03 00 49 00 00 00 37 af 30 0e 02 00 03 00 00 00 # 0x0000048f -00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000049f -00 c8 01 00 00 00 00 00 00 b8 01 00 00 00 00 00 # 0x000004af -00 03 02 00 84 01 00 00 00 00 00 00 d0 01 00 00 # 0x000004bf -00 00 00 00 03 01 00 94 01 00 00 00 00 00 00 00 # 0x000004cf -00 00 00 # 0x000004df +# offset 0x0000047a: page record, payload 38 bytes +03 00 26 00 00 00 45 61 97 32 02 00 03 00 00 00 # 0x0000047a +00 00 80 00 c0 00 00 02 00 00 00 08 00 00 03 02 # 0x0000048a +00 72 6e 03 02 00 61 74 03 01 00 65 00 00 00 00 # 0x0000049a -# offset 0x000004e2: continuation record, payload 0 bytes -07 00 00 00 00 00 27 80 63 d1 # 0x000004e2 +# offset 0x000004aa: continuation record, payload 0 bytes +07 00 00 00 00 00 27 80 63 d1 # 0x000004aa -# offset 0x000004ec: ready record, payload 32 bytes -05 00 20 00 00 00 5d f3 97 80 41 a2 18 18 cb 82 # 0x000004ec -8c 97 b6 58 14 60 26 9a 88 2e be d4 c3 4b 83 c9 # 0x000004fc -9d 32 9d de e6 70 17 4a 5d bc # 0x0000050c +# offset 0x000004b4: ready record, payload 32 bytes +05 00 20 00 00 00 4d 17 72 ed dd 87 26 75 cf 8e # 0x000004b4 +e5 1f 37 e3 05 92 0a e2 f8 ef c1 16 54 be 49 e7 # 0x000004c4 +b2 6c df 40 d2 77 fe 05 82 ea # 0x000004d4 -# offset 0x00000516: history record, payload 6 bytes -04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x00000516 +# offset 0x000004de: history record, payload 6 bytes +04 00 06 00 00 00 20 32 ed e1 00 00 02 00 00 00 # 0x000004de -# offset 0x00000526: page record, payload 38 bytes -03 00 26 00 00 00 9e 10 a2 93 02 00 02 00 00 00 # 0x00000526 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000536 -00 08 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000546 +# offset 0x000004ee: page record, payload 31 bytes +03 00 1f 00 00 00 4a ed 2f c2 02 00 02 00 00 00 # 0x000004ee +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x000004fe +00 42 00 00 00 00 00 00 00 # 0x0000050e -# offset 0x00000556: page record, payload 38 bytes -03 00 26 00 00 00 70 e1 36 3a 02 00 02 00 00 00 # 0x00000556 -00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000566 -00 04 01 00 00 00 00 00 00 00 00 00 00 00 00 00 # 0x00000576 +# offset 0x00000517: page record, payload 31 bytes +03 00 1f 00 00 00 23 6a 6b 19 02 00 02 00 00 00 # 0x00000517 +00 00 10 00 c0 00 00 04 00 00 00 08 00 00 00 01 # 0x00000527 +00 41 00 00 00 00 00 00 00 # 0x00000537 -# offset 0x00000586: history record, payload 6 bytes -04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000586 +# offset 0x00000540: history record, payload 6 bytes +04 00 06 00 00 00 b8 7a ba b1 01 00 00 00 00 00 # 0x00000540 -# offset 0x00000596: finish record, payload 32 bytes -06 00 20 00 00 00 20 0f e0 7e 10 4f 2c da 90 fa # 0x00000596 -65 19 a9 73 26 75 2a 5b 33 cd 9d fa 73 81 86 07 # 0x000005a6 -9e 57 d4 83 41 9f a9 81 64 e7 # 0x000005b6 +# offset 0x00000550: finish record, payload 32 bytes +06 00 20 00 00 00 5a 2d a8 f4 25 b2 49 b4 3e 64 # 0x00000550 +5a 2e d6 7d 7f 3d 60 5b db 9b 4c 3a 29 00 97 ec # 0x00000560 +d6 c7 1d ea da d3 a9 fd 3d 68 # 0x00000570 diff --git a/src/terminal/snapshot/testdata/grid-v1.hex b/src/terminal/snapshot/testdata/grid-v1.hex index b266f5438..b9a74af32 100644 --- a/src/terminal/snapshot/testdata/grid-v1.hex +++ b/src/terminal/snapshot/testdata/grid-v1.hex @@ -1,14 +1,15 @@ # Ghostty snapshot fixture # Kaitai type: grid -# Kaitai params: 2 3 +# Kaitai params: 4 3 # Kaitai offset: 0 # Wire version: 1 # Generated by its snapshot test; review before replacing. # On mismatch, the candidate is copied to the repository root. # offset 0x00000000: encoded bytes -04 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 -48 00 00 e1 01 00 00 00 00 00 00 0b 03 00 1e 00 # 0x00000010 +34 03 00 04 01 00 00 00 94 00 00 00 00 00 00 00 # 0x00000000 +48 00 00 e1 01 00 00 00 00 00 00 3b 03 00 1e 00 # 0x00000010 00 00 00 00 00 00 ab ee 32 03 00 10 00 00 00 00 # 0x00000020 -00 00 00 0c 00 00 01 00 00 00 00 00 02 00 02 00 # 0x00000030 -01 03 00 00 02 03 00 00 # 0x00000040 +00 00 00 0c 00 00 00 03 00 68 00 69 10 02 00 16 # 0x00000030 +04 21 00 01 00 00 00 00 00 02 00 02 00 01 03 00 # 0x00000040 +00 02 03 00 00 # 0x00000050 diff --git a/src/terminal/snapshot/testdata/page-v1.hex b/src/terminal/snapshot/testdata/page-v1.hex index c572decc8..78018ebb6 100644 --- a/src/terminal/snapshot/testdata/page-v1.hex +++ b/src/terminal/snapshot/testdata/page-v1.hex @@ -15,8 +15,8 @@ 00 00 03 00 00 00 00 00 01 2a 00 00 00 00 00 00 # 0x00000024 00 00 00 00 01 00 02 01 00 00 00 61 05 00 00 00 # 0x00000034 61 6c 70 68 61 03 00 01 04 03 02 01 04 00 00 00 # 0x00000044 -62 65 74 61 04 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 -00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 0b # 0x00000064 +62 65 74 61 34 03 00 04 01 00 04 00 b4 01 00 00 # 0x00000054 +00 00 0c 00 68 03 00 1e 00 00 04 00 20 01 00 3b # 0x00000064 03 00 e1 01 00 00 00 00 00 00 ab ee 32 03 00 10 # 0x00000074 00 00 00 00 00 00 00 0c 00 00 01 00 00 00 01 00 # 0x00000084 00 00 02 00 01 03 00 00 02 03 00 00 # 0x00000094