From 446f80f4edd16d217e8ec928664d86a529b3a223 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 6 Jul 2026 14:04:22 -0700 Subject: [PATCH] terminal: render state update optimizations (~2.7x to ~11x less lock hold) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This optimizes `RenderState.update`, the per-frame call that snapshots terminal state for the renderer and is the main reason the renderer thread holds the terminal lock. Lock hold time is reduced ~2.7x to ~11x depending on the frame. ## The changes 1. iterate page chunks instead of rows in `update` 2. classify cells with masked vector compares. 3. split the update into `beginUpdate`/`endUpdate` phases. There's a lot to be gained by accumulating data with the lock held and then processing it out of the lock. 4. generalize the masked-compare scans into `page.Mask`. This is just a really common pattern we're doing now and it yields a ton of great value. Its error prone so lets make it a tested helper. ## Benchmarks Measured with the new `ghostty-bench +screen-clone` modes (`render`, `render-locked`, `render-clean`, `render-partial`), 120x80 terminal, M4 Max, macOS 26, ReleaseFast, hyperfine means of 10+ runs, per-update times derived from fixed-count update loops with process startup subtracted. "Lock held" is the time the terminal lock must be held per update; "before" held the lock for the entire update. | scenario | before (lock held) | after (lock held) | after (total) | lock change | |----------|--------------------|-------------------|---------------|-------------| | clean frame (nothing dirty) | 202 ns | 19 ns | 19 ns | 10.9x | | partial frame (1 dirty row) | 290 ns | 54 ns | 54 ns | 5.4x | | full rebuild, lightly styled | 6.9 µs | 2.5 µs | 3.0 µs | 2.7x | | full rebuild, fully styled | 9.3 µs | 2.4 µs | 8.0 µs | 3.8x | | full rebuild, fully styled, 250x150 | 49.9 µs | 9.4 µs | 31.6 µs | 5.3x | | full rebuild, plain text | 1.9 µs | 1.9 µs | 1.9 µs | 1.0x (memcpy floor) | The clean and partial cases are the steady-state frame costs (cursor blink, mouse movement, typing). The full-rebuild cases are the contended ones: colored scrolling output (build logs, htop, vim) moves the viewport pin every frame, forcing a full rebuild exactly when the IO thread is busiest, so that row of the table is where lock contention actually hurts. Plain text was already at the memcpy floor and is unchanged. ## LLM Notes This work was driven by Fable 5: benchmarks, optimizations, the property test, and the measurements above. I reviewed every line, simplified the design in a few places (API naming, the Mask helper shape), and re-ran the verifications myself. --- include/ghostty/vt/render.h | 66 +++ src/benchmark/ScreenClone.zig | 113 ++++- src/lib_vt.zig | 2 + src/renderer/generic.zig | 16 +- src/terminal/Terminal.zig | 91 ++-- src/terminal/c/main.zig | 2 + src/terminal/c/render.zig | 74 +++ src/terminal/page.zig | 274 ++++++++++ src/terminal/render.zig | 915 ++++++++++++++++++++++++++++------ 9 files changed, 1346 insertions(+), 207 deletions(-) diff --git a/include/ghostty/vt/render.h b/include/ghostty/vt/render.h index c5b1d0d4f..2e3f2f71e 100644 --- a/include/ghostty/vt/render.h +++ b/include/ghostty/vt/render.h @@ -39,6 +39,18 @@ extern "C" { * 2. Update it from a terminal instance whenever you need. * 3. Read from the render state to get the data needed to draw your frame. * + * ## Two-Phase Updates + * + * For callers that synchronize terminal access (e.g. a renderer thread + * sharing a lock with an IO thread), the update can be split into two + * phases to minimize the time the terminal must be held exclusively: + * ghostty_render_state_begin_update requires terminal access, while + * ghostty_render_state_end_update completes any deferred work using only + * memory owned by the render state. A typical renderer would lock, begin + * the update, unlock, and then end the update while the IO thread is free + * to continue modifying the terminal. ghostty_render_state_update is a + * convenience that performs both phases in one call. + * * ## Dirty Tracking * * Dirty tracking is a key feature of the render state that allows renderers @@ -331,6 +343,12 @@ GHOSTTY_API void ghostty_render_state_free(GhosttyRenderState state); * This consumes terminal/screen dirty state in the same way as the internal * render state update path. * + * This is a convenience function that performs a full update in one call, + * equivalent to ghostty_render_state_begin_update immediately followed by + * ghostty_render_state_end_update. Callers that hold a lock over the + * terminal state should prefer calling the two phases directly so that the + * lock is only held for the begin phase. + * * @param state The render state handle (NULL returns GHOSTTY_INVALID_VALUE) * @param terminal The terminal handle to read from (NULL returns GHOSTTY_INVALID_VALUE) * @return GHOSTTY_SUCCESS on success, GHOSTTY_INVALID_VALUE if `state` or @@ -342,6 +360,54 @@ GHOSTTY_API void ghostty_render_state_free(GhosttyRenderState state); GHOSTTY_API GhosttyResult ghostty_render_state_update(GhosttyRenderState state, GhosttyTerminal terminal); +/** + * Begin an update of a render state instance from a terminal. + * + * Every begin must be completed with a ghostty_render_state_end_update call + * before the render state is read. + * + * This two-phase structure exists for callers that synchronize access to the + * terminal state (e.g. with a lock shared with an IO thread): only this + * function requires terminal access, so a caller can hold its lock for this + * call only and then call ghostty_render_state_end_update after releasing + * it. The end phase exclusively reads and writes memory owned by the render + * state, so it is safe to call while the terminal is being modified. + * + * Work that doesn't require terminal access may be deferred to the end phase + * to keep this call (and therefore lock hold time) as short as possible. + * Callers must treat the render state as incomplete until + * ghostty_render_state_end_update is called. + * + * This consumes terminal/screen dirty state in the same way as the internal + * render state update path. + * + * @param state The render state handle (NULL returns GHOSTTY_INVALID_VALUE) + * @param terminal The terminal handle to read from (NULL returns GHOSTTY_INVALID_VALUE) + * @return GHOSTTY_SUCCESS on success, GHOSTTY_INVALID_VALUE if `state` or + * `terminal` is NULL, GHOSTTY_OUT_OF_MEMORY if updating the state requires + * allocation and that allocation fails + * + * @ingroup render + */ +GHOSTTY_API GhosttyResult ghostty_render_state_begin_update(GhosttyRenderState state, + GhosttyTerminal terminal); + +/** + * Complete a prior ghostty_render_state_begin_update call by performing any + * deferred work. + * + * This only reads and writes memory owned by the render state, so it is safe + * to call while the terminal is being modified (no terminal synchronization + * is required). Calling this without a prior begin is a safe no-op. + * + * @param state The render state handle (NULL returns GHOSTTY_INVALID_VALUE) + * @return GHOSTTY_SUCCESS on success, GHOSTTY_INVALID_VALUE if `state` is + * NULL + * + * @ingroup render + */ +GHOSTTY_API GhosttyResult ghostty_render_state_end_update(GhosttyRenderState state); + /** * Get a value from a render state. * diff --git a/src/benchmark/ScreenClone.zig b/src/benchmark/ScreenClone.zig index 108eaa0c6..5176ce26c 100644 --- a/src/benchmark/ScreenClone.zig +++ b/src/benchmark/ScreenClone.zig @@ -21,6 +21,10 @@ pub const Options = struct { /// The type of codepoint width calculation to use. mode: Mode = .clone, + /// Multiplier on the number of iterations each step runs. This is + /// useful to make a benchmark run long enough for profiling. + loops: u32 = 1, + /// The size of the terminal. This affects benchmarking when /// dealing with soft line wrapping and the memory impact /// of page sizes. @@ -48,6 +52,22 @@ pub const Mode = enum { /// RenderState rather than a screen clone. render, + + /// Like render, but only the portion of the render state update + /// that requires holding a terminal lock (beginUpdate). The + /// deferred work (endUpdate) is excluded since it happens outside + /// of any locks. + @"render-locked", + + /// RenderState update with no changes to the terminal. This is + /// the common case for a renderer that is redrawing frames (e.g. + /// cursor blink, mouse movement) without terminal changes. + @"render-clean", + + /// RenderState update where a single row is dirty. This models the + /// common case of a shell prompt or TUI updating a small portion + /// of the screen between frames. + @"render-partial", }; pub fn create( @@ -79,6 +99,9 @@ pub fn benchmark(self: *ScreenClone) Benchmark { .noop => stepNoop, .clone => stepClone, .render => stepRender, + .@"render-locked" => stepRenderLocked, + .@"render-clean" => stepRenderClean, + .@"render-partial" => stepRenderPartial, }, .setupFn = setup, .teardownFn = teardown, @@ -178,16 +201,102 @@ fn stepRender(ptr: *anyopaque) Benchmark.Error!void { // We loop because its so fast that a single benchmark run doesn't // properly capture our speeds. - for (0..1000) |_| { + for (0..50_000 * @as(u64, self.opts.loops)) |_| { // Forces a full rebuild because it thinks our screen changed state.screen = .alternate; state.update(alloc, &self.terminal) catch |err| { log.warn("error cloning screen err={}", .{err}); return error.BenchmarkFailed; }; - std.mem.doNotOptimizeAway(state); + std.mem.doNotOptimizeAway(&state); // Note: we purposely do not free memory because we don't want // to benchmark that. We'll free when the benchmark exits. } } + +fn stepRenderLocked(ptr: *anyopaque) Benchmark.Error!void { + const self: *ScreenClone = @ptrCast(@alignCast(ptr)); + + // We do this once out of the loop because a significant slowdown + // on the first run is allocation. After that first run, even with + // a full rebuild, it is much faster. Let's ignore that first run + // slowdown. + const alloc = self.terminal.screens.active.alloc; + var state: terminalpkg.RenderState = .empty; + state.update(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + + // We loop because its so fast that a single benchmark run doesn't + // properly capture our speeds. + for (0..50_000 * @as(u64, self.opts.loops)) |_| { + // Forces a full rebuild because it thinks our screen changed + state.screen = .alternate; + state.beginUpdate(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(&state); + + // Note: we purposely do not free memory because we don't want + // to benchmark that. We'll free when the benchmark exits. + } +} + +fn stepRenderClean(ptr: *anyopaque) Benchmark.Error!void { + const self: *ScreenClone = @ptrCast(@alignCast(ptr)); + + // Initial update so that subsequent updates are clean (nothing + // dirty, no rebuilds). + const alloc = self.terminal.screens.active.alloc; + var state: terminalpkg.RenderState = .empty; + state.update(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + + // We loop because its so fast that a single benchmark run doesn't + // properly capture our speeds. + for (0..3_000_000 * @as(u64, self.opts.loops)) |_| { + state.update(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(&state); + } +} + +fn stepRenderPartial(ptr: *anyopaque) Benchmark.Error!void { + const self: *ScreenClone = @ptrCast(@alignCast(ptr)); + + // Initial update so that subsequent updates are incremental. + const alloc = self.terminal.screens.active.alloc; + var state: terminalpkg.RenderState = .empty; + state.update(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + + // Grab a pin roughly in the middle of the active area that we + // dirty on every iteration to simulate a small screen update. + const pages = &self.terminal.screens.active.pages; + const pin = pages.pin(.{ .active = .{ + .x = 0, + .y = self.terminal.rows / 2, + } }).?; + + // We loop because its so fast that a single benchmark run doesn't + // properly capture our speeds. + for (0..2_000_000 * @as(u64, self.opts.loops)) |_| { + // Mark a single row dirty. `update` clears this so each + // iteration rebuilds exactly one row. + pin.markDirty(); + state.update(alloc, &self.terminal) catch |err| { + log.warn("error cloning screen err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(&state); + } +} diff --git a/src/lib_vt.zig b/src/lib_vt.zig index 1a5bf6162..ab13254df 100644 --- a/src/lib_vt.zig +++ b/src/lib_vt.zig @@ -235,6 +235,8 @@ comptime { @export(&c.terminal_selection_format_alloc, .{ .name = "ghostty_terminal_selection_format_alloc" }); @export(&c.render_state_new, .{ .name = "ghostty_render_state_new" }); @export(&c.render_state_update, .{ .name = "ghostty_render_state_update" }); + @export(&c.render_state_begin_update, .{ .name = "ghostty_render_state_begin_update" }); + @export(&c.render_state_end_update, .{ .name = "ghostty_render_state_end_update" }); @export(&c.render_state_get, .{ .name = "ghostty_render_state_get" }); @export(&c.render_state_get_multi, .{ .name = "ghostty_render_state_get_multi" }); @export(&c.render_state_set, .{ .name = "ghostty_render_state_set" }); diff --git a/src/renderer/generic.zig b/src/renderer/generic.zig index 0f4a294bc..5102ad4d4 100644 --- a/src/renderer/generic.zig +++ b/src/renderer/generic.zig @@ -1199,8 +1199,15 @@ pub fn Renderer(comptime GraphicsAPI: type) type { state.terminal.scrollViewport(.bottom); } - // Update our terminal state - try self.terminal_state.update(self.alloc, state.terminal); + // Begin the update of our terminal state. Work that + // doesn't require terminal access (e.g. style + // denormalization) is deferred to the endUpdate call + // outside of this critical section, keeping our lock + // hold time as short as possible. + try self.terminal_state.beginUpdate( + self.alloc, + state.terminal, + ); // If our terminal state is dirty at all we need to redo // the viewport search. @@ -1279,6 +1286,11 @@ pub fn Renderer(comptime GraphicsAPI: type) type { }; }; + // Outside the critical area, complete the update we began + // within it. This must be done before anything reads the + // render state (e.g. rebuildCells). + self.terminal_state.endUpdate(); + // Outside the critical area we can update our links to contain // our regex results. self.config.links.renderCellMap( diff --git a/src/terminal/Terminal.zig b/src/terminal/Terminal.zig index 83a6df60b..a6a044afa 100644 --- a/src/terminal/Terminal.zig +++ b/src/terminal/Terminal.zig @@ -512,12 +512,12 @@ fn printSliceFill( // a codepoint cell (no grapheme data or bg-color tag), narrow, and // not a hyperlink. The mask covers every field that must match // the expected value (see printSliceCheckExpected) exactly. - const simple_mask = comptime fieldMask(Cell, &.{ + const SimpleMask = pagepkg.Mask(Cell, &.{ "content_tag", "style_id", "wide", "hyperlink", - }); + }, 4); // The bit offset of the codepoint content within a Cell, used to // construct cell values from a template without field assignments. @@ -619,8 +619,7 @@ fn printSliceFill( // spacer head (or a blank narrow cell if we're inside // a right margin) and wraps. We require a simple cell, // otherwise fall back to print() for the cleanup. - const bits: u64 = @bitCast(cells[0]); - if ((bits & simple_mask) != check_expected) break; + if (!SimpleMask.eqlScalar(cells[0], check_expected)) break; var spacer = template; if (right_limit == self.cols) { @@ -661,24 +660,20 @@ fn printSliceFill( // several cells at a time manually. var simple = k; simple: { - const lanes = 4; - const V = @Vector(lanes, u64); - const mask_v: V = @splat(simple_mask); - const expect_v: V = @splat(check_expected); - const cells64: [*]const u64 = @ptrCast(cells); - while (simple + lanes <= cell_count) { - const v: V = cells64[simple..][0..lanes].*; - const ok = (v & mask_v) == expect_v; - if (!@reduce(.And, ok)) { - const bits: std.meta.Int(.unsigned, lanes) = @bitCast(ok); - simple += @ctz(~bits); - break :simple; - } - simple += lanes; + while (simple + SimpleMask.group_len <= cell_count) { + const p = SimpleMask.eqlPrefix( + cells[0..cell_count], + simple, + check_expected, + ); + simple += p; + if (p != SimpleMask.group_len) break :simple; } while (simple < cell_count) : (simple += 1) { - const bits: u64 = @bitCast(cells[simple]); - if ((bits & simple_mask) != check_expected) break; + if (!SimpleMask.eqlScalar( + cells[simple], + check_expected, + )) break; } } @@ -718,8 +713,7 @@ fn printSliceFill( // run of identical old styles, two ref-count updates, // and a branch-free fill. if (comptime width == .narrow) bulk: { - const cells64: [*]const u64 = @ptrCast(cells); - const first = cells64[k] & simple_mask; + const first = SimpleMask.pattern(cells[k]); // The old cell must be a plain narrow codepoint cell // with no hyperlink whose only difference is the @@ -733,22 +727,17 @@ fn printSliceFill( // Find the run of cells with identical masked bits. var m = k + 1; scan: { - const lanes = 4; - const V = @Vector(lanes, u64); - const mask_v: V = @splat(simple_mask); - const first_v: V = @splat(first); - while (m + lanes <= cell_count) { - const v: V = cells64[m..][0..lanes].*; - const ok = (v & mask_v) == first_v; - if (!@reduce(.And, ok)) { - const bits: std.meta.Int(.unsigned, lanes) = @bitCast(ok); - m += @ctz(~bits); - break :scan; - } - m += lanes; + while (m + SimpleMask.group_len <= cell_count) { + const p = SimpleMask.eqlPrefix( + cells[0..cell_count], + m, + first, + ); + m += p; + if (p != SimpleMask.group_len) break :scan; } while (m < cell_count) : (m += 1) { - if ((cells64[m] & simple_mask) != first) break; + if (!SimpleMask.eqlScalar(cells[m], first)) break; } } @@ -835,9 +824,9 @@ fn printSliceFill( return printed; } -/// The expected value of a simple cell (per the check mask built from -/// fieldMask in printSliceFill) that already has the given style -/// (so no ref-counting is needed). +/// The expected value of a simple cell (per SimpleMask in +/// printSliceFill) that already has the given style (so no +/// ref-counting is needed). inline fn printSliceCheckExpected(style_id: style.Id) u64 { var e: Cell = @bitCast(@as(u64, 0)); e.style_id = style_id; @@ -3725,30 +3714,6 @@ fn clearDirty(t: *Terminal) void { t.screens.active.pages.clearDirty(); } -/// Returns a mask with all bits set for the given fields of the packed -/// struct T, used for masked compares of raw backing-integer values. -fn fieldMask( - comptime T: type, - comptime fields: []const []const u8, -) @typeInfo(T).@"struct".backing_integer.? { - // Backing int of the packed struct - const Int = @typeInfo(T).@"struct".backing_integer.?; - - var mask: Int = 0; - inline for (fields) |field| { - // The type that fits all the bits we need to set. - const Ones = std.meta.Int( - .unsigned, - @bitSizeOf(@FieldType(T, field)), - ); - - // Mask out the ones - mask |= @as(Int, std.math.maxInt(Ones)) << @bitOffsetOf(T, field); - } - - return mask; -} - test "Terminal: input with no control characters" { const alloc = testing.allocator; var t = try init(alloc, .{ .cols = 40, .rows = 40 }); diff --git a/src/terminal/c/main.zig b/src/terminal/c/main.zig index ec0f4691b..ae91565e0 100644 --- a/src/terminal/c/main.zig +++ b/src/terminal/c/main.zig @@ -83,6 +83,8 @@ pub const formatter_free = formatter.free; pub const render_state_new = render.new; pub const render_state_free = render.free; pub const render_state_update = render.update; +pub const render_state_begin_update = render.begin_update; +pub const render_state_end_update = render.end_update; pub const render_state_get = render.get; pub const render_state_get_multi = render.get_multi; pub const render_state_set = render.set; diff --git a/src/terminal/c/render.zig b/src/terminal/c/render.zig index e199b0e3a..6cf78ec54 100644 --- a/src/terminal/c/render.zig +++ b/src/terminal/c/render.zig @@ -188,6 +188,25 @@ pub fn update( return .success; } +pub fn begin_update( + state_: RenderState, + terminal_: terminal_c.Terminal, +) callconv(lib.calling_conv) Result { + const state = state_ orelse return .invalid_value; + const t: *ZigTerminal = (terminal_ orelse return .invalid_value).terminal; + + state.state.beginUpdate(state.alloc, t) catch return .out_of_memory; + return .success; +} + +pub fn end_update( + state_: RenderState, +) callconv(lib.calling_conv) Result { + const state = state_ orelse return .invalid_value; + state.state.endUpdate(); + return .success; +} + pub fn get( state_: RenderState, data: Data, @@ -787,6 +806,61 @@ test "render: update invalid value" { try testing.expectEqual(Result.invalid_value, update(state, null)); } +test "render: begin/end update invalid value" { + var state: RenderState = null; + try testing.expectEqual(Result.success, new( + &lib.alloc.test_allocator, + &state, + )); + defer free(state); + + try testing.expectEqual(Result.invalid_value, begin_update(null, null)); + try testing.expectEqual(Result.invalid_value, begin_update(state, null)); + try testing.expectEqual(Result.invalid_value, end_update(null)); + + // End without a begin is safe. + try testing.expectEqual(Result.success, end_update(state)); +} + +test "render: begin/end update" { + var terminal: terminal_c.Terminal = null; + try testing.expectEqual(Result.success, terminal_c.new( + &lib.alloc.test_allocator, + &terminal, + .{ + .cols = 10, + .rows = 3, + .max_scrollback = 10_000, + }, + )); + defer terminal_c.free(terminal); + + // Write some styled text so that the update has deferred work. + const t = terminal.?.terminal; + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("\x1b[1mAB"); // Bold + + var state: RenderState = null; + try testing.expectEqual(Result.success, new( + &lib.alloc.test_allocator, + &state, + )); + defer free(state); + + // Begin should record pending work, end should complete it. + try testing.expectEqual(Result.success, begin_update(state, terminal)); + try testing.expect(state.?.state.pending_styles.items.len > 0); + try testing.expectEqual(Result.success, end_update(state)); + try testing.expectEqual(0, state.?.state.pending_styles.items.len); + + // The cell styles should be complete. + const row_data = state.?.state.row_data.slice(); + const cells = row_data.items(.cells); + try testing.expect(cells[0].get(0).style.flags.bold); + try testing.expect(cells[0].get(1).style.flags.bold); +} + test "render: get invalid value" { var cols: size.CellCountInt = 0; try testing.expectEqual(Result.invalid_value, get(null, .cols, @ptrCast(&cols))); diff --git a/src/terminal/page.zig b/src/terminal/page.zig index 0942e260c..803e67c8f 100644 --- a/src/terminal/page.zig +++ b/src/terminal/page.zig @@ -1988,6 +1988,11 @@ pub const Row = packed struct(u64) { prompt_continuation = 2, }; + /// The backing integer of this packed struct. Prefer this over + /// hardcoding the integer type so that code is resilient to the + /// size changing. + pub const Backing = @typeInfo(Row).@"struct".backing_integer.?; + /// C ABI type. pub const C = u64; @@ -2100,6 +2105,11 @@ pub const Cell = packed struct(u64) { prompt = 2, }; + /// The backing integer of this packed struct. Prefer this over + /// hardcoding the integer type so that code is resilient to the + /// size changing. + pub const Backing = @typeInfo(Cell).@"struct".backing_integer.?; + /// C ABI type. pub const C = u64; @@ -2194,6 +2204,270 @@ pub const Cell = packed struct(u64) { } }; +/// Returns a mask with all bits set for the given fields of the packed +/// struct T, used for masked compares of raw backing-integer values +/// (e.g. `Row.Backing`, `Cell.Backing`). This is an implementation +/// detail of `Mask`, which is the public API built on top of this. +fn fieldMask( + comptime T: type, + comptime fields: []const []const u8, +) @typeInfo(T).@"struct".backing_integer.? { + // Backing int of the packed struct + const Int = @typeInfo(T).@"struct".backing_integer.?; + + var mask: Int = 0; + inline for (fields) |field| { + // The type that fits all the bits we need to set. + const Ones = std.meta.Int( + .unsigned, + @bitSizeOf(@FieldType(T, field)), + ); + + // Mask out the ones + mask |= @as(Int, std.math.maxInt(Ones)) << @bitOffsetOf(T, field); + } + + return mask; +} + +/// A comptime-generated helper for classifying and comparing packed +/// struct values (e.g. Row, Cell) in bulk, using masked compares of +/// their raw backing integers. +/// +/// Masked compares are the key to making bulk row/cell processing fast. +/// Rows and cells are small packed structs specifically so that a single +/// integer load observes every field at once. A masked compare can then +/// answer a multi-field question with one AND and one compare, instead +/// of extracting and branching on each field individually (each packed +/// field access compiles to its own shift/mask). Just as importantly, +/// the integer form vectorizes trivially: `@splat` the mask and expected +/// value, and whole groups of rows or cells can be classified with a +/// few SIMD instructions. +/// +/// Some real examples of this in use: +/// +/// - Terminal print fast path: a cell can be overwritten by the +/// simple/fast path only if its content tag is a plain codepoint, +/// it has no style, isn't wide, and isn't a hyperlink. Masking +/// with those fields and comparing against a template answers all +/// four questions in one compare per cell. +/// +/// - Render state updates: a cell whose masked +/// `{content_tag, style_id}` bits are zero is a plain cell that +/// needs no managed-memory handling, so a vector OR-reduce can +/// skip entire groups of plain cells at once. Similarly, cells +/// whose masked bits equal the first cell's form a run sharing one +/// style, letting the update record one style lookup per run +/// rather than per cell. +/// +/// - Dirty scans: OR-reducing groups of rows against the `dirty` +/// field mask finds whether any row in the group needs a rebuild +/// without touching each row's flag individually. +/// +/// T is the packed struct type, fields are the fields covered by the +/// mask, and group_len is the number of values processed at once by +/// the group (vectorized) operations. Callers typically scan a slice +/// with the group operations and fall back to the scalar variants for +/// the remainder and for pinpointing values within a matched group. +pub fn Mask( + comptime T: type, + comptime fields: []const []const u8, + comptime group_len_param: comptime_int, +) type { + return struct { + const Backing = @typeInfo(T).@"struct".backing_integer.?; + const mask: Backing = fieldMask(T, fields); + + /// The number of values processed at once by group operations. + pub const group_len = group_len_param; + + /// A group of raw values for the vectorized operations. + const Group = @Vector(group_len, Backing); + + /// Load a group of values from the slice starting at index i. + /// Asserts that at least group_len values are available. + inline fn load(values: []const T, i: usize) Group { + return @bitCast(values[i..][0..group_len].*); + } + + /// Returns the raw backing bits of a single value. + pub inline fn bits(v: T) Backing { + return @bitCast(v); + } + + /// Returns the masked bits of a single value: the bits of the + /// masked fields with all other fields zeroed. Use this to + /// build the expected value for the eql functions. + pub inline fn pattern(v: T) Backing { + return bits(v) & mask; + } + + /// Returns the backing bits of a single value with the masked + /// fields zeroed: the complement of `pattern`. Use this to + /// compare values while ignoring the masked fields. + pub inline fn strip(v: T) Backing { + return bits(v) & ~mask; + } + + /// Returns true if every value in the group of group_len + /// values starting at index i matches, where a value matches + /// when none of the masked fields have any bits set: false + /// for bools, zero for ints, the zero tag for enums, and so + /// on. Asserts that at least group_len values are available. + pub inline fn match(values: []const T, i: usize) bool { + return @reduce(.Or, load(values, i)) & mask == 0; + } + + /// Scalar variant of `match` for a single value. + pub inline fn matchScalar(v: T) bool { + return bits(v) & mask == 0; + } + + /// Returns true if the masked fields of every value in the + /// group of group_len values starting at index i equal the + /// expected pattern (see `pattern`). + /// + /// This is a masked compare: fields outside the mask may vary + /// freely. Use this to detect runs of values that share the + /// masked field contents while other fields differ, e.g. a run + /// of cells with the same style ID but different codepoints. + /// If the result you derive from a run depends on fields + /// outside the mask, use `eqlExact` instead. + pub inline fn eql( + values: []const T, + i: usize, + expected: Backing, + ) bool { + const masked = load(values, i) & @as(Group, @splat(mask)); + return @reduce(.And, masked == @as(Group, @splat(expected))); + } + + /// Scalar variant of `eql` for a single value. + pub inline fn eqlScalar(v: T, expected: Backing) bool { + return pattern(v) == expected; + } + + /// Like `eql` but returns the number of leading values whose + /// masked fields equal the expected pattern, i.e. group_len if + /// the entire group matches. This is useful for early-exit run + /// scans that need to pinpoint exactly where a run ends rather + /// than only whether the whole group matches. + pub inline fn eqlPrefix( + values: []const T, + i: usize, + expected: Backing, + ) usize { + const masked = load(values, i) & @as(Group, @splat(mask)); + const ok = masked == @as(Group, @splat(expected)); + + // Test the whole group before extracting the prefix + // count: turning a vector compare into a scalar bitmask + // is expensive on some targets (e.g. NEON has no movemask + // instruction) and run scans overwhelmingly see fully + // matching groups, so we only pay for the extraction on + // the final group of a run. + if (@reduce(.And, ok)) { + @branchHint(.likely); + return group_len; + } + + const ok_bits: std.meta.Int( + .unsigned, + group_len, + ) = @bitCast(ok); + return @ctz(~ok_bits); + } + + /// Returns true if every value in the group of group_len + /// values starting at index i is bit-identical to the expected + /// value. Note: this compares entire values; it is NOT + /// affected by the field mask. + /// + /// This exists alongside `eql` for run detection where the + /// derived result depends on fields outside the mask, so a + /// masked compare would incorrectly extend the run. For + /// example, the background color of a bg-color cell lives in + /// the content field: two such cells only share a background + /// if their content bits are identical, not merely their + /// content tag. Values in such runs are typically produced by + /// bulk fills (e.g. erase with a pending background) and are + /// bit-identical in practice, so exact equality is both + /// correct and cheap. + pub inline fn eqlExact( + values: []const T, + i: usize, + expected: Backing, + ) bool { + const group = load(values, i); + return @reduce(.And, group == @as(Group, @splat(expected))); + } + }; +} + +test "Mask" { + const M = Mask(Cell, &.{ "content_tag", "style_id" }, 4); + + const plain: Cell = .init('A'); + var styled: Cell = .init('B'); + styled.style_id = 5; + var styled2: Cell = .init('C'); + styled2.style_id = 5; + var other: Cell = .init('D'); + other.style_id = 6; + + // match: plain cells only + { + var cells: [4]Cell = .{ plain, plain, plain, plain }; + try testing.expect(M.match(&cells, 0)); + try testing.expect(M.matchScalar(plain)); + + cells[2] = styled; + try testing.expect(!M.match(&cells, 0)); + try testing.expect(!M.matchScalar(styled)); + } + + // eql: runs of matching masked fields, other fields may vary + { + const expected = M.pattern(styled); + var cells: [4]Cell = .{ styled, styled2, styled, styled2 }; + try testing.expect(M.eql(&cells, 0, expected)); + try testing.expect(M.eqlScalar(styled2, expected)); + + cells[1] = other; + try testing.expect(!M.eql(&cells, 0, expected)); + try testing.expect(!M.eqlScalar(other, expected)); + } + + // eqlPrefix: count of leading values matching the pattern + { + const expected = M.pattern(styled); + var cells: [4]Cell = .{ styled, styled2, other, styled }; + try testing.expectEqual(2, M.eqlPrefix(&cells, 0, expected)); + + cells[2] = styled; + try testing.expectEqual(4, M.eqlPrefix(&cells, 0, expected)); + } + + // eqlExact: bit-identical values only + { + const expected = M.bits(styled); + var cells: [4]Cell = .{ styled, styled, styled, styled }; + try testing.expect(M.eqlExact(&cells, 0, expected)); + + // Same masked fields but different codepoint is not exact. + cells[3] = styled2; + try testing.expect(!M.eqlExact(&cells, 0, expected)); + } + + // strip: compare values while ignoring the masked fields + { + var styled_other: Cell = .init('B'); + styled_other.style_id = 6; + try testing.expectEqual(M.strip(styled), M.strip(styled_other)); + try testing.expect(M.strip(styled) != M.strip(styled2)); + } +} + // Uncomment this when you want to do some math. // test "Page size calculator" { // const total_size = alignForward( diff --git a/src/terminal/render.zig b/src/terminal/render.zig index 98e142245..78906a663 100644 --- a/src/terminal/render.zig +++ b/src/terminal/render.zig @@ -41,6 +41,29 @@ const Terminal = @import("Terminal.zig"); /// defer state.deinit(alloc); /// state.update(alloc, &terminal); /// +/// ## Two-Phase Updates +/// +/// For callers that synchronize terminal access (e.g. a renderer thread +/// sharing a lock with an IO thread), the update can be split into two +/// phases to minimize the time the terminal must be held exclusively: +/// `beginUpdate` requires terminal access, while `endUpdate` completes +/// any deferred work using only memory owned by the render state. +/// +/// { +/// mutex.lock(); +/// defer mutex.unlock(); +/// try state.beginUpdate(alloc, &terminal); +/// } +/// +/// // The IO thread is free to modify the terminal while we +/// // complete the update. +/// state.endUpdate(); +/// +/// The render state must be treated as incomplete between the two calls. +/// `update` is a convenience that performs both phases in one call. +/// +/// ## Memory +/// /// Note: the render state retains as much memory as possible between updates /// to prevent future allocations. If a very large frame is rendered once, /// the render state will retain that much memory until deinit. To avoid @@ -91,6 +114,16 @@ pub const RenderState = struct { /// if possible. selection_cache: ?SelectionCache = null, + /// The pending style runs requiring an endUpdate call, in the + /// order they were recorded. If multiple begins happen without an + /// endUpdate call, runs accumulate; rows rebuilt more than once + /// may then have superseded (stale) runs in this list, which is + /// harmless: newer runs are appended later so they win, and cells + /// not covered by newer runs have a default style ID in their raw + /// data so their style is undefined by contract anyway. See + /// beginUpdate. + pending_styles: std.ArrayList(StyleRun) = .empty, + /// Initial state. pub const empty: RenderState = .{ .rows = 0, @@ -244,6 +277,23 @@ pub const RenderState = struct { br_pin: PageList.Pin, }; + /// A run of cells within one row sharing one style, pending + /// denormalization into the per-cell data. This is populated by + /// `beginUpdate` and consumed by `endUpdate`. This exists so that + /// the (potentially large) denormalization of styles into cells + /// can happen outside of any terminal locks. See `beginUpdate`. + pub const StyleRun = struct { + /// The viewport row. + y: size.CellCountInt, + + /// Start (inclusive) and end (exclusive) x coordinates. + start: size.CellCountInt, + end: size.CellCountInt, + + /// The style for this cell range. + style: Style, + }; + pub fn deinit(self: *RenderState, alloc: Allocator) void { for ( self.row_data.items(.arena), @@ -254,16 +304,53 @@ pub const RenderState = struct { cells.deinit(alloc); } self.row_data.deinit(alloc); + self.pending_styles.deinit(alloc); } /// Update the render state to the latest terminal state. /// + /// This is a convenience function that performs a full update in + /// one call, equivalent to `beginUpdate` immediately followed by + /// `endUpdate`. Callers that hold a lock over the terminal state + /// should prefer calling the two phases directly so that the lock + /// is only held for `beginUpdate`. + /// /// This will reset the terminal dirty state since it is consumed /// by this render state update. pub fn update( self: *RenderState, alloc: Allocator, t: *Terminal, + ) Allocator.Error!void { + try self.beginUpdate(alloc, t); + self.endUpdate(); + } + + /// Begin an update of the render state to the latest terminal + /// state. Every begin must be completed with an `endUpdate` call + /// before the render state is read. + /// + /// This two-phase structure exists for callers that lock the + /// terminal state: only this function requires terminal access, so + /// a caller can hold its lock for this call only and then call + /// `endUpdate` after releasing it. `endUpdate` exclusively reads + /// and writes memory owned by the render state. + /// + /// Work that doesn't require terminal access may be deferred to + /// `endUpdate` to keep this call (and therefore lock hold time) as + /// short as possible. At the time of writing, the deferred work is + /// the per-cell style denormalization, so between this call and + /// `endUpdate` the per-cell `style` data of any updated rows is + /// stale and must not be read. More work may be deferred in the + /// future; callers should treat the render state as incomplete + /// until `endUpdate` is called. + /// + /// This will reset the terminal dirty state since it is consumed + /// by this render state update. + pub fn beginUpdate( + self: *RenderState, + alloc: Allocator, + t: *Terminal, ) Allocator.Error!void { const s: *Screen = t.screens.active; const viewport_pin = s.pages.getTopLeft(.viewport); @@ -322,7 +409,14 @@ pub const RenderState = struct { // Colors. self.colors.cursor = t.colors.cursor.get(); - self.colors.palette = t.colors.palette.current; + + // The palette is a relatively large copy (768 bytes at the time + // of writing) so we only copy it when it could have changed. All + // palette modifications set a terminal-level dirty flag (see + // Terminal.Dirty.palette), and any terminal-level dirty flag + // forces a redraw, so checking redraw is sufficient. + if (redraw) self.colors.palette = t.colors.palette.current; + bg_fg: { // Background/foreground can be unset initially which would // depend on "default" background/foreground. The expected use @@ -389,27 +483,54 @@ pub const RenderState = struct { const row_highlights = row_data.items(.highlights); const row_dirties = row_data.items(.dirty); - // Track the last page that we know was dirty. This lets us - // more quickly do the full-page dirty check. - var last_dirty_page: ?*page.Page = null; + // If we're redrawing then every row will be rebuilt, superseding + // any pending style runs from prior updates. Clearing also + // guarantees pending runs always match the current dimensions + // (dimension changes force a redraw). + if (redraw) self.pending_styles.clearRetainingCapacity(); - // Go through and setup our rows. - var row_it = s.pages.rowIterator( - .right_down, - .{ .viewport = .{} }, - null, - ); - var y: size.CellCountInt = 0; + // Go through and setup our rows. We iterate page chunks rather + // than individual rows so that per-page work (dirty flags, cursor + // detection, memory pointers) is hoisted out of the row loop. This + // makes the common case of a clean (or mostly clean) frame very + // cheap: a contiguous scan of row dirty flags. + const builder: RowBuilder = .{ + .alloc = alloc, + .cols = self.cols, + .arenas = row_arenas, + .raws = row_rows, + .cells = row_cells, + .sels = row_sels, + .highlights = row_highlights, + .dirties = row_dirties, + .pending_styles = &self.pending_styles, + }; + var y: usize = 0; var any_dirty: bool = false; - while (row_it.next()) |row_pin| : (y = y + 1) { + var page_it = viewport_pin.pageIterator(.right_down, null); + while (y < self.rows) { + const chunk = page_it.next() orelse break; + const node = chunk.node; + const p: *page.Page = &node.data; + + // The number of rows we consume from this chunk. The chunk + // may extend beyond the viewport (the viewport is always + // exactly `rows` tall) so we clamp. + const take: usize = @min( + @as(usize, chunk.end - chunk.start), + self.rows - y, + ); + // Find our cursor if we haven't found it yet. We do this even - // if the row is not dirty because the cursor is unrelated. + // if rows are not dirty because the cursor is unrelated. We + // can check the chunk bounds once rather than every row. if (self.cursor.viewport == null and - row_pin.node == s.cursor.page_pin.node and - row_pin.y == s.cursor.page_pin.y) - { + node == s.cursor.page_pin.node) + cursor: { + const cy = s.cursor.page_pin.y; + if (cy < chunk.start or cy >= chunk.start + take) break :cursor; self.cursor.viewport = .{ - .y = y, + .y = @intCast(y + (cy - chunk.start)), .x = s.cursor.x, // Future: we should use our own state here to look this @@ -421,135 +542,73 @@ pub const RenderState = struct { }; } - // Store our pin. We have to store these even if we're not dirty - // because dirty is only a renderer optimization. It doesn't - // apply to memory movement. This will let us remap any cell - // pins back to an exact entry in our RenderState. - row_pins[y] = row_pin; + // The page-level dirty flag applies to every row in the chunk. + // We consume (clear) it now; each node appears at most once in + // this iteration and we're the only consumer of dirty state. + const page_dirty = p.dirty; + if (page_dirty) p.dirty = false; - // Get all our cells in the page. - const p: *page.Page = &row_pin.node.data; - const page_rac = row_pin.rowAndCell(); + // Get our contiguous rows for this chunk. + const page_rows: []page.Row = p.rows.ptr(p.memory)[chunk.start..][0..take]; + assert(p.size.cols == self.cols); - dirty: { - // If we're redrawing then we're definitely dirty. - if (redraw) break :dirty; - - // If our page is the same as last time then its dirty. - if (p == last_dirty_page) break :dirty; - if (p.dirty) { - // If this page is dirty then clear the dirty flag - // of the last page and then store this one. This benchmarks - // faster than iterating pages again later. - if (last_dirty_page) |last_p| last_p.dirty = false; - last_dirty_page = p; - break :dirty; + // Store our pins. We have to store these even for rows that + // aren't dirty because dirty is only a renderer optimization; + // it doesn't apply to memory movement. This lets us remap any + // cell pins back to an exact entry in our RenderState. + // + // We can skip the writes when the pins are unchanged: if we're + // not redrawing, every pin was stored by a prior update (row + // count changes force a redraw). Within a single update a node + // appears at most once and its stored pins have consecutive y + // values, so if the first and last pins of this chunk's range + // already match then every pin in between matches too. + if (redraw or + row_pins[y].node != node or + row_pins[y].y != chunk.start or + row_pins[y + take - 1].node != node or + row_pins[y + take - 1].y != chunk.start + take - 1) + { + for (row_pins[y..][0..take], chunk.start..) |*pin, py| { + pin.* = .{ .node = node, .y = @intCast(py) }; } - - // If our row is dirty then we're dirty. - if (page_rac.row.dirty) break :dirty; - - // Not dirty! - continue; } - // Set that at least one row was dirty. - any_dirty = true; - - // Clear our row dirty, we'll clear our page dirty later. - // We can't clear it now because we have more rows to go through. - page_rac.row.dirty = false; - - // Promote our arena. State is copied by value so we need to - // restore it on all exit paths so we don't leak memory. - var arena = row_arenas[y].promote(alloc); - defer row_arenas[y] = arena.state; - - // Reset our cells if we're rebuilding this row. - if (row_cells[y].len > 0) { - _ = arena.reset(.retain_capacity); - row_cells[y].clearRetainingCapacity(); - row_sels[y] = null; - row_highlights[y] = .empty; - } - row_dirties[y] = true; - - // Get all our cells in the page. - const page_cells: []const page.Cell = p.getCells(page_rac.row); - assert(page_cells.len == self.cols); - - // Copy our raw row data - row_rows[y] = page_rac.row.*; - - // Note: our cells MultiArrayList uses our general allocator. - // We do this on purpose because as rows become dirty, we do - // not want to reallocate space for cells (which are large). This - // was a source of huge slowdown. - // - // Our per-row arena is only used for temporary allocations - // pertaining to cells directly (e.g. graphemes, hyperlinks). - const cells: *std.MultiArrayList(Cell) = &row_cells[y]; - try cells.resize(alloc, self.cols); - - // We always copy our raw cell data. In the case we have no - // managed memory, we can skip setting any other fields. - // - // This is an important optimization. For plain-text screens - // this ends up being something around 300% faster based on - // the `screen-clone` benchmark. - const cells_slice = cells.slice(); - fastmem.copy( - page.Cell, - cells_slice.items(.raw), - page_cells, - ); - if (!page_rac.row.managedMemory()) continue; - - const arena_alloc = arena.allocator(); - const cells_grapheme = cells_slice.items(.grapheme); - const cells_style = cells_slice.items(.style); - for (page_cells, 0..) |*page_cell, x| { - // Append assuming its a single-codepoint, styled cell - // (most common by far). - if (page_cell.style_id > 0) cells_style[x] = p.styles.get( - p.memory, - page_cell.style_id, - ).*; - - // Switch on our content tag to handle less likely cases. - switch (page_cell.content_tag) { - .codepoint => { + if (!redraw and !page_dirty) { + // Only dirty rows (usually none) need a rebuild. Scan the + // dirty flags a group at a time; the dirty bit is directly + // testable on the packed row representation. + var i: usize = 0; + while (take - i >= RowDirtyMask.group_len) : (i += RowDirtyMask.group_len) { + if (RowDirtyMask.match(page_rows, i)) { @branchHint(.likely); - // Primary codepoint goes into `raw` field. - }, + continue; + } - // If we have a multi-codepoint grapheme, look it up and - // set our content type. - .codepoint_grapheme => { - @branchHint(.unlikely); - cells_grapheme[x] = try arena_alloc.dupe( - u21, - p.lookupGrapheme(page_cell) orelse &.{}, - ); - }, - - .bg_color_rgb => { - @branchHint(.unlikely); - cells_style[x] = .{ .bg_color = .{ .rgb = .{ - .r = page_cell.content.color_rgb.r, - .g = page_cell.content.color_rgb.g, - .b = page_cell.content.color_rgb.b, - } } }; - }, - - .bg_color_palette => { - @branchHint(.unlikely); - cells_style[x] = .{ .bg_color = .{ - .palette = page_cell.content.color_palette, - } }; - }, + for (page_rows[i..][0..RowDirtyMask.group_len], i..) |*page_row, j| { + if (!page_row.dirty) continue; + page_row.dirty = false; + any_dirty = true; + try builder.row(p, page_row, y + j); + } + } + while (i < take) : (i += 1) { + const page_row = &page_rows[i]; + if (!page_row.dirty) continue; + page_row.dirty = false; + any_dirty = true; + try builder.row(p, page_row, y + i); + } + } else { + // Rebuild every row in the chunk. + any_dirty = true; + for (page_rows, 0..) |*page_row, i| { + page_row.dirty = false; + try builder.row(p, page_row, y + i); } } + + y += take; } assert(y == self.rows); @@ -640,14 +699,40 @@ pub const RenderState = struct { self.dirty = .partial; } - // Finalize our final dirty page - if (last_dirty_page) |last_p| last_p.dirty = false; - // Clear our dirty flags t.flags.dirty = .{}; s.dirty = .{}; } + /// Complete a prior `beginUpdate` call by performing any deferred + /// work. At the time of writing, this denormalizes the pending + /// style runs into the per-cell style data. + /// + /// This only reads and writes memory owned by the render state, so + /// it is safe to call while the terminal is being modified (no + /// terminal lock is required). + pub fn endUpdate(self: *RenderState) void { + // Common case: no styled rows were rebuilt. + if (self.pending_styles.items.len == 0) return; + + const row_data = self.row_data.slice(); + const row_cells = row_data.items(.cells); + for (self.pending_styles.items) |run| { + // Defensive: the row data may have changed shape if the + // caller violated ordering (e.g. an error path skipped an + // endUpdate between updates). Any update that changes + // dimensions clears the pending list (redraw), so this + // should never actually trigger, but the cost is trivial. + if (run.y >= row_cells.len) continue; + const styles = row_cells[run.y].slice().items(.style); + const end = @min(run.end, styles.len); + const start = @min(run.start, end); + + @memset(styles[start..end], run.style); + } + self.pending_styles.clearRetainingCapacity(); + } + /// Update the highlights in the render state from the given flattened /// highlights. Because this uses flattened highlights, it does not require /// reading from the terminal state so it should be done outside of @@ -879,6 +964,231 @@ pub const RenderState = struct { } }; +/// The number of rows/cells we scan as a single group when looking for +/// dirty rows or special cells. Rows and cells are small packed structs +/// so a group is scanned with a handful of vector operations. +const scan_group_len = 8; + +/// Group scan helper for the row dirty flag. A row that matches has +/// its dirty flag unset. +const RowDirtyMask = page.Mask( + page.Row, + &.{"dirty"}, + scan_group_len, +); + +/// Group scan helper for the cell fields that require managed memory +/// handling. A cell that matches is a plain (possibly zero) codepoint +/// with a default style, requiring no work beyond the raw copy. See +/// RowBuilder.row. +const CellSpecialMask = page.Mask(page.Cell, &.{ + "content_tag", + "style_id", +}, scan_group_len); + +/// Internal helper for RenderState.update that rebuilds a single row of +/// the render state from the current page contents. +const RowBuilder = struct { + alloc: Allocator, + cols: usize, + arenas: []ArenaAllocator.State, + raws: []page.Row, + cells: []std.MultiArrayList(RenderState.Cell), + sels: []?[2]size.CellCountInt, + highlights: []std.ArrayList(RenderState.Highlight), + dirties: []bool, + pending_styles: *std.ArrayList(RenderState.StyleRun), + + fn row( + b: *const RowBuilder, + p: *page.Page, + page_row: *const page.Row, + vy: usize, + ) Allocator.Error!void { + // Promote our arena. State is copied by value so we need to + // restore it on all exit paths so we don't leak memory. + var arena = b.arenas[vy].promote(b.alloc); + defer b.arenas[vy] = arena.state; + + // Reset our per-row state if we're rebuilding this row. A + // non-zero cell length means the row was populated by a prior + // update. + if (b.cells[vy].len > 0) { + _ = arena.reset(.retain_capacity); + b.sels[vy] = null; + b.highlights[vy] = .empty; + } + b.dirties[vy] = true; + + // Get all our cells in the page. + const page_cells: []const page.Cell = page_row.cells.ptr(p.memory)[0..b.cols]; + + // Copy our raw row data + b.raws[vy] = page_row.*; + + // Note: our cells MultiArrayList uses our general allocator. + // We do this on purpose because as rows become dirty, we do + // not want to reallocate space for cells (which are large). This + // was a source of huge slowdown. + // + // Our per-row arena is only used for temporary allocations + // pertaining to cells directly (e.g. graphemes, hyperlinks). + const cells: *std.MultiArrayList(RenderState.Cell) = &b.cells[vy]; + if (cells.len != b.cols) try cells.resize(b.alloc, b.cols); + + // We always copy our raw cell data. In the case we have no + // managed memory, we can skip setting any other fields. + // + // This is an important optimization. For plain-text screens + // this ends up being something around 300% faster based on + // the `screen-clone` benchmark. + const cells_slice = cells.slice(); + fastmem.copy( + page.Cell, + cells_slice.items(.raw), + page_cells, + ); + if (!page_row.managedMemory()) return; + + const arena_alloc = arena.allocator(); + const cells_grapheme = cells_slice.items(.grapheme); + const n = page_cells.len; + var x: usize = 0; + scan: while (x < n) { + // Skip runs of plain cells a group at a time. Cells that + // need managed handling are often rare even within rows that + // have managed memory (e.g. a row is "styled" if a single + // cell has a style) so groups are skipped with a few vector + // operations. + while (n - x >= CellSpecialMask.group_len) { + if (!CellSpecialMask.match(page_cells, x)) break; + x += CellSpecialMask.group_len; + } + + // Scalar scan to the next special cell. + while (true) { + if (x >= n) break :scan; + if (!CellSpecialMask.matchScalar(page_cells[x])) break; + x += 1; + } + + const page_cell = &page_cells[x]; + + switch (page_cell.content_tag) { + // Single-codepoint styled cells are by far the most + // common special cells, and they usually come in long + // runs sharing one style ID (e.g. a fully styled row + // usually uses a single style). Find the run and record + // it: this does one style lookup per run and defers the + // (large) per-cell fill to endUpdate, outside of any + // terminal locks. + .codepoint => { + @branchHint(.likely); + const sid = page_cell.style_id; + assert(sid > 0); // special + codepoint implies styled + const style_val: Style = p.styles.get(p.memory, sid).*; + + // A cell continues the run if its masked special + // bits are exactly the style ID of the run (in + // particular the content tag must be a plain + // codepoint). We can check groups of cells at a + // time this way. + const pattern = CellSpecialMask.pattern(page_cell.*); + const start = x; + x += 1; + while (n - x >= CellSpecialMask.group_len) { + if (!CellSpecialMask.eql( + page_cells, + x, + pattern, + )) break; + x += CellSpecialMask.group_len; + } + while (x < n) : (x += 1) { + if (!CellSpecialMask.eqlScalar( + page_cells[x], + pattern, + )) break; + } + + try b.pending_styles.append(b.alloc, .{ + .y = @intCast(vy), + .start = @intCast(start), + .end = @intCast(x), + .style = style_val, + }); + }, + + // If we have a multi-codepoint grapheme, look it up and + // set our content type. Note grapheme cells may also + // be styled. The style must be recorded as a run (rather + // than written directly) so that it is ordered correctly + // relative to possibly-stale runs from prior updates. + .codepoint_grapheme => { + if (page_cell.style_id > 0) { + try b.pending_styles.append(b.alloc, .{ + .y = @intCast(vy), + .start = @intCast(x), + .end = @intCast(x + 1), + .style = p.styles.get( + p.memory, + page_cell.style_id, + ).*, + }); + } + cells_grapheme[x] = try arena_alloc.dupe( + u21, + p.lookupGrapheme(page_cell) orelse &.{}, + ); + x += 1; + }, + + // Background-color-only cells. The style is derived + // entirely from the cell contents. Consecutive cleared + // cells with the same background are bit-identical, so + // we run-detect on full equality (e.g. a line cleared + // with a background color pending is one run). + .bg_color_rgb, .bg_color_palette => { + const style_val: Style = switch (page_cell.content_tag) { + .bg_color_rgb => .{ .bg_color = .{ .rgb = .{ + .r = page_cell.content.color_rgb.r, + .g = page_cell.content.color_rgb.g, + .b = page_cell.content.color_rgb.b, + } } }, + .bg_color_palette => .{ .bg_color = .{ + .palette = page_cell.content.color_palette, + } }, + else => unreachable, + }; + + const first_bits = CellSpecialMask.bits(page_cell.*); + const start = x; + x += 1; + while (n - x >= CellSpecialMask.group_len) { + if (!CellSpecialMask.eqlExact( + page_cells, + x, + first_bits, + )) break; + x += CellSpecialMask.group_len; + } + while (x < n) : (x += 1) { + if (CellSpecialMask.bits(page_cells[x]) != first_bits) + break; + } + + try b.pending_styles.append(b.alloc, .{ + .y = @intCast(vy), + .start = @intCast(start), + .end = @intCast(x), + .style = style_val, + }); + }, + } + } + } +}; + test "styled" { const testing = std.testing; const alloc = testing.allocator; @@ -979,6 +1289,331 @@ test "styled text" { try testing.expectEqual(0, cells[0].get(3).raw.codepoint()); } +/// Verifies that an incrementally updated render state has identical +/// contents to a from-scratch rebuild. This is the load-bearing check +/// for our dirty tracking: if any terminal operation changes row +/// contents without setting a dirty signal that `update` honors +/// (terminal dirty, screen dirty, page dirty, row dirty, viewport pin, +/// or dimensions), the incremental state will contain stale rows and +/// this comparison will fail. +fn testCompareStates( + incremental: *const RenderState, + fresh: *const RenderState, +) !void { + const testing = std.testing; + + // Row metadata that is allowed to be stale in an incremental + // update. Dirty tracking only guarantees that VISUAL changes are + // flagged (see page.Row.dirty); these fields are non-visual + // metadata that the terminal may change without dirtying the row + // (e.g. Screen.cursorResetWrap clears wrap flags without a dirty + // mark). This staleness predates the chunked update + // implementation; it is present in the row-iterator implementation + // as well. + const StaleOkMask = page.Mask(page.Row, &.{ + "wrap", + "wrap_continuation", + "semantic_prompt", + "dirty", + }, 1); + + try testing.expectEqual(fresh.rows, incremental.rows); + try testing.expectEqual(fresh.cols, incremental.cols); + try testing.expectEqual(fresh.cursor.active, incremental.cursor.active); + try testing.expectEqual(fresh.cursor.viewport, incremental.cursor.viewport); + try testing.expectEqual( + @as(page.Cell.Backing, @bitCast(fresh.cursor.cell)), + @as(page.Cell.Backing, @bitCast(incremental.cursor.cell)), + ); + + const inc_data = incremental.row_data.slice(); + const new_data = fresh.row_data.slice(); + try testing.expectEqual(new_data.len, inc_data.len); + for (0..new_data.len) |y| { + errdefer std.log.warn("mismatch on row y={}", .{y}); + + // Pins must match exactly. + const inc_pin = inc_data.items(.pin)[y]; + const new_pin = new_data.items(.pin)[y]; + try testing.expectEqual(new_pin.node, inc_pin.node); + try testing.expectEqual(new_pin.y, inc_pin.y); + + // Raw row data must match, except for non-visual metadata + // fields which may legitimately be stale (see StaleOkMask). + const inc_row = inc_data.items(.raw)[y]; + const new_row = new_data.items(.raw)[y]; + try testing.expectEqual( + StaleOkMask.strip(new_row), + StaleOkMask.strip(inc_row), + ); + + const inc_cells = inc_data.items(.cells)[y].slice(); + const new_cells = new_data.items(.cells)[y].slice(); + try testing.expectEqual(new_cells.len, inc_cells.len); + const managed = new_row.managedMemory(); + for (0..new_cells.len) |x| { + errdefer std.log.warn("mismatch on cell x={}", .{x}); + + // Raw cell contents must match. + const inc_cell = inc_cells.items(.raw)[x]; + const new_cell = new_cells.items(.raw)[x]; + try testing.expectEqual( + @as(page.Cell.Backing, @bitCast(new_cell)), + @as(page.Cell.Backing, @bitCast(inc_cell)), + ); + + // The style is only defined if the cell is styled or is + // a bg-color cell within a row that has managed memory. + if (new_cell.style_id != 0 or + (managed and switch (new_cell.content_tag) { + .bg_color_rgb, .bg_color_palette => true, + else => false, + })) + { + try testing.expect(std.meta.eql( + new_cells.items(.style)[x], + inc_cells.items(.style)[x], + )); + } + + // Graphemes are only defined for grapheme cells. + if (new_cell.content_tag == .codepoint_grapheme) { + try testing.expectEqualSlices( + u21, + new_cells.items(.grapheme)[x], + inc_cells.items(.grapheme)[x], + ); + } + } + } +} + +test "incremental updates match full rebuild" { + const testing = std.testing; + const alloc = testing.allocator; + + // Deterministic so failures are reproducible. + var prng = std.Random.DefaultPrng.init(0xB0BA_CAFE); + const rand = prng.random(); + + var t = try Terminal.init(alloc, .{ + .cols = 20, + .rows = 8, + .max_scrollback = 500, + }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + + var inc: RenderState = .empty; + defer inc.deinit(alloc); + + var buf: [64]u8 = undefined; + for (0..300) |_| { + // Perform a random batch of operations between updates. + for (0..rand.intRangeAtMost(usize, 1, 6)) |_| { + switch (rand.intRangeAtMost(u8, 0, 18)) { + // Plain text (possibly wrapping and scrolling). + 0, 1, 2 => for (0..rand.intRangeAtMost(usize, 1, 30)) |_| { + s.nextSlice(&.{rand.intRangeAtMost(u8, 'A', 'Z')}); + }, + + // Newlines to build scrollback and trigger pruning. + 3, 4 => for (0..rand.intRangeAtMost(usize, 1, 10)) |_| { + s.nextSlice("x\r\n"); + }, + + // Cursor movement. + 5 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{};{}H", .{ + rand.intRangeAtMost(u16, 1, 8), + rand.intRangeAtMost(u16, 1, 20), + })), + + // Styling: bold, truecolor bg, palette fg, reset. + 6 => s.nextSlice(switch (rand.intRangeAtMost(u8, 0, 3)) { + 0 => "\x1b[1m", + 1 => "\x1b[48;2;30;60;90m", + 2 => "\x1b[38;5;120m", + else => "\x1b[0m", + }), + + // Erase ops (EL, ED variants including scrollback). + 7 => s.nextSlice(switch (rand.intRangeAtMost(u8, 0, 4)) { + 0 => "\x1b[K", + 1 => "\x1b[1K", + 2 => "\x1b[J", + 3 => "\x1b[2J", + else => "\x1b[3J", + }), + + // Insert/delete lines (row rotations within regions). + 8 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}L", .{ + rand.intRangeAtMost(u16, 1, 4), + })), + 9 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}M", .{ + rand.intRangeAtMost(u16, 1, 4), + })), + + // Scroll up/down (page-dirty row rotations). + 10 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}S", .{ + rand.intRangeAtMost(u16, 1, 4), + })), + 11 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}T", .{ + rand.intRangeAtMost(u16, 1, 4), + })), + + // Set/reset scroll regions to exercise bounded scrolls. + 12 => { + const top = rand.intRangeAtMost(u16, 1, 4); + const bot = rand.intRangeAtMost(u16, top + 1, 8); + s.nextSlice(try std.fmt.bufPrint( + &buf, + "\x1b[{};{}r", + .{ top, bot }, + )); + }, + + // Insert/delete/erase chars within a row. + 13 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}@", .{ + rand.intRangeAtMost(u16, 1, 5), + })), + 14 => s.nextSlice(try std.fmt.bufPrint(&buf, "\x1b[{}P", .{ + rand.intRangeAtMost(u16, 1, 5), + })), + + // Reverse index (scroll down at top). + 15 => s.nextSlice("\x1bM"), + + // Wide chars and multi-codepoint graphemes. + 16 => s.nextSlice("字👨‍👩‍👧"), + + // Alternate screen switching (screen key redraw path). + 17 => s.nextSlice(if (rand.boolean()) + "\x1b[?1049h" + else + "\x1b[?1049l"), + + // DECALN full-screen fill. + 18 => s.nextSlice("\x1b#8"), + + else => unreachable, + } + } + + // Occasionally scroll the viewport into scrollback and back. + switch (rand.intRangeAtMost(u8, 0, 9)) { + 0 => t.scrollViewport(.{ .delta = -3 }), + 1 => t.scrollViewport(.{ .delta = 2 }), + 2 => t.scrollViewport(.bottom), + 3 => t.scrollViewport(.top), + else => {}, + } + + // Update our incremental state first: it must consume the dirty + // state. The fresh state always fully rebuilds (its dimensions + // start empty so it always redraws) and so does not depend on + // any dirty flags. + try inc.update(alloc, &t); + + var fresh: RenderState = .empty; + defer fresh.deinit(alloc); + try fresh.update(alloc, &t); + + try testCompareStates(&inc, &fresh); + } +} + +test "begin and end update" { + const testing = std.testing; + const alloc = testing.allocator; + + var t = try Terminal.init(alloc, .{ + .cols = 10, + .rows = 3, + }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + s.nextSlice("\x1b[1mAB"); // Bold + s.nextSlice("\x1b[0;3mC"); // Italic + + var state: RenderState = .empty; + defer state.deinit(alloc); + try state.beginUpdate(alloc, &t); + + // We should have pending style runs on row 0: one for the bold + // run and one for the italic run. + { + const runs = state.pending_styles.items; + try testing.expectEqual(2, runs.len); + try testing.expectEqual(0, runs[0].y); + try testing.expectEqual(0, runs[0].start); + try testing.expectEqual(2, runs[0].end); + try testing.expect(runs[0].style.flags.bold); + try testing.expectEqual(0, runs[1].y); + try testing.expectEqual(2, runs[1].start); + try testing.expectEqual(3, runs[1].end); + try testing.expect(runs[1].style.flags.italic); + } + + // End our update. This should denormalize the runs into cells + // and clear the pending runs. + state.endUpdate(); + { + try testing.expectEqual(0, state.pending_styles.items.len); + + const row_data = state.row_data.slice(); + const cells = row_data.items(.cells); + try testing.expect(cells[0].get(0).style.flags.bold); + try testing.expect(cells[0].get(1).style.flags.bold); + try testing.expect(cells[0].get(2).style.flags.italic); + } +} + +test "bg color cells" { + const testing = std.testing; + const alloc = testing.allocator; + + var t = try Terminal.init(alloc, .{ + .cols = 10, + .rows = 3, + }); + defer t.deinit(alloc); + + var s = t.vtStream(); + defer s.deinit(); + + // Write a styled cell (so the row has managed memory) then erase + // the rest of the line with a palette background pending. The + // erase produces bg_color content cells rather than styled cells. + s.nextSlice("\x1b[1mA\x1b[48;5;1m\x1b[K"); + + var state: RenderState = .empty; + defer state.deinit(alloc); + try state.update(alloc, &t); + + const row_data = state.row_data.slice(); + const cells = row_data.items(.cells); + { + const cell = cells[0].get(0); + try testing.expectEqual('A', cell.raw.codepoint()); + try testing.expect(cell.style.flags.bold); + } + for (1..10) |x| { + const cell = cells[0].get(x); + try testing.expectEqual( + page.Cell.ContentTag.bg_color_palette, + cell.raw.content_tag, + ); + try testing.expectEqual( + Style.Color{ .palette = 1 }, + cell.style.bg_color, + ); + } +} + test "grapheme" { const testing = std.testing; const alloc = testing.allocator;