diff --git a/src/benchmark/PageCompression.zig b/src/benchmark/PageCompression.zig index 04dc7ef32..21153659e 100644 --- a/src/benchmark/PageCompression.zig +++ b/src/benchmark/PageCompression.zig @@ -4,7 +4,8 @@ //! This benchmark is intentionally independent of terminal page ownership and //! lifecycle. It treats its input as opaque bytes and calls only the standalone //! LZ4 block codec. In particular, it does not compress pages owned by a live -//! terminal and is not evidence that compression is enabled in production. +//! terminal or measure the scheduling and state transitions used by automatic +//! scrollback compression. //! //! ## Input //! diff --git a/src/benchmark/ScrollbackCompression.zig b/src/benchmark/ScrollbackCompression.zig index 3684a91b3..85ca62fcd 100644 --- a/src/benchmark/ScrollbackCompression.zig +++ b/src/benchmark/ScrollbackCompression.zig @@ -38,9 +38,10 @@ //! //! A fully historical page is a node strictly before the node containing the //! top of the active area. The boundary node is deliberately excluded because -//! it can contain both history and active rows. Compression is currently an -//! on-demand PageList operation only; this benchmark does not enable it in -//! normal terminal execution. +//! it can contain both history and active rows. Normal terminal execution +//! compresses these pages incrementally after output becomes idle. The +//! benchmark invokes PageList operations directly so its timed regions exclude +//! the production scheduler's idle delay and renderer-thread coordination. //! //! ## Examples //! diff --git a/src/config/Config.zig b/src/config/Config.zig index fa491e49c..77043647c 100644 --- a/src/config/Config.zig +++ b/src/config/Config.zig @@ -1373,10 +1373,16 @@ input: RepeatableReadableIO = .{}, /// When this limit is reached, the oldest lines are removed from the /// scrollback. /// -/// Scrollback currently exists completely in memory. This means that the -/// larger this value, the larger potential memory usage. Scrollback is -/// allocated lazily up to this limit, so if you set this to a very large -/// value, it will not immediately consume a lot of memory. +/// Scrollback is stored in memory and allocated lazily up to this limit, so +/// setting a very large limit does not immediately consume that amount of +/// memory. On supported systems, Ghostty attempts to compress fully historical +/// pages while the terminal is idle. This can reduce physical memory usage, +/// depending on the contents of the scrollback. +/// +/// This limit always measures the uncompressed logical size of the terminal +/// pages. Compression does not allow Ghostty to retain more history than the +/// configured limit. Accessing compressed history restores it transparently +/// and may increase the terminal's physical memory usage again. /// /// This size is per terminal surface, not for the entire application. /// diff --git a/src/inspector/widgets/pagelist.zig b/src/inspector/widgets/pagelist.zig index 4041ba332..fac6afbed 100644 --- a/src/inspector/widgets/pagelist.zig +++ b/src/inspector/widgets/pagelist.zig @@ -1,12 +1,24 @@ const std = @import("std"); const cimgui = @import("dcimgui"); const terminal = @import("../../terminal/main.zig"); +const pagepkg = @import("../../terminal/page.zig"); const stylepkg = @import("../../terminal/style.zig"); const widgets = @import("../widgets.zig"); const units = @import("../units.zig"); const PageList = terminal.PageList; +/// Cell pointers resolved against a Node.PreservedPage rather than the +/// PageList node. +/// Omitting the node makes it impossible for inspector helpers to +/// accidentally restore the original compressed page. +const InspectedCell = struct { + row: *pagepkg.Row, + cell: *pagepkg.Cell, + row_idx: terminal.size.CellCountInt, + col_idx: terminal.size.CellCountInt, +}; + /// PageList inspector widget. pub const Inspector = struct { pub const empty: Inspector = .{}; @@ -82,11 +94,18 @@ pub const Inspector = struct { })) continue; defer cimgui.c.ImGui_TreePop(); - // Opening a compressed entry is an explicit request for its - // contents, so restoration is appropriate here. Collapsed - // entries use only metadata and remain compressed. - const page = resident orelse page_node.page(); - widgets.page.inspector(page); + // Decode compressed contents into a temporary page. The + // original node stays compressed while its entry is open. + var preserved = page_node.pagePreservingState( + std.heap.page_allocator, + ) catch { + cimgui.c.ImGui_TextDisabled( + "(unable to copy compressed page)", + ); + continue; + }; + defer preserved.deinit(); + widgets.page.inspector(preserved.page()); } } } @@ -627,16 +646,35 @@ pub const CellChooser = struct { .history => terminal.Point{ .history = self.lookup_coord }, }; - const cell = pages.getCell(pt) orelse { + const pin = pages.pin(pt) orelse { cimgui.c.ImGui_TextDisabled("(cell out of range)"); return; }; - self.cell_info.draw(cell, pt); + // Cell pointers must come from the same page whose auxiliary tables + // we inspect. For compressed nodes this is an independent preserved + // page, leaving the PageList node compressed across inspector frames. + var preserved = pin.node.pagePreservingState( + std.heap.page_allocator, + ) catch { + cimgui.c.ImGui_TextDisabled("(unable to copy compressed page)"); + return; + }; + defer preserved.deinit(); + + const page = preserved.page(); + const rac = page.getRowAndCell(pin.x, pin.y); + const cell: InspectedCell = .{ + .row = rac.row, + .cell = rac.cell, + .row_idx = pin.y, + .col_idx = pin.x, + }; + + self.cell_info.draw(cell, pt, page); if (cell.cell.style_id != stylepkg.default_id) { cimgui.c.ImGui_SeparatorText("Style"); - const page = cell.node.page(); const style = page.styles.get( page.memory, cell.cell.style_id, @@ -646,12 +684,12 @@ pub const CellChooser = struct { if (cell.cell.hyperlink) { cimgui.c.ImGui_SeparatorText("Hyperlink"); - hyperlinkTable(cell); + hyperlinkTable(cell, page); } if (cell.cell.hasGrapheme()) { cimgui.c.ImGui_SeparatorText("Grapheme"); - graphemeTable(cell); + graphemeTable(cell, page); } } }; @@ -665,7 +703,7 @@ fn maxCoord( return br_point.coord(); } -fn hyperlinkTable(cell: PageList.Cell) void { +fn hyperlinkTable(cell: InspectedCell, page: *const terminal.Page) void { if (!cimgui.c.ImGui_BeginTable( "cell_hyperlink", 2, @@ -673,7 +711,6 @@ fn hyperlinkTable(cell: PageList.Cell) void { )) return; defer cimgui.c.ImGui_EndTable(); - const page = cell.node.page(); const link_id = page.lookupHyperlink(cell.cell) orelse { cimgui.c.ImGui_TableNextRow(); _ = cimgui.c.ImGui_TableSetColumnIndex(0); @@ -720,7 +757,7 @@ fn hyperlinkTable(cell: PageList.Cell) void { cimgui.c.ImGui_Text("%d", refs); } -fn graphemeTable(cell: PageList.Cell) void { +fn graphemeTable(cell: InspectedCell, page: *const terminal.Page) void { if (!cimgui.c.ImGui_BeginTable( "cell_grapheme", 2, @@ -728,7 +765,6 @@ fn graphemeTable(cell: PageList.Cell) void { )) return; defer cimgui.c.ImGui_EndTable(); - const page = cell.node.page(); const cps = page.lookupGrapheme(cell.cell) orelse { cimgui.c.ImGui_TableNextRow(); _ = cimgui.c.ImGui_TableSetColumnIndex(0); @@ -768,8 +804,9 @@ pub const CellInfo = struct { pub fn draw( _: *const CellInfo, - cell: PageList.Cell, + cell: InspectedCell, point: terminal.Point, + page: *const terminal.Page, ) void { if (!cimgui.c.ImGui_BeginTable( "cell_info", @@ -835,7 +872,7 @@ pub const CellInfo = struct { _ = cimgui.c.ImGui_TableSetColumnIndex(2); if (cimgui.c.ImGui_BeginListBox("##cell_grapheme", .{ .x = 0, .y = 0 })) { defer cimgui.c.ImGui_EndListBox(); - if (cell.node.page().lookupGrapheme(cell.cell)) |cps| { + if (page.lookupGrapheme(cell.cell)) |cps| { var buf: [96]u8 = undefined; for (cps) |cp| { const label = std.fmt.bufPrintZ(&buf, "U+{X}", .{cp}) catch "U+?"; @@ -933,7 +970,7 @@ pub const CellInfo = struct { widgets.helpMarker("OSC8 hyperlink ID associated with this cell."); _ = cimgui.c.ImGui_TableSetColumnIndex(2); - const link_id = cell.node.page().lookupHyperlink(cell.cell) orelse 0; + const link_id = page.lookupHyperlink(cell.cell) orelse 0; cimgui.c.ImGui_Text("id=%d", link_id); } } diff --git a/src/renderer/Thread.zig b/src/renderer/Thread.zig index 488642199..bb07028ad 100644 --- a/src/renderer/Thread.zig +++ b/src/renderer/Thread.zig @@ -34,6 +34,91 @@ const must_draw_from_app_thread = /// the future if we want it configurable. pub const Mailbox = BlockingQueue(rendererpkg.Message, 64); +/// Schedules incremental terminal compression after renderer activity stops. +/// +/// This owns all renderer-specific compression state. The terminal decides +/// whether compression is required and performs the actual work; the renderer +/// only provides idle scheduling and avoids waiting for the terminal lock. +const Compression = struct { + const idle_interval = 250; + const step_interval = 1; + + timer: xev.Timer, + completion: xev.Completion = .{}, + reset_completion: xev.Completion = .{}, + + fn init() !Compression { + return .{ .timer = try xev.Timer.init() }; + } + + fn deinit(self: *Compression) void { + self.timer.deinit(); + } + + /// Start or postpone compression after a renderer wake. + fn wake(self: *Compression, thread: *Thread) void { + // An active timer already proves work was pending. Otherwise consult + // the terminal, without waiting if rendering or parsing holds its lock. + if (self.completion.state() != .active) { + if (thread.state.mutex.tryLock()) { + defer thread.state.mutex.unlock(); + if (!thread.state.terminal.compressionRequired()) return; + } + } + + self.schedule(thread, idle_interval); + } + + /// Start the one-shot timer, or move its deadline if it is already active. + fn schedule(self: *Compression, thread: *Thread, delay_ms: u64) void { + self.timer.reset( + &thread.loop, + &self.completion, + &self.reset_completion, + delay_ms, + Thread, + thread, + timerCallback, + ); + } + + fn timerCallback( + thread_: ?*Thread, + _: *xev.Loop, + _: *xev.Completion, + result: xev.Timer.RunError!void, + ) xev.CallbackAction { + _ = result catch |err| switch (err) { + error.Canceled => return .disarm, + else => { + log.warn("error in compression timer err={}", .{err}); + return .disarm; + }, + }; + + const thread = thread_ orelse return .disarm; + const self = &thread.compression; + + if (step(thread.state)) |delay| self.schedule(thread, delay); + return .disarm; + } + + /// Try one bounded step without waiting for the terminal lock. The return + /// value is the delay before another attempt, or null when work is done. + fn step(state: *rendererpkg.State) ?u64 { + if (!state.mutex.tryLock()) return idle_interval; + defer state.mutex.unlock(); + + return switch (state.terminal.compress(.incremental)) { + .pending => step_interval, + .unsupported, + .deferred, + .complete, + => null, + }; + } +}; + /// Allocator used for some state alloc: std.mem.Allocator, @@ -72,6 +157,9 @@ cursor_h: xev.Timer, cursor_c: xev.Completion = .{}, cursor_c_cancel: xev.Completion = .{}, +/// Incremental scrollback compression scheduling. +compression: Compression, + /// The surface we're rendering to. surface: *apprt.Surface, @@ -158,6 +246,9 @@ pub fn init( var cursor_timer = try xev.Timer.init(); errdefer cursor_timer.deinit(); + var compression = try Compression.init(); + errdefer compression.deinit(); + // The mailbox for messaging this thread var mailbox = try Mailbox.create(alloc); errdefer mailbox.destroy(alloc); @@ -172,6 +263,7 @@ pub fn init( .draw_h = draw_h, .draw_now = draw_now, .cursor_h = cursor_timer, + .compression = compression, .surface = surface, .renderer = renderer_impl, .state = state, @@ -189,6 +281,7 @@ pub fn deinit(self: *Thread) void { self.draw_h.deinit(); self.draw_now.deinit(); self.cursor_h.deinit(); + self.compression.deinit(); self.loop.deinit(); // Nothing can possibly access the mailbox anymore, destroy it. @@ -537,6 +630,10 @@ fn wakeupCallback( // Render immediately _ = renderCallback(t, undefined, undefined, {}); + // PageList mutations maintain their own compression dirty state. Checking + // it here covers output, resize, and viewport scrolling uniformly. + t.compression.wake(t); + // The below is not used anymore but if we ever want to introduce // a configuration to introduce a delay to coalesce renders, we can // use this. @@ -723,3 +820,63 @@ fn cursorBlinkInterval() u64 { return CURSOR_BLINK_INTERVAL; } + +test "compression timer advances primary history" { + const testing = std.testing; + const terminalpkg = @import("../terminal/main.zig"); + + var terminal = try terminalpkg.Terminal.init(testing.allocator, .{ + .cols = 80, + .rows = 24, + .max_scrollback = 10 * 1024 * 1024, + }); + defer terminal.deinit(testing.allocator); + + const primary = terminal.screens.get(.primary).?; + primary.cursorAbsolute(0, primary.pages.rows - 1); + while (primary.pages.pages.first.? == + primary.pages.getTopLeft(.active).node) + { + try primary.cursorDownScroll(); + } + + // Compression always targets primary history, even while an alternate + // screen is active. + _ = try terminal.screens.getInit(testing.allocator, .alternate, .{ + .cols = 80, + .rows = 24, + .max_scrollback = 0, + }); + terminal.screens.switchTo(.alternate); + + var mutex: std.Thread.Mutex = .{}; + var state: rendererpkg.State = .{ + .mutex = &mutex, + .terminal = &terminal, + }; + try testing.expect(terminal.compressionRequired()); + + // Contention postpones background work instead of blocking output. + mutex.lock(); + try testing.expectEqual( + @as(?u64, Compression.idle_interval), + Compression.step(&state), + ); + mutex.unlock(); + + try testing.expectEqual( + @as(?u64, Compression.step_interval), + Compression.step(&state), + ); + try testing.expectEqual( + @as(usize, 1), + primary.pages.memoryStats().compressed_pages, + ); + + // The no-work verification pass completes and leaves the timer disarmed. + try testing.expectEqual( + @as(?u64, null), + Compression.step(&state), + ); + try testing.expect(!terminal.compressionRequired()); +}