diff --git a/src/terminal/PageList.zig b/src/terminal/PageList.zig index 338aad4c0..001df75a6 100644 --- a/src/terminal/PageList.zig +++ b/src/terminal/PageList.zig @@ -14,6 +14,7 @@ const DoublyLinkedList = @import("../datastruct/main.zig").IntrusiveDoublyLinked const color = @import("color.zig"); const highlight = @import("highlight.zig"); const kitty = @import("kitty.zig"); +const terminal_mem = @import("mem.zig"); const point = @import("point.zig"); const pagepkg = @import("page.zig"); const stylepkg = @import("style.zig"); @@ -481,7 +482,7 @@ fn initPages( const page_buf = if (pooled) buf: { try tw.check(.page_buf_std); const buf = try pool.pages.create(); - recommitPoolItem(buf); + terminal_mem.recommit(buf); break :buf buf; } else buf: { try tw.check(.page_buf_non_std); @@ -3487,7 +3488,7 @@ inline fn createPageExt( // dispenses. Otherwise, we use the heap allocator to allocate. const page_buf = if (pooled) buf: { const buf = try pool.pages.create(); - recommitPoolItem(buf); + terminal_mem.recommit(buf); break :buf buf; } else try page_alloc.alignedAlloc( u8, @@ -3565,7 +3566,7 @@ fn destroyNodeExt( // supported) so it can be reused. const item: *align(std.heap.page_size_min) [std_size]u8 = @ptrCast(@alignCast(page.memory.ptr)); - decommitPoolItem(item, page.memory.len); + _ = terminal_mem.decommit(.zero, item, page.memory.len); pool.pages.destroy(item); }, @@ -3578,100 +3579,6 @@ fn destroyNodeExt( pool.nodes.destroy(node); } -/// Zero a pool item buffer that is about to be returned to the pool -/// free list and, where supported, tell the OS it can reclaim the -/// backing memory while the buffer sits unused. Without the decommit, -/// a pool that shrinks (scrollback clears, pruning churn, resets) -/// retains its high-water RSS forever, since MemoryPool free lists -/// never return memory to the OS. -/// -/// dirty_len is the length of the page that lived in this item, which -/// may be smaller than the item; bytes beyond it are already zero. -/// -/// The invariant required for page reuse (see createPageExt): after -/// this call the entire item reads back as zeroes. The pool writes its -/// free list node into the first pointer-size bytes afterwards, which -/// is safe because page initialization always overwrites them (see the -/// comptime assert in Page). -fn decommitPoolItem( - item: *align(std.heap.page_size_min) [std_size]u8, - dirty_len: usize, -) void { - // In test builds the buffer comes from std.testing.allocator, not - // the OS page allocator, so madvise is neither safe nor meaningful. - if (comptime builtin.is_test) { - @memset(item[0..dirty_len], 0); - return; - } - - // MADV_DONTNEED on a private anonymous mapping immediately - // reclaims the pages and guarantees they read back zero-filled - // on the next access, so we can skip the memset entirely. - // - // We use MADV_DONTNEED rather than MADV_FREE deliberately: - // MADV_FREE doesn't reduce RSS until memory pressure (invisible - // to users watching memory usage) and doesn't guarantee zeroes - // on the next read. MADV_DONTNEED's synchronous TLB-shootdown - // cost only matters at allocator-level call frequency; page - // destroys here are rare (clears, reflow, capacity changes), and - // the hottest reuse path (grow's prune-reuse) doesn't go through - // this function at all. - if (comptime builtin.os.tag == .linux) { - if (std.posix.madvise( - item, - std_size, - std.posix.MADV.DONTNEED, - )) |_| return else |err| { - log.warn("madvise(DONTNEED) failed err={}", .{err}); - // Fall through to the memset below. - } - } - - // On Darwin we zero in place and then mark the memory as - // reusable, which removes it from the process footprint until - // it is touched again. Contents of reusable memory are either - // preserved (our zeroes) or reclaimed and zero-filled on the - // next access, so the invariant holds either way. The reuse - // side calls MADV_FREE_REUSE to fix up footprint accounting - // (see recommitPoolItem). - if (comptime builtin.os.tag.isDarwin()) { - @memset(item[0..dirty_len], 0); - std.posix.madvise( - item, - std_size, - std.posix.MADV.FREE_REUSABLE, - ) catch { - // Best-effort: plain MADV_FREE reclaims under memory - // pressure only and needs no reuse pairing. - std.posix.madvise( - item, - std_size, - std.posix.MADV.FREE, - ) catch {}; - }; - return; - } - - @memset(item[0..dirty_len], 0); -} - -/// The counterpart to decommitPoolItem for buffers handed back out by -/// the pool. Only Darwin requires this: MADV_FREE_REUSE re-accounts -/// previously reusable memory to the process footprint. Without it the -/// memory still works (touching reusable pages revives them) but -/// footprint reporting can undercount. Harmless on buffers that were -/// never marked reusable (fresh or preheated items). -fn recommitPoolItem(item: *align(std.heap.page_size_min) [std_size]u8) void { - if (comptime builtin.is_test) return; - if (comptime builtin.os.tag.isDarwin()) { - std.posix.madvise( - item, - std_size, - std.posix.MADV.FREE_REUSE, - ) catch {}; - } -} - /// Fast-path function to erase exactly 1 row. Erasing means that the row /// is completely REMOVED, not just cleared. All rows following the removed /// row will be shifted up by 1 to fill the empty space. @@ -14444,34 +14351,6 @@ test "PageList compact oversized page" { } } -test "PageList decommitPoolItem zeroes the dirty region" { - const testing = std.testing; - const alloc = testing.allocator; - - // Note: in test builds decommitPoolItem always takes the memset - // path; the madvise-based paths rely on kernel zero-fill semantics - // that unit tests cannot exercise (see decommitPoolItem). - const buf = try alloc.alignedAlloc( - u8, - .fromByteUnits(std.heap.page_size_min), - std_size, - ); - defer alloc.free(buf); - const item: *align(std.heap.page_size_min) [std_size]u8 = @ptrCast(buf.ptr); - - // Fully dirty item. - @memset(item, 0xAA); - decommitPoolItem(item, std_size); - try testing.expect(std.mem.allEqual(u8, item, 0)); - - // Partially dirty item: an item that hosted a page smaller than - // the item is only dirty up to the page length; the rest is zero - // by invariant and must remain zero. - @memset(item[0..1024], 0xAA); - decommitPoolItem(item, 1024); - try testing.expect(std.mem.allEqual(u8, item, 0)); -} - test "PageList destroyed pool page reuse is zeroed" { const testing = std.testing; const alloc = testing.allocator; diff --git a/src/terminal/main.zig b/src/terminal/main.zig index 12a43083a..0a2e7434f 100644 --- a/src/terminal/main.zig +++ b/src/terminal/main.zig @@ -85,6 +85,7 @@ test { _ = @import("bitmap_allocator.zig"); _ = @import("compress.zig"); _ = @import("hash_map.zig"); + _ = @import("mem.zig"); _ = @import("ref_counted_set.zig"); _ = @import("size.zig"); } diff --git a/src/terminal/mem.zig b/src/terminal/mem.zig new file mode 100644 index 000000000..940985766 --- /dev/null +++ b/src/terminal/mem.zig @@ -0,0 +1,171 @@ +//! Virtual-memory operations shared by terminal page owners. +//! +//! Terminal pages use page-aligned, page-multiple mappings. This module can +//! discard the physical pages behind one of those mappings without releasing +//! its virtual address range, then prepare the same range for reuse. It does +//! not allocate memory or decide which terminal pages should be discarded. +const std = @import("std"); +const builtin = @import("builtin"); +const assert = @import("../quirks.zig").inlineAssert; + +const log = std.log.scoped(.terminal_mem); + +/// What guarantee decommit must provide when the OS cannot discard a mapping. +pub const DecommitMode = enum { + /// The dirty prefix must read as zero after this call, even when physical + /// reclamation is unavailable. Bytes after the prefix must already be zero. + zero, + + /// Physical-memory reclamation is required. Do not touch the mapping when + /// reclamation is unsupported or fails; report the failure to the caller. + strict, +}; + +/// Discard physical pages while retaining a mapping's virtual address range. +/// +/// The complete mapping must be page-aligned and a multiple of the minimum +/// system page size. `dirty_len` identifies the prefix whose contents may be +/// nonzero. Strict mode requires the complete mapping to be dirty because a +/// successful discard invalidates all of its contents. +/// +/// The return value reports whether the OS accepted the reclamation request. +/// Test builds return true after simulating reclamation by zeroing dirty bytes. +/// In zero mode, the requested bytes are guaranteed to be zero regardless of +/// the return value. +pub fn decommit( + comptime mode: DecommitMode, + memory: []align(std.heap.page_size_min) u8, + dirty_len: usize, +) bool { + assert(memory.len > 0); + assert(@intFromPtr(memory.ptr) % std.heap.page_size_min == 0); + assert(memory.len % std.heap.page_size_min == 0); + assert(dirty_len <= memory.len); + if (comptime mode == .strict) assert(dirty_len == memory.len); + + // Testing allocator ranges may share an OS mapping with unrelated memory, + // so madvise is not safe. Zeroing models the only content guarantee callers + // have after a successful discard. + if (comptime builtin.is_test) { + @memset(memory[0..dirty_len], 0); + return true; + } + + // DONTNEED immediately reclaims private anonymous pages on Linux and + // faults them back as zero-filled pages. We deliberately avoid MADV_FREE: + // it does not reduce RSS until memory pressure and does not guarantee that + // the next read is zero. + if (comptime builtin.os.tag == .linux) { + if (std.posix.madvise( + memory.ptr, + memory.len, + std.posix.MADV.DONTNEED, + )) |_| return true else |err| { + log.warn("madvise(DONTNEED) failed err={}", .{err}); + if (comptime mode == .strict) return false; + // Zero mode falls through to the memset below. + } + } + + // FREE_REUSABLE removes the range from the Darwin process footprint while + // retaining its mapping. Zero mode clears its dirty prefix first because + // the kernel may preserve the contents. Strict mode avoids that write + // because its caller will replace the entire mapping after recommit. + if (comptime builtin.os.tag.isDarwin()) { + if (comptime mode == .zero) @memset(memory[0..dirty_len], 0); + + if (std.posix.madvise( + memory.ptr, + memory.len, + std.posix.MADV.FREE_REUSABLE, + )) |_| return true else |err| { + switch (mode) { + .strict => { + log.warn("madvise(FREE_REUSABLE) failed err={}", .{err}); + return false; + }, + + .zero => { + // Plain FREE can still reclaim the already-zero mapping + // under pressure and does not require a reuse pairing. + std.posix.madvise( + memory.ptr, + memory.len, + std.posix.MADV.FREE, + ) catch {}; + return false; + }, + } + } + } + + if (comptime mode == .zero) @memset(memory[0..dirty_len], 0); + return false; +} + +/// Prepare a mapping previously passed to decommit for reuse. +/// +/// Linux and test builds need no explicit operation. Darwin pairs +/// FREE_REUSABLE with FREE_REUSE so pages touched by the caller are accounted +/// to the process again. Failure does not invalidate the retained mapping, so +/// reuse can continue after logging the accounting failure. +pub fn recommit(memory: []align(std.heap.page_size_min) u8) void { + assert(memory.len > 0); + assert(@intFromPtr(memory.ptr) % std.heap.page_size_min == 0); + assert(memory.len % std.heap.page_size_min == 0); + + if (comptime builtin.is_test) return; + if (comptime builtin.os.tag.isDarwin()) { + std.posix.madvise( + memory.ptr, + memory.len, + std.posix.MADV.FREE_REUSE, + ) catch |err| { + log.warn("madvise(FREE_REUSE) failed err={}", .{err}); + }; + } +} + +test "decommit with zero fallback clears the dirty prefix" { + const testing = std.testing; + const memory_len = 2 * std.heap.page_size_min; + const memory = try testing.allocator.alignedAlloc( + u8, + .fromByteUnits(std.heap.page_size_min), + memory_len, + ); + defer testing.allocator.free(memory); + + @memset(memory, 0xAA); + _ = decommit(.zero, memory, memory.len); + try testing.expect(std.mem.allEqual(u8, memory, 0)); + + // The tail is already zero by contract, so a partially dirty mapping only + // needs its dirty prefix cleared. + @memset(memory[0..1024], 0xAA); + _ = decommit(.zero, memory, 1024); + try testing.expect(std.mem.allEqual(u8, memory, 0)); +} + +test "strict decommit retains the mapping for recommit" { + const testing = std.testing; + const memory_len = 2 * std.heap.page_size_min; + const memory = try testing.allocator.alignedAlloc( + u8, + .fromByteUnits(std.heap.page_size_min), + memory_len, + ); + defer testing.allocator.free(memory); + @memset(memory, 0xAA); + + const original_ptr = memory.ptr; + const original_len = memory.len; + try testing.expect(decommit(.strict, memory, memory.len)); + try testing.expectEqual(original_ptr, memory.ptr); + try testing.expectEqual(original_len, memory.len); + try testing.expect(std.mem.allEqual(u8, memory, 0)); + + recommit(memory); + @memset(memory, 0xBB); + try testing.expect(std.mem.allEqual(u8, memory, 0xBB)); +}