diff --git a/src/datastruct/main.zig b/src/datastruct/main.zig index 73e9afba9..32c5ff609 100644 --- a/src/datastruct/main.zig +++ b/src/datastruct/main.zig @@ -14,6 +14,7 @@ pub const IntrusiveDoublyLinkedList = intrusive_linked_list.DoublyLinkedList; pub const LimitedAllocator = @import("limited_allocator.zig").LimitedAllocator; pub const MessageData = @import("message_data.zig").MessageData; pub const SplitTree = split_tree.SplitTree; +pub const WasmPagePool = @import("wasm_page_pool.zig").WasmPagePool; test { @import("std").testing.refAllDecls(@This()); diff --git a/src/datastruct/wasm_page_pool.zig b/src/datastruct/wasm_page_pool.zig new file mode 100644 index 000000000..7f1193825 --- /dev/null +++ b/src/datastruct/wasm_page_pool.zig @@ -0,0 +1,157 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +/// A memory pool of wasm-page-multiple-sized items backed directly by +/// wasm linear memory, for wasm targets only. +/// +/// This is necessary for two reasons: the std.heap.MemoryPool grows 1.5x +/// at each growth point. The backing allocator for that is usually a GPA +/// which is the BrkAllocator for wasm. This grows by power-of-two +/// big-allocation slots. If you pair these together you get a massive +/// permanent linear memory growth. Native (non-wasm) targets don't care +/// because unused virtual mappings are effectively free, but this isn't +/// exactly true for Wasm runtimes. +/// +/// This pool instead grows exactly @sizeOf(Item) bytes of fresh linear +/// memory per item with memory.grow (which also guarantees wasm-page +/// alignment and zeroing) and recycles freed items through a free list +/// that is container-level, i.e. shared by every pool of the same Item +/// type in the module instance. Every freed item is immediately +/// reusable by every pool, and each item costs exactly @sizeOf(Item) +/// reserved bytes, ever. +/// +/// The "shared by every pool of the same Item type" is really important: +/// the normal Ghostty memory pool is per-terminal. This one is per-module. +/// The Ghostty wasm modules are not multi-threaded so this doesn't require +/// any synchronization. But this means that all terminals share one pool +/// so memory doesn't balloon like crazy that way either. +/// +/// Requirements on Item: +/// +/// - @sizeOf(Item) must be a nonzero multiple of the wasm page size +/// (64KiB), since memory.grow allocates in whole pages. This is +/// also what makes the pool exact-fit: any other size would strand +/// the remainder of the last page. +/// - Natural alignment must be at most the wasm page size. +/// +/// Free-list items are dirty except that their first pointer-size bytes +/// hold the intrusive free-list node, exactly like the std MemoryPool. +/// Callers that need zeroed items must zero them (fresh items from +/// memory.grow are zero by the wasm spec; recycled items retain +/// whatever the caller left, so zero either on destroy or on create). +/// +/// The API mirrors the subset of std.heap.memory_pool.AlignedManaged +/// that callers use (initCapacity/deinit/reset/create/destroy plus the +/// managed allocator field), so it can be swapped in comptime. +pub fn WasmPagePool(comptime Item: type) type { + return struct { + const Self = @This(); + + comptime { + if (builtin.target.cpu.arch.isWasm()) { + // The shared free list (and wasm's single linear + // memory) requires a single-threaded target. + assert(builtin.single_threaded); + // Items are whole wasm pages + assert(item_size > 0); + assert(item_size % std.heap.page_size_min == 0); + assert(@alignOf(Item) <= std.heap.page_size_min); + // Free items store the intrusive node in their bytes. + assert(item_size >= @sizeOf(std.SinglyLinkedList.Node)); + } + } + + /// The allocator this managed pool was initialized with, + /// carried for interface compatibility with the std managed + /// memory pools. Pool items never come from it (they come + /// from memory.grow), but callers may use it for related + /// allocations that don't fit the pool. + allocator: Allocator, + + pub const item_size = @sizeOf(Item); + pub const ItemPtr = *align(std.heap.page_size_min) Item; + + /// Freed items, shared by every pool of this Item type in the + /// module instance. + var free_list: std.SinglyLinkedList = .{}; + + pub fn initCapacity( + allocator: Allocator, + preheat: usize, + ) Allocator.Error!Self { + // Preheating is intentionally a no-op: item creation is a + // cheap memory.grow or free-list pop. And we want to avoid + // the preallocation mem cost. + _ = preheat; + return .{ .allocator = allocator }; + } + + pub fn deinit(self: *Self) void { + // Items are global to the instance and outlive the pool so + // other pools can reuse them. Callers must destroy() any + // items they still hold before deinit or they leak. + self.* = undefined; + } + + pub fn reset( + self: *Self, + mode: std.heap.ArenaAllocator.ResetMode, + ) bool { + // There is nothing to trim: linear memory cannot shrink, + // so retaining every freed item for reuse is always the + // right policy on wasm. As with deinit, live items must be + // destroyed by the caller first. + _ = self; + _ = mode; + return true; + } + + pub fn create(self: *Self) Allocator.Error!ItemPtr { + _ = self; + + // If we have a free item, use it. + if (free_list.popFirst()) |node| return @ptrCast(@alignCast(node)); + + // Grow linear memory by exactly one item. + const pages = comptime item_size / std.heap.page_size_min; + const page_index = @wasmMemoryGrow(0, pages); + if (page_index == -1) return error.OutOfMemory; + return @ptrFromInt(@as(usize, @intCast(page_index)) * std.heap.page_size_min); + } + + pub fn destroy(self: *Self, ptr: ItemPtr) void { + _ = self; + const node: *std.SinglyLinkedList.Node = @ptrCast(ptr); + node.* = .{}; + free_list.prepend(node); + } + }; +} + +test WasmPagePool { + if (!comptime builtin.target.cpu.arch.isWasm()) return error.SkipZigTest; + + const testing = std.testing; + const Pool = WasmPagePool([64 * 1024]u8); + var pool: Pool = try .initCapacity(testing.allocator, 0); + defer pool.deinit(); + + const a = try pool.create(); + const b = try pool.create(); + try testing.expect(a != b); + + // Freed items are recycled, most recent first. + pool.destroy(b); + try testing.expectEqual(b, try pool.create()); + + // The free list is shared across pools of the same Item type. + var pool2: Pool = try .initCapacity(testing.allocator, 0); + defer pool2.deinit(); + pool.destroy(a); + try testing.expectEqual(a, try pool2.create()); + + pool.destroy(a); + pool.destroy(b); +} diff --git a/src/terminal/PageList.zig b/src/terminal/PageList.zig index 672370af2..f380bd42c 100644 --- a/src/terminal/PageList.zig +++ b/src/terminal/PageList.zig @@ -12,6 +12,7 @@ const fastmem = @import("../fastmem.zig"); const simd = @import("../simd/main.zig"); const tripwire = @import("../tripwire.zig"); const DoublyLinkedList = @import("../datastruct/main.zig").IntrusiveDoublyLinkedList; +const WasmPagePool = @import("../datastruct/main.zig").WasmPagePool; const color = @import("color.zig"); const compression = @import("compress.zig"); const highlight = @import("highlight.zig"); @@ -306,13 +307,24 @@ const std_capacity = pagepkg.std_capacity; /// The byte size required for a standard page. const std_size = Page.layout(std_capacity).total_size; +/// True when the page pool is the wasm page pool, which recycles items +/// through a free list shared by the whole module instance instead of +/// dying with the pool. +/// +/// Test builds use the std pool even on wasm so that pool memory goes +/// through the testing allocator and participates in leak detection. +const wasm_page_pool = builtin.target.cpu.arch.isWasm() and !builtin.is_test; + /// The memory pool we use for page memory buffers. We use a separate pool /// so we can allocate these with a page allocator. We have to use a page /// allocator because we need memory that is zero-initialized and page-aligned. -const PagePool = std.heap.memory_pool.AlignedManaged( - [std_size]u8, - .fromByteUnits(std.heap.page_size_min), -); +const PagePool = if (wasm_page_pool) + WasmPagePool([std_size]u8) +else + std.heap.memory_pool.AlignedManaged( + [std_size]u8, + .fromByteUnits(std.heap.page_size_min), + ); /// List of pins, known as "tracked" pins. These are pins that are kept /// up to date automatically through page-modifying operations. @@ -677,13 +689,13 @@ fn initPages( // redundant here for safety. assert(layout.total_size <= size.max_page_size); - // If we have an error, we need to clean up our heap-owned pages - // since they're not in the pool. + // If we have an error, we need to clean up our pages: heap-owned + // pages are freed directly and pool-owned pages are reclaimed. errdefer { var it = page_list.first; while (it) |node| : (it = node.next) { switch (node.owned) { - .pool => {}, + .pool => reclaimPoolPage(pool, node.page()), .heap => page_alloc.free(node.page().memory), } } @@ -876,6 +888,20 @@ fn verifyIntegrity(self: *const PageList) IntegrityError!void { } } +/// Return a pool-owned page buffer to the page pool during a teardown +/// walk, zeroed for reuse (mirroring destroyNodeExt). Teardown walks +/// call this unconditionally for every pool-owned page. +fn reclaimPoolPage(pool: *MemoryPool, page: *const Page) void { + // Only wasm requires this + if (comptime !wasm_page_pool) return; + + const item: *align(std.heap.page_size_min) [std_size]u8 = + @ptrCast(@alignCast(page.memory.ptr)); + // We have to zero the item + _ = terminal_mem.decommit(.zero, item, page.memory.len); + pool.pages.destroy(item); +} + /// Deinit the pagelist, freeing all page memory and the memory pool. pub fn deinit(self: *PageList) void { // Verify integrity before cleanup @@ -884,14 +910,14 @@ pub fn deinit(self: *PageList) void { // Always deallocate our hashmap. self.tracked_pins.deinit(self.pool.alloc); - // Go through our linked list and deallocate all pages that are - // heap-owned (not in the pool). + // Go through our linked list and release every page: heap-owned + // pages are freed directly and pool-owned pages are reclaimed. const page_alloc = self.pool.pages.allocator; var it = self.pages.first; while (it) |node| : (it = node.next) { const page = node.restore(.discard); switch (node.owned) { - .pool => {}, + .pool => reclaimPoolPage(&self.pool, page), .heap => page_alloc.free(page.memory), } } @@ -933,15 +959,16 @@ pub fn reset(self: *PageList) void { cap.rows, ) catch unreachable; - // Before resetting our pools we need to free any pages that - // are heap-owned since those were allocated outside the pool. + // Before resetting our pools we need to release our pages: + // heap-owned pages are freed since they were allocated outside + // the pool, and pool-owned pages are reclaimed. { const page_alloc = self.pool.pages.allocator; var it = self.pages.first; while (it) |node| : (it = node.next) { const page = node.restore(.discard); switch (node.owned) { - .pool => {}, + .pool => reclaimPoolPage(&self.pool, page), .heap => page_alloc.free(page.memory), } } @@ -962,46 +989,51 @@ pub fn reset(self: *PageList) void { // retaining a certain amount of memory, it won't use mmap and won't // be zeroed. This block zeroes out all the memory in the pool arena. // + // The wasm page pool has no arena to scrub: its free-list items were + // zeroed by reclaimPoolPage above. + // // Note: we only have to do this for the page pool because the nodes are // always fully overwritten on each allocation. - inline for (.{ - self.pool.pages.unmanaged.arena_state.used_list, - self.pool.pages.unmanaged.arena_state.free_list, - }) |first| { - var node_ = first; - while (node_) |node| : (node_ = node.next) { - // NOTE: Zig 0.16.0's arenas don't use the linked list types - // anymore, so we can just reference fields directly. The node - // type is still private though, so we have to parse out some - // of the internal methods to work with the buffer - namely - // Node.loadBuf and Node.Size.toInt. They are combined below. - // - // PS: My (vancluever's) reading of the code gives me the - // impression that we no longer need to offset the data by the - // header, because there's no linked list overhead anymore. But - // I'm sure we'll see pretty quick when I run the tests. :) - // - const BufNode = struct { - size: Size, - end_index: usize, - next: ?*@This(), + if (comptime !wasm_page_pool) { + inline for (.{ + self.pool.pages.unmanaged.arena_state.used_list, + self.pool.pages.unmanaged.arena_state.free_list, + }) |first| { + var node_ = first; + while (node_) |node| : (node_ = node.next) { + // NOTE: Zig 0.16.0's arenas don't use the linked list types + // anymore, so we can just reference fields directly. The node + // type is still private though, so we have to parse out some + // of the internal methods to work with the buffer - namely + // Node.loadBuf and Node.Size.toInt. They are combined below. + // + // PS: My (vancluever's) reading of the code gives me the + // impression that we no longer need to offset the data by the + // header, because there's no linked list overhead anymore. But + // I'm sure we'll see pretty quick when I run the tests. :) + // + const BufNode = struct { + size: Size, + end_index: usize, + next: ?*@This(), - const Size = packed struct(usize) { - resizing: bool, - _: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0, + const Size = packed struct(usize) { + resizing: bool, + _: @Int(.unsigned, @bitSizeOf(usize) - 1) = 0, - fn toInt(s: Size) usize { - var int = s; - int.resizing = false; - return @bitCast(int); - } + fn toInt(s: Size) usize { + var int = s; + int.resizing = false; + return @bitCast(int); + } + }; }; - }; - const buf_node_ptr: *BufNode = @ptrCast(node); - const buf_node_size = @atomicLoad(BufNode.Size, &buf_node_ptr.size, .monotonic); - const buf = @as([*]u8, @ptrCast(node))[0..buf_node_size.toInt()][@sizeOf(BufNode)..]; - @memset(buf, 0); + const buf_node_ptr: *BufNode = @ptrCast(node); + const buf_node_size = @atomicLoad(BufNode.Size, &buf_node_ptr.size, .monotonic); + const buf = @as([*]u8, @ptrCast(node))[0..buf_node_size.toInt()][@sizeOf(BufNode)..]; + @memset(buf, 0); + } } } @@ -1104,7 +1136,7 @@ pub fn clone( var page_it = page_list.first; while (page_it) |node| : (page_it = node.next) { switch (node.owned) { - .pool => {}, + .pool => reclaimPoolPage(&pool, node.page()), .heap => page_alloc.free(node.page().memory), } }