From ffe015ee55d1ab39cbf1525823ff18092433eab9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 3 Sep 2026 09:10:17 -0700 Subject: [PATCH 1/4] terminal: bitmap allocator marks free chunks with zero bits --- src/terminal/bitmap_allocator.zig | 126 +++++++++++++++++------------- 1 file changed, 70 insertions(+), 56 deletions(-) diff --git a/src/terminal/bitmap_allocator.zig b/src/terminal/bitmap_allocator.zig index a300cf2e8..eed0bdd86 100644 --- a/src/terminal/bitmap_allocator.zig +++ b/src/terminal/bitmap_allocator.zig @@ -38,9 +38,8 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { pub const bitmap_bit_size = @bitSizeOf(u64); /// The bitmap of available chunks. Each bit represents a chunk. A - /// 1 means the chunk is free and a 0 means it's used. We use 1 - /// for free since it makes it very slightly faster to find free - /// chunks. + /// 0 means the chunk is free and a 1 means it's used, so an + /// all-zero bitmap is a fully free allocator. bitmap: Offset(u64), bitmap_count: usize, @@ -56,13 +55,23 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { pub fn init(buf: OffsetBuf, l: Layout) Self { assert(base_align.check(@intFromPtr(buf.start()))); - // Initialize our bitmaps to all 1s to note that all chunks are free. + // Clear our bitmaps to note that all chunks are free. const bitmap = buf.member(u64, l.bitmap_start); - const bitmap_ptr = bitmap.ptr(buf); - @memset(bitmap_ptr[0..l.bitmap_count], std.math.maxInt(u64)); + @memset(bitmap.ptr(buf)[0..l.bitmap_count], 0); + return initAssumeZeroed(buf, l); + } + + /// Initialize the allocator map over memory that the caller + /// guarantees is already zero-filled. + /// + /// This writes nothing to the buffer: an all-zero bitmap already + /// marks every chunk as free. Behavior is undefined if the bitmap + /// region is not zero. + pub fn initAssumeZeroed(buf: OffsetBuf, l: Layout) Self { + assert(base_align.check(@intFromPtr(buf.start()))); return .{ - .bitmap = bitmap, + .bitmap = buf.member(u64, l.bitmap_start), .bitmap_count = l.bitmap_count, .chunks = buf.member(u8, l.chunks_start), }; @@ -112,7 +121,11 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { // next scan starts at the first word with a free bit. self.search_start = start: { var new_start = start; - while (new_start < self.bitmap_count and bitmaps[new_start] == 0) new_start += 1; + while (new_start < self.bitmap_count and + bitmaps[new_start] == std.math.maxInt(u64)) + { + new_start += 1; + } break :start new_start; }; @@ -153,20 +166,20 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { // Number of bits we need to mark in this bitmap. const bits = @min(rem, 64 - bit); - bitmaps[i] |= ~@as(u64, 0) >> @intCast(64 - bits) << @intCast(bit); + bitmaps[i] &= ~(~@as(u64, 0) >> @intCast(64 - bits) << @intCast(bit)); rem -= bits; } // Mark any full bitmaps worth of bits that need to be marked. i += 1; while (rem > 64) : (i += 1) { - bitmaps[i] = std.math.maxInt(u64); + bitmaps[i] = 0; rem -= 64; } // Mark any bits at the start of this last bitmap if it needs it. if (rem > 0) { - bitmaps[i] |= ~@as(u64, 0) >> @intCast(64 - rem); + bitmaps[i] &= ~(~@as(u64, 0) >> @intCast(64 - rem)); } } @@ -178,10 +191,9 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { /// Returns the number of bytes currently in use. pub fn usedBytes(self: Self, base: anytype) usize { const bitmaps = self.bitmap.ptr(base); - var free_chunks: usize = 0; - for (bitmaps[0..self.bitmap_count]) |bitmap| free_chunks += @popCount(bitmap); - const total_chunks = self.bitmap_count * bitmap_bit_size; - return (total_chunks - free_chunks) * chunk_size; + var used_chunks: usize = 0; + for (bitmaps[0..self.bitmap_count]) |bitmap| used_chunks += @popCount(bitmap); + return used_chunks * chunk_size; } /// For testing only. @@ -200,7 +212,7 @@ pub fn BitmapAllocator(comptime chunk_size: comptime_int) type { for (chunk_idx..chunk_idx + chunk_count) |i| { const bitmap = @divFloor(i, bitmap_bit_size); const bit = i % bitmap_bit_size; - if (bitmaps[bitmap] & (@as(u64, 1) << @intCast(bit)) != 0) { + if (bitmaps[bitmap] & (@as(u64, 1) << @intCast(bit)) == 0) { return false; } } @@ -275,7 +287,7 @@ fn findFreeChunks(bitmaps: []u64, n: usize) ?usize { var i: usize = 0; search: while (i < bitmaps.len) { // Number of chunks available at the end of this bitmap. - const prefix = @clz(~bitmaps[i]); + const prefix = @clz(bitmaps[i]); // If there are no chunks available at the end of this bitmap // then we can't start in it, so we'll try the next one. @@ -298,7 +310,7 @@ fn findFreeChunks(bitmaps: []u64, n: usize) ?usize { // There's more than 64 remaining chunks and this bitmap has // content in it, so we try starting again with this bitmap. - if (bitmaps[i] != std.math.maxInt(u64)) continue :search; + if (bitmaps[i] != 0) continue :search; // This bitmap is completely free, we can subtract 64 from // our remaining number. @@ -307,17 +319,17 @@ fn findFreeChunks(bitmaps: []u64, n: usize) ?usize { // If the number of available chunks at the start of this bitmap // is less than the remaining required, we have to try again. - if (@ctz(~bitmaps[i]) < rem) continue; + if (@ctz(bitmaps[i]) < rem) continue; const suffix = (n - prefix) % 64; - // Found! Mark everything between our start and end as full. - bitmaps[start_bitmap] ^= ~@as(u64, 0) >> @intCast(start_bit) << @intCast(start_bit); + // Found! Mark everything between our start and end as used. + bitmaps[start_bitmap] |= ~@as(u64, 0) >> @intCast(start_bit) << @intCast(start_bit); const full_bitmaps = @divFloor(n - prefix - suffix, 64); for (bitmaps[start_bitmap + 1 ..][0..full_bitmaps]) |*bitmap| { - bitmap.* = 0; + bitmap.* = std.math.maxInt(u64); } - if (suffix > 0) bitmaps[i] ^= ~@as(u64, 0) >> @intCast(64 - suffix); + if (suffix > 0) bitmaps[i] |= ~@as(u64, 0) >> @intCast(64 - suffix); return start_bitmap * 64 + start_bit; } @@ -337,8 +349,10 @@ fn findFreeChunks(bitmaps: []u64, n: usize) ?usize { // = 000001000000010000 // ^ ^ // In this example there are 2 places with at least 4 sequential 1s. - var shifted: u64 = bitmap.*; - for (1..n) |i| shifted &= bitmap.* >> @intCast(i); + // Work on the inverted word so that free chunks are 1 bits. + const free = ~bitmap.*; + var shifted: u64 = free; + for (1..n) |i| shifted &= free >> @intCast(i); // If we have zero then we have no matches if (shifted == 0) continue; @@ -349,7 +363,7 @@ fn findFreeChunks(bitmaps: []u64, n: usize) ?usize { // Calculate the mask so we can mark it as used const mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - n)) << @intCast(bit); - bitmap.* ^= mask; + bitmap.* |= mask; return (idx * 64) + bit; } @@ -361,12 +375,12 @@ test "findFreeChunks single found" { const testing = std.testing; var bitmaps = [_]u64{ - 0b10000000_00000000_00000000_00000000_00000000_00000000_00001110_00000000, + 0b01111111_11111111_11111111_11111111_11111111_11111111_11110001_11111111, }; const idx = findFreeChunks(&bitmaps, 2).?; try testing.expectEqual(@as(usize, 9), idx); try testing.expectEqual( - 0b10000000_00000000_00000000_00000000_00000000_00000000_00001000_00000000, + 0b01111111_11111111_11111111_11111111_11111111_11111111_11110111_11111111, bitmaps[0], ); } @@ -374,7 +388,7 @@ test "findFreeChunks single found" { test "findFreeChunks single not found" { const testing = std.testing; - var bitmaps = [_]u64{0b10000111_00000000_00000000_00000000_00000000_00000000_00000000_00000000}; + var bitmaps = [_]u64{0b01111000_11111111_11111111_11111111_11111111_11111111_11111111_11111111}; const idx = findFreeChunks(&bitmaps, 4); try testing.expect(idx == null); } @@ -383,13 +397,13 @@ test "findFreeChunks multiple found" { const testing = std.testing; var bitmaps = [_]u64{ - 0b10000111_00000000_00000000_00000000_00000000_00000000_00000000_01110000, - 0b10000000_00111110_00000000_00000000_00000000_00000000_00111110_00000000, + 0b01111000_11111111_11111111_11111111_11111111_11111111_11111111_10001111, + 0b01111111_11000001_11111111_11111111_11111111_11111111_11000001_11111111, }; const idx = findFreeChunks(&bitmaps, 4).?; try testing.expectEqual(@as(usize, 73), idx); try testing.expectEqual( - 0b10000000_00111110_00000000_00000000_00000000_00000000_00100000_00000000, + 0b01111111_11000001_11111111_11111111_11111111_11111111_11011111_11111111, bitmaps[1], ); } @@ -398,11 +412,11 @@ test "findFreeChunks exactly 64 chunks" { const testing = std.testing; var bitmaps = [_]u64{ - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, }; const idx = findFreeChunks(&bitmaps, 64).?; try testing.expectEqual( - 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[0], ); try testing.expectEqual(@as(usize, 0), idx); @@ -412,16 +426,16 @@ test "findFreeChunks larger than 64 chunks" { const testing = std.testing; var bitmaps = [_]u64{ - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, }; const idx = findFreeChunks(&bitmaps, 65).?; try testing.expectEqual( - 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[0], ); try testing.expectEqual( - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111110, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000001, bitmaps[1], ); try testing.expectEqual(@as(usize, 0), idx); @@ -431,21 +445,21 @@ test "findFreeChunks larger than 64 chunks not at beginning" { const testing = std.testing; var bitmaps = [_]u64{ - 0b11111111_00000000_00000000_00000000_00000000_00000000_00000000_00000000, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, }; const idx = findFreeChunks(&bitmaps, 65).?; try testing.expectEqual( - 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[0], ); try testing.expectEqual( - 0b11111110_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b00000001_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[1], ); try testing.expectEqual( - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, bitmaps[2], ); try testing.expectEqual(@as(usize, 56), idx); @@ -455,16 +469,16 @@ test "findFreeChunks larger than 64 chunks exact" { const testing = std.testing; var bitmaps = [_]u64{ - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, }; const idx = findFreeChunks(&bitmaps, 128).?; try testing.expectEqual( - 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[0], ); try testing.expectEqual( - 0b00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, bitmaps[1], ); try testing.expectEqual(@as(usize, 0), idx); @@ -677,7 +691,7 @@ test "BitmapAllocator alloc and free one bitmap" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -798,7 +812,7 @@ test "BitmapAllocator alloc and free half bitmap" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -853,7 +867,7 @@ test "BitmapAllocator alloc and free two half bitmaps" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -890,7 +904,7 @@ test "BitmapAllocator alloc and free 1.5 bitmaps" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -945,7 +959,7 @@ test "BitmapAllocator alloc and free two 1.5 bitmaps" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -1002,7 +1016,7 @@ test "BitmapAllocator alloc and free 1.5 bitmaps offset by 0.75" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -1080,7 +1094,7 @@ test "BitmapAllocator alloc and free three 0.75 bitmaps" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([3]u64, @splat(~@as(u64, 0))), + &@as([3]u64, @splat(0)), bm.bitmap.ptr(buf)[0..3], ); } @@ -1159,7 +1173,7 @@ test "BitmapAllocator alloc and free two 1.5 bitmaps offset 0.75" { // All of our bitmaps should be free. try testing.expectEqualSlices( u64, - &@as([4]u64, @splat(~@as(u64, 0))), + &@as([4]u64, @splat(0)), bm.bitmap.ptr(buf)[0..4], ); } From c0a4f80d80d75f7d3250d12554501fd7197e9bb0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 3 Sep 2026 09:13:36 -0700 Subject: [PATCH 2/4] terminal: hash map and ref counted set can initialize from zeroed memory --- src/terminal/hash_map.zig | 27 ++++++++++++++++++++++++--- src/terminal/ref_counted_set.zig | 29 ++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/terminal/hash_map.zig b/src/terminal/hash_map.zig index 6f3b8b3aa..777a3559c 100644 --- a/src/terminal/hash_map.zig +++ b/src/terminal/hash_map.zig @@ -152,8 +152,19 @@ pub fn OffsetHashMap( /// memory. The backing memory must be aligned to base_align. pub fn init(buf: OffsetBuf, l: Layout) Self { assert(base_align.check(@intFromPtr(buf.start()))); + return fromUnmanaged(buf, Unmanaged.init(buf, l)); + } - const m = Unmanaged.init(buf, l); + /// Like `init`, but for backing memory that the caller guarantees + /// is already zero-filled (e.g. fresh OS pages). Only the header + /// is written: all-zero slot metadata already means every slot is + /// free, so the metadata array is left untouched. + pub fn initAssumeZeroed(buf: OffsetBuf, l: Layout) Self { + assert(base_align.check(@intFromPtr(buf.start()))); + return fromUnmanaged(buf, Unmanaged.initAssumeZeroed(buf, l)); + } + + fn fromUnmanaged(buf: OffsetBuf, m: Unmanaged) Self { return .{ .metadata = getOffset( Unmanaged.Metadata, buf, @@ -344,6 +355,17 @@ fn HashMapUnmanaged( /// Initialize a hash map with a given capacity and a buffer. The /// buffer must fit within the size defined by `layoutForCapacity`. pub fn init(buf: OffsetBuf, layout: Layout) Self { + var map = initAssumeZeroed(buf, layout); + map.initMetadatas(); + return map; + } + + /// Like `init`, but for a buffer that the caller guarantees is + /// already zero-filled. Only the header is written: an all-zero + /// metadata byte is a free slot (see `Metadata.isFree`), so the + /// slot metadata is left untouched. Behavior is undefined if the + /// metadata region is not zero. + pub fn initAssumeZeroed(buf: OffsetBuf, layout: Layout) Self { assert(base_align.check(@intFromPtr(buf.start()))); // Get all our main pointers @@ -351,13 +373,12 @@ fn HashMapUnmanaged( const metadata_ptr: [*]Metadata = @ptrCast(metadata_buf.start()); // Build our map - var map: Self = .{ .metadata = metadata_ptr }; + const map: Self = .{ .metadata = metadata_ptr }; const hdr = map.header(); hdr.capacity = layout.capacity; hdr.size = 0; if (@sizeOf([*]K) != 0) hdr.keys = metadata_buf.member(K, layout.keys_start); if (@sizeOf([*]V) != 0) hdr.values = metadata_buf.member(V, layout.vals_start); - map.initMetadatas(); return map; } diff --git a/src/terminal/ref_counted_set.zig b/src/terminal/ref_counted_set.zig index a8e64fcc7..669c2c575 100644 --- a/src/terminal/ref_counted_set.zig +++ b/src/terminal/ref_counted_set.zig @@ -212,6 +212,30 @@ pub fn RefCountedSet( @memset(table.ptr(base)[0..l.table_cap], 0); @memset(items.ptr(base)[0..l.cap], .{}); + return initFromParts(table, items, l, context); + } + + /// Like `init`, but for backing memory that the caller guarantees + /// is already zero-filled (e.g. fresh OS pages). This writes + /// nothing to the backing buffer, so the OS pages behind the table + /// and items stay untouched until the first `add`. + /// + /// Behavior is undefined if the backing memory is not zero. + pub fn initAssumeZeroed(base: OffsetBuf, l: Layout, context: Context) Self { + return initFromParts( + base.member(Id, l.table_start), + base.member(Item, l.items_start), + l, + context, + ); + } + + fn initFromParts( + table: Offset(Id), + items: Offset(Item), + l: Layout, + context: Context, + ) Self { return .{ .table = table, .items = items, @@ -726,7 +750,9 @@ pub fn RefCountedSet( var psl_stats: [32]Id = @splat(0); - for (items[0..self.layout.cap], 0..) |item, id| { + // Start at item 1 because item 0 is reserved and never + // assigned to. Its metadata doesn't matter. + for (items[1..self.next_id], 1..) |item, id| { if (item.meta.bucket < std.math.maxInt(Id)) { assert(table[item.meta.bucket] == id); psl_stats[item.meta.psl] += 1; @@ -740,6 +766,7 @@ pub fn RefCountedSet( psl_stats = @splat(0); for (table[0..self.layout.table_cap], 0..) |id, bucket| { + if (id == 0) continue; const item = items[id]; if (item.meta.bucket < std.math.maxInt(Id)) { assert(item.meta.bucket == bucket); From 6112935a2fe9b7f5fd6c7595225c53406bee3bb1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 3 Sep 2026 10:20:52 -0700 Subject: [PATCH 3/4] terminal: hash map keeps its capacity and entry pointers in the struct --- src/terminal/hash_map.zig | 272 ++++++++++++++++++-------------------- 1 file changed, 128 insertions(+), 144 deletions(-) diff --git a/src/terminal/hash_map.zig b/src/terminal/hash_map.zig index 777a3559c..050bea63f 100644 --- a/src/terminal/hash_map.zig +++ b/src/terminal/hash_map.zig @@ -139,8 +139,18 @@ pub fn OffsetHashMap( /// This is the alignment that the base pointer must have. pub const base_align = Unmanaged.base_align; + /// The slot metadata in the backing memory. The map's size counter + /// sits immediately before it (see `Unmanaged.Header`). metadata: Offset(Unmanaged.Metadata) = .{}, + /// The key and value arrays in the backing memory. + keys: Offset(K) = .{}, + values: Offset(V) = .{}, + + /// The number of slots. This never changes after init, so it is + /// kept here rather than in the backing memory. + capacity: Unmanaged.Size = 0, + /// Returns the total size of the backing memory required for a /// HashMap with the given capacity. The base ptr must also be /// aligned to base_align. @@ -151,30 +161,35 @@ pub fn OffsetHashMap( /// Initialize a new HashMap with the given capacity and backing /// memory. The backing memory must be aligned to base_align. pub fn init(buf: OffsetBuf, l: Layout) Self { - assert(base_align.check(@intFromPtr(buf.start()))); - return fromUnmanaged(buf, Unmanaged.init(buf, l)); + const self = initAssumeZeroed(buf, l); + var m = self.map(buf); + m.clearRetainingCapacity(); + return self; } /// Like `init`, but for backing memory that the caller guarantees - /// is already zero-filled (e.g. fresh OS pages). Only the header - /// is written: all-zero slot metadata already means every slot is - /// free, so the metadata array is left untouched. + /// is already zero-filled (e.g. fresh OS pages). This writes + /// nothing to the backing memory: all-zero slot metadata means + /// every slot is free and a zero size counter means empty, so the + /// OS pages behind the map stay untouched until the first insert. pub fn initAssumeZeroed(buf: OffsetBuf, l: Layout) Self { assert(base_align.check(@intFromPtr(buf.start()))); - return fromUnmanaged(buf, Unmanaged.initAssumeZeroed(buf, l)); - } - - fn fromUnmanaged(buf: OffsetBuf, m: Unmanaged) Self { - return .{ .metadata = getOffset( - Unmanaged.Metadata, - buf, - @ptrCast(m.metadata.?), - ) }; + return .{ + .metadata = buf.member(Unmanaged.Metadata, l.metadata_start), + .keys = buf.member(K, l.keys_start), + .values = buf.member(V, l.vals_start), + .capacity = l.capacity, + }; } /// Returns the pointer-based map from a base pointer. pub fn map(self: Self, base: anytype) Unmanaged { - return .{ .metadata = self.metadata.ptr(base) }; + return .{ + .metadata = self.metadata.ptr(base), + .keys = self.keys.ptr(base), + .values = self.values.ptr(base), + .cap = self.capacity, + }; } }; } @@ -207,15 +222,21 @@ fn HashMapUnmanaged( val_align, )); - // This is actually a midway pointer to the single buffer containing - // a `Header` field, the `Metadata`s and `Entry`s. - // At `-@sizeOf(Header)` is the Header field. - // At `sizeOf(Metadata) * capacity + offset`, which is pointed to by - // self.header().entries, is the array of entries. - // This means that the hashmap only holds one live allocation, to - // reduce memory fragmentation and struct size. - /// Pointer to the metadata. - metadata: ?[*]Metadata = null, + // The backing buffer holds a `Header` (the size counter) followed + // by the `Metadata`s, then the keys and values arrays. Everything + // the map needs that does not change after init, the capacity and + // the entry pointers, lives in this struct instead of the buffer, + // so that a zero-filled buffer is a valid empty map and init never + // has to write to it. + /// Pointer to the slot metadata. The header sits right before it. + metadata: [*]Metadata, + + /// The key and value arrays. + keys: [*]K, + values: [*]V, + + /// The number of slots. Always zero or a power of two. + cap: Size, // This hashmap is specially designed for sizes that fit in a u32. pub const Size = u32; @@ -234,11 +255,10 @@ fn HashMapUnmanaged( value: V, }; + /// The part of the map's state that changes after init. It lives + /// in the backing buffer so that the map can be handed around by + /// value; its zero value is the empty map. const Header = struct { - /// The keys/values offset are relative to the metadata - values: Offset(V), - keys: Offset(K), - capacity: Size, size: Size, }; @@ -298,20 +318,20 @@ fn HashMapUnmanaged( index: Size = 0, pub fn next(it: *Iterator) ?Entry { - assert(it.index <= it.hm.capacity()); + assert(it.index <= it.hm.cap); if (it.hm.header().size == 0) return null; - const cap = it.hm.capacity(); - const end = it.hm.metadata.? + cap; - var metadata = it.hm.metadata.? + it.index; + const cap = it.hm.cap; + const end = it.hm.metadata + cap; + var metadata = it.hm.metadata + it.index; while (metadata != end) : ({ metadata += 1; it.index += 1; }) { if (metadata[0].isUsed()) { - const key = &it.hm.keys()[it.index]; - const value = &it.hm.values()[it.index]; + const key = &it.hm.keys[it.index]; + const value = &it.hm.values[it.index]; it.index += 1; return Entry{ .key_ptr = key, .value_ptr = value }; } @@ -356,31 +376,23 @@ fn HashMapUnmanaged( /// buffer must fit within the size defined by `layoutForCapacity`. pub fn init(buf: OffsetBuf, layout: Layout) Self { var map = initAssumeZeroed(buf, layout); - map.initMetadatas(); + map.clearRetainingCapacity(); return map; } /// Like `init`, but for a buffer that the caller guarantees is - /// already zero-filled. Only the header is written: an all-zero - /// metadata byte is a free slot (see `Metadata.isFree`), so the - /// slot metadata is left untouched. Behavior is undefined if the + /// already zero-filled. Nothing is written: an all-zero metadata + /// byte is a free slot (see `Metadata.isFree`) and a zero header + /// is an empty map. Behavior is undefined if the header and /// metadata region is not zero. pub fn initAssumeZeroed(buf: OffsetBuf, layout: Layout) Self { assert(base_align.check(@intFromPtr(buf.start()))); - - // Get all our main pointers - const metadata_buf = buf.rebase(@sizeOf(Header)); - const metadata_ptr: [*]Metadata = @ptrCast(metadata_buf.start()); - - // Build our map - const map: Self = .{ .metadata = metadata_ptr }; - const hdr = map.header(); - hdr.capacity = layout.capacity; - hdr.size = 0; - if (@sizeOf([*]K) != 0) hdr.keys = metadata_buf.member(K, layout.keys_start); - if (@sizeOf([*]V) != 0) hdr.values = metadata_buf.member(V, layout.vals_start); - - return map; + return .{ + .metadata = @ptrCast(buf.start() + layout.metadata_start), + .keys = buf.member(K, layout.keys_start).ptr(buf), + .values = buf.member(V, layout.vals_start).ptr(buf), + .cap = layout.capacity, + }; } pub fn ensureTotalCapacity(self: *Self, new_size: Size) Allocator.Error!void { @@ -394,10 +406,8 @@ fn HashMapUnmanaged( } pub fn clearRetainingCapacity(self: *Self) void { - if (self.metadata) |_| { - self.initMetadatas(); - self.header().size = 0; - } + self.initMetadatas(); + self.header().size = 0; } pub fn count(self: *const Self) Size { @@ -405,28 +415,18 @@ fn HashMapUnmanaged( } fn header(self: *const Self) *Header { - return @ptrCast(@as([*]Header, @ptrCast(@alignCast(self.metadata.?))) - 1); - } - - fn keys(self: *const Self) [*]K { - return self.header().keys.ptr(self.metadata.?); - } - - fn values(self: *const Self) [*]V { - return self.header().values.ptr(self.metadata.?); + return @ptrCast(@as([*]Header, @ptrCast(@alignCast(self.metadata))) - 1); } pub fn capacity(self: *const Self) Size { - if (self.metadata == null) return 0; - - return self.header().capacity; + return self.cap; } /// Maximum number of entries the map will hold. This is less than /// capacity when max_load_percentage is below 100, which keeps free /// slots in every probe chain and bounds probe lengths. pub fn maxLoad(self: *const Self) Size { - return maxLoadForCapacity(self.capacity()); + return maxLoadForCapacity(self.cap); } pub fn iterator(self: *const Self) Iterator { @@ -434,35 +434,19 @@ fn HashMapUnmanaged( } pub fn keyIterator(self: *const Self) KeyIterator { - if (self.metadata) |metadata| { - return .{ - .len = self.capacity(), - .metadata = metadata, - .items = self.keys(), - }; - } else { - return .{ - .len = 0, - .metadata = undefined, - .items = undefined, - }; - } + return .{ + .len = self.cap, + .metadata = self.metadata, + .items = self.keys, + }; } pub fn valueIterator(self: *const Self) ValueIterator { - if (self.metadata) |metadata| { - return .{ - .len = self.capacity(), - .metadata = metadata, - .items = self.values(), - }; - } else { - return .{ - .len = 0, - .metadata = undefined, - .items = undefined, - }; - } + return .{ + .len = self.cap, + .metadata = self.metadata, + .items = self.values, + }; } /// Insert an entry in the map. Assumes it is not already present. @@ -502,21 +486,21 @@ fn HashMapUnmanaged( assert(!self.containsContext(key, ctx)); // A free slot must exist for the probe below to terminate. - assert(self.header().size < self.capacity()); + assert(self.header().size < self.cap); const hash = ctx.hash(key); - const mask = self.capacity() - 1; + const mask = self.cap - 1; var idx = @as(usize, @truncate(hash & mask)); - var metadata = self.metadata.? + idx; + var metadata = self.metadata + idx; while (metadata[0].isUsed()) { idx = (idx + 1) & mask; - metadata = self.metadata.? + idx; + metadata = self.metadata + idx; } metadata[0].fill(Metadata.takeFingerprint(hash)); - self.keys()[idx] = key; - self.values()[idx] = value; + self.keys[idx] = key; + self.values[idx] = value; self.header().size += 1; } @@ -571,8 +555,8 @@ fn HashMapUnmanaged( pub fn fetchRemoveContext(self: *Self, key: K, ctx: Context) ?KV { const idx = self.getIndex(key, ctx) orelse return null; const result = KV{ - .key = self.keys()[idx], - .value = self.values()[idx], + .key = self.keys[idx], + .value = self.values[idx], }; self.removeByIndexContext(idx, ctx); return result; @@ -597,16 +581,16 @@ fn HashMapUnmanaged( if (@TypeOf(hash) != Hash) { @compileError("Context " ++ @typeName(@TypeOf(ctx)) ++ " has a generic hash function that returns the wrong type! " ++ @typeName(Hash) ++ " was expected, but found " ++ @typeName(@TypeOf(hash))); } - const mask = self.capacity() - 1; + const mask = self.cap - 1; const fingerprint = Metadata.takeFingerprint(hash); // Don't loop indefinitely when there are no free slots. - var limit = self.capacity(); + var limit = self.cap; var idx = @as(usize, @truncate(hash & mask)); - var metadata = self.metadata.? + idx; + var metadata = self.metadata + idx; while (!metadata[0].isFree() and limit != 0) { if (metadata[0].isUsed() and metadata[0].fingerprint == fingerprint) { - const test_key = &self.keys()[idx]; + const test_key = &self.keys[idx]; // If you get a compile error on this line, it means that your generic eql // function is invalid for these parameters. const eql = ctx.eql(key, test_key.*); @@ -622,7 +606,7 @@ fn HashMapUnmanaged( limit -= 1; idx = (idx + 1) & mask; - metadata = self.metadata.? + idx; + metadata = self.metadata + idx; } return null; @@ -639,8 +623,8 @@ fn HashMapUnmanaged( pub fn getEntryAdapted(self: Self, key: anytype, ctx: anytype) ?Entry { if (self.getIndex(key, ctx)) |idx| { return Entry{ - .key_ptr = &self.keys()[idx], - .value_ptr = &self.values()[idx], + .key_ptr = &self.keys[idx], + .value_ptr = &self.values[idx], }; } return null; @@ -668,7 +652,7 @@ fn HashMapUnmanaged( } pub fn getKeyPtrAdapted(self: Self, key: anytype, ctx: anytype) ?*K { if (self.getIndex(key, ctx)) |idx| { - return &self.keys()[idx]; + return &self.keys[idx]; } return null; } @@ -684,7 +668,7 @@ fn HashMapUnmanaged( } pub fn getKeyAdapted(self: Self, key: anytype, ctx: anytype) ?K { if (self.getIndex(key, ctx)) |idx| { - return self.keys()[idx]; + return self.keys[idx]; } return null; } @@ -700,7 +684,7 @@ fn HashMapUnmanaged( } pub fn getPtrAdapted(self: Self, key: anytype, ctx: anytype) ?*V { if (self.getIndex(key, ctx)) |idx| { - return &self.values()[idx]; + return &self.values[idx]; } return null; } @@ -716,7 +700,7 @@ fn HashMapUnmanaged( } pub fn getAdapted(self: Self, key: anytype, ctx: anytype) ?V { if (self.getIndex(key, ctx)) |idx| { - return self.values()[idx]; + return self.values[idx]; } return null; } @@ -745,8 +729,8 @@ fn HashMapUnmanaged( // error, we could not add another. const index = self.getIndex(key, key_ctx) orelse return err; return GetOrPutResult{ - .key_ptr = &self.keys()[index], - .value_ptr = &self.values()[index], + .key_ptr = &self.keys[index], + .value_ptr = &self.values[index], .found_existing = true, }; }; @@ -774,15 +758,15 @@ fn HashMapUnmanaged( if (@TypeOf(hash) != Hash) { @compileError("Context " ++ @typeName(@TypeOf(ctx)) ++ " has a generic hash function that returns the wrong type! " ++ @typeName(Hash) ++ " was expected, but found " ++ @typeName(@TypeOf(hash))); } - const mask = self.capacity() - 1; + const mask = self.cap - 1; const fingerprint = Metadata.takeFingerprint(hash); - var limit = self.capacity(); + var limit = self.cap; var idx = @as(usize, @truncate(hash & mask)); - var metadata = self.metadata.? + idx; + var metadata = self.metadata + idx; while (!metadata[0].isFree() and limit != 0) { if (metadata[0].isUsed() and metadata[0].fingerprint == fingerprint) { - const test_key = &self.keys()[idx]; + const test_key = &self.keys[idx]; // If you get a compile error on this line, it means that your generic eql // function is invalid for these parameters. const eql = ctx.eql(key, test_key.*); @@ -794,7 +778,7 @@ fn HashMapUnmanaged( if (eql) { return GetOrPutResult{ .key_ptr = test_key, - .value_ptr = &self.values()[idx], + .value_ptr = &self.values[idx], .found_existing = true, }; } @@ -802,7 +786,7 @@ fn HashMapUnmanaged( limit -= 1; idx = (idx + 1) & mask; - metadata = self.metadata.? + idx; + metadata = self.metadata + idx; } // The caller guaranteed capacity for at least one new entry, so @@ -812,8 +796,8 @@ fn HashMapUnmanaged( assert(metadata[0].isFree()); metadata[0].fill(fingerprint); - const new_key = &self.keys()[idx]; - const new_value = &self.values()[idx]; + const new_key = &self.keys[idx]; + const new_value = &self.values[idx]; new_key.* = undefined; new_value.* = undefined; self.header().size += 1; @@ -860,10 +844,10 @@ fn HashMapUnmanaged( /// hole further along the cluster, until the cluster ends at a free /// slot. fn removeByIndexContext(self: *Self, idx: usize, ctx: Context) void { - const mask: usize = self.capacity() - 1; - const metadata = self.metadata.?; - const keys_ptr = self.keys(); - const values_ptr = self.values(); + const mask: usize = self.cap - 1; + const metadata = self.metadata; + const keys_ptr = self.keys; + const values_ptr = self.values; // A completely full table has no free slot to terminate the // scan, so bound it to one full cycle. That is sufficient: the @@ -871,7 +855,7 @@ fn HashMapUnmanaged( // visited, so each entry needs to be considered exactly once. var hole = idx; var j = idx; - var limit = self.capacity() - 1; + var limit = self.cap - 1; while (limit != 0) : (limit -= 1) { j = (j + 1) & mask; if (metadata[j].isFree()) break; @@ -925,7 +909,7 @@ fn HashMapUnmanaged( // map, which is assumed to exist as key_ptr must be valid. This // item must be at index 0. const idx = if (@sizeOf(K) > 0) - (@intFromPtr(key_ptr) - @intFromPtr(self.keys())) / @sizeOf(K) + (@intFromPtr(key_ptr) - @intFromPtr(self.keys)) / @sizeOf(K) else 0; @@ -933,7 +917,7 @@ fn HashMapUnmanaged( } fn initMetadatas(self: *Self) void { - @memset(@as([*]u8, @ptrCast(self.metadata.?))[0 .. @sizeOf(Metadata) * self.capacity()], 0); + @memset(@as([*]u8, @ptrCast(self.metadata))[0 .. @sizeOf(Metadata) * self.cap], 0); } /// Returns an error if the map cannot hold `new_count` more entries. @@ -953,11 +937,16 @@ fn HashMapUnmanaged( } /// The memory layout for the underlying buffer for a given capacity. + /// All offsets are from the start of the buffer. const Layout = struct { /// The total size of the buffer required. The buffer is expected /// to be aligned to `base_align`. total_size: usize, + /// The offset to the start of the slot metadata. The header + /// occupies the bytes before it. + metadata_start: usize, + /// The offset to the start of the keys data. keys_start: usize, @@ -979,9 +968,9 @@ fn HashMapUnmanaged( // See: https://github.com/ziglang/zig/pull/19048 const cap: usize = new_capacity; - // Pack our metadata, keys, and values. + // Pack our header, metadata, keys, and values. const meta_start = @sizeOf(Header); - const meta_end = @sizeOf(Header) + cap * @sizeOf(Metadata); + const meta_end = meta_start + cap * @sizeOf(Metadata); const keys_start = std.mem.alignForward(usize, meta_end, key_align); const keys_end = keys_start + cap * @sizeOf(K); const vals_start = std.mem.alignForward(usize, keys_end, val_align); @@ -995,16 +984,11 @@ fn HashMapUnmanaged( base_align.toByteUnits(), ); - // The offsets we actually store in the map are from the - // metadata pointer so that we can use self.metadata as - // the base. - const keys_offset = keys_start - meta_start; - const vals_offset = vals_start - meta_start; - return .{ .total_size = total_size, - .keys_start = keys_offset, - .vals_start = vals_offset, + .metadata_start = meta_start, + .keys_start = keys_start, + .vals_start = vals_start, .capacity = new_capacity, }; } @@ -1054,13 +1038,13 @@ fn expectCanonical(map: anytype, ctx: anytype) !void { const mask = cap - 1; var used: usize = 0; for (0..cap) |idx| { - const metadata = map.metadata.?[idx]; + const metadata = map.metadata[idx]; if (!metadata.isUsed()) continue; used += 1; - var probe: usize = @truncate(ctx.hash(map.keys()[idx]) & mask); + var probe: usize = @truncate(ctx.hash(map.keys[idx]) & mask); while (probe != idx) : (probe = (probe + 1) & mask) { - try expect(map.metadata.?[probe].isUsed()); + try expect(map.metadata[probe].isUsed()); } } try expectEqual(map.count(), used); From d2ff6d77a05d9e9b247684fad6bcb7979d0b2aca Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 3 Sep 2026 10:34:46 -0700 Subject: [PATCH 4/4] terminal: pages initialize from zeroed memory and cache-line align their cells --- src/terminal/page.zig | 275 +++++++++++++++++++++++++++--------------- 1 file changed, 176 insertions(+), 99 deletions(-) diff --git a/src/terminal/page.zig b/src/terminal/page.zig index 94898f83d..ad2b1a7e5 100644 --- a/src/terminal/page.zig +++ b/src/terminal/page.zig @@ -24,7 +24,6 @@ const BitmapAllocator = @import("bitmap_allocator.zig").BitmapAllocator; const hash_map = @import("hash_map.zig"); const AutoOffsetHashMap = hash_map.AutoOffsetHashMap; const alignForward = std.mem.alignForward; -const alignBackward = std.mem.alignBackward; const log = std.log.scoped(.page); @@ -127,6 +126,19 @@ const hyperlink_count_default = 4; const hyperlink_bytes_default = hyperlink_count_default * @sizeOf(hyperlink.Set.Item); const hyperlink_cell_multiplier = 16; +/// The alignment of the start of a page's cell array. Align it to a cache +/// line so that row cells always start on a cache line. This avoids +/// a scenario where cells in every row always start mid-cache line and +/// straddle an extra. +const cells_align: usize = @max( + @alignOf(Cell), + @min( + std.atomic.cache_line, + // Cap it at page size for freestanding targets. + std.heap.page_size_min, + ), +); + /// A page represents a specific section of terminal screen. The primary /// idea of a page is that it is a fully self-contained unit that can be /// serialized, copied, etc. as a convenient way to represent a section @@ -148,8 +160,8 @@ pub const Page = struct { // alignment is always divisible by this. assert(std.heap.page_size_min % @max( @alignOf(Row), - @alignOf(Cell), - StyleSet.base_align.toByteUnits(), + cells_align, + MetaLayout.alignment, ) == 0); } @@ -242,6 +254,12 @@ pub const Page = struct { /// Initialize a new page using the given backing memory. /// It is up to the caller to not call deinit on these pages. + /// + /// The backing memory must be zero-filled. A page treats zero as the + /// empty state everywhere: cells are blank when zero, and every + /// metadata member initializes from zeroed memory without writing + /// anything. Only the row headers are written, so the OS pages behind + /// everything else stay untouched until first use. pub inline fn initBuf(buf: OffsetBuf, l: Layout) Page { const cap = l.capacity; @@ -267,28 +285,28 @@ pub const Page = struct { .memory = @alignCast(buf.start()[0..l.total_size]), .rows = rows, .cells = cells, - .styles = StyleSet.init( + .styles = StyleSet.initAssumeZeroed( buf.add(l.styles_start), l.styles_layout, .{}, ), - .string_alloc = .init( + .string_alloc = .initAssumeZeroed( buf.add(l.string_alloc_start), l.string_alloc_layout, ), - .grapheme_alloc = .init( + .grapheme_alloc = .initAssumeZeroed( buf.add(l.grapheme_alloc_start), l.grapheme_alloc_layout, ), - .grapheme_map = .init( + .grapheme_map = .initAssumeZeroed( buf.add(l.grapheme_map_start), l.grapheme_map_layout, ), - .hyperlink_map = .init( + .hyperlink_map = .initAssumeZeroed( buf.add(l.hyperlink_map_start), l.hyperlink_map_layout, ), - .hyperlink_set = .init( + .hyperlink_set = .initAssumeZeroed( buf.add(l.hyperlink_set_start), l.hyperlink_set_layout, .{}, @@ -1730,57 +1748,32 @@ pub const Page = struct { /// The memory layout for a page given a desired minimum cols /// and rows size. + /// + /// The backing memory is laid out as the row headers, the cell + /// array, and then the metadata block (see `MetaLayout`): + /// + /// [rows][cells][styles, graphemes, strings, hyperlinks] + /// + /// Row headers always start at offset zero. The cell array is aligned + /// to a cache line (see `cells_align`). Initializing a page writes only + /// the row headers: the cells and every metadata member treat zero as + /// their empty state, so the OS pages behind everything past the row + /// headers stay untouched until something is stored in them. pub inline fn layout(cap: Capacity) Layout { - const rows_count: usize = @intCast(cap.rows); + const meta: MetaLayout = .init(cap); + const rows_count: usize = @intCast(cap.rows); const rows_start = 0; const rows_end: usize = rows_start + (rows_count * @sizeOf(Row)); const cells_count: usize = @as(usize, cap.cols) * @as(usize, cap.rows); - const cells_start = alignForward(usize, rows_end, @alignOf(Cell)); + const cells_start = alignForward(usize, rows_end, cells_align); const cells_end = cells_start + (cells_count * @sizeOf(Cell)); - const styles_layout: StyleSet.Layout = .init(cap.styles); - const styles_start = alignForward(usize, cells_end, StyleSet.base_align.toByteUnits()); - const styles_end = styles_start + styles_layout.total_size; + const meta_start = alignForward(usize, cells_end, MetaLayout.alignment); + const meta_end = meta_start + meta.total_size; - const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes); - const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits()); - const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size; - - const grapheme_count: usize = count: { - if (cap.grapheme_bytes == 0) break :count 0; - // Use divCeil to match GraphemeAlloc.layout() which uses alignForward, - // ensuring grapheme_map has capacity when grapheme_alloc has chunks. - const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable; - break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable; - }; - const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count)); - const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits()); - const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size; - - const string_layout = StringAlloc.layout(cap.string_bytes); - const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits()); - const string_end = string_start + string_layout.total_size; - - const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item)); - const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count)); - const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits()); - const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size; - - const hyperlink_map_count: u32 = count: { - if (hyperlink_count == 0) break :count 0; - const mult = std.math.cast( - u32, - hyperlink_count * hyperlink_cell_multiplier, - ) orelse break :count std.math.maxInt(u32); - break :count mult; - }; - const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count); - const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits()); - const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size; - - const total_size = alignForward(usize, hyperlink_map_end, std.heap.page_size_min); + const total_size = alignForward(usize, meta_end, std.heap.page_size_min); return .{ .total_size = total_size, @@ -1788,21 +1781,111 @@ pub const Page = struct { .rows_size = rows_end - rows_start, .cells_start = cells_start, .cells_size = cells_end - cells_start, - .styles_start = styles_start, - .styles_layout = styles_layout, - .grapheme_alloc_start = grapheme_alloc_start, - .grapheme_alloc_layout = grapheme_alloc_layout, - .grapheme_map_start = grapheme_map_start, - .grapheme_map_layout = grapheme_map_layout, - .string_alloc_start = string_start, - .string_alloc_layout = string_layout, - .hyperlink_map_start = hyperlink_map_start, - .hyperlink_map_layout = hyperlink_map_layout, - .hyperlink_set_start = hyperlink_set_start, - .hyperlink_set_layout = hyperlink_set_layout, + .styles_start = meta_start + meta.styles_start, + .styles_layout = meta.styles_layout, + .grapheme_alloc_start = meta_start + meta.grapheme_alloc_start, + .grapheme_alloc_layout = meta.grapheme_alloc_layout, + .grapheme_map_start = meta_start + meta.grapheme_map_start, + .grapheme_map_layout = meta.grapheme_map_layout, + .string_alloc_start = meta_start + meta.string_alloc_start, + .string_alloc_layout = meta.string_alloc_layout, + .hyperlink_map_start = meta_start + meta.hyperlink_map_start, + .hyperlink_map_layout = meta.hyperlink_map_layout, + .hyperlink_set_start = meta_start + meta.hyperlink_set_start, + .hyperlink_set_layout = meta.hyperlink_set_layout, .capacity = cap, }; } + + /// Meta is everything that isn't the grid, such as styles, graphemes, + /// etc. `layout` places it directly after the grid. + pub const MetaLayout = struct { + /// The size of the block including internal alignment padding. + total_size: usize, + styles_start: usize, + styles_layout: StyleSet.Layout, + grapheme_alloc_start: usize, + grapheme_alloc_layout: GraphemeAlloc.Layout, + grapheme_map_start: usize, + grapheme_map_layout: GraphemeMap.Layout, + string_alloc_start: usize, + string_alloc_layout: StringAlloc.Layout, + hyperlink_set_start: usize, + hyperlink_set_layout: hyperlink.Set.Layout, + hyperlink_map_start: usize, + hyperlink_map_layout: hyperlink.Map.Layout, + + /// The alignment of the block's start: the largest alignment any + /// member requires, so that the member offsets above don't depend + /// on where the block is placed. + pub const alignment = @max( + StyleSet.base_align.toByteUnits(), + GraphemeAlloc.base_align.toByteUnits(), + GraphemeMap.base_align.toByteUnits(), + StringAlloc.base_align.toByteUnits(), + hyperlink.Set.base_align.toByteUnits(), + hyperlink.Map.base_align.toByteUnits(), + ); + + /// Compute the layout of the block for the given capacity. + pub fn init(cap: Capacity) MetaLayout { + const styles_layout: StyleSet.Layout = .init(cap.styles); + const styles_start = 0; + const styles_end = styles_start + styles_layout.total_size; + + const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes); + const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits()); + const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size; + + const grapheme_count: usize = count: { + if (cap.grapheme_bytes == 0) break :count 0; + // Use divCeil to match GraphemeAlloc.layout() which uses alignForward, + // ensuring grapheme_map has capacity when grapheme_alloc has chunks. + const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable; + break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable; + }; + const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count)); + const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits()); + const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size; + + const string_layout = StringAlloc.layout(cap.string_bytes); + const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits()); + const string_end = string_start + string_layout.total_size; + + const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item)); + const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count)); + const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits()); + const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size; + + const hyperlink_map_count: u32 = count: { + if (hyperlink_count == 0) break :count 0; + const mult = std.math.cast( + u32, + hyperlink_count * hyperlink_cell_multiplier, + ) orelse break :count std.math.maxInt(u32); + break :count mult; + }; + const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count); + const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits()); + const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size; + + return .{ + .total_size = hyperlink_map_end, + .styles_start = styles_start, + .styles_layout = styles_layout, + .grapheme_alloc_start = grapheme_alloc_start, + .grapheme_alloc_layout = grapheme_alloc_layout, + .grapheme_map_start = grapheme_map_start, + .grapheme_map_layout = grapheme_map_layout, + .string_alloc_start = string_start, + .string_alloc_layout = string_layout, + .hyperlink_set_start = hyperlink_set_start, + .hyperlink_set_layout = hyperlink_set_layout, + .hyperlink_map_start = hyperlink_map_start, + .hyperlink_map_layout = hyperlink_map_layout, + }; + } + }; }; /// The standard capacity for a page that doesn't have special @@ -1864,15 +1947,16 @@ pub const Capacity = struct { /// the amount of memory the original capacity will take. If you modify /// the original capacity to add rows, then you can fit more columns. pub fn maxCols(self: Capacity) ?size.CellCountInt { - const available_bits = self.availableBitsForGrid(); + const available = self.availableBytesForGrid(); - // If we can't even fit the row metadata, return null - if (available_bits <= @bitSizeOf(Row)) return null; + // A single row's header occupies a whole cell-aligned region + // ahead of the cells. If we can't even fit that, return null. + const row_region = alignForward(usize, @sizeOf(Row), cells_align); + if (available <= row_region) return null; // We do the math of how many columns we can fit in the remaining - // bits ignoring the metadata of a row. - const remaining_bits = available_bits - @bitSizeOf(Row); - const max_cols = remaining_bits / @bitSizeOf(Cell); + // bytes ignoring the metadata of a row. + const max_cols = (available - row_region) / @sizeOf(Cell); // Clamp to CellCountInt max return @min(std.math.maxInt(size.CellCountInt), max_cols); @@ -1886,51 +1970,44 @@ pub const Capacity = struct { pub fn adjust(self: Capacity, req: Adjustment) Allocator.Error!Capacity { var adjusted = self; if (req.cols) |cols| { - const available_bits = self.availableBitsForGrid(); + const total_size = Page.layout(self).total_size; + const available = self.availableBytesForGrid(); // The size per row is: // - The row metadata itself // - The cells per row (n=cols) - const bits_per_row: usize = @bitSizeOf(Row) + @bitSizeOf(Cell) * @as(usize, @intCast(cols)); - const new_rows: usize = @divFloor(available_bits, bits_per_row); + const bytes_per_row: usize = @sizeOf(Row) + @sizeOf(Cell) * @as(usize, @intCast(cols)); + var new_rows: usize = @divFloor(available, bytes_per_row); + + // The cell array is aligned to a cache line, so the padding + // between the row headers and the cells depends on the row + // count. Trim rows until the layout fits the original size. + // The padding is less than a cache line so this takes a + // handful of iterations at most. + adjusted.cols = cols; + while (new_rows > 0) : (new_rows -= 1) { + adjusted.rows = @intCast(new_rows); + if (Page.layout(adjusted).total_size <= total_size) break; + } // If our rows go to zero then we can't fit any row metadata // for the desired number of columns. if (new_rows == 0) return error.OutOfMemory; - - adjusted.cols = cols; - adjusted.rows = @intCast(new_rows); } return adjusted; } - /// Computes the number of bits available for rows and cells in the page. - /// - /// This is done by laying out the "meta" members (styles, graphemes, - /// hyperlinks, strings) from the end of the page and finding where they - /// start, which gives us the space available for rows and cells. - fn availableBitsForGrid(self: Capacity) usize { - // The math below only works if there is no alignment gap between - // the end of the rows array and the start of the cells array. - // - // To guarantee this, we assert that Row's size is a multiple of - // Cell's alignment, so that any length array of Rows will end on - // a valid alignment for the start of the Cell array. - assert(@sizeOf(Row) % @alignOf(Cell) == 0); + /// Computes the number of bytes available for the row headers and + /// cells in the page: the page size minus the metadata block. + fn availableBytesForGrid(self: Capacity) usize { + comptime { + assert(cells_align % Page.MetaLayout.alignment == 0); + assert(@sizeOf(Cell) % Page.MetaLayout.alignment == 0); + } const l = Page.layout(self); - - // Layout meta members from the end to find styles_start - const hyperlink_map_start = alignBackward(usize, l.total_size - l.hyperlink_map_layout.total_size, hyperlink.Map.base_align.toByteUnits()); - const hyperlink_set_start = alignBackward(usize, hyperlink_map_start - l.hyperlink_set_layout.total_size, hyperlink.Set.base_align.toByteUnits()); - const string_alloc_start = alignBackward(usize, hyperlink_set_start - l.string_alloc_layout.total_size, StringAlloc.base_align.toByteUnits()); - const grapheme_map_start = alignBackward(usize, string_alloc_start - l.grapheme_map_layout.total_size, GraphemeMap.base_align.toByteUnits()); - const grapheme_alloc_start = alignBackward(usize, grapheme_map_start - l.grapheme_alloc_layout.total_size, GraphemeAlloc.base_align.toByteUnits()); - const styles_start = alignBackward(usize, grapheme_alloc_start - l.styles_layout.total_size, StyleSet.base_align.toByteUnits()); - - // Multiply by 8 to convert bytes to bits - return styles_start * 8; + return l.total_size - Page.MetaLayout.init(self).total_size; } };