terminal: pages initialize from zeroed memory and cache-line align their cells

src/terminal/page.zig

@@ -24,7 +24,6 @@ const BitmapAllocator = @import("bitmap_allocator.zig").BitmapAllocator;
 const hash_map = @import("hash_map.zig");
 const AutoOffsetHashMap = hash_map.AutoOffsetHashMap;
 const alignForward = std.mem.alignForward;
-const alignBackward = std.mem.alignBackward;
 
 const log = std.log.scoped(.page);
 
@@ -127,6 +126,19 @@ const hyperlink_count_default = 4;
 const hyperlink_bytes_default = hyperlink_count_default * @sizeOf(hyperlink.Set.Item);
 const hyperlink_cell_multiplier = 16;
 
+/// The alignment of the start of a page's cell array. Align it to a cache
+/// line so that row cells always start on a cache line. This avoids
+/// a scenario where cells in every row always start mid-cache line and
+/// straddle an extra.
+const cells_align: usize = @max(
+    @alignOf(Cell),
+    @min(
+        std.atomic.cache_line,
+        // Cap it at page size for freestanding targets.
+        std.heap.page_size_min,
+    ),
+);
+
 /// A page represents a specific section of terminal screen. The primary
 /// idea of a page is that it is a fully self-contained unit that can be
 /// serialized, copied, etc. as a convenient way to represent a section
@@ -148,8 +160,8 @@ pub const Page = struct {
         // alignment is always divisible by this.
         assert(std.heap.page_size_min % @max(
             @alignOf(Row),
-            @alignOf(Cell),
-            StyleSet.base_align.toByteUnits(),
+            cells_align,
+            MetaLayout.alignment,
         ) == 0);
     }
 
@@ -242,6 +254,12 @@ pub const Page = struct {
 
     /// Initialize a new page using the given backing memory.
     /// It is up to the caller to not call deinit on these pages.
+    ///
+    /// The backing memory must be zero-filled. A page treats zero as the
+    /// empty state everywhere: cells are blank when zero, and every
+    /// metadata member initializes from zeroed memory without writing
+    /// anything. Only the row headers are written, so the OS pages behind
+    /// everything else stay untouched until first use.
     pub inline fn initBuf(buf: OffsetBuf, l: Layout) Page {
         const cap = l.capacity;
 
@@ -267,28 +285,28 @@ pub const Page = struct {
             .memory = @alignCast(buf.start()[0..l.total_size]),
             .rows = rows,
             .cells = cells,
-            .styles = StyleSet.init(
+            .styles = StyleSet.initAssumeZeroed(
                 buf.add(l.styles_start),
                 l.styles_layout,
                 .{},
             ),
-            .string_alloc = .init(
+            .string_alloc = .initAssumeZeroed(
                 buf.add(l.string_alloc_start),
                 l.string_alloc_layout,
             ),
-            .grapheme_alloc = .init(
+            .grapheme_alloc = .initAssumeZeroed(
                 buf.add(l.grapheme_alloc_start),
                 l.grapheme_alloc_layout,
             ),
-            .grapheme_map = .init(
+            .grapheme_map = .initAssumeZeroed(
                 buf.add(l.grapheme_map_start),
                 l.grapheme_map_layout,
             ),
-            .hyperlink_map = .init(
+            .hyperlink_map = .initAssumeZeroed(
                 buf.add(l.hyperlink_map_start),
                 l.hyperlink_map_layout,
             ),
-            .hyperlink_set = .init(
+            .hyperlink_set = .initAssumeZeroed(
                 buf.add(l.hyperlink_set_start),
                 l.hyperlink_set_layout,
                 .{},
@@ -1730,57 +1748,32 @@ pub const Page = struct {
 
     /// The memory layout for a page given a desired minimum cols
     /// and rows size.
+    ///
+    /// The backing memory is laid out as the row headers, the cell
+    /// array, and then the metadata block (see `MetaLayout`):
+    ///
+    ///   [rows][cells][styles, graphemes, strings, hyperlinks]
+    ///
+    /// Row headers always start at offset zero. The cell array is aligned
+    /// to a cache line (see `cells_align`). Initializing a page writes only
+    /// the row headers: the cells and every metadata member treat zero as
+    /// their empty state, so the OS pages behind everything past the row
+    /// headers stay untouched until something is stored in them.
     pub inline fn layout(cap: Capacity) Layout {
-        const rows_count: usize = @intCast(cap.rows);
+        const meta: MetaLayout = .init(cap);
 
+        const rows_count: usize = @intCast(cap.rows);
         const rows_start = 0;
         const rows_end: usize = rows_start + (rows_count * @sizeOf(Row));
 
         const cells_count: usize = @as(usize, cap.cols) * @as(usize, cap.rows);
-        const cells_start = alignForward(usize, rows_end, @alignOf(Cell));
+        const cells_start = alignForward(usize, rows_end, cells_align);
         const cells_end = cells_start + (cells_count * @sizeOf(Cell));
 
-        const styles_layout: StyleSet.Layout = .init(cap.styles);
-        const styles_start = alignForward(usize, cells_end, StyleSet.base_align.toByteUnits());
-        const styles_end = styles_start + styles_layout.total_size;
+        const meta_start = alignForward(usize, cells_end, MetaLayout.alignment);
+        const meta_end = meta_start + meta.total_size;
 
-        const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes);
-        const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits());
-        const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size;
-
-        const grapheme_count: usize = count: {
-            if (cap.grapheme_bytes == 0) break :count 0;
-            // Use divCeil to match GraphemeAlloc.layout() which uses alignForward,
-            // ensuring grapheme_map has capacity when grapheme_alloc has chunks.
-            const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable;
-            break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable;
-        };
-        const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count));
-        const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits());
-        const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size;
-
-        const string_layout = StringAlloc.layout(cap.string_bytes);
-        const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits());
-        const string_end = string_start + string_layout.total_size;
-
-        const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item));
-        const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count));
-        const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits());
-        const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size;
-
-        const hyperlink_map_count: u32 = count: {
-            if (hyperlink_count == 0) break :count 0;
-            const mult = std.math.cast(
-                u32,
-                hyperlink_count * hyperlink_cell_multiplier,
-            ) orelse break :count std.math.maxInt(u32);
-            break :count mult;
-        };
-        const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count);
-        const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits());
-        const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size;
-
-        const total_size = alignForward(usize, hyperlink_map_end, std.heap.page_size_min);
+        const total_size = alignForward(usize, meta_end, std.heap.page_size_min);
 
         return .{
             .total_size = total_size,
@@ -1788,21 +1781,111 @@ pub const Page = struct {
             .rows_size = rows_end - rows_start,
             .cells_start = cells_start,
             .cells_size = cells_end - cells_start,
-            .styles_start = styles_start,
-            .styles_layout = styles_layout,
-            .grapheme_alloc_start = grapheme_alloc_start,
-            .grapheme_alloc_layout = grapheme_alloc_layout,
-            .grapheme_map_start = grapheme_map_start,
-            .grapheme_map_layout = grapheme_map_layout,
-            .string_alloc_start = string_start,
-            .string_alloc_layout = string_layout,
-            .hyperlink_map_start = hyperlink_map_start,
-            .hyperlink_map_layout = hyperlink_map_layout,
-            .hyperlink_set_start = hyperlink_set_start,
-            .hyperlink_set_layout = hyperlink_set_layout,
+            .styles_start = meta_start + meta.styles_start,
+            .styles_layout = meta.styles_layout,
+            .grapheme_alloc_start = meta_start + meta.grapheme_alloc_start,
+            .grapheme_alloc_layout = meta.grapheme_alloc_layout,
+            .grapheme_map_start = meta_start + meta.grapheme_map_start,
+            .grapheme_map_layout = meta.grapheme_map_layout,
+            .string_alloc_start = meta_start + meta.string_alloc_start,
+            .string_alloc_layout = meta.string_alloc_layout,
+            .hyperlink_map_start = meta_start + meta.hyperlink_map_start,
+            .hyperlink_map_layout = meta.hyperlink_map_layout,
+            .hyperlink_set_start = meta_start + meta.hyperlink_set_start,
+            .hyperlink_set_layout = meta.hyperlink_set_layout,
             .capacity = cap,
         };
     }
+
+    /// Meta is everything that isn't the grid, such as styles, graphemes,
+    /// etc. `layout` places it directly after the grid.
+    pub const MetaLayout = struct {
+        /// The size of the block including internal alignment padding.
+        total_size: usize,
+        styles_start: usize,
+        styles_layout: StyleSet.Layout,
+        grapheme_alloc_start: usize,
+        grapheme_alloc_layout: GraphemeAlloc.Layout,
+        grapheme_map_start: usize,
+        grapheme_map_layout: GraphemeMap.Layout,
+        string_alloc_start: usize,
+        string_alloc_layout: StringAlloc.Layout,
+        hyperlink_set_start: usize,
+        hyperlink_set_layout: hyperlink.Set.Layout,
+        hyperlink_map_start: usize,
+        hyperlink_map_layout: hyperlink.Map.Layout,
+
+        /// The alignment of the block's start: the largest alignment any
+        /// member requires, so that the member offsets above don't depend
+        /// on where the block is placed.
+        pub const alignment = @max(
+            StyleSet.base_align.toByteUnits(),
+            GraphemeAlloc.base_align.toByteUnits(),
+            GraphemeMap.base_align.toByteUnits(),
+            StringAlloc.base_align.toByteUnits(),
+            hyperlink.Set.base_align.toByteUnits(),
+            hyperlink.Map.base_align.toByteUnits(),
+        );
+
+        /// Compute the layout of the block for the given capacity.
+        pub fn init(cap: Capacity) MetaLayout {
+            const styles_layout: StyleSet.Layout = .init(cap.styles);
+            const styles_start = 0;
+            const styles_end = styles_start + styles_layout.total_size;
+
+            const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes);
+            const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits());
+            const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size;
+
+            const grapheme_count: usize = count: {
+                if (cap.grapheme_bytes == 0) break :count 0;
+                // Use divCeil to match GraphemeAlloc.layout() which uses alignForward,
+                // ensuring grapheme_map has capacity when grapheme_alloc has chunks.
+                const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable;
+                break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable;
+            };
+            const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count));
+            const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits());
+            const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size;
+
+            const string_layout = StringAlloc.layout(cap.string_bytes);
+            const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits());
+            const string_end = string_start + string_layout.total_size;
+
+            const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item));
+            const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count));
+            const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits());
+            const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size;
+
+            const hyperlink_map_count: u32 = count: {
+                if (hyperlink_count == 0) break :count 0;
+                const mult = std.math.cast(
+                    u32,
+                    hyperlink_count * hyperlink_cell_multiplier,
+                ) orelse break :count std.math.maxInt(u32);
+                break :count mult;
+            };
+            const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count);
+            const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits());
+            const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size;
+
+            return .{
+                .total_size = hyperlink_map_end,
+                .styles_start = styles_start,
+                .styles_layout = styles_layout,
+                .grapheme_alloc_start = grapheme_alloc_start,
+                .grapheme_alloc_layout = grapheme_alloc_layout,
+                .grapheme_map_start = grapheme_map_start,
+                .grapheme_map_layout = grapheme_map_layout,
+                .string_alloc_start = string_start,
+                .string_alloc_layout = string_layout,
+                .hyperlink_set_start = hyperlink_set_start,
+                .hyperlink_set_layout = hyperlink_set_layout,
+                .hyperlink_map_start = hyperlink_map_start,
+                .hyperlink_map_layout = hyperlink_map_layout,
+            };
+        }
+    };
 };
 
 /// The standard capacity for a page that doesn't have special
@@ -1864,15 +1947,16 @@ pub const Capacity = struct {
     /// the amount of memory the original capacity will take. If you modify
     /// the original capacity to add rows, then you can fit more columns.
     pub fn maxCols(self: Capacity) ?size.CellCountInt {
-        const available_bits = self.availableBitsForGrid();
+        const available = self.availableBytesForGrid();
 
-        // If we can't even fit the row metadata, return null
-        if (available_bits <= @bitSizeOf(Row)) return null;
+        // A single row's header occupies a whole cell-aligned region
+        // ahead of the cells. If we can't even fit that, return null.
+        const row_region = alignForward(usize, @sizeOf(Row), cells_align);
+        if (available <= row_region) return null;
 
         // We do the math of how many columns we can fit in the remaining
-        // bits ignoring the metadata of a row.
-        const remaining_bits = available_bits - @bitSizeOf(Row);
-        const max_cols = remaining_bits / @bitSizeOf(Cell);
+        // bytes ignoring the metadata of a row.
+        const max_cols = (available - row_region) / @sizeOf(Cell);
 
         // Clamp to CellCountInt max
         return @min(std.math.maxInt(size.CellCountInt), max_cols);
@@ -1886,51 +1970,44 @@ pub const Capacity = struct {
     pub fn adjust(self: Capacity, req: Adjustment) Allocator.Error!Capacity {
         var adjusted = self;
         if (req.cols) |cols| {
-            const available_bits = self.availableBitsForGrid();
+            const total_size = Page.layout(self).total_size;
+            const available = self.availableBytesForGrid();
 
             // The size per row is:
             //   - The row metadata itself
             //   - The cells per row (n=cols)
-            const bits_per_row: usize = @bitSizeOf(Row) + @bitSizeOf(Cell) * @as(usize, @intCast(cols));
-            const new_rows: usize = @divFloor(available_bits, bits_per_row);
+            const bytes_per_row: usize = @sizeOf(Row) + @sizeOf(Cell) * @as(usize, @intCast(cols));
+            var new_rows: usize = @divFloor(available, bytes_per_row);
+
+            // The cell array is aligned to a cache line, so the padding
+            // between the row headers and the cells depends on the row
+            // count. Trim rows until the layout fits the original size.
+            // The padding is less than a cache line so this takes a
+            // handful of iterations at most.
+            adjusted.cols = cols;
+            while (new_rows > 0) : (new_rows -= 1) {
+                adjusted.rows = @intCast(new_rows);
+                if (Page.layout(adjusted).total_size <= total_size) break;
+            }
 
             // If our rows go to zero then we can't fit any row metadata
             // for the desired number of columns.
             if (new_rows == 0) return error.OutOfMemory;
-
-            adjusted.cols = cols;
-            adjusted.rows = @intCast(new_rows);
         }
 
         return adjusted;
     }
 
-    /// Computes the number of bits available for rows and cells in the page.
-    ///
-    /// This is done by laying out the "meta" members (styles, graphemes,
-    /// hyperlinks, strings) from the end of the page and finding where they
-    /// start, which gives us the space available for rows and cells.
-    fn availableBitsForGrid(self: Capacity) usize {
-        // The math below only works if there is no alignment gap between
-        // the end of the rows array and the start of the cells array.
-        //
-        // To guarantee this, we assert that Row's size is a multiple of
-        // Cell's alignment, so that any length array of Rows will end on
-        // a valid alignment for the start of the Cell array.
-        assert(@sizeOf(Row) % @alignOf(Cell) == 0);
+    /// Computes the number of bytes available for the row headers and
+    /// cells in the page: the page size minus the metadata block.
+    fn availableBytesForGrid(self: Capacity) usize {
+        comptime {
+            assert(cells_align % Page.MetaLayout.alignment == 0);
+            assert(@sizeOf(Cell) % Page.MetaLayout.alignment == 0);
+        }
 
         const l = Page.layout(self);
-
-        // Layout meta members from the end to find styles_start
-        const hyperlink_map_start = alignBackward(usize, l.total_size - l.hyperlink_map_layout.total_size, hyperlink.Map.base_align.toByteUnits());
-        const hyperlink_set_start = alignBackward(usize, hyperlink_map_start - l.hyperlink_set_layout.total_size, hyperlink.Set.base_align.toByteUnits());
-        const string_alloc_start = alignBackward(usize, hyperlink_set_start - l.string_alloc_layout.total_size, StringAlloc.base_align.toByteUnits());
-        const grapheme_map_start = alignBackward(usize, string_alloc_start - l.grapheme_map_layout.total_size, GraphemeMap.base_align.toByteUnits());
-        const grapheme_alloc_start = alignBackward(usize, grapheme_map_start - l.grapheme_alloc_layout.total_size, GraphemeAlloc.base_align.toByteUnits());
-        const styles_start = alignBackward(usize, grapheme_alloc_start - l.styles_layout.total_size, StyleSet.base_align.toByteUnits());
-
-        // Multiply by 8 to convert bytes to bits
-        return styles_start * 8;
+        return l.total_size - Page.MetaLayout.init(self).total_size;
     }
 };