diff --git a/src/terminal/page.zig b/src/terminal/page.zig index 94898f83d..ad2b1a7e5 100644 --- a/src/terminal/page.zig +++ b/src/terminal/page.zig @@ -24,7 +24,6 @@ const BitmapAllocator = @import("bitmap_allocator.zig").BitmapAllocator; const hash_map = @import("hash_map.zig"); const AutoOffsetHashMap = hash_map.AutoOffsetHashMap; const alignForward = std.mem.alignForward; -const alignBackward = std.mem.alignBackward; const log = std.log.scoped(.page); @@ -127,6 +126,19 @@ const hyperlink_count_default = 4; const hyperlink_bytes_default = hyperlink_count_default * @sizeOf(hyperlink.Set.Item); const hyperlink_cell_multiplier = 16; +/// The alignment of the start of a page's cell array. Align it to a cache +/// line so that row cells always start on a cache line. This avoids +/// a scenario where cells in every row always start mid-cache line and +/// straddle an extra. +const cells_align: usize = @max( + @alignOf(Cell), + @min( + std.atomic.cache_line, + // Cap it at page size for freestanding targets. + std.heap.page_size_min, + ), +); + /// A page represents a specific section of terminal screen. The primary /// idea of a page is that it is a fully self-contained unit that can be /// serialized, copied, etc. as a convenient way to represent a section @@ -148,8 +160,8 @@ pub const Page = struct { // alignment is always divisible by this. assert(std.heap.page_size_min % @max( @alignOf(Row), - @alignOf(Cell), - StyleSet.base_align.toByteUnits(), + cells_align, + MetaLayout.alignment, ) == 0); } @@ -242,6 +254,12 @@ pub const Page = struct { /// Initialize a new page using the given backing memory. /// It is up to the caller to not call deinit on these pages. + /// + /// The backing memory must be zero-filled. A page treats zero as the + /// empty state everywhere: cells are blank when zero, and every + /// metadata member initializes from zeroed memory without writing + /// anything. Only the row headers are written, so the OS pages behind + /// everything else stay untouched until first use. pub inline fn initBuf(buf: OffsetBuf, l: Layout) Page { const cap = l.capacity; @@ -267,28 +285,28 @@ pub const Page = struct { .memory = @alignCast(buf.start()[0..l.total_size]), .rows = rows, .cells = cells, - .styles = StyleSet.init( + .styles = StyleSet.initAssumeZeroed( buf.add(l.styles_start), l.styles_layout, .{}, ), - .string_alloc = .init( + .string_alloc = .initAssumeZeroed( buf.add(l.string_alloc_start), l.string_alloc_layout, ), - .grapheme_alloc = .init( + .grapheme_alloc = .initAssumeZeroed( buf.add(l.grapheme_alloc_start), l.grapheme_alloc_layout, ), - .grapheme_map = .init( + .grapheme_map = .initAssumeZeroed( buf.add(l.grapheme_map_start), l.grapheme_map_layout, ), - .hyperlink_map = .init( + .hyperlink_map = .initAssumeZeroed( buf.add(l.hyperlink_map_start), l.hyperlink_map_layout, ), - .hyperlink_set = .init( + .hyperlink_set = .initAssumeZeroed( buf.add(l.hyperlink_set_start), l.hyperlink_set_layout, .{}, @@ -1730,57 +1748,32 @@ pub const Page = struct { /// The memory layout for a page given a desired minimum cols /// and rows size. + /// + /// The backing memory is laid out as the row headers, the cell + /// array, and then the metadata block (see `MetaLayout`): + /// + /// [rows][cells][styles, graphemes, strings, hyperlinks] + /// + /// Row headers always start at offset zero. The cell array is aligned + /// to a cache line (see `cells_align`). Initializing a page writes only + /// the row headers: the cells and every metadata member treat zero as + /// their empty state, so the OS pages behind everything past the row + /// headers stay untouched until something is stored in them. pub inline fn layout(cap: Capacity) Layout { - const rows_count: usize = @intCast(cap.rows); + const meta: MetaLayout = .init(cap); + const rows_count: usize = @intCast(cap.rows); const rows_start = 0; const rows_end: usize = rows_start + (rows_count * @sizeOf(Row)); const cells_count: usize = @as(usize, cap.cols) * @as(usize, cap.rows); - const cells_start = alignForward(usize, rows_end, @alignOf(Cell)); + const cells_start = alignForward(usize, rows_end, cells_align); const cells_end = cells_start + (cells_count * @sizeOf(Cell)); - const styles_layout: StyleSet.Layout = .init(cap.styles); - const styles_start = alignForward(usize, cells_end, StyleSet.base_align.toByteUnits()); - const styles_end = styles_start + styles_layout.total_size; + const meta_start = alignForward(usize, cells_end, MetaLayout.alignment); + const meta_end = meta_start + meta.total_size; - const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes); - const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits()); - const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size; - - const grapheme_count: usize = count: { - if (cap.grapheme_bytes == 0) break :count 0; - // Use divCeil to match GraphemeAlloc.layout() which uses alignForward, - // ensuring grapheme_map has capacity when grapheme_alloc has chunks. - const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable; - break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable; - }; - const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count)); - const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits()); - const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size; - - const string_layout = StringAlloc.layout(cap.string_bytes); - const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits()); - const string_end = string_start + string_layout.total_size; - - const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item)); - const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count)); - const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits()); - const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size; - - const hyperlink_map_count: u32 = count: { - if (hyperlink_count == 0) break :count 0; - const mult = std.math.cast( - u32, - hyperlink_count * hyperlink_cell_multiplier, - ) orelse break :count std.math.maxInt(u32); - break :count mult; - }; - const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count); - const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits()); - const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size; - - const total_size = alignForward(usize, hyperlink_map_end, std.heap.page_size_min); + const total_size = alignForward(usize, meta_end, std.heap.page_size_min); return .{ .total_size = total_size, @@ -1788,21 +1781,111 @@ pub const Page = struct { .rows_size = rows_end - rows_start, .cells_start = cells_start, .cells_size = cells_end - cells_start, - .styles_start = styles_start, - .styles_layout = styles_layout, - .grapheme_alloc_start = grapheme_alloc_start, - .grapheme_alloc_layout = grapheme_alloc_layout, - .grapheme_map_start = grapheme_map_start, - .grapheme_map_layout = grapheme_map_layout, - .string_alloc_start = string_start, - .string_alloc_layout = string_layout, - .hyperlink_map_start = hyperlink_map_start, - .hyperlink_map_layout = hyperlink_map_layout, - .hyperlink_set_start = hyperlink_set_start, - .hyperlink_set_layout = hyperlink_set_layout, + .styles_start = meta_start + meta.styles_start, + .styles_layout = meta.styles_layout, + .grapheme_alloc_start = meta_start + meta.grapheme_alloc_start, + .grapheme_alloc_layout = meta.grapheme_alloc_layout, + .grapheme_map_start = meta_start + meta.grapheme_map_start, + .grapheme_map_layout = meta.grapheme_map_layout, + .string_alloc_start = meta_start + meta.string_alloc_start, + .string_alloc_layout = meta.string_alloc_layout, + .hyperlink_map_start = meta_start + meta.hyperlink_map_start, + .hyperlink_map_layout = meta.hyperlink_map_layout, + .hyperlink_set_start = meta_start + meta.hyperlink_set_start, + .hyperlink_set_layout = meta.hyperlink_set_layout, .capacity = cap, }; } + + /// Meta is everything that isn't the grid, such as styles, graphemes, + /// etc. `layout` places it directly after the grid. + pub const MetaLayout = struct { + /// The size of the block including internal alignment padding. + total_size: usize, + styles_start: usize, + styles_layout: StyleSet.Layout, + grapheme_alloc_start: usize, + grapheme_alloc_layout: GraphemeAlloc.Layout, + grapheme_map_start: usize, + grapheme_map_layout: GraphemeMap.Layout, + string_alloc_start: usize, + string_alloc_layout: StringAlloc.Layout, + hyperlink_set_start: usize, + hyperlink_set_layout: hyperlink.Set.Layout, + hyperlink_map_start: usize, + hyperlink_map_layout: hyperlink.Map.Layout, + + /// The alignment of the block's start: the largest alignment any + /// member requires, so that the member offsets above don't depend + /// on where the block is placed. + pub const alignment = @max( + StyleSet.base_align.toByteUnits(), + GraphemeAlloc.base_align.toByteUnits(), + GraphemeMap.base_align.toByteUnits(), + StringAlloc.base_align.toByteUnits(), + hyperlink.Set.base_align.toByteUnits(), + hyperlink.Map.base_align.toByteUnits(), + ); + + /// Compute the layout of the block for the given capacity. + pub fn init(cap: Capacity) MetaLayout { + const styles_layout: StyleSet.Layout = .init(cap.styles); + const styles_start = 0; + const styles_end = styles_start + styles_layout.total_size; + + const grapheme_alloc_layout = GraphemeAlloc.layout(cap.grapheme_bytes); + const grapheme_alloc_start = alignForward(usize, styles_end, GraphemeAlloc.base_align.toByteUnits()); + const grapheme_alloc_end = grapheme_alloc_start + grapheme_alloc_layout.total_size; + + const grapheme_count: usize = count: { + if (cap.grapheme_bytes == 0) break :count 0; + // Use divCeil to match GraphemeAlloc.layout() which uses alignForward, + // ensuring grapheme_map has capacity when grapheme_alloc has chunks. + const base = std.math.divCeil(usize, cap.grapheme_bytes, grapheme_chunk) catch unreachable; + break :count std.math.ceilPowerOfTwo(usize, base) catch unreachable; + }; + const grapheme_map_layout = GraphemeMap.layout(@intCast(grapheme_count)); + const grapheme_map_start = alignForward(usize, grapheme_alloc_end, GraphemeMap.base_align.toByteUnits()); + const grapheme_map_end = grapheme_map_start + grapheme_map_layout.total_size; + + const string_layout = StringAlloc.layout(cap.string_bytes); + const string_start = alignForward(usize, grapheme_map_end, StringAlloc.base_align.toByteUnits()); + const string_end = string_start + string_layout.total_size; + + const hyperlink_count = @divFloor(cap.hyperlink_bytes, @sizeOf(hyperlink.Set.Item)); + const hyperlink_set_layout: hyperlink.Set.Layout = .init(@intCast(hyperlink_count)); + const hyperlink_set_start = alignForward(usize, string_end, hyperlink.Set.base_align.toByteUnits()); + const hyperlink_set_end = hyperlink_set_start + hyperlink_set_layout.total_size; + + const hyperlink_map_count: u32 = count: { + if (hyperlink_count == 0) break :count 0; + const mult = std.math.cast( + u32, + hyperlink_count * hyperlink_cell_multiplier, + ) orelse break :count std.math.maxInt(u32); + break :count mult; + }; + const hyperlink_map_layout = hyperlink.Map.layout(hyperlink_map_count); + const hyperlink_map_start = alignForward(usize, hyperlink_set_end, hyperlink.Map.base_align.toByteUnits()); + const hyperlink_map_end = hyperlink_map_start + hyperlink_map_layout.total_size; + + return .{ + .total_size = hyperlink_map_end, + .styles_start = styles_start, + .styles_layout = styles_layout, + .grapheme_alloc_start = grapheme_alloc_start, + .grapheme_alloc_layout = grapheme_alloc_layout, + .grapheme_map_start = grapheme_map_start, + .grapheme_map_layout = grapheme_map_layout, + .string_alloc_start = string_start, + .string_alloc_layout = string_layout, + .hyperlink_set_start = hyperlink_set_start, + .hyperlink_set_layout = hyperlink_set_layout, + .hyperlink_map_start = hyperlink_map_start, + .hyperlink_map_layout = hyperlink_map_layout, + }; + } + }; }; /// The standard capacity for a page that doesn't have special @@ -1864,15 +1947,16 @@ pub const Capacity = struct { /// the amount of memory the original capacity will take. If you modify /// the original capacity to add rows, then you can fit more columns. pub fn maxCols(self: Capacity) ?size.CellCountInt { - const available_bits = self.availableBitsForGrid(); + const available = self.availableBytesForGrid(); - // If we can't even fit the row metadata, return null - if (available_bits <= @bitSizeOf(Row)) return null; + // A single row's header occupies a whole cell-aligned region + // ahead of the cells. If we can't even fit that, return null. + const row_region = alignForward(usize, @sizeOf(Row), cells_align); + if (available <= row_region) return null; // We do the math of how many columns we can fit in the remaining - // bits ignoring the metadata of a row. - const remaining_bits = available_bits - @bitSizeOf(Row); - const max_cols = remaining_bits / @bitSizeOf(Cell); + // bytes ignoring the metadata of a row. + const max_cols = (available - row_region) / @sizeOf(Cell); // Clamp to CellCountInt max return @min(std.math.maxInt(size.CellCountInt), max_cols); @@ -1886,51 +1970,44 @@ pub const Capacity = struct { pub fn adjust(self: Capacity, req: Adjustment) Allocator.Error!Capacity { var adjusted = self; if (req.cols) |cols| { - const available_bits = self.availableBitsForGrid(); + const total_size = Page.layout(self).total_size; + const available = self.availableBytesForGrid(); // The size per row is: // - The row metadata itself // - The cells per row (n=cols) - const bits_per_row: usize = @bitSizeOf(Row) + @bitSizeOf(Cell) * @as(usize, @intCast(cols)); - const new_rows: usize = @divFloor(available_bits, bits_per_row); + const bytes_per_row: usize = @sizeOf(Row) + @sizeOf(Cell) * @as(usize, @intCast(cols)); + var new_rows: usize = @divFloor(available, bytes_per_row); + + // The cell array is aligned to a cache line, so the padding + // between the row headers and the cells depends on the row + // count. Trim rows until the layout fits the original size. + // The padding is less than a cache line so this takes a + // handful of iterations at most. + adjusted.cols = cols; + while (new_rows > 0) : (new_rows -= 1) { + adjusted.rows = @intCast(new_rows); + if (Page.layout(adjusted).total_size <= total_size) break; + } // If our rows go to zero then we can't fit any row metadata // for the desired number of columns. if (new_rows == 0) return error.OutOfMemory; - - adjusted.cols = cols; - adjusted.rows = @intCast(new_rows); } return adjusted; } - /// Computes the number of bits available for rows and cells in the page. - /// - /// This is done by laying out the "meta" members (styles, graphemes, - /// hyperlinks, strings) from the end of the page and finding where they - /// start, which gives us the space available for rows and cells. - fn availableBitsForGrid(self: Capacity) usize { - // The math below only works if there is no alignment gap between - // the end of the rows array and the start of the cells array. - // - // To guarantee this, we assert that Row's size is a multiple of - // Cell's alignment, so that any length array of Rows will end on - // a valid alignment for the start of the Cell array. - assert(@sizeOf(Row) % @alignOf(Cell) == 0); + /// Computes the number of bytes available for the row headers and + /// cells in the page: the page size minus the metadata block. + fn availableBytesForGrid(self: Capacity) usize { + comptime { + assert(cells_align % Page.MetaLayout.alignment == 0); + assert(@sizeOf(Cell) % Page.MetaLayout.alignment == 0); + } const l = Page.layout(self); - - // Layout meta members from the end to find styles_start - const hyperlink_map_start = alignBackward(usize, l.total_size - l.hyperlink_map_layout.total_size, hyperlink.Map.base_align.toByteUnits()); - const hyperlink_set_start = alignBackward(usize, hyperlink_map_start - l.hyperlink_set_layout.total_size, hyperlink.Set.base_align.toByteUnits()); - const string_alloc_start = alignBackward(usize, hyperlink_set_start - l.string_alloc_layout.total_size, StringAlloc.base_align.toByteUnits()); - const grapheme_map_start = alignBackward(usize, string_alloc_start - l.grapheme_map_layout.total_size, GraphemeMap.base_align.toByteUnits()); - const grapheme_alloc_start = alignBackward(usize, grapheme_map_start - l.grapheme_alloc_layout.total_size, GraphemeAlloc.base_align.toByteUnits()); - const styles_start = alignBackward(usize, grapheme_alloc_start - l.styles_layout.total_size, StyleSet.base_align.toByteUnits()); - - // Multiply by 8 to convert bytes to bits - return styles_start * 8; + return l.total_size - Page.MetaLayout.init(self).total_size; } };