Renderer: Debug Overlay Framework (#10503)

This adds a cross-platform ability to draw an overlay of debug
information directly on top of our terminal.

The debug overlay is drawn on the CPU using z2d and then rendered as an
image on the GPU (using the same subsystem and shaders as things like
the Kitty Graphics Protocol). This lets us iterate more quickly on
overlays since it's all simple imperative 2D drawing operations, and
also lets it be cross-platform immediately.

The goal of this PR is to **introduce the framework.** To enable the
overlays, you have to modify code and recompile it, but of course in the
future this will be runtime toggle-able via the inspector. Additionally,
the dummy overlay in this PR (OSC8 highlighting) isn't particularly
useful and has limitations; it's only meant to be a demo. More realistic
overlays will come later.

## Demo

Here is a demo of an overlay highlighting OSC8 hyperlinks:

<img width="1746" height="1648" alt="CleanShot_2026-01-30_at_15 04 582x"
src="https://github.com/user-attachments/assets/4c490d11-5baa-4ef6-b97b-1cfd321235d8"
/>

## Performance

```
Empty but overlay active:
generic_renderer: [rebuildOverlay time] start_micro=1769835575905814 duration=12ns
generic_renderer: [updateFrame time] start_micro=1769835575905766 duration=199ns

Hyperlink on every line:
generic_renderer: [rebuildOverlay time] start_micro=1769835573560797 duration=452ns
generic_renderer: [updateFrame time] start_micro=1769835573560682 duration=891ns
```

Pretty great! 🎉

With the overlay active but nothing to draw, the overlay adds ~6%
overhead. With the overlay active and hyperlinks on every line, the
overlay adds ~50% overhead. These are big percentages, but the absolute
numbers are miniscule. And, for a high value debug feature that is only
on some of the time, it's totally acceptable.

Also, as a reminder, for 120fps we have _8.3 million nanoseconds_ per
frame (including the time to draw, of course). If we're spending
sub-1000ns we're in the realm of 1M FPS. So, it's fine. lol.

(Edit: I previously had debug timings which are significantly worse but
have a lot more safety checks.)

## Other Notes

* **Damage/dirty tracking.** I don't do any for the overlay, and we just
rebuild it on every frame. I'm not convinced the complexity is worth it
for a debug overlay and the performance numbers at this point do not
necessitate it in any way.

* **Update vs draw times.** We call `draw` _far_ more often than
`update` so putting the overlay rebuild into `update` made way more
sense. I did measure both.
This commit is contained in:
Mitchell Hashimoto
2026-01-31 08:46:52 -08:00
committed by GitHub
3 changed files with 451 additions and 28 deletions

229
src/renderer/Overlay.zig Normal file
View File

@@ -0,0 +1,229 @@
/// The debug overlay that can be drawn on top of the terminal
/// during the rendering process.
///
/// This is implemented by doing all the drawing on the CPU via z2d,
/// since the debug overlay isn't that common, z2d is pretty fast, and
/// it simplifies our implementation quite a bit by not relying on us
/// having a bunch of shaders that we have to write per-platform.
///
/// Initialize the overlay, apply features with `applyFeatures`, then
/// get the resulting image with `pendingImage` to upload to the GPU.
/// This works in concert with `renderer.image.State` to simplify. Draw
/// it on the GPU as an image composited on top of the terminal output.
const Overlay = @This();
const std = @import("std");
const Allocator = std.mem.Allocator;
const z2d = @import("z2d");
const terminal = @import("../terminal/main.zig");
const size = @import("size.zig");
const Size = size.Size;
const CellSize = size.CellSize;
const Image = @import("image.zig").Image;
/// The surface we're drawing our overlay to.
surface: z2d.Surface,
/// Cell size information so we can map grid coordinates to pixels.
cell_size: CellSize,
/// The set of available features and their configuration.
pub const Feature = union(enum) {
highlight_hyperlinks,
};
pub const InitError = Allocator.Error || error{
// The terminal dimensions are invalid to support an overlay.
// Either too small or too big.
InvalidDimensions,
};
/// Initialize a new, blank overlay.
pub fn init(alloc: Allocator, sz: Size) InitError!Overlay {
// Our surface does NOT need to take into account padding because
// we render the overlay using the image subsystem and shaders which
// already take that into account.
const term_size = sz.terminal();
var sfc = z2d.Surface.initPixel(
.{ .rgba = .{ .r = 0, .g = 0, .b = 0, .a = 0 } },
alloc,
std.math.cast(i32, term_size.width) orelse
return error.InvalidDimensions,
std.math.cast(i32, term_size.height) orelse
return error.InvalidDimensions,
) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.InvalidWidth, error.InvalidHeight => return error.InvalidDimensions,
};
errdefer sfc.deinit(alloc);
return .{
.surface = sfc,
.cell_size = sz.cell,
};
}
pub fn deinit(self: *Overlay, alloc: Allocator) void {
self.surface.deinit(alloc);
}
/// Returns a pending image that can be used to copy, convert, upload, etc.
pub fn pendingImage(self: *const Overlay) Image.Pending {
return .{
.width = @intCast(self.surface.getWidth()),
.height = @intCast(self.surface.getHeight()),
.pixel_format = .rgba,
.data = @ptrCast(self.surface.image_surface_rgba.buf.ptr),
};
}
/// Clear the overlay.
pub fn reset(self: *Overlay) void {
self.surface.paintPixel(.{ .rgba = .{
.r = 0,
.g = 0,
.b = 0,
.a = 0,
} });
}
/// Apply the given features to this overlay. This will draw on top of
/// any pre-existing content in the overlay.
pub fn applyFeatures(
self: *Overlay,
alloc: Allocator,
state: *const terminal.RenderState,
features: []const Feature,
) void {
for (features) |f| switch (f) {
.highlight_hyperlinks => self.highlightHyperlinks(
alloc,
state,
),
};
}
/// Add rectangles around contiguous hyperlinks in the render state.
///
/// Note: this currently doesn't take into account unique hyperlink IDs
/// because the render state doesn't contain this. This will be added
/// later.
fn highlightHyperlinks(
self: *Overlay,
alloc: Allocator,
state: *const terminal.RenderState,
) void {
const border_fill_rgb: z2d.pixel.RGB = .{ .r = 180, .g = 180, .b = 255 };
const border_color = border_fill_rgb.asPixel();
const fill_color: z2d.Pixel = px: {
var rgba: z2d.pixel.RGBA = .fromPixel(border_color);
rgba.a = 128;
break :px rgba.multiply().asPixel();
};
const row_slice = state.row_data.slice();
const row_raw = row_slice.items(.raw);
const row_cells = row_slice.items(.cells);
for (row_raw, row_cells, 0..) |row, cells, y| {
if (!row.hyperlink) continue;
const cells_slice = cells.slice();
const raw_cells = cells_slice.items(.raw);
var x: usize = 0;
while (x < raw_cells.len) {
// Skip cells without hyperlinks
if (!raw_cells[x].hyperlink) {
x += 1;
continue;
}
// Found start of a hyperlink run
const start_x = x;
// Find end of contiguous hyperlink cells
while (x < raw_cells.len and raw_cells[x].hyperlink) x += 1;
const end_x = x;
self.highlightRect(
alloc,
start_x,
y,
end_x - start_x,
1,
border_color,
fill_color,
) catch |err| {
std.log.warn("Error drawing hyperlink border: {}", .{err});
};
}
}
}
/// Creates a rectangle for highlighting a grid region. x/y/width/height
/// are all in grid cells.
fn highlightRect(
self: *Overlay,
alloc: Allocator,
x: usize,
y: usize,
width: usize,
height: usize,
border_color: z2d.Pixel,
fill_color: z2d.Pixel,
) !void {
// All math below uses checked arithmetic to avoid overflows. The
// inputs aren't trusted and the path this is in isn't hot enough
// to wrarrant unsafe optimizations.
// Calculate our width/height in pixels.
const px_width = std.math.cast(i32, try std.math.mul(
usize,
width,
self.cell_size.width,
)) orelse return error.Overflow;
const px_height = std.math.cast(i32, try std.math.mul(
usize,
height,
self.cell_size.height,
)) orelse return error.Overflow;
// Calculate pixel coordinates
const start_x: f64 = @floatFromInt(std.math.cast(i32, try std.math.mul(
usize,
x,
self.cell_size.width,
)) orelse return error.Overflow);
const start_y: f64 = @floatFromInt(std.math.cast(i32, try std.math.mul(
usize,
y,
self.cell_size.height,
)) orelse return error.Overflow);
const end_x: f64 = start_x + @as(f64, @floatFromInt(px_width));
const end_y: f64 = start_y + @as(f64, @floatFromInt(px_height));
// Grab our context to draw
var ctx: z2d.Context = .init(alloc, &self.surface);
defer ctx.deinit();
// Don't need AA because we use sharp edges
ctx.setAntiAliasingMode(.none);
// Can use hairline since we have 1px borders
ctx.setHairline(true);
// Draw rectangle path
try ctx.moveTo(start_x, start_y);
try ctx.lineTo(end_x, start_y);
try ctx.lineTo(end_x, end_y);
try ctx.lineTo(start_x, end_y);
try ctx.closePath();
// Fill
ctx.setSourceToPixel(fill_color);
try ctx.fill();
// Border
ctx.setLineWidth(1);
ctx.setSourceToPixel(border_color);
try ctx.stroke();
}

View File

@@ -17,6 +17,7 @@ const noMinContrast = cellpkg.noMinContrast;
const constraintWidth = cellpkg.constraintWidth;
const isCovering = cellpkg.isCovering;
const rowNeverExtendBg = @import("row.zig").neverExtendBg;
const Overlay = @import("Overlay.zig");
const imagepkg = @import("image.zig");
const ImageState = imagepkg.State;
const shadertoy = @import("shadertoy.zig");
@@ -221,6 +222,16 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
/// a large screen.
terminal_state_frame_count: usize = 0,
/// Our overlay state, if any.
overlay: ?Overlay = null,
// Right now, the debug overlay is turned on and configured by
// modifying these and recompiling. In the future, we will expose
// all of this at runtime via the inspector.
const overlay_features: []const Overlay.Feature = &.{
//.highlight_hyperlinks,
};
const HighlightTag = enum(u8) {
search_match,
search_match_selected,
@@ -781,6 +792,7 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
}
pub fn deinit(self: *Self) void {
if (self.overlay) |*overlay| overlay.deinit(self.alloc);
self.terminal_state.deinit(self.alloc);
if (self.search_selected_match) |*m| m.arena.deinit();
if (self.search_matches) |*m| m.arena.deinit();
@@ -1107,6 +1119,16 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
state: *renderer.State,
cursor_blink_visible: bool,
) Allocator.Error!void {
// const start = std.time.Instant.now() catch unreachable;
// const start_micro = std.time.microTimestamp();
// defer {
// const end = std.time.Instant.now() catch unreachable;
// log.warn(
// "[updateFrame time] start_micro={} duration={}ns",
// .{ start_micro, end.since(start) / std.time.ns_per_us },
// );
// }
// We fully deinit and reset the terminal state every so often
// so that a particularly large terminal state doesn't cause
// the renderer to hold on to retained memory.
@@ -1282,6 +1304,15 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
// Reset our dirty state after updating.
defer self.terminal_state.dirty = .false;
// Rebuild the overlay image if we have one. We can do this
// outside of any critical areas.
self.rebuildOverlay() catch |err| {
log.warn(
"error rebuilding overlay surface err={}",
.{err},
);
};
// Acquire the draw mutex for all remaining state updates.
{
self.draw_mutex.lock();
@@ -1331,6 +1362,16 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
else => {},
};
// Prepare our overlay image for upload (or unload). This
// has to use our general allocator since it modifies
// state that survives frames.
self.images.overlayUpdate(
self.alloc,
self.overlay,
) catch |err| {
log.warn("error updating overlay images err={}", .{err});
};
// Update custom shader uniforms that depend on terminal state.
self.updateCustomShaderUniformsFromState();
}
@@ -1348,6 +1389,16 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
self: *Self,
sync: bool,
) !void {
// const start = std.time.Instant.now() catch unreachable;
// const start_micro = std.time.microTimestamp();
// defer {
// const end = std.time.Instant.now() catch unreachable;
// log.warn(
// "[drawFrame time] start_micro={} duration={}ns",
// .{ start_micro, end.since(start) / std.time.ns_per_us },
// );
// }
// We hold a the draw mutex to prevent changes to any
// data we access while we're in the middle of drawing.
self.draw_mutex.lock();
@@ -1585,6 +1636,15 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
&pass,
.kitty_above_text,
);
// Debug overlay. We do this before any custom shader state
// because our debug overlay is aligned with the grid.
self.images.draw(
&self.api,
self.shaders.pipelines.image,
&pass,
.overlay,
);
}
// If we have custom shaders, then we render them.
@@ -2179,6 +2239,48 @@ pub fn Renderer(comptime GraphicsAPI: type) type {
}
}
/// Build the overlay as configured. Returns null if there is no
/// overlay currently configured.
fn rebuildOverlay(self: *Self) Overlay.InitError!void {
// const start = std.time.Instant.now() catch unreachable;
// const start_micro = std.time.microTimestamp();
// defer {
// const end = std.time.Instant.now() catch unreachable;
// log.warn(
// "[rebuildOverlay time] start_micro={} duration={}ns",
// .{ start_micro, end.since(start) / std.time.ns_per_us },
// );
// }
const alloc = self.alloc;
// If we have no features enabled, don't build an overlay.
// If we had a previous overlay, deallocate it.
if (overlay_features.len == 0) {
if (self.overlay) |*old| {
old.deinit(alloc);
self.overlay = null;
}
return;
}
// If we had a previous overlay, clear it. Otherwise, init.
const overlay: *Overlay = if (self.overlay) |*v| overlay: {
v.reset();
break :overlay v;
} else overlay: {
const new: Overlay = try .init(alloc, self.size);
self.overlay = new;
break :overlay &self.overlay.?;
};
overlay.applyFeatures(
alloc,
&self.terminal_state,
overlay_features,
);
}
const PreeditRange = struct {
y: terminal.size.CellCountInt,
x: [2]terminal.size.CellCountInt,

View File

@@ -8,6 +8,7 @@ const Renderer = @import("../renderer.zig").Renderer;
const GraphicsAPI = Renderer.API;
const Texture = GraphicsAPI.Texture;
const CellSize = @import("size.zig").CellSize;
const Overlay = @import("Overlay.zig");
const log = std.log.scoped(.renderer_image);
@@ -32,12 +33,16 @@ pub const State = struct {
/// on frame builds and are generally more expensive to handle.
kitty_virtual: bool,
/// Overlays
overlay_placements: std.ArrayListUnmanaged(Placement),
pub const empty: State = .{
.images = .empty,
.kitty_placements = .empty,
.kitty_bg_end = 0,
.kitty_text_end = 0,
.kitty_virtual = false,
.overlay_placements = .empty,
};
pub fn deinit(self: *State, alloc: Allocator) void {
@@ -47,6 +52,7 @@ pub const State = struct {
self.images.deinit(alloc);
}
self.kitty_placements.deinit(alloc);
self.overlay_placements.deinit(alloc);
}
/// Upload any images to the GPU that need to be uploaded,
@@ -88,6 +94,7 @@ pub const State = struct {
kitty_below_bg,
kitty_below_text,
kitty_above_text,
overlay,
};
/// Draw the given named set of placements.
@@ -105,6 +112,7 @@ pub const State = struct {
.kitty_below_bg => self.kitty_placements.items[0..self.kitty_bg_end],
.kitty_below_text => self.kitty_placements.items[self.kitty_bg_end..self.kitty_text_end],
.kitty_above_text => self.kitty_placements.items[self.kitty_text_end..],
.overlay => self.overlay_placements.items,
};
for (placements) |p| {
@@ -170,6 +178,57 @@ pub const State = struct {
}
}
/// Update our overlay state. Null value deletes any existing overlay.
pub fn overlayUpdate(
self: *State,
alloc: Allocator,
overlay_: ?Overlay,
) !void {
const overlay = overlay_ orelse {
// If we don't have an overlay, remove any existing one.
if (self.images.getPtr(.overlay)) |data| {
data.image.markForUnload();
}
return;
};
// For transmit time we always just use the current time
// and overwrite the overlay.
const transmit_time = try std.time.Instant.now();
// Ensure we have space for our overlay placement. Do this before
// we upload our image so we don't have to deal with cleaning
// that up.
self.overlay_placements.clearRetainingCapacity();
try self.overlay_placements.ensureUnusedCapacity(alloc, 1);
// Setup our image.
const pending = overlay.pendingImage();
try self.prepImage(
alloc,
.overlay,
transmit_time,
pending,
);
errdefer comptime unreachable;
// Setup our placement
self.overlay_placements.appendAssumeCapacity(.{
.image_id = .overlay,
.x = 0,
.y = 0,
.z = 0,
.width = pending.width,
.height = pending.height,
.cell_offset_x = 0,
.cell_offset_y = 0,
.source_x = 0,
.source_y = 0,
.source_width = pending.width,
.source_height = pending.height,
});
}
/// Returns true if the Kitty graphics state requires an update based
/// on the terminal state and our internal state.
///
@@ -220,6 +279,8 @@ pub const State = struct {
.kitty => |id| if (storage.imageById(id) == null) {
kv.value_ptr.image.markForUnload();
},
.overlay => {},
}
}
}
@@ -330,7 +391,7 @@ pub const State = struct {
text_end orelse @intCast(self.kitty_placements.items.len);
}
const PrepKittyImageError = error{
const PrepImageError = error{
OutOfMemory,
ImageConversionError,
};
@@ -345,7 +406,7 @@ pub const State = struct {
bot_y: u32,
image: *const terminal.kitty.graphics.Image,
p: *const terminal.kitty.graphics.ImageStorage.Placement,
) PrepKittyImageError!void {
) PrepImageError!void {
// Get the rect for the placement. If this placement doesn't have
// a rect then its virtual or something so skip it.
const rect = p.rect(image.*, t) orelse return;
@@ -407,7 +468,7 @@ pub const State = struct {
t: *const terminal.Terminal,
p: *const terminal.kitty.graphics.unicode.Placement,
cell_size: CellSize,
) PrepKittyImageError!void {
) PrepImageError!void {
const storage = &t.screens.active.kitty_images;
const image = storage.imageById(p.image_id) orelse {
log.warn(
@@ -459,34 +520,32 @@ pub const State = struct {
});
}
/// Prepare the provided image for upload to the GPU by copying its
/// data with our allocator and setting it to the pending state.
fn prepKittyImage(
/// Prepare an image for upload to the GPU.
fn prepImage(
self: *State,
alloc: Allocator,
image: *const terminal.kitty.graphics.Image,
) PrepKittyImageError!void {
id: Id,
transmit_time: std.time.Instant,
pending: Image.Pending,
) PrepImageError!void {
// If this image exists and its transmit time is the same we assume
// it is the identical image so we don't need to send it to the GPU.
const gop = try self.images.getOrPut(
alloc,
.{ .kitty = image.id },
);
const gop = try self.images.getOrPut(alloc, id);
if (gop.found_existing and
gop.value_ptr.transmit_time.order(image.transmit_time) == .eq)
gop.value_ptr.transmit_time.order(transmit_time) == .eq)
{
return;
}
// Copy the data into the pending state.
// Copy the data so we own it.
const data = if (alloc.dupe(
u8,
image.data,
pending.dataSlice(),
)) |v| v else |_| {
if (!gop.found_existing) {
// If this is a new entry we can just remove it since it
// was never sent to the GPU.
_ = self.images.remove(.{ .kitty = image.id });
_ = self.images.remove(id);
} else {
// If this was an existing entry, it is invalid and
// we must unload it.
@@ -502,15 +561,9 @@ pub const State = struct {
// Store it in the map
const new_image: Image = .{
.pending = .{
.width = image.width,
.height = image.height,
.pixel_format = switch (image.format) {
.gray => .gray,
.gray_alpha => .gray_alpha,
.rgb => .rgb,
.rgba => .rgba,
.png => unreachable, // should be decoded by now
},
.width = pending.width,
.height = pending.height,
.pixel_format = pending.pixel_format,
.data = data.ptr,
},
};
@@ -530,10 +583,40 @@ pub const State = struct {
errdefer gop.value_ptr.image.markForUnload();
gop.value_ptr.image.prepForUpload(alloc) catch |err| {
log.warn("error preparing kitty image for upload err={}", .{err});
log.warn("error preparing image for upload err={}", .{err});
return error.ImageConversionError;
};
gop.value_ptr.transmit_time = image.transmit_time;
gop.value_ptr.transmit_time = transmit_time;
}
/// Prepare the provided Kitty image for upload to the GPU by copying its
/// data with our allocator and setting it to the pending state.
fn prepKittyImage(
self: *State,
alloc: Allocator,
image: *const terminal.kitty.graphics.Image,
) PrepImageError!void {
try self.prepImage(
alloc,
.{ .kitty = image.id },
image.transmit_time,
.{
.width = image.width,
.height = image.height,
.pixel_format = switch (image.format) {
.gray => .gray,
.gray_alpha => .gray_alpha,
.rgb => .rgb,
.rgba => .rgba,
.png => unreachable, // should be decoded by now
},
// constCasts are always gross but this one is safe is because
// the data is only read from here and copied into its own
// buffer.
.data = @constCast(image.data.ptr),
},
);
}
};
@@ -574,13 +657,19 @@ pub const Id = union(enum) {
/// The value is the ID assigned by the terminal.
kitty: u32,
/// Debug overlay. This is always composited down to a single
/// image for now. In the future we can support layers here if we want.
overlay,
/// Z-ordering tie-breaker for images with the same z value.
pub fn zLessThan(lhs: Id, rhs: Id) bool {
// If our tags aren't the same, we sort by tag.
if (std.meta.activeTag(lhs) != std.meta.activeTag(rhs)) {
return switch (lhs) {
// Kitty images always sort before (lower z) non-kitty images.
.kitty => false,
.kitty => true,
.overlay => false,
};
}
@@ -589,6 +678,9 @@ pub const Id = union(enum) {
const rhs_id = rhs.kitty;
return lhs_id < rhs_id;
},
// No sensical ordering
.overlay => return false,
}
}
};