terminal: fill style-only cell runs in bulk in printSliceFill

Profiling terminal-stream on a 2.6 GB recording of real terminal
sessions showed printSliceFill as the single largest item (~25% of
total time), and disassembly showed the time split across three
scalar loops: the run-eligibility scan over codepoints, the
simple-cell check that guards the branch-free fill, and the general
path that fixes up style ref counts one cell at a time. The store
loop itself was already auto-vectorized by LLVM, but the two scans
are early-exit search loops that LLVM does not vectorize, and the
general path turns out to be the common case in real traffic:
styled text constantly overwrites cells holding a different style
(TUI redraws, scrolling colored output), so every such cell failed
the simple check and paid a release/use pair.

Three changes, which only pay off together (vectorizing the scans
without the bulk path makes mismatch-heavy rows slower because the
wider check re-runs for every cell the general path consumes):

The run-eligibility scan handles the narrow class, codepoints in
[0x10, 0xFF], eight lanes at a time. The simple-cell check compares
four masked cells per iteration. And a new bulk path handles runs
of cells that differ from the expected simple cell only by style
id: one vector scan finds the extent of the uniformly-styled run,
the ref counts are fixed with a single releaseMultiple/useMultiple
pair, and the cells are filled with the same branch-free store
loop as the simple case. Cells with graphemes, hyperlinks, or wide
content still fall back to print().

Measured with ghostty-bench terminal-stream (120x80, M4 Max,
ReleaseFast, hyperfine means of 5 runs). The redraw corpus is a
full-screen 80-row styled repaint whose span color rotates every
frame, so every cell is overwritten with a different style:

| stream                     | before  | after   | change |
|----------------------------|---------|---------|--------|
| real 2.6 GB session corpus | 8.826 s | 7.955 s | +11%   |
| TUI redraw (100 MB)        | 348 ms  | 287 ms  | +21%   |
This commit is contained in:
Mitchell Hashimoto
2026-07-06 12:41:27 -07:00
parent 300f42c7a9
commit cb2d785871

View File

@@ -528,7 +528,32 @@ fn printSliceFill(
// in the run is always written as a fresh, single-codepoint cell,
// so the grapheme break check against it is exact.
const run_len: usize = run: {
for (1..cps.len) |idx| {
var idx: usize = 1;
// Vectorized scan for the narrow class: codepoints in
// [0x10, 0xFF] are always eligible with no further checks
// and dominate real-world input, so scan for the first
// codepoint outside that range several lanes at a time.
// Anything else (including eligible unicode) proceeds via
// the scalar loop below.
if (comptime width == .narrow) {
const lanes = 8;
const V = @Vector(lanes, u32);
const lo: V = @splat(0x10);
const hi: V = @splat(0xFF);
while (idx + lanes <= cps.len) {
const v: V = cps[idx..][0..lanes].*;
const in_range = (v >= lo) & (v <= hi);
if (!@reduce(.And, in_range)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(in_range);
idx += @ctz(~bits);
break;
}
idx += lanes;
}
}
while (idx < cps.len) : (idx += 1) {
const cp = cps[idx];
if (comptime width == .narrow) {
if (cp >= 0x10 and cp <= 0xFF) continue;
@@ -630,11 +655,31 @@ fn printSliceFill(
var k: usize = 0; // cells written
fill: while (k < cell_count) {
// Find the run of simple cells so the store loop below is
// branch-free (and vectorizable).
// branch-free (and vectorizable). This is an early-exit
// search loop that LLVM won't auto-vectorize, and reused
// rows typically match the whole way through, so scan
// several cells at a time manually.
var simple = k;
while (simple < cell_count) : (simple += 1) {
const bits: u64 = @bitCast(cells[simple]);
if ((bits & simple_mask) != check_expected) break;
simple: {
const lanes = 4;
const V = @Vector(lanes, u64);
const mask_v: V = @splat(simple_mask);
const expect_v: V = @splat(check_expected);
const cells64: [*]const u64 = @ptrCast(cells);
while (simple + lanes <= cell_count) {
const v: V = cells64[simple..][0..lanes].*;
const ok = (v & mask_v) == expect_v;
if (!@reduce(.And, ok)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(ok);
simple += @ctz(~bits);
break :simple;
}
simple += lanes;
}
while (simple < cell_count) : (simple += 1) {
const bits: u64 = @bitCast(cells[simple]);
if ((bits & simple_mask) != check_expected) break;
}
}
if (comptime width == .wide) {
@@ -665,6 +710,68 @@ fn printSliceFill(
}
if (k >= cell_count) break;
// Bulk path for runs of cells that differ from the
// expected simple cell only by their style: this is the
// common case when styled text overwrites previously
// styled (or default-styled) rows, e.g. TUI redraws.
// These runs are handled wholesale: one scan to find the
// run of identical old styles, two ref-count updates,
// and a branch-free fill.
if (comptime width == .narrow) bulk: {
const cells64: [*]const u64 = @ptrCast(cells);
const first = cells64[k] & simple_mask;
// The old cell must be a plain narrow codepoint cell
// with no hyperlink whose only difference is the
// style id (see printSliceCheckExpected: every other
// masked field must be zero).
const style_shift = @bitOffsetOf(Cell, "style_id");
const old_style: style.Id = @truncate(first >> style_shift);
if (first != printSliceCheckExpected(old_style)) break :bulk;
assert(old_style != style_id); // it failed the simple check
// Find the run of cells with identical masked bits.
var m = k + 1;
scan: {
const lanes = 4;
const V = @Vector(lanes, u64);
const mask_v: V = @splat(simple_mask);
const first_v: V = @splat(first);
while (m + lanes <= cell_count) {
const v: V = cells64[m..][0..lanes].*;
const ok = (v & mask_v) == first_v;
if (!@reduce(.And, ok)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(ok);
m += @ctz(~bits);
break :scan;
}
m += lanes;
}
while (m < cell_count) : (m += 1) {
if ((cells64[m] & simple_mask) != first) break;
}
}
// Fix up the style ref counts for the whole run at
// once. Each of the old cells held a reference to
// old_style so the release is safe by construction.
const n = m - k;
if (old_style != style.default_id) {
page.styles.releaseMultiple(page.memory, old_style, @intCast(n));
}
if (style_id != style.default_id) {
page.styles.useMultiple(page.memory, style_id, @intCast(n));
}
for (k..m) |idx| {
cells[idx] = @bitCast(
template_bits | (@as(u64, cps[printed + idx]) << cp_shift),
);
}
k = m;
continue :fill;
}
// General path for cells that failed the masked check:
// style-only mismatches are handled inline; anything that
// needs cleanup (wide chars and their spacers, grapheme