diff --git a/src/benchmark/TerminalFormatter.zig b/src/benchmark/TerminalFormatter.zig new file mode 100644 index 000000000..fb763ab52 --- /dev/null +++ b/src/benchmark/TerminalFormatter.zig @@ -0,0 +1,482 @@ +//! Benchmarks the terminal formatter (`terminal/formatter.zig`). +//! +//! The formatter is the hot path for clipboard copy (plain/VT/HTML), +//! `write_screen_file`, `selectionString`, terminal search window +//! encoding, and the libghostty-vt formatter C API. This benchmark +//! measures formatting terminal contents that were built during setup +//! (outside the timed region) from a pre-generated VT stream. +//! +//! ## Input +//! +//! `--data` names a pre-generated VT byte stream (for example from +//! `ghostty-gen styled`). The stream is fed to a terminal of the +//! requested dimensions with unlimited scrollback during setup. The +//! resulting screen contents (scrollback included) are what each step +//! formats. +//! +//! ## Modes +//! +//! * `noop` performs no formatting and establishes loop/setup overhead. +//! Subtract this from `format` timings when using hyperfine. +//! * `format` formats the configured region once per loop into a +//! reusable buffer. Buffer growth happens on the first iteration only. +//! * `report` formats once and prints content/output sizes. It is for +//! computing throughput (cells/s, bytes/s), not timing comparisons. +//! +//! ## Examples +//! +//! Build benchmarks in ReleaseFast mode: +//! +//! zig build -Demit-bench -Doptimize=ReleaseFast -Demit-macos-app=false +//! +//! Generate a deterministic corpus, then measure: +//! +//! ghostty-gen styled --seed=42 | head -c 640000 > /tmp/plain.vt +//! hyperfine --warmup 3 \ +//! 'ghostty-bench +terminal-formatter --mode=noop --data=/tmp/plain.vt' \ +//! 'ghostty-bench +terminal-formatter --emit=plain --loops=50 --data=/tmp/plain.vt' \ +//! 'ghostty-bench +terminal-formatter --emit=vt --loops=50 --data=/tmp/plain.vt' +const TerminalFormatter = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const terminalpkg = @import("../terminal/main.zig"); +const formatterpkg = terminalpkg.formatter; +const Benchmark = @import("Benchmark.zig"); +const options = @import("options.zig"); +const Terminal = terminalpkg.Terminal; +const Selection = terminalpkg.Selection; +const global = @import("../global.zig"); + +const log = std.log.scoped(.@"terminal-formatter-bench"); + +alloc: Allocator, +opts: Options, +terminal: ?Terminal = null, + +/// Reused across steps so buffer growth is a one-time setup cost. +output: std.Io.Writer.Allocating, + +/// Reused pin map storage for `--pin-map=true`. +pins: formatterpkg.PinMap.Map = .empty, + +pub const Options = struct { + /// Set by the shared CLI parser for string option ownership. + _arena: ?std.heap.ArenaAllocator = null, + + /// Select the operation performed inside the timed benchmark step. + mode: Mode = .format, + + /// The output format to emit. + emit: Emit = .vt, + + /// The region of the screen to format. + region: Region = .screen, + + /// Unwrap soft-wrapped lines. + unwrap: bool = false, + + /// Track the source pin of every emitted byte. This exercises the + /// (documented as expensive) pin_map path used by selectionString + /// and search. + @"pin-map": bool = false, + + /// Number of format operations per benchmark step. Increase this + /// when the content is too small for stable hyperfine measurements. + loops: u32 = 25, + + /// The size of the terminal. This affects wrapping and page sizes. + @"terminal-rows": u16 = 24, + @"terminal-cols": u16 = 80, + + /// Pre-generated VT stream fed to the terminal during setup. `-` + /// reads stdin, although a regular file is recommended so identical + /// state can be reused across runs. When unset, the terminal is + /// empty. + data: ?[]const u8 = null, + + pub fn deinit(self: *Options) void { + if (self._arena) |arena| arena.deinit(); + self.* = undefined; + } +}; + +pub const Mode = enum { + /// Establish the benchmark loop and setup overhead. + noop, + + /// Format the configured region once per loop. + format, + + /// Print content and output sizes. Not a timing benchmark. + report, + + /// Semantic verification, not a timing benchmark: format the + /// terminal (must be `--emit=vt`), feed the output into a fresh + /// terminal of the same dimensions, format that, and verify the + /// two outputs converge to identical bytes. This proves the VT + /// output faithfully reconstructs the terminal content even when + /// the exact byte encoding changes. + roundtrip, +}; + +pub const Emit = enum { + plain, + vt, + html, + + fn format(self: Emit) formatterpkg.Format { + return switch (self) { + .plain => .plain, + .vt => .vt, + .html => .html, + }; + } +}; + +pub const Region = enum { + /// Everything: scrollback and active screen. + screen, + + /// Only the active screen (bottom rows). + active, + + /// Only the scrollback. + history, +}; + +pub fn create( + alloc: Allocator, + opts: Options, +) !*TerminalFormatter { + const ptr = try alloc.create(TerminalFormatter); + errdefer alloc.destroy(ptr); + ptr.* = .{ + .alloc = alloc, + .opts = opts, + .output = .init(alloc), + }; + return ptr; +} + +pub fn destroy(self: *TerminalFormatter, alloc: Allocator) void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.output.deinit(); + self.pins.deinit(self.alloc); + alloc.destroy(self); +} + +pub fn benchmark(self: *TerminalFormatter) Benchmark { + return .init(self, .{ + .stepFn = switch (self.opts.mode) { + .noop => stepNoop, + .format => stepFormat, + .report => stepReport, + .roundtrip => stepRoundtrip, + }, + .setupFn = setup, + .teardownFn = teardown, + }); +} + +/// Build the terminal state every mode shares. All of this is outside +/// the timed region for `Benchmark`, but is included in whole-process +/// timings, hence the `noop` mode. +fn setup(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + self.setupImpl() catch |err| { + log.warn("failed to prepare formatter benchmark err={}", .{err}); + return error.BenchmarkFailed; + }; +} + +fn setupImpl(self: *TerminalFormatter) !void { + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.terminal = try Terminal.init(global.io(), self.alloc, .{ + .cols = self.opts.@"terminal-cols", + .rows = self.opts.@"terminal-rows", + .max_scrollback_bytes = null, + .max_scrollback_lines = null, + }); + const terminal = &self.terminal.?; + + // Feed the input corpus through the standard VT stream. + if (try options.dataFile(self.opts.data)) |data_f| { + defer data_f.close(global.io()); + + var stream = terminal.vtStream(); + defer stream.deinit(); + + var read_buf: [4096]u8 align(std.atomic.cache_line) = undefined; + var f_reader = data_f.reader(global.io(), &read_buf); + const r = &f_reader.interface; + + var buf: [4096]u8 = undefined; + while (true) { + const n = try r.readSliceShort(&buf); + if (n == 0) break; // EOF reached + stream.nextSlice(buf[0..n]); + } + } +} + +fn teardown(ptr: *anyopaque) void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + if (self.terminal) |*t| t.deinit(self.alloc); + self.terminal = null; + self.output.shrinkRetainingCapacity(0); + self.pins.clearRetainingCapacity(); +} + +/// Build the screen formatter matching our options. This mirrors how +/// Surface clipboard copy and write_screen_file construct formatters. +fn formatter(self: *TerminalFormatter) ?formatterpkg.ScreenFormatter { + const screen = self.terminal.?.screens.active; + + var f: formatterpkg.ScreenFormatter = .init(screen, .{ + .emit = self.opts.emit.format(), + .unwrap = self.opts.unwrap, + }); + + f.content = switch (self.opts.region) { + .screen => .{ .selection = null }, + + inline .active, .history => |region| content: { + const tag: terminalpkg.point.Tag = switch (region) { + .active => .active, + .history => .history, + .screen => unreachable, + }; + const tl = screen.pages.getTopLeft(tag); + const br = screen.pages.getBottomRight(tag) orelse return null; + break :content .{ .selection = Selection.init(tl, br, false) }; + }, + }; + + return f; +} + +fn stepNoop(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + for (0..self.opts.loops) |_| { + std.mem.doNotOptimizeAway(self.output.written()); + } +} + +fn stepFormat(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + for (0..self.opts.loops) |_| { + self.output.shrinkRetainingCapacity(0); + + var f = self.formatter() orelse continue; + if (self.opts.@"pin-map") { + self.pins.clearRetainingCapacity(); + f.pin_map = .{ .alloc = self.alloc, .map = &self.pins }; + } + + f.format(&self.output.writer) catch |err| { + log.warn("formatting failed err={}", .{err}); + return error.BenchmarkFailed; + }; + std.mem.doNotOptimizeAway(self.output.written()); + } +} + +/// Print the content dimensions and emitted output size. This shares +/// the formatting code with format mode but deliberately makes no +/// timing claims. Use it to compute cells/s and bytes/s from timings. +fn stepReport(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + + self.output.shrinkRetainingCapacity(0); + if (self.formatter()) |f_init| { + var f = f_init; + if (self.opts.@"pin-map") { + self.pins.clearRetainingCapacity(); + f.pin_map = .{ .alloc = self.alloc, .map = &self.pins }; + } + f.format(&self.output.writer) catch |err| { + log.warn("formatting failed err={}", .{err}); + return error.BenchmarkFailed; + }; + } + + // Count the total pages and rows in the pagelist. + const screen = self.terminal.?.screens.active; + var pages: usize = 0; + var rows: usize = 0; + { + var node = screen.pages.pages.first; + while (node) |n| : (node = n.next) { + pages += 1; + rows += n.page().size.rows; + } + } + + // Hash the output (and the pin coordinates, which are stable across + // runs unlike the node pointers) so different implementations can be + // checked for identical output. + const out_hash = std.hash.Wyhash.hash(0, self.output.written()); + var pin_hasher = std.hash.Wyhash.init(0); + for (self.pins.points.items) |coord| { + const x: u16 = coord.x; + const y: u16 = @intCast(coord.y); + pin_hasher.update(std.mem.asBytes(&x)); + pin_hasher.update(std.mem.asBytes(&y)); + } + + std.debug.print( + "terminal-formatter emit={s} region={s} pages={d} rows={d} " ++ + "cols={d} cells={d} out_bytes={d} pin_bytes={d} " ++ + "out_hash={x} pin_hash={x}\n", + .{ + @tagName(self.opts.emit), + @tagName(self.opts.region), + pages, + rows, + self.opts.@"terminal-cols", + rows * self.opts.@"terminal-cols", + self.output.written().len, + self.pins.count(), + out_hash, + pin_hasher.final(), + }, + ); + + // Pin map storage details on a separate line so the main report + // line remains comparable across implementations. + if (self.opts.@"pin-map") { + std.debug.print( + "terminal-formatter-pins points={d} nodes={d}\n", + .{ self.pins.points.items.len, self.pins.nodes.items.len }, + ); + } +} + +fn stepRoundtrip(ptr: *anyopaque) Benchmark.Error!void { + const self: *TerminalFormatter = @ptrCast(@alignCast(ptr)); + self.stepRoundtripImpl() catch |err| { + log.warn("roundtrip failed err={}", .{err}); + return error.BenchmarkFailed; + }; +} + +/// Format the terminal, replay the output into a fresh terminal of the +/// same dimensions, format that, and require both outputs to be +/// identical. This verifies that the emitted VT sequences faithfully +/// reconstruct the terminal contents (text, styles, wrapping) without +/// requiring any specific byte encoding of the first output. +fn stepRoundtripImpl(self: *TerminalFormatter) !void { + // Format the original terminal. + self.output.shrinkRetainingCapacity(0); + if (self.formatter()) |f_init| { + var f = f_init; + try f.format(&self.output.writer); + } + const first = self.output.written(); + + // Replay into a fresh terminal. + var t2 = try Terminal.init(global.io(), self.alloc, .{ + .cols = self.opts.@"terminal-cols", + .rows = self.opts.@"terminal-rows", + .max_scrollback_bytes = null, + .max_scrollback_lines = null, + }); + defer t2.deinit(self.alloc); + { + var stream = t2.vtStream(); + defer stream.deinit(); + stream.nextSlice(first); + } + + // Format the replayed terminal identically. + var out2: std.Io.Writer.Allocating = .init(self.alloc); + defer out2.deinit(); + var f2: formatterpkg.ScreenFormatter = .init(t2.screens.active, .{ + .emit = self.opts.emit.format(), + .unwrap = self.opts.unwrap, + }); + try f2.format(&out2.writer); + const second = out2.written(); + + const equal = std.mem.eql(u8, first, second); + std.debug.print( + "terminal-formatter roundtrip emit={s} bytes={d} replay_bytes={d} equal={}\n", + .{ @tagName(self.opts.emit), first.len, second.len, equal }, + ); + + if (!equal) { + // Find the first differing offset to ease debugging. + const n = @min(first.len, second.len); + var i: usize = 0; + while (i < n and first[i] == second[i]) i += 1; + std.debug.print( + "terminal-formatter roundtrip mismatch offset={d} " ++ + "first={f} second={f}\n", + .{ + i, + std.zig.fmtString(first[i -| 32..@min(first.len, i + 32)]), + std.zig.fmtString(second[i -| 32..@min(second.len, i + 32)]), + }, + ); + return error.RoundtripMismatch; + } +} + +test TerminalFormatter { + const testing = std.testing; + const impl: *TerminalFormatter = try .create(testing.allocator, .{}); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} + +test "TerminalFormatter roundtrip" { + const testing = std.testing; + const impl: *TerminalFormatter = try .create(testing.allocator, .{ + .mode = .roundtrip, + .@"terminal-rows" = 4, + .@"terminal-cols" = 8, + }); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} + +test "TerminalFormatter formats all emit formats and regions" { + const testing = std.testing; + + inline for (.{ Emit.plain, Emit.vt, Emit.html }) |emit| { + inline for (.{ Region.screen, Region.active, Region.history }) |region| { + const impl: *TerminalFormatter = try .create(testing.allocator, .{ + .emit = emit, + .region = region, + .loops = 1, + .@"terminal-rows" = 4, + .@"terminal-cols" = 8, + }); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); + } + } +} + +test "TerminalFormatter pin map" { + const testing = std.testing; + const impl: *TerminalFormatter = try .create(testing.allocator, .{ + .@"pin-map" = true, + .loops = 1, + .@"terminal-rows" = 4, + .@"terminal-cols" = 8, + }); + defer impl.destroy(testing.allocator); + + const bench = impl.benchmark(); + _ = try bench.run(.once); +} diff --git a/src/benchmark/cli.zig b/src/benchmark/cli.zig index f6a31f2c5..f76dfc4b7 100644 --- a/src/benchmark/cli.zig +++ b/src/benchmark/cli.zig @@ -14,6 +14,7 @@ pub const Action = enum { @"page-compression", @"scrollback-compression", @"screen-clone", + @"terminal-formatter", @"terminal-parser", @"terminal-resize", @"terminal-snapshot", @@ -39,6 +40,7 @@ pub const Action = enum { .@"terminal-stream" => @import("TerminalStream.zig"), .@"codepoint-width" => @import("CodepointWidth.zig"), .@"grapheme-break" => @import("GraphemeBreak.zig"), + .@"terminal-formatter" => @import("TerminalFormatter.zig"), .@"terminal-parser" => @import("TerminalParser.zig"), .@"terminal-resize" => @import("TerminalResize.zig"), .@"terminal-snapshot" => @import("TerminalSnapshot.zig"), diff --git a/src/benchmark/main.zig b/src/benchmark/main.zig index 666ffb0f6..2c1f31f52 100644 --- a/src/benchmark/main.zig +++ b/src/benchmark/main.zig @@ -6,6 +6,7 @@ pub const CodepointWidth = @import("CodepointWidth.zig"); pub const GraphemeBreak = @import("GraphemeBreak.zig"); pub const HyperlinkMap = @import("HyperlinkMap.zig"); pub const ScreenClone = @import("ScreenClone.zig"); +pub const TerminalFormatter = @import("TerminalFormatter.zig"); pub const TerminalParser = @import("TerminalParser.zig"); pub const TerminalResize = @import("TerminalResize.zig"); pub const TerminalSnapshot = @import("TerminalSnapshot.zig"); diff --git a/src/fastprint.zig b/src/fastprint.zig new file mode 100644 index 000000000..e42030c20 --- /dev/null +++ b/src/fastprint.zig @@ -0,0 +1,68 @@ +//! Fastprint has fast printing routines that are significantly +//! faster than going through std.fmt. + +const std = @import("std"); + +/// Print a decimal type T. The buffer is expected to be large enough so if +/// necessary use comptime with a buffer-too-large to determine your +/// max size needed. Returns the length written. +pub fn printDecimal(comptime T: type, buf: []u8, v: T) usize { + // Note this only supports types as we need them. + switch (T) { + u8 => { + if (v >= 100) { + buf[0] = '0' + v / 100; + buf[1..3].* = std.fmt.digits2(v % 100); + return 3; + } + if (v >= 10) { + buf[0..2].* = std.fmt.digits2(v); + return 2; + } + buf[0] = '0' + v; + return 1; + }, + + // This can probably be generalized... + u21 => { + // Maximum u21 is 2097151: at most 7 digits. + var tmp: [7]u8 = undefined; + var val: u32 = v; + var i: usize = tmp.len; + while (true) { + i -= 1; + tmp[i] = '0' + @as(u8, @intCast(val % 10)); + val /= 10; + if (val == 0) break; + } + const n = tmp.len - i; + @memcpy(buf[0..n], tmp[i..]); + return n; + }, + + else => comptime unreachable, + } +} + +test printDecimal { + const testing = std.testing; + var buf: [16]u8 = undefined; + + // u8: exercise 1, 2, and 3 digit values including boundaries. + const u8_cases = [_]u8{ 0, 1, 9, 10, 99, 100, 255 }; + for (u8_cases) |v| { + var expected_buf: [3]u8 = undefined; + const expected = std.fmt.bufPrint(&expected_buf, "{d}", .{v}) catch unreachable; + const len = printDecimal(u8, &buf, v); + try testing.expectEqualStrings(expected, buf[0..len]); + } + + // u21: exercise digit count boundaries up to the maximum. + const u21_cases = [_]u21{ 0, 9, 10, 128, 65535, 1114111, std.math.maxInt(u21) }; + for (u21_cases) |v| { + var expected_buf: [8]u8 = undefined; + const expected = std.fmt.bufPrint(&expected_buf, "{d}", .{v}) catch unreachable; + const len = printDecimal(u21, &buf, v); + try testing.expectEqualStrings(expected, buf[0..len]); + } +} diff --git a/src/synthetic/cli.zig b/src/synthetic/cli.zig index dcac0d0dc..560896030 100644 --- a/src/synthetic/cli.zig +++ b/src/synthetic/cli.zig @@ -9,6 +9,7 @@ pub const Action = enum { ascii, kitty, osc, + styled, utf8, /// Returns the struct associated with the action. The struct @@ -24,6 +25,7 @@ pub const Action = enum { .ascii => @import("cli/Ascii.zig"), .kitty => @import("cli/Kitty.zig"), .osc => @import("cli/Osc.zig"), + .styled => @import("cli/Styled.zig"), .utf8 => @import("cli/Utf8.zig"), }; } diff --git a/src/synthetic/cli/Styled.zig b/src/synthetic/cli/Styled.zig new file mode 100644 index 000000000..c437a6ab5 --- /dev/null +++ b/src/synthetic/cli/Styled.zig @@ -0,0 +1,410 @@ +//! Generates styled terminal content: lines of printable text +//! interspersed with SGR sequences, optional multi-byte UTF-8 +//! codepoints, combining marks (multi-codepoint graphemes), and +//! OSC 8 hyperlinks. +//! +//! This exists primarily to build corpora for benchmarking code that +//! consumes terminal *contents* (e.g. the terminal formatter used for +//! clipboard copy and dumps), where we want workloads with controlled +//! amounts of styling and Unicode rather than raw random bytes. +//! +//! Examples: +//! +//! # Plain ASCII lines (equivalent to `ascii` with lines) +//! ghostty-gen styled --seed=42 +//! +//! # Heavily styled ASCII +//! ghostty-gen styled --seed=42 --style-rate=0.8 +//! +//! # Unicode-heavy, no styling +//! ghostty-gen styled --seed=42 --weight-two=1 --weight-three=1 \ +//! --weight-four=0.5 --grapheme-rate=0.1 +//! +//! # Mixed: styles, Unicode, and hyperlinks +//! ghostty-gen styled --seed=42 --style-rate=0.3 --weight-two=0.5 \ +//! --weight-three=0.25 --weight-four=0.1 --grapheme-rate=0.05 \ +//! --osc8-rate=0.1 +const Styled = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +pub const Options = struct { + /// Seed to use for deterministic generation. If unset, a time-based + /// seed is used by the generic synthetic CLI. + seed: ?u64 = null, + + /// Emit lines whose printable length (in characters, not columns) + /// is uniformly distributed in `[line-min, line-max]`. Each line is + /// terminated by CR LF. + @"line-min": usize = 40, + @"line-max": usize = 80, + + /// Probability that a style change (SGR sequence) is emitted at + /// each run boundary. Zero (default) produces unstyled output. + @"style-rate": f64 = 0.0, + + /// The length in characters of a "run": a sequence of characters + /// that share styling. Style/hyperlink changes only happen on run + /// boundaries. + @"run-min": usize = 4, + @"run-max": usize = 16, + + /// Relative weights for choosing the UTF-8 encoding length of each + /// generated character. Unlike the raw `utf8` generator, characters + /// are drawn from curated printable ranges so the output never + /// contains control characters: + /// + /// one: printable ASCII (with occasional spaces) + /// two: Latin-1 Supplement / Latin Extended (narrow) + /// three: Hiragana/Katakana and CJK ideographs (wide) + /// four: emoji (wide) + @"weight-one": f64 = 1.0, + @"weight-two": f64 = 0.0, + @"weight-three": f64 = 0.0, + @"weight-four": f64 = 0.0, + + /// Probability that a generated character is followed by a + /// combining diacritical mark, producing a multi-codepoint + /// grapheme cluster. + @"grapheme-rate": f64 = 0.0, + + /// Probability that an OSC 8 hyperlink is toggled (opened or + /// closed) at each run boundary. + @"osc8-rate": f64 = 0.0, +}; + +opts: Options, + +pub fn create( + alloc: Allocator, + opts: Options, +) !*Styled { + for ([_]f64{ + opts.@"style-rate", + opts.@"grapheme-rate", + opts.@"osc8-rate", + }) |rate| { + if (rate < 0 or rate > 1) return error.InvalidValue; + } + + const weights = [_]f64{ + opts.@"weight-one", + opts.@"weight-two", + opts.@"weight-three", + opts.@"weight-four", + }; + var weight_sum: f64 = 0; + for (weights) |weight| { + if (weight < 0) return error.InvalidValue; + weight_sum += weight; + } + if (weight_sum <= 0) return error.InvalidValue; + + if (opts.@"line-min" == 0) return error.InvalidValue; + if (opts.@"run-min" == 0) return error.InvalidValue; + + const ptr = try alloc.create(Styled); + errdefer alloc.destroy(ptr); + ptr.* = .{ .opts = opts }; + return ptr; +} + +pub fn destroy(self: *Styled, alloc: Allocator) void { + alloc.destroy(self); +} + +pub fn run(self: *Styled, writer: *std.Io.Writer, rand: std.Random) !void { + var prng: ?std.Random.DefaultPrng = null; + var gen_rand = rand; + if (self.opts.seed) |seed| { + prng = std.Random.DefaultPrng.init(seed); + gen_rand = prng.?.random(); + } + + var state: State = .{}; + while (true) { + self.writeLine(writer, gen_rand, &state) catch |err| { + const Error = error{ WriteFailed, BrokenPipe } || @TypeOf(err); + switch (@as(Error, err)) { + error.BrokenPipe => return, // stdout closed + error.WriteFailed => return, // fixed buffer full + } + }; + } +} + +const State = struct { + /// True when a non-default SGR style is currently active. + styled: bool = false, + + /// True when an OSC 8 hyperlink is currently open. + link: bool = false, +}; + +fn writeLine( + self: *Styled, + writer: *std.Io.Writer, + rand: std.Random, + state: *State, +) std.Io.Writer.Error!void { + const line_min = self.opts.@"line-min"; + const line_max = @max(line_min, self.opts.@"line-max"); + const run_min = self.opts.@"run-min"; + const run_max = @max(run_min, self.opts.@"run-max"); + + const line_len = rand.intRangeAtMostBiased(usize, line_min, line_max); + var remaining = line_len; + while (remaining > 0) { + if (self.opts.@"style-rate" > 0 and + rand.float(f64) < self.opts.@"style-rate") + { + try self.writeSgr(writer, rand, state); + } + + if (self.opts.@"osc8-rate" > 0 and + rand.float(f64) < self.opts.@"osc8-rate") + { + try self.toggleLink(writer, rand, state); + } + + const run_len = @min( + remaining, + rand.intRangeAtMostBiased(usize, run_min, run_max), + ); + for (0..run_len) |_| try self.writeChar(writer, rand); + remaining -= run_len; + } + + // Close any open styling/hyperlink so state never bleeds across + // lines. This mirrors what well-behaved programs do. + if (state.styled) { + try writer.writeAll("\x1b[0m"); + state.styled = false; + } + if (state.link) { + try writer.writeAll("\x1b]8;;\x1b\\"); + state.link = false; + } + + try writer.writeAll("\r\n"); +} + +fn writeChar( + self: *Styled, + writer: *std.Io.Writer, + rand: std.Random, +) std.Io.Writer.Error!void { + const weights = [_]f64{ + self.opts.@"weight-one", + self.opts.@"weight-two", + self.opts.@"weight-three", + self.opts.@"weight-four", + }; + + const cp: u21 = switch (rand.weightedIndex(f64, &weights)) { + // Printable ASCII with occasional word-ish spacing. + 0 => if (rand.float(f64) < 0.15) + ' ' + else + rand.intRangeAtMostBiased(u21, 0x21, 0x7E), + + // Latin-1 Supplement and Latin Extended-A/B (narrow). + 1 => rand.intRangeAtMostBiased(u21, 0xC0, 0x24F), + + // Kana or CJK ideographs (wide). + 2 => if (rand.boolean()) + rand.intRangeAtMostBiased(u21, 0x3041, 0x30FE) + else + rand.intRangeAtMostBiased(u21, 0x4E00, 0x9FFF), + + // Emoji (wide). + 3 => rand.intRangeAtMostBiased(u21, 0x1F600, 0x1F64F), + + else => unreachable, + }; + + try writer.print("{u}", .{cp}); + + // Optionally follow with a combining diacritical mark to form a + // multi-codepoint grapheme cluster. + if (self.opts.@"grapheme-rate" > 0 and + rand.float(f64) < self.opts.@"grapheme-rate") + { + const mark = rand.intRangeAtMostBiased(u21, 0x300, 0x36F); + try writer.print("{u}", .{mark}); + } +} + +fn writeSgr( + self: *Styled, + writer: *std.Io.Writer, + rand: std.Random, + state: *State, +) std.Io.Writer.Error!void { + _ = self; + + switch (rand.uintLessThan(u8, 8)) { + // Reset. + 0 => { + try writer.writeAll("\x1b[0m"); + state.styled = false; + return; + }, + + // 16-color foreground. + 1 => { + const base: u8 = if (rand.boolean()) 30 else 90; + try writer.print("\x1b[{d}m", .{base + rand.uintLessThan(u8, 8)}); + }, + + // 16-color background. + 2 => { + const base: u8 = if (rand.boolean()) 40 else 100; + try writer.print("\x1b[{d}m", .{base + rand.uintLessThan(u8, 8)}); + }, + + // 256-color foreground/background. + 3 => try writer.print("\x1b[38;5;{d}m", .{rand.int(u8)}), + 4 => try writer.print("\x1b[48;5;{d}m", .{rand.int(u8)}), + + // RGB foreground/background. Components are quantized so the + // number of unique styles stays bounded (6^3 per channel pair) + // rather than pathologically unique per cell. + 5, 6 => |kind| { + const layer: u8 = if (kind == 5) 38 else 48; + try writer.print("\x1b[{d};2;{d};{d};{d}m", .{ + layer, + rand.uintLessThan(u8, 6) * 51, + rand.uintLessThan(u8, 6) * 51, + rand.uintLessThan(u8, 6) * 51, + }); + }, + + // Attributes: bold, italic, underline, inverse, strikethrough. + 7 => { + const attrs = [_]u8{ 1, 3, 4, 7, 9 }; + try writer.print("\x1b[{d}m", .{ + attrs[rand.uintLessThan(usize, attrs.len)], + }); + }, + + else => unreachable, + } + + state.styled = true; +} + +fn toggleLink( + self: *Styled, + writer: *std.Io.Writer, + rand: std.Random, + state: *State, +) std.Io.Writer.Error!void { + _ = self; + + if (state.link) { + try writer.writeAll("\x1b]8;;\x1b\\"); + state.link = false; + return; + } + + try writer.print( + "\x1b]8;;http://example.com/{d}\x1b\\", + .{rand.uintLessThan(u16, 1024)}, + ); + state.link = true; +} + +test Styled { + const testing = std.testing; + const alloc = testing.allocator; + + const impl: *Styled = try .create(alloc, .{ .seed = 1 }); + defer impl.destroy(alloc); + + var prng = std.Random.DefaultPrng.init(1); + const rand = prng.random(); + + var buf: [4096]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try impl.run(&writer, rand); + const output = writer.buffered(); + try testing.expect(output.len > 0); + + // Default options: plain printable ASCII lines only. + for (output) |byte| { + try testing.expect(byte == '\r' or byte == '\n' or + (byte >= 0x20 and byte < 0x7F)); + } +} + +test "Styled styled output" { + const testing = std.testing; + const alloc = testing.allocator; + + const impl: *Styled = try .create(alloc, .{ + .seed = 1, + .@"style-rate" = 0.5, + .@"osc8-rate" = 0.2, + }); + defer impl.destroy(alloc); + + var prng = std.Random.DefaultPrng.init(1); + const rand = prng.random(); + + var buf: [16384]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try impl.run(&writer, rand); + const output = writer.buffered(); + + // Must contain SGR and OSC 8 sequences. + try testing.expect(std.mem.indexOf(u8, output, "\x1b[") != null); + try testing.expect(std.mem.indexOf(u8, output, "\x1b]8;;") != null); +} + +test "Styled unicode output" { + const testing = std.testing; + const alloc = testing.allocator; + + const impl: *Styled = try .create(alloc, .{ + .seed = 1, + .@"weight-two" = 1.0, + .@"weight-three" = 1.0, + .@"weight-four" = 1.0, + .@"grapheme-rate" = 0.25, + }); + defer impl.destroy(alloc); + + var prng = std.Random.DefaultPrng.init(1); + const rand = prng.random(); + + var buf: [16384]u8 = undefined; + var writer: std.Io.Writer = .fixed(&buf); + try impl.run(&writer, rand); + const output = writer.buffered(); + + // No escapes, valid UTF-8 (modulo a possibly truncated tail from + // the fixed buffer filling up mid-sequence). + try testing.expect(std.mem.indexOfScalar(u8, output, 0x1B) == null); + var end = output.len; + while (end > 0 and output[end - 1] & 0xC0 == 0x80) end -= 1; + if (end > 0 and output[end - 1] >= 0xC0) end -= 1; + try testing.expect(std.unicode.utf8ValidateSlice(output[0..end])); +} + +test "Styled invalid options" { + const testing = std.testing; + const alloc = testing.allocator; + + try testing.expectError(error.InvalidValue, Styled.create(alloc, .{ + .@"style-rate" = 1.5, + })); + try testing.expectError(error.InvalidValue, Styled.create(alloc, .{ + .@"weight-one" = 0, + })); + try testing.expectError(error.InvalidValue, Styled.create(alloc, .{ + .@"line-min" = 0, + })); +} diff --git a/src/terminal/Screen.zig b/src/terminal/Screen.zig index 8cc2787e1..304168899 100644 --- a/src/terminal/Screen.zig +++ b/src/terminal/Screen.zig @@ -17,6 +17,7 @@ const PageList = @import("PageList.zig"); const selection_codepoints = @import("selection_codepoints.zig"); const StringMap = @import("StringMap.zig"); const ScreenFormatter = @import("formatter.zig").ScreenFormatter; +const PinMap = @import("formatter.zig").PinMap; const osc = @import("osc.zig"); const pagepkg = @import("page.zig"); const point = @import("point.zig"); @@ -2910,7 +2911,7 @@ pub fn selectionString( formatter.content = .{ .selection = opts.sel }; // If we have a string map, we need to set that up. - var pins: std.ArrayList(Pin) = .empty; + var pins: PinMap.Map = .empty; defer pins.deinit(alloc); if (opts.map != null) formatter.pin_map = .{ .alloc = alloc, @@ -2928,11 +2929,13 @@ pub fn selectionString( const map_string = try alloc.dupeZ(u8, text); errdefer alloc.free(map_string); try selectionString_tw.check(.copy_map); - const map_pins = try pins.toOwnedSlice(alloc); map.* = .{ .string = map_string, - .map = map_pins, + .map = pins, }; + + // Ownership of the pin map moved to the string map. + pins = .empty; } return text; diff --git a/src/terminal/StringMap.zig b/src/terminal/StringMap.zig index 9b2fa3ca8..b06a6d176 100644 --- a/src/terminal/StringMap.zig +++ b/src/terminal/StringMap.zig @@ -6,9 +6,9 @@ const std = @import("std"); const build_options = @import("terminal_options"); const oni = @import("oniguruma"); const point = @import("point.zig"); +const PinMap = @import("formatter.zig").PinMap; const Selection = @import("Selection.zig"); const Screen = @import("Screen.zig"); -const Pin = @import("PageList.zig").Pin; const Allocator = std.mem.Allocator; // Retry budget for StringMap regex searches. @@ -18,11 +18,15 @@ const Allocator = std.mem.Allocator; const oni_search_retry_limit = 100_000; string: [:0]const u8, -map: []Pin, + +/// Mapping of string byte offsets to pins. See PinMap for the +/// storage details. +map: PinMap.Map, pub fn deinit(self: StringMap, alloc: Allocator) void { alloc.free(self.string); - alloc.free(self.map); + var map = self.map; + map.deinit(alloc); } /// Returns an iterator that yields the next match of the given regex. @@ -106,8 +110,8 @@ pub const Match = struct { pub fn selection(self: Match) Selection { const start_idx: usize = @intCast(self.region.starts()[0]); const end_idx: usize = @intCast(self.region.ends()[0] - 1); - const start_pt = self.map.map[self.offset + start_idx]; - const end_pt = self.map.map[self.offset + end_idx]; + const start_pt = self.map.map.get(self.offset + start_idx).?; + const end_pt = self.map.map.get(self.offset + end_idx).?; return .init(start_pt, end_pt, false); } }; diff --git a/src/terminal/formatter.zig b/src/terminal/formatter.zig index 7cde65560..99a632a62 100644 --- a/src/terminal/formatter.zig +++ b/src/terminal/formatter.zig @@ -1,5 +1,6 @@ const std = @import("std"); const assert = @import("../quirks.zig").inlineAssert; +const fastprint = @import("../fastprint.zig"); const lib = @import("lib.zig"); const Allocator = std.mem.Allocator; const color = @import("color.zig"); @@ -119,9 +120,117 @@ pub const Options = struct { /// /// Used by formatters that operate on PageLists to track the source position /// of each byte written. The caller is responsible for freeing the map. +/// +/// The mapping is stored in two parts: a per-byte x/y coordinate (8 +/// bytes per output byte, half the size of a Pin) and a tiny table of +/// page nodes covering byte ranges (there are only ever a handful of +/// pages). This also lets page formatters write coordinates directly +/// into the map without a separate coordinate-to-pin conversion pass. pub const PinMap = struct { alloc: Allocator, - map: *std.ArrayList(Pin), + map: *Map, + + /// The type of the page node referenced by pins. + pub const Node = @FieldType(Pin, "node"); + + /// A page node covering output bytes starting at `offset` + /// (inclusive) until the next entry's offset (or the end of the + /// output). + pub const NodeRun = struct { + offset: usize, + node: Node, + }; + + pub const Map = struct { + /// The x/y coordinate within its page for every output byte. + points: std.ArrayList(Coordinate) = .empty, + + /// The page node for ranges of output bytes, ordered by offset. + nodes: std.ArrayList(NodeRun) = .empty, + + pub const empty: Map = .{}; + + pub fn deinit(self: *Map, alloc: Allocator) void { + self.points.deinit(alloc); + self.nodes.deinit(alloc); + } + + pub fn clearRetainingCapacity(self: *Map) void { + self.points.clearRetainingCapacity(); + self.nodes.clearRetainingCapacity(); + } + + /// The total number of bytes mapped. + pub fn count(self: *const Map) usize { + return self.points.items.len; + } + + /// Set the page node for all bytes appended from here on, + /// until the next call. No-op if the node is unchanged. + pub fn setNode( + self: *Map, + alloc: Allocator, + node: Node, + ) Allocator.Error!void { + if (self.nodes.getLastOrNull()) |last| { + if (last.node == node) return; + } + + try self.nodes.append(alloc, .{ + .offset = self.points.items.len, + .node = node, + }); + } + + /// Append `n` bytes that map to `pin`. + pub fn append( + self: *Map, + alloc: Allocator, + pin: Pin, + n: usize, + ) Allocator.Error!void { + if (n == 0) return; + try self.setNode(alloc, pin.node); + try self.points.appendNTimes( + alloc, + .{ .x = pin.x, .y = pin.y }, + n, + ); + } + + /// Returns the pin that the byte at the given offset maps to, + /// or null if the offset is out of range. + pub fn get(self: *const Map, offset: usize) ?Pin { + if (offset >= self.points.items.len) return null; + const coord = self.points.items[offset]; + return .{ + .node = findNode(self.nodes.items, offset) orelse return null, + .x = coord.x, + .y = @intCast(coord.y), + }; + } + + /// Returns the last pin in the map, if any. + pub fn getLastOrNull(self: *const Map) ?Pin { + const len = self.points.items.len; + if (len == 0) return null; + return self.get(len - 1); + } + }; + + /// Binary search for the node covering `offset` in a slice of node + /// runs sorted by offset. Returns null only if the slice is empty + /// or the offset precedes the first run. + pub fn findNode(runs: []const NodeRun, offset: usize) ?Node { + if (runs.len == 0 or offset < runs[0].offset) return null; + var lo: usize = 0; + var hi: usize = runs.len; + while (lo + 1 < hi) { + const mid = lo + (hi - lo) / 2; + if (runs[mid].offset <= offset) lo = mid else hi = mid; + } + return runs[lo].node; + } }; /// Terminal formatter formats the active terminal screen. @@ -288,7 +397,7 @@ pub const TerminalFormatter = struct { // Map all those bytes to the same pin. Use the top left to ensure // the node pointer is always properly initialized. - m.map.appendNTimes( + m.map.append( m.alloc, self.terminal.screens.active.pages.getTopLeft(.screen), std.math.cast(usize, discarding.count) orelse return error.WriteFailed, @@ -326,7 +435,7 @@ pub const TerminalFormatter = struct { // Map all those bytes to the same pin. Use the top left to ensure // the node pointer is always properly initialized. - m.map.appendNTimes( + m.map.append( m.alloc, self.terminal.screens.active.pages.getTopLeft(.screen), std.math.cast(usize, discarding.count) orelse return error.WriteFailed, @@ -403,16 +512,10 @@ pub const TerminalFormatter = struct { extra_formatter.extra.pwd = self.extra.pwd; try extra_formatter.format(&discarding.writer); - m.map.appendNTimes( + m.map.append( m.alloc, - if (m.map.items.len > 0) pin: { - const last = m.map.items[m.map.items.len - 1]; - break :pin .{ - .node = last.node, - .x = last.x, - .y = last.y, - }; - } else self.terminal.screens.active.pages.getTopLeft(.screen), + m.map.getLastOrNull() orelse + self.terminal.screens.active.pages.getTopLeft(.screen), std.math.cast(usize, discarding.count) orelse return error.WriteFailed, ) catch return error.WriteFailed; } @@ -671,21 +774,10 @@ pub const ScreenFormatter = struct { // Map all those bytes to the same pin. Use the first page node // to ensure the node pointer is always properly initialized. - m.map.appendNTimes( + m.map.append( m.alloc, - if (m.map.items.len > 0) pin: { - // There is a weird Zig miscompilation here on 0.15.2. - // If I return the m.map.items value directly then we - // get undefined memory (even though we're copying a - // Pin struct). If we duplicate here like this we do - // not. - const last = m.map.items[m.map.items.len - 1]; - break :pin .{ - .node = last.node, - .x = last.x, - .y = last.y, - }; - } else self.screen.pages.getTopLeft(.screen), + m.map.getLastOrNull() orelse + self.screen.pages.getTopLeft(.screen), std.math.cast(usize, discarding.count) orelse return error.WriteFailed, ) catch return error.WriteFailed; } @@ -737,10 +829,6 @@ pub const PageListFormatter = struct { const tl: PageList.Pin = self.top_left orelse self.list.getTopLeft(.screen); const br: PageList.Pin = self.bottom_right orelse self.list.getBottomRight(.screen).?; - // If we keep track of pins, we'll need this. - var point_map: std.ArrayList(Coordinate) = .empty; - defer if (self.pin_map) |*m| point_map.deinit(m.alloc); - var page_state: ?PageFormatter.TrailingState = null; var iter = tl.pageIterator(.right_down, br); while (iter.next()) |chunk| { @@ -763,27 +851,19 @@ pub const PageListFormatter = struct { if (chunk.node == br.node) formatter.end_x = br.x; } - // If we're tracking pins, then we setup a point map for the - // page formatter (cause it can't track pins). And then we convert - // this to pins later. + // If we're tracking pins, the page formatter writes its + // per-byte coordinates directly into our map's point list + // and we record which page node covers those bytes. if (self.pin_map) |*m| { - point_map.clearRetainingCapacity(); - formatter.point_map = .{ .alloc = m.alloc, .map = &point_map }; + m.map.setNode(m.alloc, chunk.node) catch return error.WriteFailed; + formatter.point_map = .{ + .alloc = m.alloc, + .map = &m.map.points, + .base = m.map.points.items.len, + }; } page_state = try formatter.formatWithState(writer); - - // If we're tracking pins then grab our points and write them - // to our pin map. - if (self.pin_map) |*m| { - for (point_map.items) |coord| { - m.map.append(m.alloc, .{ - .node = chunk.node, - .x = coord.x, - .y = @intCast(coord.y), - }) catch return error.WriteFailed; - } - } } } }; @@ -831,16 +911,25 @@ pub const PageFormatter = struct { /// The x/y coordinate will be the coordinates within the page. /// /// Warning: there is a significant performance hit to track this - point_map: ?struct { - alloc: Allocator, - map: *std.ArrayList(Coordinate), - }, + point_map: ?PointMap, /// The previous trailing state from the prior page. If you're iterating /// over multiple pages this helps ensure that unwrapping and other /// accounting works properly. trailing_state: ?TrailingState, + /// See point_map. + pub const PointMap = struct { + alloc: Allocator, + map: *std.ArrayList(Coordinate), + + /// The index in `map` at which this formatter's output begins. + /// Entries before this index belong to a caller (e.g. previous + /// pages of a PageListFormatter) and aren't inspected. This + /// exists so that callers can share one list across pages. + base: usize = 0, + }; + /// Trailing state. This is used to ensure that rows wrapped across /// multiple pages are unwrapped properly, as well as other accounting /// we may do in the future. @@ -877,6 +966,18 @@ pub const PageFormatter = struct { pub fn formatWithState( self: PageFormatter, writer: *std.Io.Writer, + ) std.Io.Writer.Error!TrailingState { + // Specialize the hot path on the emitted format so that the + // per-cell loop contains no per-cell format dispatch. + switch (self.opts.emit) { + inline else => |emit| return self.formatWithStateEmit(writer, emit), + } + } + + fn formatWithStateEmit( + self: PageFormatter, + writer: *std.Io.Writer, + comptime emit: Format, ) std.Io.Writer.Error!TrailingState { var blank_rows: usize = 0; var blank_cells: usize = 0; @@ -934,7 +1035,7 @@ pub const PageFormatter = struct { } // Wrap HTML output in monospace font styling - switch (self.opts.emit) { + switch (emit) { .plain => {}, .html => { @@ -994,8 +1095,23 @@ pub const PageFormatter = struct { }, } - // Our style for non-plain formats + // Our style for non-plain formats. Alongside the style itself we + // track the page-local interned style id it corresponds to (styles + // are interned per-page so id equality implies style equality). + // The id is only a fast-path hint: it is set to `invalid_style_id` + // whenever the current style didn't come from an interned id + // (e.g. bg-color-only cells which synthesize styles). + const invalid_style_id: u32 = std.math.maxInt(u32); var style: Style = .{}; + var style_id: u32 = 0; + + // Whether the codepoint map has any entries. Hoisted out of the + // per-codepoint path so that the common no-map case can use the + // fast cell run path below. + const cp_map_empty: bool = if (self.opts.codepoint_map) |m| + m.len == 0 + else + true; // Track hyperlink state for HTML output. We need to close tags // when the hyperlink changes or ends. @@ -1050,11 +1166,12 @@ pub const PageFormatter = struct { // Reset style before emitting newlines to prevent background // colors from bleeding into the next line's leading cells. if (!style.default()) { - try self.formatStyleClose(writer); + try self.formatStyleClose(emit, writer); style = .{}; + style_id = 0; } - const sequence: []const u8 = switch (self.opts.emit) { + const sequence: []const u8 = switch (emit) { // Plaintext just uses standard newlines because newlines // on their own usually move the cursor back in anywhere // you type plaintext. @@ -1077,7 +1194,7 @@ pub const PageFormatter = struct { // in a prior page, so we just map to the first row of this // page. if (self.point_map) |*map| { - const start: Coordinate = if (map.map.items.len > 0) + const start: Coordinate = if (map.map.items.len > map.base) map.map.items[map.map.items.len - 1] else .{ .x = 0, .y = 0 }; @@ -1113,8 +1230,56 @@ pub const PageFormatter = struct { if (!row.wrap_continuation or !self.opts.unwrap) blank_cells = 0; // Go through each cell and print it - for (cells_subset, row_start_x..) |*cell, x_usize| { - const x: size.CellCountInt = @intCast(x_usize); + var cell_i: usize = 0; + while (cell_i < cells_subset.len) : (cell_i += 1) { + const cell: *const Cell = &cells_subset[cell_i]; + const x: size.CellCountInt = @intCast(row_start_x + cell_i); + + // Fast path: runs of simple cells (single codepoint, no + // style/hyperlink transitions) are encoded in batches, + // avoiding all of the per-cell bookkeeping below. This is + // only valid when we have no codepoint map and when our + // current style/hyperlink state is known-stable. + if (cp_map_empty) fast: { + if (comptime formatStyled(emit)) { + if (style_id == invalid_style_id) break :fast; + } + + // Specialized on point tracking so that the common + // non-tracking case has zero per-cell overhead. + const consumed = if (self.point_map == null) + try self.writeCellRun( + emit, + false, + writer, + cells_subset[cell_i..], + x, + y, + style_id, + current_hyperlink_id, + &blank_cells, + ) + else + try self.writeCellRun( + emit, + true, + writer, + cells_subset[cell_i..], + x, + y, + style_id, + current_hyperlink_id, + &blank_cells, + ); + + // Zero cells consumed means the first cell isn't + // eligible for the fast path; handle it below. + if (consumed == 0) break :fast; + + // The continue expression adds the final one. + cell_i += consumed - 1; + continue; + } // Skip spacers. These happen naturally when wide characters // are printed again on the screen (for well-behaved terminals!) @@ -1130,8 +1295,9 @@ pub const PageFormatter = struct { // If we're emitting styled output (not plaintext) and // the cell has some kind of styling or is not empty // then this isn't blank. - if (formatStyled(self.opts.emit) and - (!cell.isEmpty() or cell.hasStyling())) break :blank; + if (comptime formatStyled(emit)) { + if (!cell.isEmpty() or cell.hasStyling()) break :blank; + } // Cells with no text are blank if (!cell.hasText()) { @@ -1153,32 +1319,12 @@ pub const PageFormatter = struct { if (blank_cells > 0) { try writer.splatByteAll(' ', blank_cells); - if (self.point_map) |*map| { - // Map each blank cell to its coordinate. Blank cells can span - // multiple rows if they carry over from wrap continuation. - var remaining_blanks = blank_cells; - var blank_x = x; - var blank_y = y; - while (remaining_blanks > 0) : (remaining_blanks -= 1) { - if (blank_x > 0) { - // We have space in this row - blank_x -= 1; - } else if (blank_y > 0) { - // Wrap to previous row - blank_y -= 1; - blank_x = self.page.size.cols - 1; - } else { - // Can't go back further, just use (0, 0) - blank_x = 0; - blank_y = 0; - } - - map.map.append( - map.alloc, - .{ .x = blank_x, .y = blank_y }, - ) catch return error.WriteFailed; - } - } + if (self.point_map) |*map| try self.appendBlankPoints( + map, + blank_cells, + x, + y, + ); blank_cells = 0; } @@ -1186,24 +1332,44 @@ pub const PageFormatter = struct { style: { // If we aren't emitting styled output then we don't // have to worry about styles. - if (!formatStyled(self.opts.emit)) break :style; + if (!comptime formatStyled(emit)) break :style; + + // Fast path: styles are interned per-page, so if this + // cell's style id matches the id of our current style + // then the style is unchanged. + const cell_style_id: u32 = switch (cell.content_tag) { + .codepoint, .codepoint_grapheme => cell.style_id, + .bg_color_palette, .bg_color_rgb => invalid_style_id, + }; + if (cell_style_id == style_id and + cell_style_id != invalid_style_id) break :style; // Get our cell style. const cell_style = self.cellStyle(cell); // If the style hasn't changed, don't bloat output. - if (cell_style.eql(style)) break :style; + // When both ids are interned (and thus different, since + // equal ids broke out above), interning guarantees the + // styles differ so we can skip the comparison entirely. + if (cell_style_id == invalid_style_id or + style_id == invalid_style_id) + { + if (cell_style.eql(style)) { + style_id = cell_style_id; + break :style; + } + } // If we had a previous style, we need to close it, // because we've confirmed we have some new style // (which is maybe default). - if (!style.default()) switch (self.opts.emit) { - .html => try self.formatStyleClose(writer), + if (!style.default()) switch (emit) { + .html => try self.formatStyleClose(emit, writer), // For VT, we only close if we're switching to a default // style because any non-default style will emit // a \x1b[0m as the start of a VT coloring sequence. - .vt => if (cell_style.default()) try self.formatStyleClose(writer), + .vt => if (cell_style.default()) try self.formatStyleClose(emit, writer), // Unreachable because of the styled() check at the // top of this block. @@ -1212,12 +1378,14 @@ pub const PageFormatter = struct { // At this point, we can copy our style over style = cell_style; + style_id = cell_style_id; // If we're just the default style now, we're done. if (cell_style.default()) break :style; // New style, emit it. try self.formatStyleOpen( + emit, writer, &style, ); @@ -1227,16 +1395,18 @@ pub const PageFormatter = struct { if (self.point_map) |*map| { var discarding: std.Io.Writer.Discarding = .init(&.{}); try self.formatStyleOpen( + emit, &discarding.writer, &style, ); - for (0..std.math.cast( - usize, - discarding.count, - ) orelse return error.WriteFailed) |_| map.map.append(map.alloc, .{ - .x = x, - .y = y, - }) catch return error.WriteFailed; + map.map.appendNTimes( + map.alloc, + .{ .x = x, .y = y }, + std.math.cast( + usize, + discarding.count, + ) orelse return error.WriteFailed, + ) catch return error.WriteFailed; } } @@ -1245,7 +1415,7 @@ pub const PageFormatter = struct { // We currently only emit hyperlinks for HTML. In the // future we can support emitting OSC 8 hyperlinks for // VT output as well. - if (self.opts.emit != .html) break :hyperlink; + if (comptime emit != .html) break :hyperlink; // Get the hyperlink ID. This ID is our internal ID, // not necessarily the OSC8 ID. @@ -1261,7 +1431,7 @@ pub const PageFormatter = struct { // If our prior hyperlink ID was non-null, we need to // close it because the ID has changed. if (current_hyperlink_id != null) { - try self.formatHyperlinkClose(writer); + try self.formatHyperlinkClose(emit, writer); current_hyperlink_id = null; } @@ -1278,6 +1448,7 @@ pub const PageFormatter = struct { break :uri link.uri.offset.ptr(self.page.memory)[0..link.uri.len]; }; try self.formatHyperlinkOpen( + emit, writer, uri, ); @@ -1287,16 +1458,18 @@ pub const PageFormatter = struct { if (self.point_map) |*map| { var discarding: std.Io.Writer.Discarding = .init(&.{}); try self.formatHyperlinkOpen( + emit, &discarding.writer, uri, ); - for (0..std.math.cast( - usize, - discarding.count, - ) orelse return error.WriteFailed) |_| map.map.append(map.alloc, .{ - .x = x, - .y = y, - }) catch return error.WriteFailed; + map.map.appendNTimes( + map.alloc, + .{ .x = x, .y = y }, + std.math.cast( + usize, + discarding.count, + ) orelse return error.WriteFailed, + ) catch return error.WriteFailed; } } @@ -1304,20 +1477,21 @@ pub const PageFormatter = struct { // We combine codepoint and graphemes because both have // shared style handling. We use comptime to dup it. inline .codepoint, .codepoint_grapheme => |tag| { - try self.writeCell(tag, writer, cell); + try self.writeCell(tag, emit, writer, cell); // If we have a point map, all codepoints map to this // cell. if (self.point_map) |*map| { var discarding: std.Io.Writer.Discarding = .init(&.{}); - try self.writeCell(tag, &discarding.writer, cell); - for (0..std.math.cast( - usize, - discarding.count, - ) orelse return error.WriteFailed) |_| map.map.append(map.alloc, .{ - .x = x, - .y = y, - }) catch return error.WriteFailed; + try self.writeCell(tag, emit, &discarding.writer, cell); + map.map.appendNTimes( + map.alloc, + .{ .x = x, .y = y }, + std.math.cast( + usize, + discarding.count, + ) orelse return error.WriteFailed, + ) catch return error.WriteFailed; } }, @@ -1335,13 +1509,13 @@ pub const PageFormatter = struct { } // If the style is non-default, we need to close our style tag. - if (!style.default()) try self.formatStyleClose(writer); + if (!style.default()) try self.formatStyleClose(emit, writer); // Close any open hyperlink for HTML output - if (current_hyperlink_id != null) try self.formatHyperlinkClose(writer); + if (current_hyperlink_id != null) try self.formatHyperlinkClose(emit, writer); // Close the monospace wrapper for HTML output - if (self.opts.emit == .html) { + if (comptime emit == .html) { const closing = ""; try writer.writeAll(closing); if (self.point_map) |*map| { @@ -1362,9 +1536,302 @@ pub const PageFormatter = struct { return .{ .rows = blank_rows, .cells = blank_cells }; } + /// Fast path for writing runs of simple cells: single-codepoint cells + /// that require no style or hyperlink handling. Output bytes are + /// batched into a stack buffer to avoid per-cell writer dispatch. + /// Returns the number of cells consumed, which may be zero if the + /// first cell isn't eligible for the fast path (in which case the + /// caller must handle it via the slow path). + /// + /// Requirements (asserted by the caller, not here): + /// + /// - The codepoint map is empty. + /// - For styled formats, `run_style_id` is the valid interned + /// page-local id of the currently active style. + /// - For HTML, no hyperlink is currently open. + /// + /// `run_x`/`run_y` are the page coordinates of `cells[0]`, used for + /// point map tracking. + /// + /// Blank cell accounting matches the slow path: accumulated blanks + /// are only materialized once a non-blank cell is found, and any + /// remainder is written back to `blank_cells`. + fn writeCellRun( + self: *const PageFormatter, + comptime emit: Format, + comptime track_points: bool, + writer: *std.Io.Writer, + cells: []const Cell, + run_x: size.CellCountInt, + run_y: size.CellCountInt, + run_style_id: u32, + run_hyperlink_id: ?hyperlink.Id, + blank_cells: *usize, + ) std.Io.Writer.Error!usize { + assert(track_points == (self.point_map != null)); + + // The largest single-cell encoding must fit after a flush: the + // HTML entity for the maximum codepoint ("") is 10 + // bytes, escapes are up to 6. + const max_encoding_len = 16; + var buf: [512]u8 = undefined; + var len: usize = 0; + var pending: usize = blank_cells.*; + + var i: usize = 0; + while (i < cells.len) : (i += 1) { + const cell = &cells[i]; + + // Spacers produce no output, matching the slow path which + // skips them before any blank/style handling. + switch (cell.wide) { + .narrow, .wide => {}, + .spacer_head, .spacer_tail => continue, + } + + // Only text cells: bg-color cells synthesize styles and take + // the slow path. + switch (cell.content_tag) { + .codepoint, .codepoint_grapheme => {}, + .bg_color_palette, .bg_color_rgb => break, + } + + if (comptime formatStyled(emit)) { + // Style transition, take the slow path. + if (cell.style_id != run_style_id) break; + } + + const cp: u21 = cell.content.codepoint.data; + + // Blank cell accounting, matching the slow path blank block. + if (comptime formatStyled(emit)) { + // Styled formats only treat unstyled empty cells as + // blank; anything else (including spaces) is written + // so that styling is preserved. + if (cp == 0 and cell.wide == .narrow and run_style_id == 0) { + pending += 1; + continue; + } + } else { + // Cells with no text are blank. + if (cp == 0) { + pending += 1; + continue; + } + + // Trailing spaces are blank. + if (cp == ' ' and self.opts.trim) { + pending += 1; + continue; + } + } + + // Hyperlink state must be stable within a run: any non-blank + // cell must belong to the currently open hyperlink (or none). + // Transitions take the slow path. This is checked after blank + // accounting because blank cells never touch hyperlink state. + if (comptime emit == .html) { + if (cell.hyperlink) { + const run_id = run_hyperlink_id orelse break; + const cell_id = self.page.lookupHyperlink(cell) orelse break; + if (cell_id != run_id) break; + } else if (run_hyperlink_id != null) break; + } + + // The page coordinate of this cell, for point tracking. + const x: size.CellCountInt = @intCast(run_x + i); + + // This cell produces output: materialize accumulated blanks. + if (pending > 0) { + if (comptime track_points) try self.appendBlankPoints( + &self.point_map.?, + pending, + x, + run_y, + ); + + while (pending > 0) { + if (len == buf.len) { + try writer.writeAll(buf[0..len]); + len = 0; + } + const n = @min(pending, buf.len - len); + @memset(buf[len..][0..n], ' '); + len += n; + pending -= n; + } + } + + // Flush if the largest possible encoding may not fit. + if (len + max_encoding_len > buf.len) { + try writer.writeAll(buf[0..len]); + len = 0; + } + + var cell_bytes: usize = 0; + + // Empty (but styled or wide) cells emit a space, matching + // writeCell. + if (cp == 0) { + buf[len] = ' '; + len += 1; + cell_bytes = 1; + } else { + cell_bytes = encodeCodepoint(emit, &buf, &len, cp); + + // Multi-codepoint graphemes emit their extra codepoints, + // matching writeCell. This is out-of-line to keep the + // hot loop for the common single-codepoint case small. + if (cell.content_tag == .codepoint_grapheme) { + @branchHint(.unlikely); + cell_bytes += try self.writeGraphemeCps( + emit, + writer, + cell, + &buf, + &len, + ); + } + } + + // All of the cell's bytes map to the cell's coordinate. + if (comptime track_points) { + const map = &self.point_map.?; + map.map.appendNTimes( + map.alloc, + .{ .x = x, .y = run_y }, + cell_bytes, + ) catch return error.WriteFailed; + } + } + + if (len > 0) try writer.writeAll(buf[0..len]); + blank_cells.* = pending; + return i; + } + + /// Encode the extra codepoints of a multi-codepoint grapheme into + /// buf, flushing to the writer as needed. Returns the number of + /// bytes written. This is deliberately not inlined so that the + /// (rare) grapheme case doesn't bloat the writeCellRun hot loop. + noinline fn writeGraphemeCps( + self: *const PageFormatter, + comptime emit: Format, + writer: *std.Io.Writer, + cell: *const Cell, + buf: *[512]u8, + len: *usize, + ) std.Io.Writer.Error!usize { + const max_encoding_len = 16; + var bytes: usize = 0; + for (self.page.lookupGrapheme(cell).?) |gcp| { + if (len.* + max_encoding_len > buf.len) { + try writer.writeAll(buf[0..len.*]); + len.* = 0; + } + bytes += encodeCodepoint(emit, buf, len, gcp); + } + return bytes; + } + + /// Encode a single codepoint into buf at len, advancing len and + /// returning the number of bytes written. The caller must guarantee + /// enough remaining buffer space for the largest possible encoding. + inline fn encodeCodepoint( + comptime emit: Format, + buf: *[512]u8, + len: *usize, + cp: u21, + ) usize { + const start = len.*; + switch (emit) { + .plain, .vt => if (cp < 0x80) { + buf[start] = @intCast(cp); + len.* += 1; + } else { + len.* += std.unicode.utf8Encode(cp, buf[start..][0..4]) catch l: { + // Matches Writer.printUnicodeCodepoint: invalid + // codepoints become the replacement character. + buf[start..][0..3].* = std.unicode.replacement_character_utf8; + break :l 3; + }; + }, + + .html => html: { + const esc: ?[]const u8 = switch (cp) { + '<' => "<", + '>' => ">", + '&' => "&", + '"' => """, + '\'' => "'", + else => null, + }; + if (esc) |s| { + @memcpy(buf[start..][0..s.len], s); + len.* += s.len; + break :html; + } + + // ASCII is emitted directly, everything else as a + // numeric entity. See writeCodepoint. + if (cp < 0x80) { + buf[start] = @intCast(cp); + len.* += 1; + break :html; + } + + buf[start..][0..2].* = "".*; + len.* += 2; + len.* += fastprint.printDecimal(u21, buf[len.*..], cp); + buf[len.*] = ';'; + len.* += 1; + }, + } + + return len.* - start; + } + + /// Append the point map entries for a run of `count` blank cells + /// that are materialized as spaces just before the cell at (x, y). + /// Blank cells can span multiple rows if they carry over from wrap + /// continuation, so this walks backwards from (x, y). + fn appendBlankPoints( + self: *const PageFormatter, + map: *const PointMap, + count: usize, + x: size.CellCountInt, + y: size.CellCountInt, + ) std.Io.Writer.Error!void { + map.map.ensureUnusedCapacity( + map.alloc, + count, + ) catch return error.WriteFailed; + + var remaining = count; + var blank_x = x; + var blank_y = y; + while (remaining > 0) : (remaining -= 1) { + if (blank_x > 0) { + // We have space in this row + blank_x -= 1; + } else if (blank_y > 0) { + // Wrap to previous row + blank_y -= 1; + blank_x = self.page.size.cols - 1; + } else { + // Can't go back further, just use (0, 0) + blank_x = 0; + blank_y = 0; + } + + map.map.appendAssumeCapacity(.{ .x = blank_x, .y = blank_y }); + } + } + fn writeCell( self: PageFormatter, comptime tag: Cell.ContentTag, + comptime emit: Format, writer: *std.Io.Writer, cell: *const Cell, ) !void { @@ -1376,16 +1843,17 @@ pub const PageFormatter = struct { return; } - try self.writeCodepointWithReplacement(writer, cell.content.codepoint.data); + try self.writeCodepointWithReplacement(emit, writer, cell.content.codepoint.data); if (comptime tag == .codepoint_grapheme) { for (self.page.lookupGrapheme(cell).?) |cp| { - try self.writeCodepointWithReplacement(writer, cp); + try self.writeCodepointWithReplacement(emit, writer, cp); } } } fn writeCodepointWithReplacement( self: PageFormatter, + comptime emit: Format, writer: *std.Io.Writer, codepoint: u21, ) !void { @@ -1407,12 +1875,14 @@ pub const PageFormatter = struct { // If no replacement, write it directly. const r = r_ orelse return try self.writeCodepoint( + emit, writer, codepoint, ); switch (r) { .codepoint => |v| try self.writeCodepoint( + emit, writer, v, ), @@ -1421,6 +1891,7 @@ pub const PageFormatter = struct { const view = std.unicode.Utf8View.init(s) catch unreachable; var it = view.iterator(); while (it.nextCodepoint()) |cp| try self.writeCodepoint( + emit, writer, cp, ); @@ -1430,11 +1901,13 @@ pub const PageFormatter = struct { fn writeCodepoint( self: PageFormatter, + comptime emit: Format, writer: *std.Io.Writer, codepoint: u21, ) !void { - switch (self.opts.emit) { - .plain, .vt => try writer.print("{u}", .{codepoint}), + _ = self; + switch (emit) { + .plain, .vt => try writer.printUnicodeCodepoint(codepoint), .html => { switch (codepoint) { '<' => try writer.writeAll("<"), @@ -1449,9 +1922,14 @@ pub const PageFormatter = struct { // meta tag because we emit partial HTML so this ensures // proper unicode handling. if (codepoint < 0x80) { - try writer.print("{u}", .{codepoint}); + try writer.writeByte(@intCast(codepoint)); } else { - try writer.print("{d};", .{codepoint}); + var buf: [16]u8 = undefined; + buf[0..2].* = "".*; + var len: usize = 2 + fastprint.printDecimal(u21, buf[2..], codepoint); + buf[len] = ';'; + len += 1; + try writer.writeAll(buf[0..len]); } }, } @@ -1496,16 +1974,17 @@ pub const PageFormatter = struct { /// and other HTML attribute values. fn formatStyleOpen( self: PageFormatter, + comptime emit: Format, writer: *std.Io.Writer, style: *const Style, ) std.Io.Writer.Error!void { - switch (self.opts.emit) { + switch (emit) { .plain => unreachable, .vt => { var formatter = style.formatterVt(); formatter.palette = self.opts.palette; - try writer.print("{f}", .{formatter}); + try formatter.format(writer); }, // We use `display: inline` so that the div doesn't impact @@ -1513,19 +1992,19 @@ pub const PageFormatter = struct { .html => { var formatter = style.formatterHtml(); formatter.palette = self.opts.palette; - try writer.print( - "