mirrors/ghostty
1
0
Fork 0
mirror of https://github.com/ghostty-org/ghostty.git synced 2026-08-02 05:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

mitchell's touchups

Browse Source 
- benchmark: avoid buffers to avoid a memcpy
- build: keep frame pointers on macOS. There was some debug changes from
  Zig 0.15 and this helps. Also, Apple actually requires/expects x29 to
  always be a frame pointer.
- build/macos: force libSystem symbols instead of compiler-rt
- global: add InitOpts.tool so that ghostty-gen/bench can parse their
  own actions in `+action`
- quirks: provide our own vectorized memset. see the comment for more
  details why.
- synthetic: fix UB by accessing global.io before it was initialized
- terminal/hash_map: force inline for unique repr types. Zig 0.15
  inlined and 0.16 doesn't, measured a huge slowdown in hyperlink
  benchmarks.
- terminal: add explicit `@Vector` usage for storing a run of identical cells
  as well as for scanning printable cells. This auto-vectorized in Zig
  0.15 but not in Zig 0.16. This produces the same assembly.
- unicode: properties and LUT need power-of-two backing integer to avoid
  bad LLVM codegen
This commit is contained in:
Mitchell Hashimoto
2026-07-20 20:33:23 -07:00
parent e8525c0fd9
commit f2a7652aba
35 changed files with 936 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/benchmark/TerminalStream.zig
View File

@@ -113,8 +113,10 @@ fn step(ptr: *anyopaque) Benchmark.Error!void {
// aren't currently IO bound.
const f = self.data_f orelse return;
var read_buf: [64 * 1024]u8 align(std.atomic.cache_line) = undefined;
var f_reader = f.reader(global.io(), &read_buf);
// Unbuffered: readSliceShort below reads directly into `buf`,
// avoiding a per-chunk memcpy through an intermediate reader
// buffer that would pollute the measurement.
var f_reader = f.reader(global.io(), &.{});
const r = &f_reader.interface;
// This buffer size matches the read buffer size used by the

2
src/benchmark/cli.zig
View File

@@ -46,7 +46,7 @@ pub const Action = enum {
/// An entrypoint for the benchmark CLI.
pub fn main(init: std.process.Init) !void {
try global.init(.{ .main = init });
try global.init(.{ .tool = init });
const alloc = std.heap.c_allocator;
const action_ = try cli.action.detectArgs(Action, alloc, init.minimal.args);
const action = action_ orelse return error.NoAction;

11
src/build/Config.zig
View File

@@ -669,6 +669,17 @@ pub fn fromOptions() Config {
};
}
/// Whether release artifacts should omit frame pointers.
///
/// Stripped release builds omit them in general, but we always keep frame
/// pointers on Apple platforms: the Apple arm64 ABI expects x29 to
/// be a valid frame pointer and Apple's profiling and crash reporting
/// tools rely on it for backtraces.
pub fn omitFramePointer(self: *const Config) bool {
if (self.target.result.os.tag.isDarwin()) return false;
return self.strip;
}
/// Returns the minimum OS version for the given OS tag. This shouldn't
/// be used generally, it should only be used for Darwin-based OS currently.
pub fn osVersionMin(tag: std.Target.Os.Tag) ?std.Target.Query.OsVersion {

2
src/build/GhosttyExe.zig
View File

@@ -18,7 +18,7 @@ pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !Ghostty
.target = cfg.target,
.optimize = cfg.optimize,
.strip = cfg.strip,
.omit_frame_pointer = cfg.strip,
.omit_frame_pointer = cfg.omitFramePointer(),
.unwind_tables = if (cfg.strip) .none else .sync,
}),
// Crashes on x86_64 self-hosted on 0.15.1

20
src/build/GhosttyLib.zig
View File

@@ -1,9 +1,11 @@
const GhosttyLib = @This();
const std = @import("std");
const builtin = @import("builtin");
const RunStep = std.Build.Step.Run;
const CombineArchivesStep = @import("CombineArchivesStep.zig");
const Config = @import("Config.zig");
const LibsystemOverrideStep = @import("LibsystemOverrideStep.zig");
const SharedDeps = @import("SharedDeps.zig");
const LipoStep = @import("LipoStep.zig");
@@ -27,7 +29,7 @@ pub fn initStatic(
.target = deps.config.target,
.optimize = deps.config.optimize,
.strip = deps.config.strip,
.omit_frame_pointer = deps.config.strip,
.omit_frame_pointer = deps.config.omitFramePointer(),
.unwind_tables = if (deps.config.strip) .none else .sync,
.link_libc = true,
}),
@@ -57,9 +59,19 @@ pub fn initStatic(
const combined = CombineArchivesStep.create(b, deps.config.target, "ghostty-internal", lib_list.items);
combined.step.dependOn(&lib.step);
// On Darwin, prefer libSystem's libc/libm over the bundled
// compiler-rt for consumers of this archive. See
// libsystem_override.sh for details. This is a no-op elsewhere.
const override = LibsystemOverrideStep.create(
b,
deps.config.target,
combined.output,
"libghostty-internal.a",
);
return .{
.step = combined.step,
.output = combined.output,
.step = override.step orelse combined.step,
.output = override.output,
// Static libraries cannot have dSYMs because they aren't linked.
.dsym = null,
@@ -88,7 +100,7 @@ pub fn initShared(
.target = deps.config.target,
.optimize = deps.config.optimize,
.strip = deps.config.strip,
.omit_frame_pointer = deps.config.strip,
.omit_frame_pointer = deps.config.omitFramePointer(),
.unwind_tables = if (deps.config.strip) .none else .sync,
}),

31
src/build/GhosttyLibVt.zig
View File

@@ -5,6 +5,7 @@ const builtin = @import("builtin");
const assert = std.debug.assert;
const RunStep = std.Build.Step.Run;
const CombineArchivesStep = @import("CombineArchivesStep.zig");
const LibsystemOverrideStep = @import("LibsystemOverrideStep.zig");
const Config = @import("Config.zig");
const GhosttyZig = @import("GhosttyZig.zig");
const LipoStep = @import("LipoStep.zig");
@@ -295,22 +296,44 @@ fn initLib(
const combined = CombineArchivesStep.create(b, target, "ghostty-vt", sources.items);
combined.step.dependOn(&lib.step);
// On Darwin, prefer libSystem's libc/libm over the bundled
// compiler-rt for consumers of this archive. See GhosttyLib
// and libsystem_override.sh for details.
const override = LibsystemOverrideStep.create(
b,
target,
combined.output,
"libghostty-vt-fat.a",
);
return .{
.step = combined.step,
.step = override.step orelse combined.step,
.artifact = &b.addInstallArtifact(lib, .{}).step,
.kind = kind,
.output = combined.output,
.output = override.output,
.dsym = dsymutil,
.pkg_config = if (pcs) |v| v.shared else null,
.pkg_config_static = if (pcs) |v| v.static else null,
};
}
// Same libSystem preference for the plain (no vendored SIMD)
// static archive; a no-op off Darwin.
const override: LibsystemOverrideStep.Result = if (kind == .static) LibsystemOverrideStep.create(
b,
target,
lib.getEmittedBin(),
"libghostty-vt-static.a",
) else .{
.step = null,
.output = lib.getEmittedBin(),
};
return .{
.step = &lib.step,
.step = override.step orelse &lib.step,
.artifact = &b.addInstallArtifact(lib, .{}).step,
.kind = kind,
.output = lib.getEmittedBin(),
.output = override.output,
.dsym = dsymutil,
.pkg_config = if (pcs) |v| v.shared else null,
.pkg_config_static = if (pcs) |v| v.static else null,

51
src/build/LibsystemOverrideStep.zig Normal file
View File

@@ -0,0 +1,51 @@
//! Wraps a Darwin static archive with the libsystem_override.sh
//! post-processing step so that consumers linking the archive bind
//! well-known libc/libm symbols (memcpy, memmove, memset, cos, sin,
//! ...) to Apple's libSystem instead of the bundled Zig compiler-rt.
//! See src/build/libsystem_override.sh for the full rationale.
//!
//! This applies to every static archive we produce, public
//! (e.g. libghostty-vt) and app-internal alike: the compiler-rt
//! bundle must stay because the few symbols libSystem does not
//! provide (f128 conversions, *q math, sincos*, ___zig_probe_stack)
//! have live references, e.g. from the bundled ubsan runtime and
//! x86_64 stack probing in Debug builds.
//!
//! Note shared libraries can't use either approach: Zig 0.16 links
//! compiler-rt as an eagerly-included object, so its definitions are
//! already bound into the dylib at link time and cannot be repaired
//! post-hoc. That needs an upstream Zig fix.
const std = @import("std");
const builtin = @import("builtin");
pub const Result = struct {
/// The step performing the override, or null if the override
/// doesn't apply to this target/host (in which case `output` is
/// the unmodified input).
step: ?*std.Build.Step,
output: std.Build.LazyPath,
};
/// Post-process the given static archive on Darwin. Requires a Darwin
/// host for the Apple toolchain (nmedit); in all other configurations
/// this is a no-op and the input is returned unmodified, which is
/// functional (the bundled compiler-rt symbols are used), just slower.
pub fn create(
b: *std.Build,
target: std.Build.ResolvedTarget,
input: std.Build.LazyPath,
out_name: []const u8,
) Result {
if (!target.result.os.tag.isDarwin() or
comptime !builtin.os.tag.isDarwin())
{
return .{ .step = null, .output = input };
}
const run = b.addSystemCommand(&.{"/bin/sh"});
run.setName("libsystem override");
run.addFileArg(b.path("src/build/libsystem_override.sh"));
run.addFileArg(input);
const output = run.addOutputFileArg(out_name);
return .{ .step = &run.step, .output = output };
}

123
src/build/libsystem_override.sh Executable file
View File

@@ -0,0 +1,123 @@
#!/bin/sh
#
# Darwin-only: rewrite a static archive so that consumers linking it bind
# libc/libm symbols to Apple's libSystem instead of the bundled Zig
# compiler-rt. This uses the Apple toolchain (xcrun, nmedit) and libSystem
# semantics, so it must run on a Darwin host for a Darwin target.
#
# Background: our static library bundles Zig's compiler-rt, which defines
# strong global implementations of libc/libm functions (memcpy, memmove,
# memset, cos, sin, ...). When a consumer (e.g. the macOS app via the
# XCFramework) links this archive, ld64 resolves those symbols from the
# archive instead of libSystem, silently replacing Apple's highly
# optimized implementations (_platform_memmove and friends, vectorized
# libm) with compiler-rt's generic ones. We measured Zig 0.16 (LLVM 21)
# compiler-rt memmove costing several percent of PTY throughput on
# scroll-region workloads versus libSystem's.
#
# The mechanism is to localize (make non-external) the libSystem-provided
# symbols defined by the compiler_rt.o archive member. This keeps
# compiler_rt.o functional for the intrinsics that libSystem does NOT
# provide (e.g. the f128 conversions ___extenddftf2/___extendxftf2, *q
# math, sincos*) while letting every other archive member bind the
# well-known libc/libm symbols to libSystem at final link.
#
# usage: libsystem_override.sh <input.a> <output.a>
set -eu
in="$1"
out="$2"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
cp -f "$in" "$out"
chmod u+w "$out"
# The symbols to prefer from libSystem. Everything listed here is a
# stable macOS export (verified against libSystem's link surface).
# Notably NOT listed (not exported by libSystem): the *q (f128)
# variants, sincos/sincosf/sincosl, and all ___-prefixed compiler
# intrinsics except the fortify _chk wrappers below.
cat >"$tmp/localize.txt" <<'EOF'
_bcmp
_memcmp
_memcpy
_memmove
_memset
_strlen
___memcpy_chk
___memmove_chk
___memset_chk
___strcat_chk
___strcpy_chk
_ceil
_ceilf
_ceill
_cos
_cosf
_cosl
_exp
_exp2
_exp2f
_exp2l
_expf
_expl
_fabs
_fabsf
_fabsl
_floor
_floorf
_floorl
_fma
_fmaf
_fmal
_fmax
_fmaxf
_fmaxl
_fmin
_fminf
_fminl
_fmod
_fmodf
_fmodl
_log
_log10
_log10f
_log10l
_log2
_log2f
_log2l
_logf
_logl
_round
_roundf
_roundl
_sin
_sinf
_sinl
_sqrt
_sqrtf
_sqrtl
_tan
_tanf
_tanl
_trunc
_truncf
_truncl
EOF
cd "$tmp"
xcrun ar x "$out" compiler_rt.o
chmod 644 compiler_rt.o
# nmedit takes a keep-list (-s): all current global definitions except
# the ones we want to localize.
xcrun nm -g compiler_rt.o | awk '$2 ~ /^[A-TV-Z]$/ {print $3}' | sort -u >all.txt
sort -u localize.txt >loc.txt
comm -23 all.txt loc.txt >keep.txt
xcrun nmedit -s keep.txt compiler_rt.o
xcrun ar r "$out" compiler_rt.o
xcrun ranlib "$out" 2>/dev/null || true

4
src/config/path.zig
View File

@@ -204,7 +204,7 @@ pub const Path = union(enum) {
defer dir.close(global.io());
const abs = abs: {
const dir_len = dir.realPathFile(global.io(), ".", &buf) catch |err| {
const abs_len = dir.realPathFile(global.io(), path, &buf) catch |err| {
if (err == error.FileNotFound) {
// The file doesn't exist. Try to resolve the relative path
// another way.
@@ -229,7 +229,7 @@ pub const Path = union(enum) {
return;
};
break :abs buf[0..dir_len];
break :abs buf[0..abs_len];
};
log.debug(

27
src/global.zig
View File

@@ -35,6 +35,14 @@ var state: ?GlobalState = undefined;
pub const InitOpts = union(enum) {
main: std.process.Init,
/// Same as `main` but for auxiliary tool binaries (e.g. ghostty-bench
/// and ghostty-gen) that have their own CLI action namespace. This
/// skips detection of ghostty app CLI actions, since tool actions
/// (e.g. `+terminal-stream`) are not valid app actions and would
/// otherwise cause init to fail with InvalidAction.
tool: std.process.Init,
c: struct {
argc: usize,
argv: [*][*:0]u8,
@@ -52,11 +60,11 @@ pub fn init(opts: InitOpts) !void {
.gpa = null,
.alloc = undefined,
.environ = switch (opts) {
.main => |m| m.minimal.environ,
.main, .tool => |m| m.minimal.environ,
.c => |c| c.environ,
},
.args = switch (opts) {
.main => |m| m.minimal.args,
.main, .tool => |m| m.minimal.args,
// TODO: Using the C API from Windows is unsupported at this time.
//
// When do we plan on supporting Windows, it's recommended to
@@ -116,11 +124,16 @@ pub fn init(opts: InitOpts) !void {
errdefer freeTmpDir(self.alloc, self.tmp_dir_path.?);
// We first try to parse any action that we may be executing.
self.action = try cli.action.detectArgs(
cli.ghostty.Action,
self.alloc,
self.args,
);
// Tool binaries (ghostty-bench, ghostty-gen) have their own action
// namespace and detect their own actions, so we skip detection here.
self.action = switch (opts) {
.main, .c => try cli.action.detectArgs(
cli.ghostty.Action,
self.alloc,
self.args,
),
.tool => null,
};
// If we have an action executing, we disable logging by default
// since we write to stderr we don't want logs messing up our

6
src/lib_vt.zig
View File

@@ -145,6 +145,12 @@ comptime {
// If we're building the C library (vs. the Zig module) then
// we want to reference the C API so that it gets exported.
if (@import("root") == lib) {
// Force-reference our memset override so its export is
// emitted. This must stay inside the root guard so that
// downstream Zig module consumers don't get the override
// injected into their binaries. See quirks_memset.zig.
_ = @import("quirks_memset.zig");
const c = terminal.c_api;
@export(&c.key_event_new, .{ .name = "ghostty_key_event_new" });
@export(&c.key_event_free, .{ .name = "ghostty_key_event_free" });

7
src/main.zig
View File

@@ -21,6 +21,12 @@ pub const std_options: std.Options = if (@hasDecl(entrypoint, "std_options"))
else
.{};
comptime {
// Force-reference our memset override so its export is emitted.
// See quirks_memset.zig for details on why this exists.
_ = @import("quirks_memset.zig");
}
test {
// Zig 0.16.0 has made test logging more strict. Now, *anything* that gets
// printed to stderr results in a "failed command" message, even if the
@@ -28,4 +34,5 @@ test {
// log spam in general), we bump the default testing log level to error.
std.testing.log_level = std.log.Level.err;
_ = entrypoint;
_ = @import("quirks_memset.zig");
}

6
src/main_bench.zig
View File

@@ -1,3 +1,9 @@
const benchmark = @import("benchmark/main.zig");
comptime {
// Force-reference our memset override so its export is emitted.
// See quirks_memset.zig for details on why this exists.
_ = @import("quirks_memset.zig");
}
pub const main = benchmark.cli.main;

4
src/main_c.zig
View File

@@ -44,6 +44,10 @@ comptime {
// Our benchmark API. We probably want to gate this on a build
// config in the future but for now we always just export it.
_ = @import("benchmark/main.zig").CApi;
// Force-reference our memset override so its export is emitted.
// See quirks_memset.zig for details on why this exists.
_ = @import("quirks_memset.zig");
}
/// ghostty_info_s

6
src/main_gen.zig
View File

@@ -1,3 +1,9 @@
const synthetic = @import("synthetic/main.zig");
comptime {
// Force-reference our memset override so its export is emitted.
// See quirks_memset.zig for details on why this exists.
_ = @import("quirks_memset.zig");
}
pub const main = synthetic.cli.main;

322
src/quirks_memset.zig Normal file
View File

@@ -0,0 +1,322 @@
//! Provides a fast `memset` symbol that overrides the slow scalar
//! implementation provided by Zig 0.16's compiler_rt.
//!
//! Consider deleting when upstream ships an optimized compiler_rt
//! memset (tracked by ziglang/zig#32091, maybe fixed at
//! ziglang/zig#35754). To verify it is safe to delete, check that
//! the disassembly of `memset` in a ReleaseFast binary is vectorized
//! (or at least not a byte-at-a-time loop).
//!
//! ## Background
//!
//! Zig 0.16.0 globally disabled LLVM loop auto-vectorization to work
//! around an LLVM 21 miscompilation (llvm/llvm-project#186922, see
//! the 0.16.0 release notes). compiler_rt's memset is a naive
//! byte-at-a-time loop that relied entirely on auto-vectorization,
//! so it now compiles to a scalar loop roughly 24x slower than what
//! Zig 0.15 produced.
//!
//! In benchmarks this made `ghostty-bench +terminal-stream` 2.8x slower on
//! ASCII input (memset was 63% of all executed instructions!).
//!
//! Because compiler_rt's symbol is weak, exporting a strong `memset`
//! from our own code transparently overrides it everywhere. The
//! implementation below is manually vectorized with `@Vector`, which
//! does not depend on the disabled loop vectorizer.
//!
//! Other mem functions deliberately not overridden:
//!
//! - memcpy/memmove: compiler_rt's implementations are manually
//! vectorized upstream ("memcpyFast") and remain fast.
//! - memcmp/bcmp/strlen: also scalar in 0.16, but they never showed
//! up in our profiles.
//!
//! This file has no effect unless it is referenced from an artifact
//! root (e.g. `comptime { _ = @import("quirks_memset.zig"); }`).
//! It must NOT be imported from shared code, otherwise downstream
//! consumers of our Zig modules would get this export injected into
//! their binaries.
//!
//! ## References
//!
//! I referenced the musl asm + c memset implementation (MIT licensed).
//! The primary thing I took away from that is the dc zva trick for
//! aarch64. The remainder is fairly obvious memset work.
const std = @import("std");
const builtin = @import("builtin");
comptime {
// Strong linkage when we control the final link (executables,
// shared libraries), weak otherwise.
const linkage: std.builtin.GlobalLinkage = switch (builtin.output_mode) {
.Exe => .strong,
.Lib => switch (builtin.link_mode) {
.dynamic => .strong,
.static => .weak,
},
.Obj => .weak,
};
// Whether the override is emitted:
//
// 1. On targets without SIMD we disable, since compiler_rt's
// scalar operations are going to be just as good.
// 2. The C object format target uses strong linkage which
// conflits with ours and errors.
// 3. Weak COFF builds fatally error because MSVC's linker
// errors when two identical linked symbols exist. MSVC has
// CRT which links so we don't need this there anyways.
const enabled =
std.simd.suggestVectorLength(u8) != null and
builtin.object_format != .c and
!(linkage == .weak and builtin.object_format == .coff);
if (enabled) @export(&memset, .{
.name = "memset",
.linkage = linkage,
// Hidden so that shared library builds (libghostty,
// libghostty-vt) resolve this internally without exporting
// it to their host applications.
.visibility = .hidden,
});
}
/// Bytes stored per loop iteration. Twice the native vector size so
/// each iteration issues two vector stores (e.g. `stp q0, q0` on
/// aarch64). Capped at 128 both because the small path below relies
/// on len < 128 once the loop is skipped, and because wider single
/// iterations stop paying off anyway (e.g. on scalable-vector
/// targets that suggest very large lengths).
const vec_bytes = @min(128, 2 * (std.simd.suggestVectorLength(u8) orelse 8));
/// Whether the `dc zva` fast path for large zero fills is available.
/// `dc zva` zeroes a whole cacheline per instruction without moving data
/// through the store pipeline. Based on musl.
const zva_enabled = builtin.cpu.arch == .aarch64 and
builtin.os.tag != .freestanding;
/// Only use `dc zva` at or above this many bytes. Below this our
/// plain vector loop measures faster. Empirically mesaured.
const zva_threshold = 16384;
/// Matches compiler_rt's memset signature (lib/compiler_rt.zig).
fn memset(dest: ?[*]u8, c: u8, len: usize) callconv(.c) ?[*]u8 {
@setRuntimeSafety(false);
if (len == 0) return dest;
const d = dest.?;
// Large path: full-width vector stores.
if (len >= vec_bytes) {
// Very large zero fills: zero whole cachelines with `dc zva`
// instead. Only implemented for the (universal in practice)
// 64-byte block size; anything else falls through to the
// vector loop.
if (comptime zva_enabled) {
if (len >= zva_threshold and c == 0) zva: {
if (zvaSize() != 64) break :zva;
const splat64: @Vector(64, u8) = @splat(0);
// Head: one unaligned store covering every byte up
// to the first 64-aligned address (and usually a bit
// beyond it; the overlap is fine).
const addr = @intFromPtr(d);
const aligned = std.mem.alignForward(usize, addr, 64);
d[0..64].* = splat64;
// Zero whole aligned cachelines while at least one
// full line remains in range. `dc zva` requires the
// WHOLE line to be inside the buffer: it always
// zeroes all 64 bytes.
var p: [*]u8 = @ptrFromInt(aligned);
const end_addr = addr + len;
while (@intFromPtr(p) + 64 <= end_addr) : (p += 64) {
asm volatile ("dc zva, %[ptr]"
:
: [ptr] "r" (p),
: .{ .memory = true });
}
// Tail: overlapping unaligned store anchored to the
// end of the buffer, covering whatever the line loop
// could not. len >= zva_threshold >= 64 so this
// cannot underflow.
d[len - 64 ..][0..64].* = splat64;
return dest;
}
}
const splat: @Vector(vec_bytes, u8) = @splat(c);
// Fill [0, N) where N is len rounded down to a multiple of
// vec_bytes.
var i: usize = 0;
while (i + vec_bytes <= len) : (i += vec_bytes) {
d[i..][0..vec_bytes].* = splat;
// This empty asm statement prevents LLVM's
// LoopIdiomRecognize pass from replacing this loop with
// a call to memset, which would be infinite recursion
// since we ARE memset. compiler_rt is protected from
// this by being built with -fno-builtin; Ghostty is not.
asm volatile ("" ::: .{ .memory = true });
}
// Fill the remaining [N, len) tail, if any, with one more
// full-width store anchored to the END of the buffer. It
// overlaps up to vec_bytes-1 bytes that the loop already
// wrote, which is fine (same value), and cannot underflow
// because len >= vec_bytes in this path.
if (i != len) d[len - vec_bytes ..][0..vec_bytes].* = splat;
return dest;
}
// Small path (len < vec_bytes): pairs of narrower stores, one
// anchored to the start of the buffer and one to the end.
//
// The invariant for each branch below: a store of width w at
// [0..w] plus a store at [len-w..len] covers every byte exactly
// when w <= len <= 2*w. Smaller lens are handled by a later
// branch; larger lens by an earlier one.
comptime std.debug.assert(vec_bytes <= 128);
if (comptime vec_bytes > 64) {
if (len >= 64) {
// 64 <= len < vec_bytes <= 128 (asserted above): one
// pair of 64-byte stores covers len <= 128. Only emitted
// when vec_bytes > 64 (e.g. AVX-512), otherwise the
// vector loop above already handled these lengths.
const splat64: @Vector(64, u8) = @splat(c);
d[0..64].* = splat64;
d[len - 64 ..][0..64].* = splat64;
return dest;
}
}
if (len >= 16) {
// 16 <= len < @min(vec_bytes, 64), so at most 63. One pair
// of 16-byte stores covers len <= 32. For 32 < len < 64, a
// second pair extends the covered prefix to [0..32] and the
// covered suffix to [len-32..len], which meet or overlap in
// the middle.
const splat16: @Vector(16, u8) = @splat(c);
d[0..16].* = splat16;
d[len - 16 ..][0..16].* = splat16;
if (len > 32) {
d[16..32].* = splat16;
d[len - 32 ..][0..16].* = splat16;
}
return dest;
}
if (len >= 8) {
// 8 <= len <= 15: one 8-byte pair (covers len <= 16).
const splat8: @Vector(8, u8) = @splat(c);
d[0..8].* = splat8;
d[len - 8 ..][0..8].* = splat8;
return dest;
}
if (len >= 4) {
// 4 <= len <= 7: one 4-byte pair (covers len <= 8).
const splat4: @Vector(4, u8) = @splat(c);
d[0..4].* = splat4;
d[len - 4 ..][0..4].* = splat4;
return dest;
}
// 1 <= len <= 3: too short for the pair trick; write bytes one
// at a time. The asm statement again keeps LoopIdiomRecognize
// from turning this into a recursive memset call; LLVM will do
// that even for a loop this short because the trip count is not
// statically known.
var i: usize = 0;
while (i < len) : (i += 1) {
d[i] = c;
asm volatile ("" ::: .{ .memory = true });
}
return dest;
}
/// Cached `dc zva` block size: 0 = not yet queried, 1 = unavailable
/// or prohibited, otherwise the block size in bytes. Racing threads
/// store the same value so the memory ordering is irrelevant.
var zva_size: std.atomic.Value(usize) = .init(0);
fn zvaSize() usize {
const cached = zva_size.load(.monotonic);
if (cached != 0) return cached;
const dczid = asm ("mrs %[out], dczid_el0"
: [out] "=r" (-> u64),
);
// Bit 4 (DZP) prohibits DC ZVA; bits 0-3 are log2 of the block
// size in 4-byte words.
const size: usize = if (dczid & 0x10 != 0)
1
else
@as(usize, 4) << @intCast(dczid & 0xF);
zva_size.store(size, .monotonic);
return size;
}
test memset {
const testing = std.testing;
// A buffer larger than anything the small/vector paths special
// case, with room for offset (alignment) variations. Guard bytes
// around the fill region must remain untouched.
var buf: [4 * vec_bytes + 33]u8 = undefined;
for (0..vec_bytes + 1) |offset| {
for (0..buf.len - offset) |len| {
for ([_]u8{ 0x00, 0x5C, 0xFF }) |c| {
@memset(&buf, 0xAA);
const region = buf[offset..][0..len];
const ret = memset(region.ptr, c, len);
try testing.expectEqual(region.ptr, ret.?);
for (buf[0..offset]) |b| try testing.expectEqual(0xAA, b);
for (region) |b| try testing.expectEqual(c, b);
for (buf[offset + len ..]) |b| try testing.expectEqual(0xAA, b);
}
}
}
// Zero length must not touch memory and must tolerate null.
try testing.expectEqual(null, memset(null, 0x11, 0));
var one: [1]u8 = .{0xAA};
_ = memset(&one, 0x11, 0);
try testing.expectEqual(0xAA, one[0]);
}
test "memset large zero fills (dc zva path)" {
const testing = std.testing;
const alloc = testing.allocator;
// Sizes at and around the ZVA threshold, with a guard byte on
// each side and every offset within a cacheline so the head and
// tail alignment handling is fully exercised.
const buf = try alloc.alloc(u8, 2 * zva_threshold + 66);
defer alloc.free(buf);
for ([_]usize{
zva_threshold - 1,
zva_threshold,
zva_threshold + 63,
2 * zva_threshold,
}) |len| {
for (0..65) |offset| {
@memset(buf, 0xAA);
const region = buf[1 + offset ..][0..len];
_ = memset(region.ptr, 0x00, len);
for (buf[0 .. 1 + offset]) |b| try testing.expectEqual(0xAA, b);
for (region) |b| try testing.expectEqual(0x00, b);
for (buf[1 + offset + len ..]) |b| try testing.expectEqual(0xAA, b);
}
}
// Nonzero fills of ZVA-eligible sizes must not take the ZVA path
// (it can only write zeroes).
@memset(buf, 0xAA);
_ = memset(buf.ptr + 1, 0x5C, zva_threshold);
try testing.expectEqual(0xAA, buf[0]);
for (buf[1 .. 1 + zva_threshold]) |b| try testing.expectEqual(0x5C, b);
try testing.expectEqual(0xAA, buf[1 + zva_threshold]);
}

12
src/simd/main.zig
View File

@@ -9,6 +9,18 @@ pub const index_of = @import("index_of.zig");
pub const vt = @import("vt.zig");
pub const codepointWidth = codepoint_width.codepointWidth;
/// The number of vector lanes to use for manually vectorized hot
/// loops operating on elements of type T, or null if the target has
/// no usable SIMD support and a plain scalar loop should be used
/// instead (e.g. wasm32 without simd128).
///
/// This is twice the native vector width so that each iteration works
/// two vector registers, giving better instruction-level parallelism.
pub fn lanes(comptime T: type) ?comptime_int {
const native = std.simd.suggestVectorLength(T) orelse return null;
return native * 2;
}
test {
@import("std").testing.refAllDecls(@This());
}

3
src/synthetic/cli.zig
View File

@@ -1,7 +1,6 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const cli = @import("../cli.zig");
const global = @import("../global.zig");
/// The available actions for the CLI. This is the list of available
/// synthetic generators. View docs for each individual one in the
@@ -93,7 +92,7 @@ fn mainActionImpl(
// Our output always goes to stdout.
var buffer: [2048]u8 = undefined;
var stdout_writer = std.Io.File.stdout().writer(global.io(), &buffer);
var stdout_writer = std.Io.File.stdout().writer(io, &buffer);
const writer = &stdout_writer.interface;
// Create our implementation

155
src/terminal/Terminal.zig
View File

@@ -10,6 +10,7 @@ const assert = @import("../quirks.zig").inlineAssert;
const tripwire = @import("../tripwire.zig");
const testing = std.testing;
const Allocator = std.mem.Allocator;
const simd = @import("../simd/main.zig");
const unicode = @import("../unicode/main.zig");
const uucode = @import("uucode");
@@ -630,6 +631,85 @@ inline fn printSliceEligible(cp: u32, comptime width: PrintSliceWidth) bool {
});
}
/// Store a run of narrow codepoint cells built from a bit template:
/// for each `idx` in `[from, to)`, `cells[idx]` is assigned the bits
/// `template_bits | (cps[idx] << cp_shift)`.
///
/// The template is a complete Cell (content tag, style, wide state,
/// etc. already baked in by the caller) whose codepoint content bits
/// are zero. Since Cell is a packed struct(u64), OR-ing a codepoint
/// into the content field's bit position yields a finished cell as a
/// single integer, keeping the loop pure data movement: no per-cell
/// field assignments and no branches.
///
/// This loop is manually vectorized: Zig 0.16 (LLVM 21) no longer
/// auto-vectorizes the scalar form the way Zig 0.15 (LLVM 20) did.
inline fn printSliceStoreRun(
cells: [*]Cell,
cps: [*]const u32,
from: usize,
to: usize,
template_bits: u64,
) void {
// The bit position of the `content` field within the packed
// Cell. A codepoint occupies the low bits of `content`, so
// shifting a codepoint left by this amount places it exactly
// where `.content = .{ .codepoint = .{ .data = cp } }` would.
const cp_shift = @bitOffsetOf(Cell, "content");
// Since codepoints are OR'd into the content field rather than
// assigned, any nonzero content bits in the template would
// corrupt the stored codepoints.
const content_mask: u64 = comptime mask: {
const bits = @bitSizeOf(@FieldType(Cell, "content"));
break :mask ((1 << bits) - 1) << cp_shift;
};
assert(template_bits & content_mask == 0);
var idx = from;
// Vectorized bulk of the run.
if (simd.lanes(u64)) |lanes| {
// u64 due to backing integer of Cell
const V = @Vector(lanes, u64);
// The template and shift amount are loop-invariant, so
// broadcast them to every lane once up front.
const template: V = @splat(template_bits);
const shift: @Vector(
lanes,
std.math.Log2Int(u64),
) = @splat(cp_shift);
while (idx + lanes <= to) : (idx += lanes) {
// Load `lanes` decoded codepoints...
const narrow: @Vector(lanes, u32) = cps[idx..][0..lanes].*;
// ...widen each u32 lane to the u64 cell size...
const wide: V = @intCast(narrow);
// ...shift each codepoint into the content field's bit
// position and merge with the template, producing
// `lanes` finished cells (coerced from vector to array
// so they can be bitcast for the store below)...
const bits: [lanes]u64 = template | (wide << shift);
// ...and store them contiguously. Cell is a packed
// struct(u64) so an array of u64 bit patterns has
// identical layout to an array of cells.
cells[idx..][0..lanes].* = @bitCast(bits);
}
}
// Scalar tail: the final `< lanes` cells of the run, or the
// entire run on targets without SIMD. Note `idx` carries over
// from the vector loop above. This is the same computation as
// the vector body, one cell at a time.
while (idx < to) : (idx += 1) {
cells[idx] = @bitCast(template_bits | (@as(u64, cps[idx]) << cp_shift));
}
}
/// The row-filling portion of the printSlice fast path, specialized by
/// width class. The first codepoint must already be validated by the
/// caller (printSliceFast).
@@ -671,19 +751,20 @@ fn printSliceFill(
// Anything else (including eligible unicode) proceeds via
// the scalar loop below.
if (comptime width == .narrow) {
const lanes = 8;
const V = @Vector(lanes, u32);
const lo: V = @splat(0x10);
const hi: V = @splat(0xFF);
while (idx + lanes <= cps.len) {
const v: V = cps[idx..][0..lanes].*;
const in_range = (v >= lo) & (v <= hi);
if (!@reduce(.And, in_range)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(in_range);
idx += @ctz(~bits);
break;
if (simd.lanes(u32)) |lanes| {
const V = @Vector(lanes, u32);
const lo: V = @splat(0x10);
const hi: V = @splat(0xFF);
while (idx + lanes <= cps.len) {
const v: V = cps[idx..][0..lanes].*;
const in_range = (v >= lo) & (v <= hi);
if (!@reduce(.And, in_range)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(in_range);
idx += @ctz(~bits);
break;
}
idx += lanes;
}
idx += lanes;
}
}
@@ -817,12 +898,38 @@ fn printSliceFill(
// We can only write whole (wide, spacer) pairs.
const pair_end = k + (simple - k) / 2 * 2;
var idx = k;
// Manually vectorized for the same reason as
// printSliceStoreRun: build a vector of wide cells,
// interleave with spacer tails, and store (wide,
// spacer) pairs several at a time.
if (simd.lanes(u64)) |lanes| {
const pair_lanes = lanes / 2;
const Vp = @Vector(pair_lanes, u64);
const wide_v: Vp = @splat(wide_bits);
const spacer_v: Vp = @splat(spacer_bits);
const shift_v: @Vector(
pair_lanes,
std.math.Log2Int(u64),
) = @splat(cp_shift);
while (idx + 2 * pair_lanes <= pair_end) : (idx += 2 * pair_lanes) {
const narrow: @Vector(pair_lanes, u32) =
cps[printed + idx / 2 ..][0..pair_lanes].*;
const wides: Vp = wide_v | (@as(Vp, @intCast(narrow)) << shift_v);
const inter: [2 * pair_lanes]u64 = std.simd.interlace(.{
wides,
spacer_v,
});
cells[idx..][0 .. 2 * pair_lanes].* = @bitCast(inter);
}
}
while (idx < pair_end) : (idx += 2) {
cells[idx] = @bitCast(
wide_bits | (@as(u64, cps[printed + idx / 2]) << cp_shift),
);
cells[idx + 1] = @bitCast(spacer_bits);
}
// If the simple run ended mid-pair we stop at the pair
// boundary and handle the offending cell below.
k = pair_end;
@@ -832,11 +939,13 @@ fn printSliceFill(
simple = pair_end;
}
} else {
for (k..simple) |idx| {
cells[idx] = @bitCast(
template_bits | (@as(u64, cps[printed + idx]) << cp_shift),
);
}
printSliceStoreRun(
cells,
cps.ptr + printed,
k,
simple,
template_bits,
);
k = simple;
}
if (k >= cell_count) break;
@@ -888,11 +997,13 @@ fn printSliceFill(
page.styles.useMultiple(page.memory, style_id, @intCast(n));
}
for (k..m) |idx| {
cells[idx] = @bitCast(
template_bits | (@as(u64, cps[printed + idx]) << cp_shift),
);
}
printSliceStoreRun(
cells,
cps.ptr + printed,
k,
m,
template_bits,
);
k = m;
continue :fill;
}

18
src/terminal/hash_map.zig
View File

@@ -70,8 +70,24 @@ fn AutoHashMapUnmanaged(
fn AutoContext(comptime K: type) type {
return struct {
pub const hash = std.hash_map.getAutoHashFn(K, @This());
pub const eql = std.hash_map.getAutoEqlFn(K, @This());
pub fn hash(_: @This(), key: K) u64 {
if (comptime std.meta.hasUniqueRepresentation(K)) {
// LLVM 21 (Zig 0.16) failed to inline this which resulted
// in a measurable almost 2x slowdown on our hyperlink map
// benchmark. So, force it.
return @call(
.always_inline,
std.hash.Wyhash.hash,
.{ 0, std.mem.asBytes(&key) },
);
}
var hasher = std.hash.Wyhash.init(0);
std.hash.autoHash(&hasher, key);
return hasher.final();
}
};
}

24
src/terminal/stream.zig
View File

@@ -579,8 +579,30 @@ pub fn Stream(comptime H: type) type {
continue;
}
// Find the end of the printable run. This is an
// early-exit search loop that LLVM won't
// auto-vectorize, and printable runs dominate real
// input, so scan several codepoints at a time
// manually (same idiom as the printSliceFill run
// scan).
var end = i + 1;
while (end < cps.len and cps[end] > 0xF) end += 1;
scan: {
if (simd.lanes(u32)) |lanes| {
const V = @Vector(lanes, u32);
const threshold: V = @splat(0xF);
while (end + lanes <= cps.len) {
const v: V = cps[end..][0..lanes].*;
const gt = v > threshold;
if (!@reduce(.And, gt)) {
const bits: std.meta.Int(.unsigned, lanes) = @bitCast(gt);
end += @ctz(~bits);
break :scan;
}
end += lanes;
}
}
while (end < cps.len and cps[end] > 0xF) end += 1;
}
self.handler.vt(.print_slice, .{ .cps = cps[i..end] });
i = end;
}

12
src/unicode/grapheme.zig
View File

@@ -141,8 +141,8 @@ inline fn invalidCodepoint(cp: anytype) bool {
/// This is all the structures and data for the precomputed lookup table
/// for all possible permutations of state and grapheme break properties.
/// Precomputation requires 2^13 keys of 4 bit values so the whole table is
/// 8KB.
/// Precomputation requires 2^13 keys of byte-sized values so the whole
/// table is 8KB.
const Precompute = struct {
const Key = packed struct(u13) {
state: uucode.grapheme.BreakState,
@@ -154,9 +154,15 @@ const Precompute = struct {
}
};
const Value = packed struct(u4) {
const Value = packed struct(u8) {
result: bool,
state: uucode.grapheme.BreakState,
/// Explicit padding to give the struct a power-of-two backing
/// integer, for the same reason as `Properties._padding` (avoids
/// exotic-width LLVM integer accesses that defeat register
/// promotion). @sizeOf was already 1, so the table stays 8KB.
_padding: u4 = 0,
};
const data = precompute: {

12
src/unicode/props.zig
View File

@@ -7,7 +7,7 @@
const std = @import("std");
const uucode = @import("uucode");
pub const Properties = packed struct {
pub const Properties = packed struct(u16) {
/// Codepoint width. We clamp to [0, 2] since Ghostty handles control
/// characters and we max out at 2 for wide characters (i.e. 3-em dash
/// becomes a 2-em dash).
@@ -23,6 +23,16 @@ pub const Properties = packed struct {
/// Emoji VS compatibility
emoji_vs_base: bool = false,
/// Explicit padding to give the struct a power-of-two backing integer.
/// This is a performance workaround: with a non-power-of-two backing
/// integer (u9), Zig 0.16 lowers loads/stores of this struct through
/// mismatched exotic-width LLVM integer types (store i9 / load i16),
/// which blocks LLVM from promoting temporaries to registers and forces
/// stack round-trips in hot paths like `graphemeBreak`. See
/// https://github.com/ziglang/zig/issues/21891 for the upstream issue.
/// The size of the lookup table is unchanged (@sizeOf was already 2).
_padding: u7 = 0,
// Needed for lut.Generator
pub fn eql(a: Properties, b: Properties) bool {
return a.width == b.width and