diff --git a/src/build/GhosttyZig.zig b/src/build/GhosttyZig.zig index 4671219e0..eb39f7a96 100644 --- a/src/build/GhosttyZig.zig +++ b/src/build/GhosttyZig.zig @@ -137,6 +137,15 @@ fn initVt( // We need uucode for grapheme break support vt.addImport("uucode", deps.uucode_mod); + // We need for Kitty graphics. If Kitty graphics is disabled then + // z2d isn't referenced and it produces no code, so its safe. + if (b.lazyDependency("z2d", .{ + .target = cfg.target, + .optimize = cfg.optimize, + })) |dep| { + vt.addImport("z2d", dep.module("z2d")); + } + // If SIMD is enabled, add all our SIMD dependencies. if (cfg.simd) { try SharedDeps.addSimd(b, vt, simd_libs); diff --git a/src/terminal/kitty/graphics.zig b/src/terminal/kitty/graphics.zig index 1f639fab8..ef45ee8d0 100644 --- a/src/terminal/kitty/graphics.zig +++ b/src/terminal/kitty/graphics.zig @@ -3,11 +3,6 @@ //! Documentation: //! https://sw.kovidgoyal.net/kitty/graphics-protocol //! -//! Unimplemented features that are still todo: -//! - shared memory transmit -//! - virtual placement w/ unicode -//! - animation -//! //! Performance: //! The performance of this particular subsystem of Ghostty is not great. //! We can avoid a lot more allocations, we can replace some C code (which @@ -21,7 +16,10 @@ const command = @import("graphics_command.zig"); const exec = @import("graphics_exec.zig"); const image = @import("graphics_image.zig"); const storage = @import("graphics_storage.zig"); +pub const animation = @import("graphics_animation.zig"); +pub const pixel = @import("graphics_pixel.zig"); pub const unicode = @import("graphics_unicode.zig"); +pub const Animation = animation.Animation; pub const Command = command.Command; pub const CommandParser = command.Parser; pub const Image = image.Image; diff --git a/src/terminal/kitty/graphics_animation.zig b/src/terminal/kitty/graphics_animation.zig new file mode 100644 index 000000000..0822a451c --- /dev/null +++ b/src/terminal/kitty/graphics_animation.zig @@ -0,0 +1,149 @@ +//! Kitty graphics protocol animation support. +//! +//! https://sw.kovidgoyal.net/kitty/graphics-protocol/#animation +//! +//! An animation is a set of frames attached to an existing image. +//! Frame 1 (the "root frame") is the image's own base data. Frames +//! 2..N are stored here. Unlike Kitty, which stores frames as delta +//! rectangles in a disk cache and composes them lazily, we compose +//! every frame eagerly into a full image-sized RGBA buffer at load +//! time. This trades some memory for a much simpler model: a frame is +//! always ready to display and there are no reference chains to +//! maintain, coalesce, or garbage collect. Frame storage is bounded +//! by the same byte limit as image storage. +//! +//! To keep composition to a single pixel format, an image is +//! converted to RGBA the first time an animation command composes +//! into it, and all transmitted frame data is converted to RGBA +//! before composition. Blending an opaque (alpha=255) source pixel +//! degenerates to a copy, so this loses no information relative to +//! Kitty's separate RGB/RGBA paths. The pixel-level primitives +//! (conversion, fill, rectangle composition) live in +//! graphics_pixel.zig. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +/// The gap assigned to a newly created frame when the command doesn't +/// specify one (z omitted or z=0). Taken from Kitty (DEFAULT_GAP). +pub const default_gap_ms: u32 = 40; + +/// Animation state for a single image. Heap-allocated and owned by +/// the Image. This is created lazily by the first animation command that +/// needs it. +pub const Animation = struct { + /// The frames following the root frame: `frames.items[i]` is + /// protocol frame number `i + 2`. The root frame (frame 1) is the + /// image's base data and is not stored here. + frames: std.ArrayListUnmanaged(Frame) = .empty, + + /// The gap of the root frame in milliseconds. Zero means gapless: + /// the frame is skipped during playback. Kitty creates the root + /// frame with a zero gap; the client gives it one with a=a,r=1,z=N. + root_gap_ms: u32 = 0, + + /// The zero-based index of the frame currently displayed: 0 is + /// the root frame, i >= 1 is frames.items[i - 1]. + current_index: u32 = 0, + + /// Playback state. Every image starts stopped (client-driven). + state: State = .stopped, + + /// Maximum number of loops to play (0 = infinite). Set from the + /// a=a v key as v-1, per the protocol's off-by-one encoding. + max_loops: u32 = 0, + + /// Number of completed loops since playback started. + current_loop: u32 = 0, + + /// Timestamp in milliseconds (on the ticker's clock, see + /// ImageStorage.animationTick) when the current frame was shown. + /// Null means "not yet shown"; the next tick stamps it. + frame_shown_at_ms: ?u64 = null, + + pub const State = enum { + /// Not advancing; frames change only via a=a c=N (client-driven). + stopped, + /// Advancing, but playback parks on the last frame waiting + /// for more frames instead of looping (a=a s=2). + loading, + /// Advancing and looping (a=a s=3). + running, + }; + + pub const Frame = struct { + /// Fully composed pixel data, always image width * height * 4 + /// bytes of RGBA. + data: []u8, + + /// Milliseconds this frame is displayed before advancing. + /// Zero means gapless: skipped during playback. + gap_ms: u32, + }; + + pub fn deinit(self: *Animation, alloc: Allocator) void { + for (self.frames.items) |frame| alloc.free(frame.data); + self.frames.deinit(alloc); + } + + /// The total number of frames including the root frame. + pub fn frameCount(self: *const Animation) u32 { + return @intCast(self.frames.items.len + 1); + } + + /// The gap of the frame at the given zero-based index. + pub fn gapAt(self: *const Animation, index: u32) u32 { + if (index == 0) return self.root_gap_ms; + return self.frames.items[index - 1].gap_ms; + } + + /// Set the gap of the frame at the given zero-based index. + pub fn setGapAt(self: *Animation, index: u32, gap_ms: u32) void { + if (index == 0) { + self.root_gap_ms = gap_ms; + } else { + self.frames.items[index - 1].gap_ms = gap_ms; + } + } + + /// The sum of all frame gaps. An animation with a zero duration + /// (every frame gapless) never advances; this doubles as the + /// guard that keeps the gapless-skip loop in animationTick from + /// spinning forever, exactly like Kitty's animation_duration. + pub fn durationMs(self: *const Animation) u64 { + var total: u64 = self.root_gap_ms; + for (self.frames.items) |frame| total += frame.gap_ms; + return total; + } + + /// Bytes of frame data held by this animation, counted against + /// the image storage limit. + pub fn frameBytes(self: *const Animation) usize { + var total: usize = 0; + for (self.frames.items) |frame| total += frame.data.len; + return total; + } +}; + +test "animation gap helpers" { + const testing = std.testing; + const alloc = testing.allocator; + + var anim: Animation = .{}; + defer anim.deinit(alloc); + try anim.frames.append(alloc, .{ + .data = try alloc.alloc(u8, 4), + .gap_ms = 100, + }); + + try testing.expectEqual(@as(u32, 2), anim.frameCount()); + try testing.expectEqual(@as(u32, 0), anim.gapAt(0)); + try testing.expectEqual(@as(u32, 100), anim.gapAt(1)); + try testing.expectEqual(@as(u64, 100), anim.durationMs()); + try testing.expectEqual(@as(usize, 4), anim.frameBytes()); + + anim.setGapAt(0, 40); + anim.setGapAt(1, 60); + try testing.expectEqual(@as(u64, 100), anim.durationMs()); + try testing.expectEqual(@as(u32, 40), anim.root_gap_ms); +} diff --git a/src/terminal/kitty/graphics_exec.zig b/src/terminal/kitty/graphics_exec.zig index e27500aab..293659279 100644 --- a/src/terminal/kitty/graphics_exec.zig +++ b/src/terminal/kitty/graphics_exec.zig @@ -521,6 +521,7 @@ const EncodeableError = Image.Error || Allocator.Error; fn encodeError(r: *Response, err: EncodeableError) void { switch (err) { error.OutOfMemory => r.message = "ENOMEM: out of memory", + error.InsufficientData => r.message = "ENODATA: insufficient data", error.InvalidData => r.message = "EINVAL: invalid data", error.DecompressionFailed => r.message = "EINVAL: decompression failed", error.FilePathTooLong => r.message = "EINVAL: file path too long", diff --git a/src/terminal/kitty/graphics_image.zig b/src/terminal/kitty/graphics_image.zig index 9cce19966..489b672e3 100644 --- a/src/terminal/kitty/graphics_image.zig +++ b/src/terminal/kitty/graphics_image.zig @@ -6,6 +6,7 @@ const ArenaAllocator = std.heap.ArenaAllocator; const posix = std.posix; const fastmem = @import("../../fastmem.zig"); +const animation = @import("graphics_animation.zig"); const command = @import("graphics_command.zig"); const PageList = @import("../PageList.zig"); const sys = @import("../sys.zig"); @@ -35,6 +36,12 @@ pub const LoadingImage = struct { /// so that we display the image after it is fully loaded. display: ?command.Display = null, + /// This is non-null when this load is an animation frame + /// transmission (a=f) rather than a new image. On completion the + /// data is composed into the target image's animation instead of + /// being stored as an image. + frame: ?FrameContext = null, + /// Quiet is the quiet settings for the initial load command. This is /// used if q isn't set on subsequent chunks. quiet: command.Command.Quiet, @@ -47,6 +54,20 @@ pub const LoadingImage = struct { /// temporary directory transmission is disabled). temporary_directory: ?[]const u8, + pub const FrameContext = struct { + /// The frame parameters from the initial a=f command. Chunked + /// continuations only contribute payload bytes; all parameters + /// come from the command that started the load, matching the + /// protocol's requirement that chunks repeat a=f. + cmd: command.AnimationFrameLoading, + + /// The generation of the target image when the load began. + /// A different generation at completion means the image was + /// replaced or evicted mid-transmission and the frame must be + /// discarded rather than composed onto the wrong image. + image_generation: u64, + }; + /// The limits of the Kitty Graphics protocol we should allow. /// /// This can be used to restrict the type of images and other @@ -501,11 +522,23 @@ pub const LoadingImage = struct { if (img.width == 0 or img.height == 0) return error.DimensionsRequired; if (img.width > max_dimension or img.height > max_dimension) return error.DimensionsTooLarge; - // Data length must be what we expect + // Data length must be what we expect. const bpp = command.Transmission.formatBpp(img.format); const expected_len = img.width * img.height * bpp; const actual_len = self.data.items.len; - if (actual_len != expected_len) { + if (self.frame != null) { + // Kitty allows animation frames to exceed their expected length + // and just truncates it. Not sure if thats expected but lets + // allow it too. + if (actual_len < expected_len) { + std.log.warn( + "insufficient frame data image id={} expected_len={} actual_len={}", + .{ img.id, expected_len, actual_len }, + ); + return error.InsufficientData; + } + self.data.items.len = expected_len; + } else if (actual_len != expected_len) { std.log.warn( "unexpected length image id={} width={} height={} bpp={} expected_len={} actual_len={}", .{ img.id, img.width, img.height, bpp, expected_len, actual_len }, @@ -653,9 +686,25 @@ pub const Image = struct { /// have changed, even if the dimensions and byte length are the /// same (e.g. a retransmission of the same ID). Stamps order by /// transmission time. Zero means "never stored". + /// + /// For animated images this also changes whenever the frame that + /// should be displayed changes (advance, edit, or delete of the + /// current frame), since consumers key texture caches off it. generation: u64 = 0, + /// Animation state, non-null once any animation command (a=f, + /// a=a) has attached animation state to this image. Owned by the + /// image; replaced/retransmitted images drop it, which implements + /// the protocol's "retransmission resets the animation" rule. + /// + /// This is only ever attached to images stored in an ImageStorage + /// and must only be mutated through the storage's own pointer + /// (Image values are copied around freely; copies share this + /// pointer and never own it). + animation: ?*animation.Animation = null, + pub const Error = error{ + InsufficientData, InvalidData, DecompressionFailed, DimensionsRequired, @@ -705,6 +754,54 @@ pub const Image = struct { pub fn deinit(self: *Image, alloc: Allocator) void { self.data.deinit(alloc); + if (self.animation) |anim| { + anim.deinit(alloc); + alloc.destroy(anim); + self.animation = null; + } + } + + /// The pixel data that should be displayed for this image. For an + /// animated image this is the current animation frame; otherwise + /// (and for the root frame) it is the image's own data. + pub fn renderData(self: *const Image) Data { + if (self.animation) |anim| { + if (anim.current_index > 0) { + return .{ .complete = anim.frames.items[anim.current_index - 1].data }; + } + } + + return self.data; + } + + /// The pixel data of the given 1-based animation frame number, or + /// null if the frame doesn't exist. Frame 1 (the root frame) + /// always exists as long as the image data is complete, even for + /// images without animation state. + /// + /// The returned slice is owned by the image (or its animation) + /// and remains valid until the image or frame is mutated. + pub fn frameData(self: *const Image, number: u32) ?[]const u8 { + switch (number) { + 0 => return null, + 1 => return self.data.bytes(), + else => { + const anim = self.animation orelse return null; + // Minus 2 because frame is 1-based and frame 1 is the + // image base data, so the animation frames start at frame 2. + const idx = number - 2; + if (idx >= anim.frames.items.len) return null; + return anim.frames.items[idx].data; + }, + } + } + + /// Total bytes of pixel data reserved against the storage limit + /// for this image: the base data plus any animation frames. + pub fn storageSize(self: *const Image) usize { + var total: usize = self.data.len(); + if (self.animation) |anim| total += anim.frameBytes(); + return total; } /// Mostly for logging diff --git a/src/terminal/kitty/graphics_pixel.zig b/src/terminal/kitty/graphics_pixel.zig new file mode 100644 index 000000000..1b543c3a9 --- /dev/null +++ b/src/terminal/kitty/graphics_pixel.zig @@ -0,0 +1,298 @@ +//! Pixel-buffer operations for the Kitty graphics protocol: format +//! conversion to RGBA, background fill, and rectangle composition. +//! These are the primitives behind animation frame loading (a=f) and +//! frame composition (a=c). See graphics_animation.zig for the +//! animation model built on top of them. +//! +//! All buffers are straight (non-premultiplied) alpha RGBA, as the +//! protocol and our consumers (renderer, C API) expect. +//! Since z2d composites in premultiplied alpha, blending round-trips +//! each pixel through multiply/demultiply. +//! +//! Note, there's a lot here that is suboptimal (non-vectorized) and we +//! should use something like wuffs probably too, but wuffs has a libc +//! dependency we don't want to force. We can also optimize this later +//! once we prove it all works. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const z2d = @import("z2d"); +const command = @import("graphics_command.zig"); + +/// Convert pixel data in the given format to a freshly allocated RGBA +/// buffer. The caller owns the result; the input is not freed. +/// +/// These stay hand-rolled rather than using wuffs' swizzler because +/// wuffs requires libc and libghostty-vt must remain buildable fully +/// freestanding (see the module doc). They produce identical values. +pub fn rgbaFromFormat( + alloc: Allocator, + format: command.Transmission.Format, + data: []const u8, +) Allocator.Error![]u8 { + switch (format) { + .rgba => return try alloc.dupe(u8, data), + + .rgb => { + const pixels = data.len / 3; + const result = try alloc.alloc(u8, pixels * 4); + for (0..pixels) |i| { + result[i * 4 + 0] = data[i * 3 + 0]; + result[i * 4 + 1] = data[i * 3 + 1]; + result[i * 4 + 2] = data[i * 3 + 2]; + result[i * 4 + 3] = 255; + } + return result; + }, + + .gray => { + const result = try alloc.alloc(u8, data.len * 4); + for (data, 0..) |v, i| { + result[i * 4 + 0] = v; + result[i * 4 + 1] = v; + result[i * 4 + 2] = v; + result[i * 4 + 3] = 255; + } + return result; + }, + + .gray_alpha => { + const pixels = data.len / 2; + const result = try alloc.alloc(u8, pixels * 4); + for (0..pixels) |i| { + const v = data[i * 2]; + result[i * 4 + 0] = v; + result[i * 4 + 1] = v; + result[i * 4 + 2] = v; + result[i * 4 + 3] = data[i * 2 + 1]; + } + return result; + }, + + // PNG is decoded to RGBA during image loading. + .png => unreachable, + } +} + +/// Fill an RGBA buffer with the given background color. +pub fn fillBackground( + data: []u8, + bg: command.AnimationFrameLoading.Background, +) void { + const raw: u32 = @bitCast(bg); + if (raw == 0) { + @memset(data, 0); + return; + } + var i: usize = 0; + while (i + 4 <= data.len) : (i += 4) { + data[i + 0] = bg.r; + data[i + 1] = bg.g; + data[i + 2] = bg.b; + data[i + 3] = bg.a; + } +} + +/// Compose the `src` rectangle (src_width x src_height RGBA pixels) +/// onto the `dst` canvas (dst_width x dst_height RGBA pixels) with +/// its top-left corner at (x, y). Portions of the rectangle outside +/// the canvas are silently clipped, matching Kitty's a=f behavior. +pub fn composeRect( + dst: []u8, + dst_width: u32, + dst_height: u32, + src: []const u8, + src_width: u32, + src_height: u32, + x: u32, + y: u32, + mode: command.CompositionMode, +) void { + if (x >= dst_width or y >= dst_height) return; + const width: usize = @min(src_width, dst_width - x); + const height: usize = @min(src_height, dst_height - y); + for (0..height) |row| { + const dst_off = ((y + row) * dst_width + x) * 4; + const src_off = row * @as(usize, src_width) * 4; + composeRow( + dst[dst_off..][0 .. width * 4], + src[src_off..][0 .. width * 4], + mode, + ); + } +} + +/// Compose a width x height rectangle between two full-size RGBA +/// canvases sharing the same `canvas_width` stride, reading from +/// (src_x, src_y) in `src` and writing at (dst_x, dst_y) in `dst`. +/// The caller must have validated that both rectangles are within +/// bounds; a=c reports out-of-bounds rectangles as errors rather +/// than clipping. +pub fn composeCanvasRect( + dst: []u8, + src: []const u8, + canvas_width: u32, + width: u32, + height: u32, + src_x: u32, + src_y: u32, + dst_x: u32, + dst_y: u32, + mode: command.CompositionMode, +) void { + for (0..height) |row| { + const dst_off = ((dst_y + row) * @as(usize, canvas_width) + dst_x) * 4; + const src_off = ((src_y + row) * @as(usize, canvas_width) + src_x) * 4; + composeRow( + dst[dst_off..][0 .. @as(usize, width) * 4], + src[src_off..][0 .. @as(usize, width) * 4], + mode, + ); + } +} + +fn composeRow(dst: []u8, src: []const u8, mode: command.CompositionMode) void { + switch (mode) { + .overwrite => @memcpy(dst, src), + .alpha_blend => { + var i: usize = 0; + while (i < dst.len) : (i += 4) { + blendPixel(dst[i..][0..4], src[i..][0..4]); + } + }, + } +} + +/// Source-over blend of one straight-alpha RGBA pixel onto another, +/// via z2d's compositor. z2d composites in premultiplied alpha, so +/// the pixels round-trip through multiply/demultiply; see the module +/// doc for how that compares to Kitty. +fn blendPixel(dst: *[4]u8, src: *const [4]u8) void { + // A fully transparent source pixel leaves the destination + // untouched, exactly like Kitty. This also keeps the no-op case + // free of the premultiply round-trip's rounding. + if (src[3] == 0) return; + + const src_px: z2d.pixel.RGBA = .{ .r = src[0], .g = src[1], .b = src[2], .a = src[3] }; + const dst_px: z2d.pixel.RGBA = .{ .r = dst[0], .g = dst[1], .b = dst[2], .a = dst[3] }; + const out = z2d.compositor.runPixel( + .integer, + dst_px.multiply().asPixel(), + src_px.multiply().asPixel(), + .src_over, + ).rgba.demultiply(); + dst.* = .{ out.r, out.g, out.b, out.a }; +} + +test "rgba conversion" { + const testing = std.testing; + const alloc = testing.allocator; + + { + const rgb = [_]u8{ 1, 2, 3, 4, 5, 6 }; + const result = try rgbaFromFormat(alloc, .rgb, &rgb); + defer alloc.free(result); + try testing.expectEqualSlices(u8, &.{ 1, 2, 3, 255, 4, 5, 6, 255 }, result); + } + + { + const gray = [_]u8{ 7, 8 }; + const result = try rgbaFromFormat(alloc, .gray, &gray); + defer alloc.free(result); + try testing.expectEqualSlices(u8, &.{ 7, 7, 7, 255, 8, 8, 8, 255 }, result); + } + + { + const ga = [_]u8{ 7, 100, 8, 200 }; + const result = try rgbaFromFormat(alloc, .gray_alpha, &ga); + defer alloc.free(result); + try testing.expectEqualSlices(u8, &.{ 7, 7, 7, 100, 8, 8, 8, 200 }, result); + } + + { + const rgba = [_]u8{ 1, 2, 3, 4 }; + const result = try rgbaFromFormat(alloc, .rgba, &rgba); + defer alloc.free(result); + try testing.expectEqualSlices(u8, &rgba, result); + } +} + +test "fill background" { + var buf: [8]u8 = undefined; + fillBackground(&buf, .{ .r = 1, .g = 2, .b = 3, .a = 4 }); + try std.testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4, 1, 2, 3, 4 }, &buf); + + fillBackground(&buf, .{}); + try std.testing.expectEqualSlices(u8, &(.{0} ** 8), &buf); +} + +test "compose rect overwrite with clipping" { + // 2x2 canvas, compose a 2x1 rect at (1, 1): only the first pixel + // of the rect fits, the rest clips off the right edge. + var dst = [_]u8{0} ** 16; + const src = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 }; + composeRect(&dst, 2, 2, &src, 2, 1, 1, 1, .overwrite); + + var expect = [_]u8{0} ** 16; + expect[12] = 1; + expect[13] = 2; + expect[14] = 3; + expect[15] = 4; + try std.testing.expectEqualSlices(u8, &expect, &dst); +} + +test "compose rect entirely out of bounds" { + var dst = [_]u8{9} ** 16; + const src = [_]u8{1} ** 4; + composeRect(&dst, 2, 2, &src, 1, 1, 2, 0, .overwrite); + composeRect(&dst, 2, 2, &src, 1, 1, 0, 2, .overwrite); + try std.testing.expectEqualSlices(u8, &(.{9} ** 16), &dst); +} + +test "alpha blend source-over semantics" { + const testing = std.testing; + + // Opaque source overwrites exactly. + { + var dst = [4]u8{ 10, 20, 30, 40 }; + blendPixel(&dst, &.{ 100, 110, 120, 255 }); + try testing.expectEqualSlices(u8, &.{ 100, 110, 120, 255 }, &dst); + } + + // Fully transparent source leaves the destination untouched + // exactly (no premultiply round-trip; matches Kitty). + { + var dst = [4]u8{ 10, 20, 30, 40 }; + blendPixel(&dst, &.{ 100, 110, 120, 0 }); + try testing.expectEqualSlices(u8, &.{ 10, 20, 30, 40 }, &dst); + } + + // 50% source over opaque destination. + { + var dst = [4]u8{ 0, 0, 0, 255 }; + blendPixel(&dst, &.{ 255, 255, 255, 128 }); + try testing.expectEqual(@as(u8, 255), dst[3]); + try testing.expectEqual(@as(u8, 128), dst[0]); + } + + // Blending over a transparent destination yields the source, + // minus one bit of rounding on the color channels from z2d's + // premultiply round-trip (Kitty's float math yields the source + // exactly here; see the module doc). + { + var dst = [4]u8{ 0, 0, 0, 0 }; + blendPixel(&dst, &.{ 200, 100, 50, 128 }); + try testing.expectEqualSlices(u8, &.{ 199, 99, 49, 128 }, &dst); + } +} + +test "compose canvas rect" { + // 3x1 canvas: copy pixel at x=0 onto x=2. + var canvas = [_]u8{ 1, 2, 3, 4, 0, 0, 0, 0, 9, 9, 9, 9 }; + composeCanvasRect(&canvas, &canvas, 3, 1, 1, 0, 0, 2, 0, .overwrite); + try std.testing.expectEqualSlices( + u8, + &.{ 1, 2, 3, 4, 0, 0, 0, 0, 1, 2, 3, 4 }, + &canvas, + ); +} diff --git a/src/terminal/kitty/graphics_storage.zig b/src/terminal/kitty/graphics_storage.zig index 329de74d5..897c8944a 100644 --- a/src/terminal/kitty/graphics_storage.zig +++ b/src/terminal/kitty/graphics_storage.zig @@ -6,6 +6,8 @@ const ArenaAllocator = std.heap.ArenaAllocator; const terminal = @import("../main.zig"); const point = @import("../point.zig"); const size = @import("../size.zig"); +const animation = @import("graphics_animation.zig"); +const pixel = @import("graphics_pixel.zig"); const command = @import("graphics_command.zig"); const PageList = @import("../PageList.zig"); const Screen = @import("../Screen.zig"); @@ -185,13 +187,14 @@ pub const ImageStorage = struct { /// Record a content mutation: marks the storage dirty and assigns a /// fresh generation stamp. Must be called by anything that changes - /// the set of images or placements (or image contents). + /// the set of images or placements (or image contents), including + /// the animation command handlers in graphics_exec.zig. /// /// Do NOT call this for geometry-only events (scrolling, resizing, /// screen switches); those must set only the dirty flag directly. /// Bumping the generation for geometry changes would break the /// contract that an unchanged generation means unchanged contents. - fn markMutated(self: *ImageStorage, io: std.Io) void { + pub fn markMutated(self: *ImageStorage, io: std.Io) void { self.dirty = true; self.generation = nextGeneration(io); } @@ -281,7 +284,7 @@ pub const ImageStorage = struct { // replacing pending snapshot metadata must be able to reuse the // reservation without evicting its own ID. const old_len = if (self.images.get(img.id)) |old| - old.data.len() + old.storageSize() else 0; assert(old_len <= self.total_bytes); @@ -315,7 +318,10 @@ pub const ImageStorage = struct { // Relative placements parented to the removed placements go too. _ = self.removeOrphans(s, null); - self.total_bytes -= gop.value_ptr.data.len(); + // Replacing an image drops its animation frames with it, + // implementing the protocol rule that retransmitting the + // base image resets the animation. + self.total_bytes -= gop.value_ptr.storageSize(); gop.value_ptr.deinit(alloc); } @@ -928,6 +934,235 @@ pub const ImageStorage = struct { return newest; } + /// Get a mutable pointer to a stored image, by ID or (newest by) + /// number, following the protocol's id/number addressing. Used by + /// the animation commands, which mutate images in place. The + /// pointer is invalidated by any operation that adds or removes + /// images. + pub fn imagePtrByIdOrNumber( + self: *const ImageStorage, + image_id: u32, + image_number: u32, + ) ?*Image { + if (image_id != 0) return self.images.getPtr(image_id); + + var newest: ?*Image = null; + var it = self.images.iterator(); + while (it.next()) |kv| { + if (kv.value_ptr.number != image_number) continue; + if (newest == null or + kv.value_ptr.generation > newest.?.generation) + { + newest = kv.value_ptr; + } + } + + return newest; + } + + /// Record that the displayed content of an image changed without + /// the image being re-added: marks the storage mutated and stamps + /// the image with the fresh generation so consumers (e.g. the + /// renderer's texture cache) replace what they hold. Used when an + /// animation changes which frame is current or edits the current + /// frame's pixels. + pub fn markImageContentChanged( + self: *ImageStorage, + io: std.Io, + img: *Image, + ) void { + self.markMutated(io); + img.generation = self.generation; + } + + /// Convert a stored image's base data to RGBA in place, adjusting + /// byte accounting. All animation composition happens in RGBA; + /// this is called before the first composition into an image. + /// + /// The pixels are unchanged visually but the stored representation + /// changed, so the image is stamped with a fresh generation. + pub fn convertImageToRgba( + self: *ImageStorage, + io: std.Io, + alloc: Allocator, + img: *Image, + ) Allocator.Error!void { + if (img.format == .rgba) return; + const old = img.data.bytes() orelse return; + + const rgba = try pixel.rgbaFromFormat(alloc, img.format, old); + self.total_bytes -= old.len; + self.total_bytes += rgba.len; + img.data.deinit(alloc); + img.data = .{ .complete = rgba }; + img.format = .rgba; + self.markImageContentChanged(io, img); + } + + /// Reserve `bytes` of storage for animation frame data belonging + /// to `image_id`, evicting other images if needed, mirroring how + /// image transmission reserves space. + /// + /// Errors if the space cannot be made available. On success the caller + /// owns the reservation and must either attach the frame data to the + /// image or call releaseAnimationBytes. + pub fn reserveAnimationBytes( + self: *ImageStorage, + io: std.Io, + alloc: Allocator, + s: *terminal.Screen, + image_id: u32, + bytes: usize, + ) Allocator.Error!void { + if (bytes > self.total_limit) return error.OutOfMemory; + + const total_bytes = self.total_bytes + bytes; + if (total_bytes > self.total_limit) { + const req_bytes = total_bytes - self.total_limit; + // Excess this large cannot be recovered by evicting other + // images (evictImageExcept also requires it). + if (req_bytes > self.total_limit) return error.OutOfMemory; + log.info("evicting images for animation frame, evicting={}", .{req_bytes}); + if (!self.evictImageExcept( + io, + alloc, + s, + req_bytes, + image_id, + )) { + log.warn("failed to evict enough images for animation frame", .{}); + return error.OutOfMemory; + } + } + + self.total_bytes += bytes; + } + + /// Release a reservation made by reserveAnimationBytes, or credit + /// bytes freed by deleting animation frame data. + pub fn releaseAnimationBytes(self: *ImageStorage, bytes: usize) void { + assert(bytes <= self.total_bytes); + self.total_bytes -= bytes; + } + + /// Advance every running animation to the frame that should be + /// displayed at `now_ms` and report when the next frame change is + /// due, as a delay in milliseconds relative to `now_ms`. Null + /// means no running animation needs a future tick. + /// + /// `now_ms` is a monotonic timestamp on a clock of the caller's + /// choosing. The same clock must be used for every call. The + /// caller is expected to be the renderer, ticking once per frame + /// build and scheduling a wakeup for the returned delay. + pub fn animationTick(self: *ImageStorage, io: std.Io, now_ms: u64) ?u64 { + var min_delay: ?u64 = null; + + var it = self.images.iterator(); + while (it.next()) |entry| { + const img: *Image = entry.value_ptr; + + // The gates below mirror Kitty's image_is_animatable. + + // No animation state was ever attached (plain image). + const anim = img.animation orelse continue; + + // Stopped is the initial state of every animation: frames + // then only change client-driven (a=a c=N), never by time. + if (anim.state == .stopped) continue; + + // Only the root frame exists; there is nothing to advance + // to yet even in the running state. + if (anim.frames.items.len == 0) continue; + + // Unplaced images don't animate. This is our simpler + // approximation of Kitty's "is actually drawn" visibility + // gate; it is what stops an image that was transmitted but + // never placed from waking the renderer forever. + if (img.metadata.placement_count == 0) continue; + + // The base pixel data hasn't arrived yet (e.g. an image + // restored from a snapshot); nothing can be displayed. + if (img.data.isPending()) continue; + + // A zero total duration means every frame is gapless and + // no frame can ever be displayed, so the animation can + // never advance. This also guards the gapless-skip loop + // below from never terminating. + if (anim.durationMs() == 0) continue; + + // A finite loop budget (a=a v=N) that ran out on an + // earlier tick froze playback on the last frame for good. + if (anim.max_loops > 0 and anim.current_loop >= anim.max_loops) continue; + + const shown_at: u64 = shown_at: { + // First tick since playback started (or since the + // current frame changed through another path, e.g. + // a=a c=N): the frame is considered shown as of now, + // and its gap starts counting from here. + const at = anim.frame_shown_at_ms orelse break :shown_at now_ms; + + // A timestamp from the future means the caller's clock + // restarted; re-anchor rather than stalling until the + // old timestamp comes around again. + if (at > now_ms) break :shown_at now_ms; + + break :shown_at at; + }; + anim.frame_shown_at_ms = shown_at; + + // The current frame is replaced once its gap has elapsed. + // We advance at most one displayed frame per tick with no + // catch-up, exactly like Kitty: if ticks lag behind the + // gaps, the animation slows down rather than skipping. + var next_at: u64 = shown_at +| anim.gapAt(anim.current_index); + if (now_ms >= next_at) advance: { + // Walk forward to the next displayable frame. This is + // a loop only because gapless (gap=0) frames are never + // displayed and are stepped over; the durationMs gate + // above guarantees a displayable frame exists. + const count: u32 = anim.frameCount(); + var idx = anim.current_index; + while (true) { + const next = (idx + 1) % count; + if (next == 0) { + // Wrapping past the last frame back to the + // root. A loading-state (a=a s=2) animation + // refuses the wrap: it parks on the last frame + // awaiting more frames from the client. + if (anim.state == .loading) break :advance; + + // Each wrap completes a loop; a finite budget + // that just ran out parks on the last frame. + anim.current_loop += 1; + if (anim.max_loops > 0 and + anim.current_loop >= anim.max_loops) break :advance; + } + idx = next; + if (anim.gapAt(idx) != 0) break; + } + + // Show the chosen frame: restart its gap timer and + // stamp a fresh generation so consumers (the renderer + // texture cache, the C API) pick up the new pixels. + anim.current_index = idx; + anim.frame_shown_at_ms = now_ms; + self.markImageContentChanged(io, img); + next_at = now_ms +| anim.gapAt(idx); + } + + // Schedule the next tick. A parked animation left next_at + // in the past and so never schedules one; it is woken by + // its trigger instead (a new frame arriving, or an a=a + // command changing the state). + if (next_at > now_ms) { + const delay = next_at - now_ms; + min_delay = if (min_delay) |m| @min(m, delay) else delay; + } + } + + return min_delay; + } + /// Clear placements intersecting the active screen, then reclaim every /// image with no remaining placement. Unlike protocol d=A, a terminal /// clear also reclaims images that were already unplaced. @@ -1148,9 +1383,12 @@ pub const ImageStorage = struct { } }, - // We don't support animation frames yet so they are successfully - // deleted! - .animation_frames => {}, + .animation_frames => |v| self.deleteAnimationFrame( + io, + alloc, + t.screens.active, + v, + ), } // Deleting placements orphans any relative placements parented @@ -1239,12 +1477,97 @@ pub const ImageStorage = struct { if (delete_unused and matched) self.deleteIfUnused(alloc, image_id); } + /// Delete an animation frame (d=f/F). Deletes never produce + /// responses, so all failures are only logged. Kitty behaviors + /// implemented here: on an image without extra frames a lowercase + /// delete is a no-op while an uppercase delete removes the entire + /// image, placements included; the frame number is clamped to the + /// last frame and zero selects the root frame; deleting the root + /// frame promotes frame 2 to be the new root. + fn deleteAnimationFrame( + self: *ImageStorage, + io: std.Io, + alloc: Allocator, + s: *terminal.Screen, + v: command.Delete.Action.AnimationFrames, + ) void { + if (v.image_id == 0 and v.image_number == 0) { + log.warn("delete animation frames requires image id or number", .{}); + return; + } + const img = self.imagePtrByIdOrNumber( + v.image_id, + v.image_number, + ) orelse { + log.warn( + "delete animation frames for unknown image id={} number={}", + .{ v.image_id, v.image_number }, + ); + return; + }; + + const anim: *animation.Animation = anim: { + if (img.animation) |anim| { + if (anim.frames.items.len > 0) break :anim anim; + } + + // The image is not (or no longer) an animation. The + // uppercase delete removes the entire image, even when it + // still has placements. + if (!v.delete) return; + self.removePlacementsByImageId(s, img.id); + const entry = self.images.getEntry(img.id).?; + self.total_bytes -= entry.value_ptr.storageSize(); + entry.value_ptr.deinit(alloc); + self.images.removeByPtr(entry.key_ptr); + return; + }; + + // Clamp the frame number: zero selects the root frame and + // values past the end select the last frame. + const count: u32 = anim.frameCount(); + var number: u32 = @min(v.frame, count); + if (number == 0) number = 1; + + if (number == 1) { + // Deleting the root frame promotes frame 2 to root. The + // promoted frame's bytes stay reserved; only the old root + // data is freed. + self.releaseAnimationBytes(img.data.len()); + img.data.deinit(alloc); + const promoted = anim.frames.orderedRemove(0); + img.data = .{ .complete = promoted.data }; + anim.root_gap_ms = promoted.gap_ms; + } else { + const removed = anim.frames.orderedRemove(number - 2); + self.releaseAnimationBytes(removed.data.len); + alloc.free(removed.data); + } + + // Fix up the current frame. + const removed_idx: u32 = if (number == 1) 0 else number - 2; + const remaining: u32 = @intCast(anim.frames.items.len); + if (anim.current_index > remaining) { + anim.current_index = remaining; + anim.frame_shown_at_ms = null; + self.markImageContentChanged(io, img); + return; + } + if (removed_idx == anim.current_index) { + anim.frame_shown_at_ms = null; + self.markImageContentChanged(io, img); + } else { + if (removed_idx < anim.current_index) anim.current_index -= 1; + self.markMutated(io); + } + } + /// Delete an image if it is unused. fn deleteIfUnused(self: *ImageStorage, alloc: Allocator, image_id: u32) void { const entry = self.images.getEntry(image_id) orelse return; if (entry.value_ptr.metadata.placement_count > 0) return; - self.total_bytes -= entry.value_ptr.data.len(); + self.total_bytes -= entry.value_ptr.storageSize(); entry.value_ptr.deinit(alloc); self.images.removeByPtr(entry.key_ptr); } @@ -1364,7 +1687,7 @@ pub const ImageStorage = struct { } const entry = self.images.getEntry(c.id).?; - const image_len = entry.value_ptr.data.len(); + const image_len = entry.value_ptr.storageSize(); log.info("evicting image id={} bytes={}", .{ c.id, image_len }); evicted += image_len; @@ -4099,3 +4422,243 @@ test "storage: placeholderTarget lookup" { try testing.expectEqual(expected, s.placeholderTarget(1, 0).?.key); } } + +test "storage: animation tick advances and schedules" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + var t = try terminal.Terminal.init(io, alloc, .{ .cols = 10, .rows = 10 }); + defer t.deinit(alloc); + t.width_px = 100; + t.height_px = 100; + + var s: ImageStorage = .{}; + defer s.deinit(alloc, t.screens.active); + + // A running 1x1 RGBA image whose animation has one extra frame. + try s.addImage(io, alloc, t.screens.active, .{ + .id = 1, + .width = 1, + .height = 1, + .format = .rgba, + .data = .{ .complete = try alloc.dupe(u8, &.{ 255, 0, 0, 255 }) }, + }); + const img = s.images.getPtr(1).?; + const anim = try alloc.create(animation.Animation); + anim.* = .{ .state = .running }; + img.animation = anim; + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 0, 255, 255 }), + .gap_ms = 40, + }); + + // Without a placement the animation doesn't advance (our + // approximation of Kitty's visibility gate). + try testing.expect(s.animationTick(io, 0) == null); + try testing.expectEqual(@as(u32, 0), anim.current_index); + + try s.addPlacement(io, alloc, t.screens.active, 1, 0, .{ + .location = .{ .pin = try trackPin(&t, .{ .x = 0, .y = 0 }) }, + }); + const gen1 = img.generation; + s.dirty = false; + + // First tick: the gapless root frame is due immediately and is + // skipped over to frame 2, which is due again in its 40ms gap. + try testing.expectEqual(@as(?u64, 40), s.animationTick(io, 0)); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expect(img.generation > gen1); + try testing.expect(s.dirty); + + // Nothing due yet: no advance, and the delay counts down. + const gen2 = img.generation; + try testing.expectEqual(@as(?u64, 30), s.animationTick(io, 10)); + try testing.expectEqual(gen2, img.generation); + + // Wrapping is fine with an infinite loop budget: the gapless + // root is skipped and frame 2 is shown again. + try testing.expectEqual(@as(?u64, 40), s.animationTick(io, 40)); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expectEqual(@as(u32, 1), anim.current_loop); +} + +test "storage: animation tick loading state parks on last frame" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + var t = try terminal.Terminal.init(io, alloc, .{ .cols = 10, .rows = 10 }); + defer t.deinit(alloc); + t.width_px = 100; + t.height_px = 100; + + var s: ImageStorage = .{}; + defer s.deinit(alloc, t.screens.active); + + // A placed 1x1 RGBA image in the loading state (a=a s=2) whose + // animation has one extra frame. + try s.addImage(io, alloc, t.screens.active, .{ + .id = 1, + .width = 1, + .height = 1, + .format = .rgba, + .data = .{ .complete = try alloc.dupe(u8, &.{ 255, 0, 0, 255 }) }, + }); + const anim = try alloc.create(animation.Animation); + anim.* = .{ .state = .loading }; + s.images.getPtr(1).?.animation = anim; + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 0, 255, 255 }), + .gap_ms = 40, + }); + try s.addPlacement(io, alloc, t.screens.active, 1, 0, .{ + .location = .{ .pin = try trackPin(&t, .{ .x = 0, .y = 0 }) }, + }); + + // Reach the last frame, then park: no wakeup is scheduled while + // waiting for more frames and the loop counter stays untouched. + try testing.expectEqual(@as(?u64, 40), s.animationTick(io, 0)); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expect(s.animationTick(io, 100) == null); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expectEqual(@as(u32, 0), anim.current_loop); + + // A new frame arriving un-parks playback. + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 255, 0, 255 }), + .gap_ms = 25, + }); + try testing.expectEqual(@as(?u64, 25), s.animationTick(io, 150)); + try testing.expectEqual(@as(u32, 2), anim.current_index); +} + +test "storage: animation tick exhausts loop budget" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + var t = try terminal.Terminal.init(io, alloc, .{ .cols = 10, .rows = 10 }); + defer t.deinit(alloc); + t.width_px = 100; + t.height_px = 100; + + var s: ImageStorage = .{}; + defer s.deinit(alloc, t.screens.active); + + // A placed, running 1x1 RGBA image with a gapped root frame, one + // extra frame, and a one-loop budget (a=a v=2). + try s.addImage(io, alloc, t.screens.active, .{ + .id = 1, + .width = 1, + .height = 1, + .format = .rgba, + .data = .{ .complete = try alloc.dupe(u8, &.{ 255, 0, 0, 255 }) }, + }); + const anim = try alloc.create(animation.Animation); + anim.* = .{ + .state = .running, + .root_gap_ms = 10, + .max_loops = 1, + }; + s.images.getPtr(1).?.animation = anim; + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 0, 255, 255 }), + .gap_ms = 40, + }); + try s.addPlacement(io, alloc, t.screens.active, 1, 0, .{ + .location = .{ .pin = try trackPin(&t, .{ .x = 0, .y = 0 }) }, + }); + + // Root shows for 10ms, frame 2 for 40ms, then the wrap exhausts + // the budget and playback freezes on the last frame for good. + try testing.expectEqual(@as(?u64, 10), s.animationTick(io, 0)); + try testing.expectEqual(@as(u32, 0), anim.current_index); + try testing.expectEqual(@as(?u64, 40), s.animationTick(io, 10)); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expect(s.animationTick(io, 50) == null); + try testing.expectEqual(@as(u32, 1), anim.current_index); + try testing.expect(s.animationTick(io, 500) == null); +} + +test "storage: animation tick ignores ineligible animations" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + var t = try terminal.Terminal.init(io, alloc, .{ .cols = 10, .rows = 10 }); + defer t.deinit(alloc); + t.width_px = 100; + t.height_px = 100; + + var s: ImageStorage = .{}; + defer s.deinit(alloc, t.screens.active); + + // A placed 1x1 RGBA image with one extra frame, in the default + // stopped state. + try s.addImage(io, alloc, t.screens.active, .{ + .id = 1, + .width = 1, + .height = 1, + .format = .rgba, + .data = .{ .complete = try alloc.dupe(u8, &.{ 255, 0, 0, 255 }) }, + }); + const anim = try alloc.create(animation.Animation); + anim.* = .{}; + s.images.getPtr(1).?.animation = anim; + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 0, 255, 255 }), + .gap_ms = 40, + }); + try s.addPlacement(io, alloc, t.screens.active, 1, 0, .{ + .location = .{ .pin = try trackPin(&t, .{ .x = 0, .y = 0 }) }, + }); + + // Stopped (the default) never advances. + try testing.expect(s.animationTick(io, 0) == null); + + // An all-gapless animation can never advance either. + anim.state = .running; + anim.frames.items[0].gap_ms = 0; + try testing.expect(s.animationTick(io, 0) == null); + try testing.expectEqual(@as(u32, 0), anim.current_index); +} + +test "storage: animation tick re-anchors a restarted clock" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + var t = try terminal.Terminal.init(io, alloc, .{ .cols = 10, .rows = 10 }); + defer t.deinit(alloc); + t.width_px = 100; + t.height_px = 100; + + var s: ImageStorage = .{}; + defer s.deinit(alloc, t.screens.active); + + // A placed, running 1x1 RGBA image displaying its extra frame, + // with a shown-at timestamp far ahead of the tick clock. + try s.addImage(io, alloc, t.screens.active, .{ + .id = 1, + .width = 1, + .height = 1, + .format = .rgba, + .data = .{ .complete = try alloc.dupe(u8, &.{ 255, 0, 0, 255 }) }, + }); + const anim = try alloc.create(animation.Animation); + anim.* = .{ + .state = .running, + .current_index = 1, + .frame_shown_at_ms = 1000, + }; + s.images.getPtr(1).?.animation = anim; + try anim.frames.append(alloc, .{ + .data = try alloc.dupe(u8, &.{ 0, 0, 255, 255 }), + .gap_ms = 40, + }); + try s.addPlacement(io, alloc, t.screens.active, 1, 0, .{ + .location = .{ .pin = try trackPin(&t, .{ .x = 0, .y = 0 }) }, + }); + + // A timestamp in the future relative to now means the caller's + // clock restarted; the animation must not stall until the old + // timestamp comes around again. + try testing.expectEqual(@as(?u64, 40), s.animationTick(io, 5)); + try testing.expectEqual(@as(?u64, 5), anim.frame_shown_at_ms); +}