feat(ui): vim.ui.img api #37914

Problem:
No builtin api to load and display images.

Solution:
Introduce vim.ui.img. Only supports kitty graphics protocol, currently.

runtime/lua/vim/ui.lua

@@ -1,4 +1,6 @@
-local M = {}
+local M = vim._defer_require('vim.ui', {
+  img = ..., --- @module 'vim.ui.img'
+})
 
 ---@class vim.ui.select.Opts
 ---@inlinedoc

runtime/lua/vim/ui/img.lua

@@ -0,0 +1,127 @@
+local M = {}
+
+---@brief
+---
+---EXPERIMENTAL: This API may change in the future. Its semantics are not yet finalized.
+---
+---This provides a functional API for displaying images in Nvim.
+---Currently supports PNG images via the Kitty graphics protocol.
+---
+---To override the image backend, replace `vim.ui.img` with your own
+---implementation providing set/get/del.
+---
+---Examples:
+---
+---```lua
+----- Load image bytes from disk and display at row 5, column 10
+---local id = vim.ui.img.set(
+---  vim.fn.readblob('/path/to/img.png'),
+---  { row = 5, col = 10, width = 40, height = 20, zindex = 50 }
+---)
+---
+----- Update the image position
+---vim.ui.img.set(id, { row = 8, col = 12 })
+---
+----- Retrieve the current image opts
+---local opts = vim.ui.img.get(id)
+---
+----- Remove the image
+---vim.ui.img.del(id)
+---```
+
+---@class vim.ui.img.Opts
+---@inlinedoc
+---@field row? integer starting row (1-indexed)
+---@field col? integer starting column (1-indexed)
+---@field width? integer width in cells
+---@field height? integer height in cells
+---@field zindex? integer stacking order (higher = on top)
+
+--- Maps user-facing ID to internal tracking info.
+---@type table<integer, { img_id: integer, opts: vim.ui.img.Opts }>
+local state = {}
+
+---Display an image or update an existing one.
+---
+---When {data_or_id} is a string, displays the image bytes at the position
+---given by {opts}. Returns an integer id for later use.
+---
+---When {data_or_id} is an integer (a previously returned id), updates
+---the image with new {opts}.
+---
+---@param data_or_id string|integer image bytes (string) or existing id (integer)
+---@param opts? vim.ui.img.Opts
+---@return integer id
+function M.set(data_or_id, opts)
+  opts = opts or {}
+  vim.validate('data_or_id', data_or_id, { 'string', 'number' })
+  vim.validate('opts', opts, 'table')
+
+  local kitty = require('vim.ui.img._kitty')
+
+  -- If given a string, this should be the bytes of a new image to display
+  if type(data_or_id) == 'string' then
+    local img_id, placement_id = kitty.set(data_or_id, opts)
+    state[placement_id] = { img_id = img_id, opts = vim.deepcopy(opts) }
+    return placement_id
+  end
+
+  -- Otherwise, we update an existing image that is actively displayed
+  local id = data_or_id
+  local entry = state[id]
+  assert(entry, 'invalid image id: ' .. tostring(id))
+
+  -- We always want to have a full set of options when passing to kitty
+  local merged = vim.tbl_extend('force', entry.opts, opts)
+  kitty.update(entry.img_id, id, merged)
+  entry.opts = merged
+  return id
+end
+
+---Get the opts for an image.
+---
+---@param id integer
+---@return vim.ui.img.Opts? opts copy of image opts, or nil if not found
+function M.get(id)
+  vim.validate('id', id, 'number')
+
+  -- Grab a copy of the most recent opts used for the image
+  local entry = state[id]
+  if not entry then
+    return nil
+  end
+
+  return vim.deepcopy(entry.opts)
+end
+
+---Delete an image, removing it from display.
+---
+---@param id integer
+---@return boolean found true if the image existed
+function M.del(id)
+  vim.validate('id', id, 'number')
+
+  -- Skip performing the deletion if we don't have an active image with the id
+  local entry = state[id]
+  if not entry then
+    return false
+  end
+
+  local kitty = require('vim.ui.img._kitty')
+  kitty.delete(entry.img_id)
+  state[id] = nil
+  return true
+end
+
+vim.api.nvim_create_autocmd('VimLeavePre', {
+  callback = function()
+    ---@type integer[]
+    local ids = vim.tbl_keys(state)
+
+    for _, id in ipairs(ids) do
+      M.del(id)
+    end
+  end,
+})
+
+return M

runtime/lua/vim/ui/img/_kitty.lua

@@ -0,0 +1,150 @@
+---Kitty graphics protocol implementation for vim.ui.img.
+local M = {}
+
+local generate_id = (function()
+  local bit = require('bit')
+  local NVIM_PID_BITS = 10
+
+  local nvim_pid = 0
+  local cnt = 30
+
+  ---@return integer
+  return function()
+    if nvim_pid == 0 then
+      local pid = vim.fn.getpid()
+      nvim_pid = bit.band(bit.bxor(pid, bit.rshift(pid, 5), bit.rshift(pid, NVIM_PID_BITS)), 0x3FF)
+    end
+    cnt = cnt + 1
+    return bit.bor(bit.lshift(nvim_pid, 24 - NVIM_PID_BITS), cnt)
+  end
+end)()
+
+---Build a Kitty graphics protocol escape sequence.
+---@param control table<string, string|number>
+---@param payload? string
+---@return string
+local function seq(control, payload)
+  local parts = { '\027_G' }
+
+  local tmp = {}
+  for k, v in pairs(control) do
+    table.insert(tmp, k .. '=' .. v)
+  end
+  if #tmp > 0 then
+    table.insert(parts, table.concat(tmp, ','))
+  end
+
+  if payload and payload ~= '' then
+    table.insert(parts, ';')
+    table.insert(parts, payload)
+  end
+
+  table.insert(parts, '\027\\')
+  return table.concat(parts)
+end
+
+---Transmit image bytes to kitty in base64 chunks using direct transmission.
+---
+---Large images may cause the terminal to hang or the escape sequence to get
+---interrupted mid-write. A future filepath option (t=f) could let the
+---terminal read the file directly, avoiding this issue for local sessions.
+---@param id integer kitty image id
+---@param data string raw image bytes
+local function transmit(id, data)
+  local chunk_size = 4096
+  local base64_data = vim.base64.encode(data)
+  local pos = 1
+  local len = #base64_data
+
+  while pos <= len do
+    local end_pos = math.min(pos + chunk_size - 1, len)
+    local chunk = base64_data:sub(pos, end_pos)
+    local is_last = end_pos >= len
+
+    local control = {}
+
+    if pos == 1 then
+      control.f = '100' -- PNG format
+      control.a = 't' -- Transmit without displaying
+      control.t = 'd' -- Direct transmission
+      control.i = id
+      control.q = '2' -- Suppress responses
+    end
+
+    control.m = is_last and '0' or '1'
+
+    vim.api.nvim_ui_send(seq(control, chunk))
+    pos = end_pos + 1
+  end
+end
+
+---Send a kitty place/display command with cursor management.
+---@param img_id integer kitty image id
+---@param placement_id integer kitty placement id
+---@param opts vim.ui.img.Opts
+local function place(img_id, placement_id, opts)
+  local cursor_save = '\0277'
+  local cursor_hide = '\027[?25l'
+  local cursor_move = string.format('\027[%d;%dH', opts.row or 1, opts.col or 1)
+  local cursor_restore = '\0278'
+  local cursor_show = '\027[?25h'
+
+  ---@type table<string, string|number>
+  local control = {
+    a = 'p',
+    i = img_id,
+    p = placement_id,
+    C = '1', -- Don't move the cursor at all
+    q = '2', -- Suppress responses
+  }
+
+  if opts.width then
+    control.c = opts.width
+  end
+  if opts.height then
+    control.r = opts.height
+  end
+  if opts.zindex then
+    control.z = opts.zindex
+  end
+
+  vim.api.nvim_ui_send(
+    cursor_save .. cursor_hide .. cursor_move .. seq(control) .. cursor_restore .. cursor_show
+  )
+end
+
+---Transmit image bytes and place the image. Returns both IDs.
+---@param data string raw image bytes
+---@param opts vim.ui.img.Opts
+---@return integer img_id
+---@return integer placement_id
+function M.set(data, opts)
+  local img_id = generate_id()
+  local placement_id = generate_id()
+
+  transmit(img_id, data)
+  place(img_id, placement_id, opts)
+
+  return img_id, placement_id
+end
+
+---Update an existing placement (flicker-free, reuses same IDs).
+---@param img_id integer
+---@param placement_id integer
+---@param opts vim.ui.img.Opts
+function M.update(img_id, placement_id, opts)
+  place(img_id, placement_id, opts)
+end
+
+---Delete an image and all its placements from the terminal.
+---@param img_id integer
+function M.delete(img_id)
+  vim.api.nvim_ui_send(seq({
+    a = 'd',
+    d = 'i',
+    i = img_id,
+    q = '2', -- Suppress responses
+  }))
+end
+
+return M

runtime/lua/vim/ui/img/health.lua

@@ -0,0 +1,52 @@
+local M = {}
+local health = vim.health
+
+local function system(cmd)
+  local result = vim.system(cmd, { text = true }):wait()
+  if not result then -- Workaround https://github.com/neovim/neovim/issues/37922
+    return false, 'command failed'
+  end
+  return result.code == 0, vim.trim(('%s\n%s'):format(result.stdout, result.stderr))
+end
+
+local function get_tmux_option(option)
+  local cmd = { 'tmux', 'show-option', '-qvg', option } -- try global scope
+  local ok, out = system(cmd)
+  local val = vim.fn.substitute(out, [[\v(\s|\r|\n)]], '', 'g')
+  if not ok then
+    health.error(('command failed: %s\n%s'):format(vim.inspect(cmd), out))
+    return 'error'
+  elseif val == '' then
+    cmd = { 'tmux', 'show-option', '-qvgs', option } -- try session scope
+    ok, out = system(cmd)
+    val = vim.fn.substitute(out, [[\v(\s|\r|\n)]], '', 'g')
+    if not ok then
+      health.error(('command failed: %s\n%s'):format(vim.inspect(cmd), out))
+      return 'error'
+    end
+  end
+  return val
+end
+
+function M.check()
+  health.start('vim.ui.img')
+
+  if not vim.env.TMUX or vim.fn.executable('tmux') == 0 then
+    health.ok('no terminal multiplexer detected')
+    return
+  end
+
+  local passthrough = get_tmux_option('allow-passthrough')
+  if passthrough ~= 'error' then
+    if passthrough == 'on' or passthrough == 'all' then
+      health.ok('allow-passthrough: ' .. passthrough)
+    else
+      health.error(
+        '`allow-passthrough` is not enabled. Images will not be displayed.',
+        { 'Add to ~/.tmux.conf:\nset-option -g allow-passthrough on' }
+      )
+    end
+  end
+end
+
+return M