feat(ui): vim.ui.img api #37914

Problem:
No builtin api to load and display images.

Solution:
Introduce vim.ui.img. Only supports kitty graphics protocol, currently.

runtime/doc/lua.txt

@@ -5285,6 +5285,82 @@ vim.uri_to_fname({uri})                                   *vim.uri_to_fname()*
         (`string`) filename or unchanged URI for non-file URIs
 
 
+==============================================================================
+Lua module: vim.ui.img                                            *vim.ui.img*
+
+EXPERIMENTAL: This API may change in the future. Its semantics are not yet
+finalized.
+
+This provides a functional API for displaying images in Nvim. Currently
+supports PNG images via the Kitty graphics protocol.
+
+To override the image backend, replace `vim.ui.img` with your own
+implementation providing set/get/del.
+
+Examples: >lua
+    -- Load image bytes from disk and display at row 5, column 10
+    local id = vim.ui.img.set(
+     vim.fn.readblob('/path/to/img.png'),
+     { row = 5, col = 10, width = 40, height = 20, zindex = 50 }
+    )
+
+    -- Update the image position
+    vim.ui.img.set(id, { row = 8, col = 12 })
+
+    -- Retrieve the current image opts
+    local opts = vim.ui.img.get(id)
+
+    -- Remove the image
+    vim.ui.img.del(id)
+<
+
+
+vim.ui.img.del({id})                                        *vim.ui.img.del()*
+    Delete an image, removing it from display.
+
+    Parameters: ~
+      • {id}  (`integer`)
+
+    Return: ~
+        (`boolean`) found true if the image existed
+
+vim.ui.img.get({id})                                        *vim.ui.img.get()*
+    Get the opts for an image.
+
+    Parameters: ~
+      • {id}  (`integer`)
+
+    Return: ~
+        (`table?`) opts copy of image opts, or nil if not found
+        • {row}? (`integer`) starting row (1-indexed)
+        • {col}? (`integer`) starting column (1-indexed)
+        • {width}? (`integer`) width in cells
+        • {height}? (`integer`) height in cells
+        • {zindex}? (`integer`) stacking order (higher = on top)
+
+vim.ui.img.set({data_or_id}, {opts})                        *vim.ui.img.set()*
+    Display an image or update an existing one.
+
+    When {data_or_id} is a string, displays the image bytes at the position
+    given by {opts}. Returns an integer id for later use.
+
+    When {data_or_id} is an integer (a previously returned id), updates the
+    image with new {opts}.
+
+    Parameters: ~
+      • {data_or_id}  (`string|integer`) image bytes (string) or existing id
+                      (integer)
+      • {opts}        (`table?`) A table with the following fields:
+                      • {row}? (`integer`) starting row (1-indexed)
+                      • {col}? (`integer`) starting column (1-indexed)
+                      • {width}? (`integer`) width in cells
+                      • {height}? (`integer`) height in cells
+                      • {zindex}? (`integer`) stacking order (higher = on top)
+
+    Return: ~
+        (`integer`) id
+
+
 ==============================================================================
 Lua module: vim.version                                          *vim.version*
 

runtime/doc/news.txt

@@ -49,6 +49,7 @@ TREESITTER
 UI
 
 • todo
+• `vim.ui.img` experimental module added to display images within neovim.
 
 VIMSCRIPT
 

runtime/lua/vim/ui.lua

@@ -1,4 +1,6 @@
-local M = {}
+local M = vim._defer_require('vim.ui', {
+  img = ..., --- @module 'vim.ui.img'
+})
 
 ---@class vim.ui.select.Opts
 ---@inlinedoc

runtime/lua/vim/ui/img.lua

@@ -0,0 +1,127 @@
+local M = {}
+
+---@brief
+---
+---EXPERIMENTAL: This API may change in the future. Its semantics are not yet finalized.
+---
+---This provides a functional API for displaying images in Nvim.
+---Currently supports PNG images via the Kitty graphics protocol.
+---
+---To override the image backend, replace `vim.ui.img` with your own
+---implementation providing set/get/del.
+---
+---Examples:
+---
+---```lua
+----- Load image bytes from disk and display at row 5, column 10
+---local id = vim.ui.img.set(
+---  vim.fn.readblob('/path/to/img.png'),
+---  { row = 5, col = 10, width = 40, height = 20, zindex = 50 }
+---)
+---
+----- Update the image position
+---vim.ui.img.set(id, { row = 8, col = 12 })
+---
+----- Retrieve the current image opts
+---local opts = vim.ui.img.get(id)
+---
+----- Remove the image
+---vim.ui.img.del(id)
+---```
+
+---@class vim.ui.img.Opts
+---@inlinedoc
+---@field row? integer starting row (1-indexed)
+---@field col? integer starting column (1-indexed)
+---@field width? integer width in cells
+---@field height? integer height in cells
+---@field zindex? integer stacking order (higher = on top)
+
+--- Maps user-facing ID to internal tracking info.
+---@type table<integer, { img_id: integer, opts: vim.ui.img.Opts }>
+local state = {}
+
+---Display an image or update an existing one.
+---
+---When {data_or_id} is a string, displays the image bytes at the position
+---given by {opts}. Returns an integer id for later use.
+---
+---When {data_or_id} is an integer (a previously returned id), updates
+---the image with new {opts}.
+---
+---@param data_or_id string|integer image bytes (string) or existing id (integer)
+---@param opts? vim.ui.img.Opts
+---@return integer id
+function M.set(data_or_id, opts)
+  opts = opts or {}
+  vim.validate('data_or_id', data_or_id, { 'string', 'number' })
+  vim.validate('opts', opts, 'table')
+
+  local kitty = require('vim.ui.img._kitty')
+
+  -- If given a string, this should be the bytes of a new image to display
+  if type(data_or_id) == 'string' then
+    local img_id, placement_id = kitty.set(data_or_id, opts)
+    state[placement_id] = { img_id = img_id, opts = vim.deepcopy(opts) }
+    return placement_id
+  end
+
+  -- Otherwise, we update an existing image that is actively displayed
+  local id = data_or_id
+  local entry = state[id]
+  assert(entry, 'invalid image id: ' .. tostring(id))
+
+  -- We always want to have a full set of options when passing to kitty
+  local merged = vim.tbl_extend('force', entry.opts, opts)
+  kitty.update(entry.img_id, id, merged)
+  entry.opts = merged
+  return id
+end
+
+---Get the opts for an image.
+---
+---@param id integer
+---@return vim.ui.img.Opts? opts copy of image opts, or nil if not found
+function M.get(id)
+  vim.validate('id', id, 'number')
+
+  -- Grab a copy of the most recent opts used for the image
+  local entry = state[id]
+  if not entry then
+    return nil
+  end
+
+  return vim.deepcopy(entry.opts)
+end
+
+---Delete an image, removing it from display.
+---
+---@param id integer
+---@return boolean found true if the image existed
+function M.del(id)
+  vim.validate('id', id, 'number')
+
+  -- Skip performing the deletion if we don't have an active image with the id
+  local entry = state[id]
+  if not entry then
+    return false
+  end
+
+  local kitty = require('vim.ui.img._kitty')
+  kitty.delete(entry.img_id)
+  state[id] = nil
+  return true
+end
+
+vim.api.nvim_create_autocmd('VimLeavePre', {
+  callback = function()
+    ---@type integer[]
+    local ids = vim.tbl_keys(state)
+
+    for _, id in ipairs(ids) do
+      M.del(id)
+    end
+  end,
+})
+
+return M

runtime/lua/vim/ui/img/_kitty.lua

@@ -0,0 +1,150 @@
+---Kitty graphics protocol implementation for vim.ui.img.
+local M = {}
+
+local generate_id = (function()
+  local bit = require('bit')
+  local NVIM_PID_BITS = 10
+
+  local nvim_pid = 0
+  local cnt = 30
+
+  ---@return integer
+  return function()
+    if nvim_pid == 0 then
+      local pid = vim.fn.getpid()
+      nvim_pid = bit.band(bit.bxor(pid, bit.rshift(pid, 5), bit.rshift(pid, NVIM_PID_BITS)), 0x3FF)
+    end
+    cnt = cnt + 1
+    return bit.bor(bit.lshift(nvim_pid, 24 - NVIM_PID_BITS), cnt)
+  end
+end)()
+
+---Build a Kitty graphics protocol escape sequence.
+---@param control table<string, string|number>
+---@param payload? string
+---@return string
+local function seq(control, payload)
+  local parts = { '\027_G' }
+
+  local tmp = {}
+  for k, v in pairs(control) do
+    table.insert(tmp, k .. '=' .. v)
+  end
+  if #tmp > 0 then
+    table.insert(parts, table.concat(tmp, ','))
+  end
+
+  if payload and payload ~= '' then
+    table.insert(parts, ';')
+    table.insert(parts, payload)
+  end
+
+  table.insert(parts, '\027\\')
+  return table.concat(parts)
+end
+
+---Transmit image bytes to kitty in base64 chunks using direct transmission.
+---
+---Large images may cause the terminal to hang or the escape sequence to get
+---interrupted mid-write. A future filepath option (t=f) could let the
+---terminal read the file directly, avoiding this issue for local sessions.
+---@param id integer kitty image id
+---@param data string raw image bytes
+local function transmit(id, data)
+  local chunk_size = 4096
+  local base64_data = vim.base64.encode(data)
+  local pos = 1
+  local len = #base64_data
+
+  while pos <= len do
+    local end_pos = math.min(pos + chunk_size - 1, len)
+    local chunk = base64_data:sub(pos, end_pos)
+    local is_last = end_pos >= len
+
+    local control = {}
+
+    if pos == 1 then
+      control.f = '100' -- PNG format
+      control.a = 't' -- Transmit without displaying
+      control.t = 'd' -- Direct transmission
+      control.i = id
+      control.q = '2' -- Suppress responses
+    end
+
+    control.m = is_last and '0' or '1'
+
+    vim.api.nvim_ui_send(seq(control, chunk))
+    pos = end_pos + 1
+  end
+end
+
+---Send a kitty place/display command with cursor management.
+---@param img_id integer kitty image id
+---@param placement_id integer kitty placement id
+---@param opts vim.ui.img.Opts
+local function place(img_id, placement_id, opts)
+  local cursor_save = '\0277'
+  local cursor_hide = '\027[?25l'
+  local cursor_move = string.format('\027[%d;%dH', opts.row or 1, opts.col or 1)
+  local cursor_restore = '\0278'
+  local cursor_show = '\027[?25h'
+
+  ---@type table<string, string|number>
+  local control = {
+    a = 'p',
+    i = img_id,
+    p = placement_id,
+    C = '1', -- Don't move the cursor at all
+    q = '2', -- Suppress responses
+  }
+
+  if opts.width then
+    control.c = opts.width
+  end
+  if opts.height then
+    control.r = opts.height
+  end
+  if opts.zindex then
+    control.z = opts.zindex
+  end
+
+  vim.api.nvim_ui_send(
+    cursor_save .. cursor_hide .. cursor_move .. seq(control) .. cursor_restore .. cursor_show
+  )
+end
+
+---Transmit image bytes and place the image. Returns both IDs.
+---@param data string raw image bytes
+---@param opts vim.ui.img.Opts
+---@return integer img_id
+---@return integer placement_id
+function M.set(data, opts)
+  local img_id = generate_id()
+  local placement_id = generate_id()
+
+  transmit(img_id, data)
+  place(img_id, placement_id, opts)
+
+  return img_id, placement_id
+end
+
+---Update an existing placement (flicker-free, reuses same IDs).
+---@param img_id integer
+---@param placement_id integer
+---@param opts vim.ui.img.Opts
+function M.update(img_id, placement_id, opts)
+  place(img_id, placement_id, opts)
+end
+
+---Delete an image and all its placements from the terminal.
+---@param img_id integer
+function M.delete(img_id)
+  vim.api.nvim_ui_send(seq({
+    a = 'd',
+    d = 'i',
+    i = img_id,
+    q = '2', -- Suppress responses
+  }))
+end
+
+return M

runtime/lua/vim/ui/img/health.lua

@@ -0,0 +1,52 @@
+local M = {}
+local health = vim.health
+
+local function system(cmd)
+  local result = vim.system(cmd, { text = true }):wait()
+  if not result then -- Workaround https://github.com/neovim/neovim/issues/37922
+    return false, 'command failed'
+  end
+  return result.code == 0, vim.trim(('%s\n%s'):format(result.stdout, result.stderr))
+end
+
+local function get_tmux_option(option)
+  local cmd = { 'tmux', 'show-option', '-qvg', option } -- try global scope
+  local ok, out = system(cmd)
+  local val = vim.fn.substitute(out, [[\v(\s|\r|\n)]], '', 'g')
+  if not ok then
+    health.error(('command failed: %s\n%s'):format(vim.inspect(cmd), out))
+    return 'error'
+  elseif val == '' then
+    cmd = { 'tmux', 'show-option', '-qvgs', option } -- try session scope
+    ok, out = system(cmd)
+    val = vim.fn.substitute(out, [[\v(\s|\r|\n)]], '', 'g')
+    if not ok then
+      health.error(('command failed: %s\n%s'):format(vim.inspect(cmd), out))
+      return 'error'
+    end
+  end
+  return val
+end
+
+function M.check()
+  health.start('vim.ui.img')
+
+  if not vim.env.TMUX or vim.fn.executable('tmux') == 0 then
+    health.ok('no terminal multiplexer detected')
+    return
+  end
+
+  local passthrough = get_tmux_option('allow-passthrough')
+  if passthrough ~= 'error' then
+    if passthrough == 'on' or passthrough == 'all' then
+      health.ok('allow-passthrough: ' .. passthrough)
+    else
+      health.error(
+        '`allow-passthrough` is not enabled. Images will not be displayed.',
+        { 'Add to ~/.tmux.conf:\nset-option -g allow-passthrough on' }
+      )
+    end
+  end
+end
+
+return M

src/gen/gen_vimdoc.lua

@@ -201,6 +201,7 @@ local config = {
       'text.lua',
       'ui.lua',
       'uri.lua',
+      'img.lua', -- ui/img.lua
       'version.lua',
 
       -- Sections at the end, in a specific order:
@@ -235,6 +236,7 @@ local config = {
       'runtime/lua/vim/snippet.lua',
       'runtime/lua/vim/text.lua',
       'runtime/lua/vim/ui.lua',
+      'runtime/lua/vim/ui/img.lua',
       'runtime/lua/vim/uri.lua',
       'runtime/lua/vim/version.lua',
     },
@@ -255,6 +257,7 @@ local config = {
     end,
     section_name = {
       ['_inspector.lua'] = 'inspector',
+      ['img.lua'] = 'ui.img',
       ['ui2.lua'] = 'ui2',
     },
     section_fmt = function(name)

test/functional/ui/img_spec.lua

@@ -0,0 +1,273 @@
+local t = require('test.testutil')
+local n = require('test.functional.testnvim')()
+
+local eq = t.eq
+
+local clear = n.clear
+local exec_lua = n.exec_lua
+
+---4x4 PNG image bytes.
+---@type string
+-- stylua: ignore
+local PNG_IMG_BYTES = string.char(unpack({
+  137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 4, 0,
+  0, 0, 4, 8, 6, 0, 0, 0, 169, 241, 158, 126, 0, 0, 0, 1, 115, 82, 71, 66, 0,
+  174, 206, 28, 233, 0, 0, 0, 39, 73, 68, 65, 84, 8, 153, 99, 252, 207, 192,
+  240, 159, 129, 129, 129, 193, 226, 63, 3, 3, 3, 3, 3, 3, 19, 3, 26, 96, 97,
+  156, 1, 145, 250, 207, 184, 12, 187, 10, 0, 36, 189, 6, 125, 75, 9, 40, 46,
+  0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
+}))
+
+---@param s string
+---@return string
+local function escape_ansi(s)
+  return (
+    string.gsub(s, '.', function(c)
+      local byte = string.byte(c)
+      if byte < 32 or byte == 127 then
+        return string.format('\\%03d', byte)
+      else
+        return c
+      end
+    end)
+  )
+end
+
+---@param s string
+---@return string
+local function base64_encode(s)
+  return exec_lua(function()
+    return vim.base64.encode(s)
+  end)
+end
+
+---Mock nvim_ui_send to capture escape sequence output.
+local function setup_img_api()
+  exec_lua(function()
+    _G.data = {}
+    local original_ui_send = vim.api.nvim_ui_send
+    vim.api.nvim_ui_send = function(d)
+      table.insert(_G.data, d)
+    end
+    _G._original_ui_send = original_ui_send
+  end)
+end
+
+---@param esc string
+---@param opts? {strict?:boolean}
+---@return {i:integer, j:integer, control:table<string, string>, data:string|nil}
+local function parse_kitty_seq(esc, opts)
+  opts = opts or {}
+  local i, j, c, d = string.find(esc, '\027_G([^;\027]+)([^\027]*)\027\\')
+  assert(c, 'invalid kitty escape sequence: ' .. escape_ansi(esc))
+
+  if opts.strict then
+    assert(i == 1, 'not starting with kitty graphics sequence: ' .. escape_ansi(esc))
+  end
+
+  ---@type table<string, string>
+  local control = {}
+  local idx = 0
+  while true do
+    local k, v, _
+    idx, _, k, v = string.find(c, '(%a+)=([^,]+),?', idx + 1)
+    if idx == nil then
+      break
+    end
+    if k and v then
+      control[k] = v
+    end
+  end
+
+  ---@type string|nil
+  local payload
+  if d and d ~= '' then
+    payload = string.sub(d, 2)
+  end
+
+  return { i = i, j = j, control = control, data = payload }
+end
+
+describe('vim.ui.img', function()
+  before_each(function()
+    clear()
+    setup_img_api()
+  end)
+
+  it('can set an image', function()
+    local esc_codes = exec_lua(function()
+      _G.data = {}
+      vim.ui.img.set(PNG_IMG_BYTES, {
+        col = 1,
+        row = 2,
+        width = 3,
+        height = 4,
+        zindex = 123,
+      })
+      return table.concat(_G.data)
+    end)
+
+    -- Transmit image bytes
+    local seq = parse_kitty_seq(esc_codes, { strict = true })
+    local image_id = seq.control.i
+    eq({
+      f = '100',
+      a = 't',
+      t = 'd',
+      i = image_id,
+      q = '2',
+      m = '0',
+    }, seq.control, 'transmit image control data')
+    eq(base64_encode(PNG_IMG_BYTES), seq.data)
+    esc_codes = string.sub(esc_codes, seq.j + 1)
+
+    -- Cursor save
+    eq(escape_ansi('\0277'), escape_ansi(string.sub(esc_codes, 1, 2)), 'cursor save')
+    esc_codes = string.sub(esc_codes, 3)
+
+    -- Cursor hide
+    eq(escape_ansi('\027[?25l'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor hide')
+    esc_codes = string.sub(esc_codes, 7)
+
+    -- Cursor move
+    eq(escape_ansi('\027[2;1H'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor movement')
+    esc_codes = string.sub(esc_codes, 7)
+
+    -- Place image
+    seq = parse_kitty_seq(esc_codes, { strict = true })
+    eq({
+      a = 'p',
+      i = image_id,
+      p = seq.control.p,
+      C = '1',
+      q = '2',
+      c = '3',
+      r = '4',
+      z = '123',
+    }, seq.control, 'display image control data')
+    esc_codes = string.sub(esc_codes, seq.j + 1)
+
+    -- Cursor restore
+    eq(escape_ansi('\0278'), escape_ansi(string.sub(esc_codes, 1, 2)), 'cursor restore')
+    esc_codes = string.sub(esc_codes, 3)
+
+    -- Cursor show
+    eq(escape_ansi('\027[?25h'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor show')
+  end)
+
+  it('can get image info', function()
+    local result = exec_lua(function()
+      local id = vim.ui.img.set(PNG_IMG_BYTES, {
+        row = 5,
+        col = 10,
+        width = 20,
+        height = 15,
+        zindex = 42,
+      })
+
+      return {
+        info = vim.ui.img.get(id),
+        missing = vim.ui.img.get(999999),
+      }
+    end)
+
+    eq({ row = 5, col = 10, width = 20, height = 15, zindex = 42 }, result.info)
+    eq(nil, result.missing)
+  end)
+
+  it('can update an image', function()
+    local result = exec_lua(function()
+      local id = vim.ui.img.set(PNG_IMG_BYTES, {
+        row = 1,
+        col = 1,
+        width = 10,
+        height = 20,
+        zindex = 99,
+      })
+
+      _G.data = {}
+      vim.ui.img.set(id, {
+        col = 5,
+        row = 6,
+        width = 7,
+        height = 8,
+        zindex = 9,
+      })
+      local esc_codes = table.concat(_G.data)
+
+      -- Partial update: only change row, other fields preserved
+      vim.ui.img.set(id, { row = 50 })
+      local info = vim.ui.img.get(id)
+
+      return { esc_codes = esc_codes, info = info }
+    end)
+
+    -- Verify partial update merged opts
+    eq({ row = 50, col = 5, width = 7, height = 8, zindex = 9 }, result.info)
+
+    local esc_codes = result.esc_codes
+
+    -- Cursor save
+    eq(escape_ansi('\0277'), escape_ansi(string.sub(esc_codes, 1, 2)), 'cursor save')
+    esc_codes = string.sub(esc_codes, 3)
+
+    -- Cursor hide
+    eq(escape_ansi('\027[?25l'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor hide')
+    esc_codes = string.sub(esc_codes, 7)
+
+    -- Cursor move to new position
+    eq(escape_ansi('\027[6;5H'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor movement')
+    esc_codes = string.sub(esc_codes, 7)
+
+    -- Place command reuses same placement ID (flicker-free update)
+    local seq = parse_kitty_seq(esc_codes, { strict = true })
+    eq({
+      a = 'p',
+      i = seq.control.i,
+      p = seq.control.p,
+      C = '1',
+      q = '2',
+      c = '7',
+      r = '8',
+      z = '9',
+    }, seq.control, 'update image control data')
+    esc_codes = string.sub(esc_codes, seq.j + 1)
+
+    -- Cursor restore
+    eq(escape_ansi('\0278'), escape_ansi(string.sub(esc_codes, 1, 2)), 'cursor restore')
+    esc_codes = string.sub(esc_codes, 3)
+
+    -- Cursor show
+    eq(escape_ansi('\027[?25h'), escape_ansi(string.sub(esc_codes, 1, 6)), 'cursor show')
+  end)
+
+  it('can delete an image', function()
+    local result = exec_lua(function()
+      local id = vim.ui.img.set(PNG_IMG_BYTES, { row = 1, col = 1 })
+
+      _G.data = {}
+      local found = vim.ui.img.del(id)
+      local after = vim.ui.img.get(id)
+      local not_found = vim.ui.img.del(id)
+
+      return {
+        esc_codes = table.concat(_G.data),
+        found = found,
+        after = after,
+        not_found = not_found,
+      }
+    end)
+
+    local seq = parse_kitty_seq(result.esc_codes, { strict = true })
+    eq({
+      a = 'd',
+      d = 'i',
+      i = seq.control.i,
+      q = '2',
+    }, seq.control, 'delete image')
+
+    eq(true, result.found)
+    eq(nil, result.after)
+    eq(false, result.not_found)
+  end)
+end)