feat(pack)!: synchronize lockfile with installed plugins when reading it

Problem: Lockfile can become out of sync with what is actually installed
  on disk when user performs (somewhat reasonable) manual actions like:
    - Delete lockfile and expect it to regenerate.
    - Delete plugin directory without `vim.pack.del()`.
    - Manually edit lock data in a bad way.

Solution: Synchronize lockfile data with installed plugins on every
  lockfile read. In particular:

    1. Install immediately all missing plugins with valid lock data.
       This helps with "manually delete plugin directory" case by
       prompting user to figure out how to properly delete a plugin.

    2. Repair lock data for properly installed plugins.
       This helps with "manually deleted lockfile", "manually edited
       lockfile in an unexpected way", "installation terminated due to
       timeout" cases.

    3. Remove unrepairable corrupted lock data and their plugins. This
       includes bad lock data for missing plugins and any lock data
       for corrupted plugins (right now this only means that plugin
       path is not a directory, but can be built upon).

  Step 1 also improves usability in case there are lazy loaded plugins
  that are rarely loaded (like on `FileType` event, for example):
    - Previously starting with config+lockfile on a new machine only
      installs rare `vim.pack.add()` plugin after it is called (while
      an entry in lockfile would still be present). This could be
      problematic if there is no Internet connection, for example.
    - Now all plugins from the lockfile are installed before actually
      executing the first `vim.pack.add()` call in 'init.lua'. And later
      they are only loaded on a rare `vim.pack.add()` call.

  ---

  Synchronizing lockfile on its every read makes it work more robustly
  if other `vim.pack` functions are called without any `vim.pack.add()`.

  ---

  Performance for a regular startup (good lockfile, everything is
  installed) is not affected and usually even increased. The bottleneck
  in this area is figuring out which plugins need to be installed.

  Previously the check was done by `vim.uv.fs_stat()` for every plugin
  in `vim.pack.add()`. Now it is replaced with a single `vim.fs.dir()`
  traversal during lockfile sync while later using lockfile data to
  figure out if plugin needs to be installed.

  The single `vim.fs.dir` approach scales better than `vim.uv.fs_stat`,
  but might be less performant if there are many plugins that will be
  not loaded via `vim.pack.add()` during startup.

  Rough estimate of how long the same steps (read lockfile and normalize
  plugin array) take with a single `vim.pack.add()` filled with 43
  plugins benchmarking:
  - Before commit: ~700 ms
  - After commit:  ~550 ms
This commit is contained in:
Evgeni Chasnovski
2025-11-13 15:45:56 +02:00
parent 60bfc741ed
commit b151aa761f
3 changed files with 375 additions and 32 deletions

View File

@@ -224,9 +224,11 @@ semver convention `v<major>.<minor>.<patch>`.
The latest state of all managed plugins is stored inside a *vim.pack-lockfile*
located at `$XDG_CONFIG_HOME/nvim/nvim-pack-lock.json`. It is a JSON file that
is used to persistently track data about plugins. For a more robust config
treat lockfile like its part: put under version control, etc. In this case
initial install prefers revision from the lockfile instead of inferring from
`version`. Should not be edited by hand or deleted.
treat lockfile like its part: put under version control, etc. In this case all
plugins from the lockfile will be installed at once and at lockfile's revision
(instead of inferring from `version`). Should not be edited by hand. Corrupted
data for installed plugins is repaired (including after deleting whole file),
but `version` fields will be missing for not yet added plugins.
Example workflows ~

View File

@@ -18,8 +18,11 @@
---located at `$XDG_CONFIG_HOME/nvim/nvim-pack-lock.json`. It is a JSON file that
---is used to persistently track data about plugins.
---For a more robust config treat lockfile like its part: put under version control, etc.
---In this case initial install prefers revision from the lockfile instead of
---inferring from `version`. Should not be edited by hand or deleted.
---In this case all plugins from the lockfile will be installed at once and
---at lockfile's revision (instead of inferring from `version`).
---Should not be edited by hand. Corrupted data for installed plugins is repaired
---(including after deleting whole file), but `version` fields will be missing
---for not yet added plugins.
---
---Example workflows ~
---
@@ -357,7 +360,7 @@ end
local function new_plug(spec, plug_dir)
local spec_resolved = normalize_spec(spec)
local path = vim.fs.joinpath(plug_dir or get_plug_dir(), spec_resolved.name)
local info = { err = '', installed = uv.fs_stat(path) ~= nil }
local info = { err = '', installed = plugin_lock.plugins[spec_resolved.name] ~= nil }
return { spec = spec_resolved, path = path, info = info }
end
@@ -652,7 +655,7 @@ local function install_list(plug_list, confirm)
-- Ensure that not fully installed plugins are absent on disk and in lockfile
for _, p in ipairs(plug_list) do
if not p.info.installed then
if not (p.info.installed and uv.fs_stat(p.path) ~= nil) then
plugin_lock.plugins[p.spec.name] = nil
vim.fs.rm(p.path, { recursive = true, force = true })
end
@@ -760,27 +763,112 @@ local function lock_write()
assert(uv.fs_close(fd))
end
local function lock_read()
--- @param names string[]
local function lock_repair(names, plug_dir)
--- @async
local function f()
for _, name in ipairs(names) do
local path = vim.fs.joinpath(plug_dir, name)
-- Try reusing existing table to preserve maybe present `version`
local data = plugin_lock.plugins[name] or {}
data.rev = git_get_hash('HEAD', path)
data.src = git_cmd({ 'remote', 'get-url', 'origin' }, path)
plugin_lock.plugins[name] = data
end
end
async.run(f):wait()
end
--- Sync lockfile data and installed plugins:
--- - Install plugins that have proper lockfile data but are not on disk.
--- - Repair corrupted lock data for installed plugins.
--- - Remove unrepairable corrupted lock data and plugins.
local function lock_sync(confirm)
if type(plugin_lock.plugins) ~= 'table' then
plugin_lock.plugins = {}
end
-- Compute installed plugins
-- NOTE: The directory traversal is done on every startup, but it is very fast.
-- Also, single `vim.fs.dir()` scales better than on demand `uv.fs_stat()` checks.
local plug_dir = get_plug_dir()
local installed = {} --- @type table<string,string>
for name, fs_type in vim.fs.dir(plug_dir) do
installed[name] = fs_type
plugin_lock.plugins[name] = plugin_lock.plugins[name] or {}
end
-- Traverse once optimizing for "regular startup" (no repair, no install)
local to_install = {} --- @type vim.pack.Plug[]
local to_repair = {} --- @type string[]
local to_remove = {} --- @type string[]
for name, data in pairs(plugin_lock.plugins) do
if type(data) ~= 'table' then
data = {} ---@diagnostic disable-line: missing-fields
plugin_lock.plugins[name] = data
end
-- Deserialize `version`
local version = data.version
if type(version) == 'string' then
data.version = version:match("^'(.+)'$") or vim.version.range(version)
end
-- Synchronize
local is_bad_lock = type(data.rev) ~= 'string' or type(data.src) ~= 'string'
local is_bad_plugin = installed[name] and installed[name] ~= 'directory'
if is_bad_lock or is_bad_plugin then
local t = installed[name] == 'directory' and to_repair or to_remove
t[#t + 1] = name
elseif not installed[name] then
local spec = { src = data.src, name = name, version = data.version }
to_install[#to_install + 1] = new_plug(spec, plug_dir)
end
end
-- Perform actions if needed
if #to_install > 0 then
table.sort(to_install, function(a, b)
return a.spec.name < b.spec.name
end)
install_list(to_install, confirm)
lock_write()
end
if #to_repair > 0 then
lock_repair(to_repair, plug_dir)
table.sort(to_repair)
notify('Repaired corrupted lock data for plugins: ' .. table.concat(to_repair, ', '), 'WARN')
lock_write()
end
if #to_remove > 0 then
for _, name in ipairs(to_remove) do
plugin_lock.plugins[name] = nil
vim.fs.rm(vim.fs.joinpath(plug_dir, name), { recursive = true, force = true })
end
table.sort(to_remove)
notify('Removed corrupted lock data for plugins: ' .. table.concat(to_remove, ', '), 'WARN')
lock_write()
end
end
local function lock_read(confirm)
if plugin_lock then
return
end
local fd = uv.fs_open(lock_get_path(), 'r', 438)
if not fd then
plugin_lock = { plugins = {} }
return
end
local stat = assert(uv.fs_fstat(fd))
local data = assert(uv.fs_read(fd, stat.size, 0))
assert(uv.fs_close(fd))
plugin_lock = vim.json.decode(data) --- @type vim.pack.Lock
-- Deserialize `version`
for _, l_data in pairs(plugin_lock.plugins) do
local version = l_data.version
if type(version) == 'string' then
l_data.version = version:match("^'(.+)'$") or vim.version.range(version)
end
local fd = uv.fs_open(lock_get_path(), 'r', 438)
if fd then
local stat = assert(uv.fs_fstat(fd))
local data = assert(uv.fs_read(fd, stat.size, 0))
assert(uv.fs_close(fd))
plugin_lock = vim.json.decode(data)
else
plugin_lock = { plugins = {} }
end
lock_sync(vim.F.if_nil(confirm, true))
end
--- @class vim.pack.keyset.add
@@ -821,6 +909,8 @@ function M.add(specs, opts)
opts = vim.tbl_extend('force', { load = vim.v.vim_did_init == 1, confirm = true }, opts or {})
vim.validate('opts', opts, 'table')
lock_read(opts.confirm)
local plug_dir = get_plug_dir()
local plugs = {} --- @type vim.pack.Plug[]
for i = 1, #specs do
@@ -829,7 +919,6 @@ function M.add(specs, opts)
plugs = normalize_plugs(plugs)
-- Pre-process
lock_read()
local plugs_to_install = {} --- @type vim.pack.Plug[]
local needs_lock_write = false
for _, p in ipairs(plugs) do

View File

@@ -327,6 +327,7 @@ local function get_lock_path()
return vim.fs.joinpath(fn.stdpath('config'), 'nvim-pack-lock.json')
end
--- @return {plugins:table<string, {rev:string, src:string, version?:string}>}
local function get_lock_tbl()
return vim.json.decode(fn.readblob(get_lock_path()))
end
@@ -393,8 +394,9 @@ describe('vim.pack', function()
exec_lua(function()
vim.pack.add({ repos_src.basic, { src = repos_src.defbranch, name = 'other-name' } })
end)
eq(false, exec_lua('return pcall(require, "basic")'))
eq(false, exec_lua('return pcall(require, "defbranch")'))
eq(false, pack_exists('basic'))
eq(false, pack_exists('defbranch'))
eq({ plugins = {} }, get_lock_tbl())
local confirm_msg_lines = ([[
These plugins will be installed:
@@ -404,6 +406,29 @@ describe('vim.pack', function()
local confirm_msg = vim.trim(vim.text.indent(0, confirm_msg_lines))
local ref_log = { { confirm_msg .. '\n', 'Proceed? &Yes\n&No\n&Always', 1, 'Question' } }
eq(ref_log, exec_lua('return _G.confirm_log'))
-- Should remove lock data if not confirmed during lockfile sync
n.clear()
exec_lua(function()
vim.pack.add({ repos_src.basic })
end)
eq(true, pack_exists('basic'))
eq('table', type(get_lock_tbl().plugins.basic))
vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
n.clear()
mock_confirm(2)
exec_lua(function()
vim.pack.add({ repos_src.basic })
end)
eq(false, pack_exists('basic'))
eq({ plugins = {} }, get_lock_tbl())
-- Should ask for confirm twice: during lockfile sync and inside
-- `vim.pack.add()` (i.e. not confirming during lockfile sync has
-- an immediate effect on whether a plugin is installed or not)
eq(2, exec_lua('return #_G.confirm_log'))
end)
it('respects `opts.confirm`', function()
@@ -413,7 +438,20 @@ describe('vim.pack', function()
end)
eq(0, exec_lua('return #_G.confirm_log'))
eq('basic main', exec_lua('return require("basic")'))
eq(true, pack_exists('basic'))
-- Should also respect `confirm` when installing during lockfile sync
vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
eq('table', type(get_lock_tbl().plugins.basic))
n.clear()
mock_confirm(1)
exec_lua(function()
vim.pack.add({}, { confirm = false })
end)
eq(0, exec_lua('return #_G.confirm_log'))
eq(true, pack_exists('basic'))
end)
it('can always confirm in current session', function()
@@ -526,24 +564,29 @@ describe('vim.pack', function()
eq(ref_lockfile, get_lock_tbl())
end)
it('uses lockfile revision during install', function()
it('uses lockfile during install', function()
exec_lua(function()
vim.pack.add({ { src = repos_src.basic, version = 'feat-branch' } })
vim.pack.add({
{ src = repos_src.basic, version = 'feat-branch' },
repos_src.defbranch,
})
end)
-- Mock clean initial install, but with lockfile present
vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
n.clear()
local basic_plug_path = vim.fs.joinpath(pack_get_dir(), 'basic')
vim.fs.rm(basic_plug_path, { force = true, recursive = true })
local basic_rev = git_get_hash('feat-branch', 'basic')
local defbranch_rev = git_get_hash('HEAD', 'defbranch')
local ref_lockfile = {
plugins = {
basic = { rev = basic_rev, src = repos_src.basic, version = "'feat-branch'" },
defbranch = { rev = defbranch_rev, src = repos_src.defbranch },
},
}
eq(ref_lockfile, get_lock_tbl())
mock_confirm(1)
exec_lua(function()
-- Should use revision from lockfile (pointing at latest 'feat-branch'
-- commit) and not use latest `main` commit
@@ -552,9 +595,17 @@ describe('vim.pack', function()
local basic_lua_file = vim.fs.joinpath(pack_get_plug_path('basic'), 'lua', 'basic.lua')
eq('return "basic feat-branch"', fn.readblob(basic_lua_file))
local confirm_log = exec_lua('return _G.confirm_log')
eq(1, #confirm_log)
matches('basic.*defbranch', confirm_log[1][1])
-- Should install `defbranch` (as it is in lockfile), but not load it
eq(true, pack_exists('defbranch'))
eq(false, exec_lua('return pcall(require, "defbranch")'))
-- Running `update()` should still update to use `main`
exec_lua(function()
vim.pack.update(nil, { force = true })
vim.pack.update({ 'basic' }, { force = true })
end)
eq('return "basic main"', fn.readblob(basic_lua_file))
@@ -585,6 +636,133 @@ describe('vim.pack', function()
eq(ref_lockfile, get_lock_tbl())
end)
it('regenerates manually deleted lockfile', function()
exec_lua(function()
vim.pack.add({
{ src = repos_src.basic, name = 'other', version = 'feat-branch' },
repos_src.defbranch,
})
end)
local lock_path = get_lock_path()
eq(true, vim.uv.fs_stat(lock_path) ~= nil)
local basic_rev = git_get_hash('feat-branch', 'basic')
local plugindirs_rev = git_get_hash('dev', 'defbranch')
-- Should try its best to regenerate lockfile based on installed plugins
fn.delete(get_lock_path())
n.clear()
exec_lua(function()
vim.pack.add({})
end)
local ref_lockfile = {
plugins = {
-- No `version = 'feat-branch'` as there is no way to get that info
-- (lockfile was the only source of that on disk)
other = { rev = basic_rev, src = repos_src.basic },
defbranch = { rev = plugindirs_rev, src = repos_src.defbranch },
},
}
eq(ref_lockfile, get_lock_tbl())
local ref_messages = 'vim.pack: Repaired corrupted lock data for plugins: defbranch, other'
eq(ref_messages, n.exec_capture('messages'))
-- Calling `add()` with `version` should still add it to lockfile
exec_lua(function()
vim.pack.add({ { src = repos_src.basic, name = 'other', version = 'feat-branch' } })
end)
eq("'feat-branch'", get_lock_tbl().plugins.other.version)
end)
it('repairs corrupted lock data for installed plugins', function()
exec_lua(function()
vim.pack.add({
-- Should preserve present `version`
{ src = repos_src.basic, version = 'feat-branch' },
repos_src.defbranch,
repos_src.semver,
repos_src.helptags,
})
end)
local lock_tbl = get_lock_tbl()
local ref_lock_tbl = vim.deepcopy(lock_tbl)
local assert = function()
exec_lua('vim.pack.add({})')
eq(ref_lock_tbl, get_lock_tbl())
eq(true, pack_exists('basic'))
eq(true, pack_exists('defbranch'))
eq(true, pack_exists('semver'))
eq(true, pack_exists('helptags'))
end
-- Missing lock data required field
lock_tbl.plugins.basic.rev = nil
-- Wrong lock data field type
lock_tbl.plugins.defbranch.src = 1 ---@diagnostic disable-line: assign-type-mismatch
-- Wrong lock data type
lock_tbl.plugins.semver = 1 ---@diagnostic disable-line: assign-type-mismatch
local lockfile_text = vim.json.encode(lock_tbl, { indent = ' ', sort_keys = true })
fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
n.clear()
assert()
local ref_messages =
'vim.pack: Repaired corrupted lock data for plugins: basic, defbranch, semver'
eq(ref_messages, n.exec_capture('messages'))
-- Should work even for badly corrupted lockfile
lockfile_text = vim.json.encode({ plugins = 1 }, { indent = ' ', sort_keys = true })
fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
n.clear()
-- Can not preserve `version` if it was deleted from the lockfile
ref_lock_tbl.plugins.basic.version = nil
assert()
end)
it('removes unrepairable corrupted data and plugins', function()
exec_lua(function()
vim.pack.add({ repos_src.basic, repos_src.defbranch, repos_src.semver, repos_src.helptags })
end)
local lock_tbl = get_lock_tbl()
local ref_lock_tbl = vim.deepcopy(lock_tbl)
-- Corrupted data for missing plugin
vim.fs.rm(pack_get_plug_path('basic'), { recursive = true, force = true })
lock_tbl.plugins.basic.rev = nil
-- Good data for corrupted plugin
local defbranch_path = pack_get_plug_path('defbranch')
vim.fs.rm(defbranch_path, { recursive = true, force = true })
fn.writefile({ 'File and not directory' }, defbranch_path)
-- Corrupted data for corrupted plugin
local semver_path = pack_get_plug_path('semver')
vim.fs.rm(semver_path, { recursive = true, force = true })
fn.writefile({ 'File and not directory' }, semver_path)
lock_tbl.plugins.semver.rev = 1 ---@diagnostic disable-line: assign-type-mismatch
local lockfile_text = vim.json.encode(lock_tbl, { indent = ' ', sort_keys = true })
fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
n.clear()
exec_lua('vim.pack.add({})')
ref_lock_tbl.plugins.basic = nil
ref_lock_tbl.plugins.defbranch = nil
ref_lock_tbl.plugins.semver = nil
eq(ref_lock_tbl, get_lock_tbl())
eq(false, pack_exists('basic'))
eq(false, pack_exists('defbranch'))
eq(false, pack_exists('semver'))
eq(true, pack_exists('helptags'))
end)
it('installs at proper version', function()
local out = exec_lua(function()
vim.pack.add({
@@ -1625,6 +1803,32 @@ describe('vim.pack', function()
eq(ref_environ, fn.environ())
end)
it('works with out of sync lockfile', function()
-- Should first autoinstall missing plugin (with confirmation)
vim.fs.rm(pack_get_plug_path('fetch'), { force = true, recursive = true })
n.clear()
mock_confirm(1)
exec_lua(function()
vim.pack.update(nil, { force = true })
end)
eq(1, exec_lua('return #_G.confirm_log'))
-- - Should checkout `version='main'` as it says in the lockfile
eq('return "fetch new 2"', fn.readblob(fetch_lua_file))
-- Should regenerate absent lockfile (from present plugins)
vim.fs.rm(get_lock_path())
n.clear()
exec_lua(function()
vim.pack.update(nil, { force = true })
end)
local lock_plugins = get_lock_tbl().plugins
eq(3, vim.tbl_count(lock_plugins))
-- - Should checkout default branch since `version='main'` info is lost
-- after lockfile is deleted.
eq(nil, lock_plugins.fetch.version)
eq('return "fetch dev"', fn.readblob(fetch_lua_file))
end)
it('validates input', function()
local function assert(err_pat, input)
local function update_input()
@@ -1774,6 +1978,29 @@ describe('vim.pack', function()
local basic_data = make_basic_data(true, true)
eq({ { defbranch_data, basic_data }, { basic_data } }, exec_lua('return _G.get_log'))
end)
it('works with out of sync lockfile', function()
exec_lua(function()
vim.pack.add({ repos_src.basic, repos_src.defbranch })
end)
eq(2, vim.tbl_count(get_lock_tbl().plugins))
local basic_lua_file = vim.fs.joinpath(pack_get_plug_path('basic'), 'lua', 'basic.lua')
-- Should first autoinstall missing plugin (with confirmation)
vim.fs.rm(pack_get_plug_path('basic'), { force = true, recursive = true })
n.clear()
mock_confirm(1)
eq(2, exec_lua('return #vim.pack.get()'))
eq(1, exec_lua('return #_G.confirm_log'))
eq('return "basic main"', fn.readblob(basic_lua_file))
-- Should regenerate absent lockfile (from present plugins)
vim.fs.rm(get_lock_path())
n.clear()
eq(2, exec_lua('return #vim.pack.get()'))
eq(2, vim.tbl_count(get_lock_tbl().plugins))
end)
end)
describe('del()', function()
@@ -1829,6 +2056,31 @@ describe('vim.pack', function()
eq({ plugins = {} }, get_lock_tbl())
end)
it('works with out of sync lockfile', function()
exec_lua(function()
vim.pack.add({ repos_src.basic, repos_src.defbranch, repos_src.plugindirs })
end)
eq(3, vim.tbl_count(get_lock_tbl().plugins))
-- Should first autoinstall missing plugin (with confirmation)
vim.fs.rm(pack_get_plug_path('basic'), { force = true, recursive = true })
n.clear()
mock_confirm(1)
exec_lua('vim.pack.del({ "defbranch" })')
eq(1, exec_lua('return #_G.confirm_log'))
eq(true, pack_exists('basic'))
eq(false, pack_exists('defbranch'))
eq(true, pack_exists('plugindirs'))
-- Should regenerate absent lockfile (from present plugins)
vim.fs.rm(get_lock_path())
n.clear()
exec_lua('vim.pack.del({ "basic" })')
eq(1, exec_lua('return #vim.pack.get()'))
eq({ 'plugindirs' }, vim.tbl_keys(get_lock_tbl().plugins))
end)
it('validates input', function()
local function assert(err_pat, input)
local function del_input()