feat(pack)!: synchronize lockfile with installed plugins when reading it

Problem: Lockfile can become out of sync with what is actually installed
  on disk when user performs (somewhat reasonable) manual actions like:
    - Delete lockfile and expect it to regenerate.
    - Delete plugin directory without `vim.pack.del()`.
    - Manually edit lock data in a bad way.

Solution: Synchronize lockfile data with installed plugins on every
  lockfile read. In particular:

    1. Install immediately all missing plugins with valid lock data.
       This helps with "manually delete plugin directory" case by
       prompting user to figure out how to properly delete a plugin.

    2. Repair lock data for properly installed plugins.
       This helps with "manually deleted lockfile", "manually edited
       lockfile in an unexpected way", "installation terminated due to
       timeout" cases.

    3. Remove unrepairable corrupted lock data and their plugins. This
       includes bad lock data for missing plugins and any lock data
       for corrupted plugins (right now this only means that plugin
       path is not a directory, but can be built upon).

  Step 1 also improves usability in case there are lazy loaded plugins
  that are rarely loaded (like on `FileType` event, for example):
    - Previously starting with config+lockfile on a new machine only
      installs rare `vim.pack.add()` plugin after it is called (while
      an entry in lockfile would still be present). This could be
      problematic if there is no Internet connection, for example.
    - Now all plugins from the lockfile are installed before actually
      executing the first `vim.pack.add()` call in 'init.lua'. And later
      they are only loaded on a rare `vim.pack.add()` call.

  ---

  Synchronizing lockfile on its every read makes it work more robustly
  if other `vim.pack` functions are called without any `vim.pack.add()`.

  ---

  Performance for a regular startup (good lockfile, everything is
  installed) is not affected and usually even increased. The bottleneck
  in this area is figuring out which plugins need to be installed.

  Previously the check was done by `vim.uv.fs_stat()` for every plugin
  in `vim.pack.add()`. Now it is replaced with a single `vim.fs.dir()`
  traversal during lockfile sync while later using lockfile data to
  figure out if plugin needs to be installed.

  The single `vim.fs.dir` approach scales better than `vim.uv.fs_stat`,
  but might be less performant if there are many plugins that will be
  not loaded via `vim.pack.add()` during startup.

  Rough estimate of how long the same steps (read lockfile and normalize
  plugin array) take with a single `vim.pack.add()` filled with 43
  plugins benchmarking:
  - Before commit: ~700 ms
  - After commit:  ~550 ms

test/functional/plugin/pack_spec.lua

@@ -327,6 +327,7 @@ local function get_lock_path()
   return vim.fs.joinpath(fn.stdpath('config'), 'nvim-pack-lock.json')
 end
 
+--- @return {plugins:table<string, {rev:string, src:string, version?:string}>}
 local function get_lock_tbl()
   return vim.json.decode(fn.readblob(get_lock_path()))
 end
@@ -393,8 +394,9 @@ describe('vim.pack', function()
       exec_lua(function()
         vim.pack.add({ repos_src.basic, { src = repos_src.defbranch, name = 'other-name' } })
       end)
-      eq(false, exec_lua('return pcall(require, "basic")'))
-      eq(false, exec_lua('return pcall(require, "defbranch")'))
+      eq(false, pack_exists('basic'))
+      eq(false, pack_exists('defbranch'))
+      eq({ plugins = {} }, get_lock_tbl())
 
       local confirm_msg_lines = ([[
         These plugins will be installed:
@@ -404,6 +406,29 @@ describe('vim.pack', function()
       local confirm_msg = vim.trim(vim.text.indent(0, confirm_msg_lines))
       local ref_log = { { confirm_msg .. '\n', 'Proceed? &Yes\n&No\n&Always', 1, 'Question' } }
       eq(ref_log, exec_lua('return _G.confirm_log'))
+
+      -- Should remove lock data if not confirmed during lockfile sync
+      n.clear()
+      exec_lua(function()
+        vim.pack.add({ repos_src.basic })
+      end)
+      eq(true, pack_exists('basic'))
+      eq('table', type(get_lock_tbl().plugins.basic))
+
+      vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
+      n.clear()
+      mock_confirm(2)
+
+      exec_lua(function()
+        vim.pack.add({ repos_src.basic })
+      end)
+      eq(false, pack_exists('basic'))
+      eq({ plugins = {} }, get_lock_tbl())
+
+      -- Should ask for confirm twice: during lockfile sync and inside
+      -- `vim.pack.add()` (i.e. not confirming during lockfile sync has
+      -- an immediate effect on whether a plugin is installed or not)
+      eq(2, exec_lua('return #_G.confirm_log'))
     end)
 
     it('respects `opts.confirm`', function()
@@ -413,7 +438,20 @@ describe('vim.pack', function()
       end)
 
       eq(0, exec_lua('return #_G.confirm_log'))
-      eq('basic main', exec_lua('return require("basic")'))
+      eq(true, pack_exists('basic'))
+
+      -- Should also respect `confirm` when installing during lockfile sync
+      vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
+      eq('table', type(get_lock_tbl().plugins.basic))
+
+      n.clear()
+      mock_confirm(1)
+
+      exec_lua(function()
+        vim.pack.add({}, { confirm = false })
+      end)
+      eq(0, exec_lua('return #_G.confirm_log'))
+      eq(true, pack_exists('basic'))
     end)
 
     it('can always confirm in current session', function()
@@ -526,24 +564,29 @@ describe('vim.pack', function()
       eq(ref_lockfile, get_lock_tbl())
     end)
 
-    it('uses lockfile revision during install', function()
+    it('uses lockfile during install', function()
       exec_lua(function()
-        vim.pack.add({ { src = repos_src.basic, version = 'feat-branch' } })
+        vim.pack.add({
+          { src = repos_src.basic, version = 'feat-branch' },
+          repos_src.defbranch,
+        })
       end)
 
       -- Mock clean initial install, but with lockfile present
+      vim.fs.rm(pack_get_dir(), { force = true, recursive = true })
       n.clear()
-      local basic_plug_path = vim.fs.joinpath(pack_get_dir(), 'basic')
-      vim.fs.rm(basic_plug_path, { force = true, recursive = true })
 
       local basic_rev = git_get_hash('feat-branch', 'basic')
+      local defbranch_rev = git_get_hash('HEAD', 'defbranch')
       local ref_lockfile = {
         plugins = {
           basic = { rev = basic_rev, src = repos_src.basic, version = "'feat-branch'" },
+          defbranch = { rev = defbranch_rev, src = repos_src.defbranch },
         },
       }
       eq(ref_lockfile, get_lock_tbl())
 
+      mock_confirm(1)
       exec_lua(function()
         -- Should use revision from lockfile (pointing at latest 'feat-branch'
         -- commit) and not use latest `main` commit
@@ -552,9 +595,17 @@ describe('vim.pack', function()
       local basic_lua_file = vim.fs.joinpath(pack_get_plug_path('basic'), 'lua', 'basic.lua')
       eq('return "basic feat-branch"', fn.readblob(basic_lua_file))
 
+      local confirm_log = exec_lua('return _G.confirm_log')
+      eq(1, #confirm_log)
+      matches('basic.*defbranch', confirm_log[1][1])
+
+      -- Should install `defbranch` (as it is in lockfile), but not load it
+      eq(true, pack_exists('defbranch'))
+      eq(false, exec_lua('return pcall(require, "defbranch")'))
+
       -- Running `update()` should still update to use `main`
       exec_lua(function()
-        vim.pack.update(nil, { force = true })
+        vim.pack.update({ 'basic' }, { force = true })
       end)
       eq('return "basic main"', fn.readblob(basic_lua_file))
 
@@ -585,6 +636,133 @@ describe('vim.pack', function()
       eq(ref_lockfile, get_lock_tbl())
     end)
 
+    it('regenerates manually deleted lockfile', function()
+      exec_lua(function()
+        vim.pack.add({
+          { src = repos_src.basic, name = 'other', version = 'feat-branch' },
+          repos_src.defbranch,
+        })
+      end)
+      local lock_path = get_lock_path()
+      eq(true, vim.uv.fs_stat(lock_path) ~= nil)
+
+      local basic_rev = git_get_hash('feat-branch', 'basic')
+      local plugindirs_rev = git_get_hash('dev', 'defbranch')
+
+      -- Should try its best to regenerate lockfile based on installed plugins
+      fn.delete(get_lock_path())
+      n.clear()
+      exec_lua(function()
+        vim.pack.add({})
+      end)
+      local ref_lockfile = {
+        plugins = {
+          -- No `version = 'feat-branch'` as there is no way to get that info
+          -- (lockfile was the only source of that on disk)
+          other = { rev = basic_rev, src = repos_src.basic },
+          defbranch = { rev = plugindirs_rev, src = repos_src.defbranch },
+        },
+      }
+      eq(ref_lockfile, get_lock_tbl())
+
+      local ref_messages = 'vim.pack: Repaired corrupted lock data for plugins: defbranch, other'
+      eq(ref_messages, n.exec_capture('messages'))
+
+      -- Calling `add()` with `version` should still add it to lockfile
+      exec_lua(function()
+        vim.pack.add({ { src = repos_src.basic, name = 'other', version = 'feat-branch' } })
+      end)
+      eq("'feat-branch'", get_lock_tbl().plugins.other.version)
+    end)
+
+    it('repairs corrupted lock data for installed plugins', function()
+      exec_lua(function()
+        vim.pack.add({
+          -- Should preserve present `version`
+          { src = repos_src.basic, version = 'feat-branch' },
+          repos_src.defbranch,
+          repos_src.semver,
+          repos_src.helptags,
+        })
+      end)
+
+      local lock_tbl = get_lock_tbl()
+      local ref_lock_tbl = vim.deepcopy(lock_tbl)
+      local assert = function()
+        exec_lua('vim.pack.add({})')
+        eq(ref_lock_tbl, get_lock_tbl())
+        eq(true, pack_exists('basic'))
+        eq(true, pack_exists('defbranch'))
+        eq(true, pack_exists('semver'))
+        eq(true, pack_exists('helptags'))
+      end
+
+      -- Missing lock data required field
+      lock_tbl.plugins.basic.rev = nil
+      -- Wrong lock data field type
+      lock_tbl.plugins.defbranch.src = 1 ---@diagnostic disable-line: assign-type-mismatch
+      -- Wrong lock data type
+      lock_tbl.plugins.semver = 1 ---@diagnostic disable-line: assign-type-mismatch
+
+      local lockfile_text = vim.json.encode(lock_tbl, { indent = '  ', sort_keys = true })
+      fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
+
+      n.clear()
+      assert()
+
+      local ref_messages =
+        'vim.pack: Repaired corrupted lock data for plugins: basic, defbranch, semver'
+      eq(ref_messages, n.exec_capture('messages'))
+
+      -- Should work even for badly corrupted lockfile
+      lockfile_text = vim.json.encode({ plugins = 1 }, { indent = '  ', sort_keys = true })
+      fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
+
+      n.clear()
+      -- Can not preserve `version` if it was deleted from the lockfile
+      ref_lock_tbl.plugins.basic.version = nil
+      assert()
+    end)
+
+    it('removes unrepairable corrupted data and plugins', function()
+      exec_lua(function()
+        vim.pack.add({ repos_src.basic, repos_src.defbranch, repos_src.semver, repos_src.helptags })
+      end)
+
+      local lock_tbl = get_lock_tbl()
+      local ref_lock_tbl = vim.deepcopy(lock_tbl)
+
+      -- Corrupted data for missing plugin
+      vim.fs.rm(pack_get_plug_path('basic'), { recursive = true, force = true })
+      lock_tbl.plugins.basic.rev = nil
+
+      -- Good data for corrupted plugin
+      local defbranch_path = pack_get_plug_path('defbranch')
+      vim.fs.rm(defbranch_path, { recursive = true, force = true })
+      fn.writefile({ 'File and not directory' }, defbranch_path)
+
+      -- Corrupted data for corrupted plugin
+      local semver_path = pack_get_plug_path('semver')
+      vim.fs.rm(semver_path, { recursive = true, force = true })
+      fn.writefile({ 'File and not directory' }, semver_path)
+      lock_tbl.plugins.semver.rev = 1 ---@diagnostic disable-line: assign-type-mismatch
+
+      local lockfile_text = vim.json.encode(lock_tbl, { indent = '  ', sort_keys = true })
+      fn.writefile(vim.split(lockfile_text, '\n'), get_lock_path())
+
+      n.clear()
+      exec_lua('vim.pack.add({})')
+      ref_lock_tbl.plugins.basic = nil
+      ref_lock_tbl.plugins.defbranch = nil
+      ref_lock_tbl.plugins.semver = nil
+      eq(ref_lock_tbl, get_lock_tbl())
+
+      eq(false, pack_exists('basic'))
+      eq(false, pack_exists('defbranch'))
+      eq(false, pack_exists('semver'))
+      eq(true, pack_exists('helptags'))
+    end)
+
     it('installs at proper version', function()
       local out = exec_lua(function()
         vim.pack.add({
@@ -1625,6 +1803,32 @@ describe('vim.pack', function()
       eq(ref_environ, fn.environ())
     end)
 
+    it('works with out of sync lockfile', function()
+      -- Should first autoinstall missing plugin (with confirmation)
+      vim.fs.rm(pack_get_plug_path('fetch'), { force = true, recursive = true })
+      n.clear()
+      mock_confirm(1)
+      exec_lua(function()
+        vim.pack.update(nil, { force = true })
+      end)
+      eq(1, exec_lua('return #_G.confirm_log'))
+      -- - Should checkout `version='main'` as it says in the lockfile
+      eq('return "fetch new 2"', fn.readblob(fetch_lua_file))
+
+      -- Should regenerate absent lockfile (from present plugins)
+      vim.fs.rm(get_lock_path())
+      n.clear()
+      exec_lua(function()
+        vim.pack.update(nil, { force = true })
+      end)
+      local lock_plugins = get_lock_tbl().plugins
+      eq(3, vim.tbl_count(lock_plugins))
+      -- - Should checkout default branch since `version='main'` info is lost
+      --   after lockfile is deleted.
+      eq(nil, lock_plugins.fetch.version)
+      eq('return "fetch dev"', fn.readblob(fetch_lua_file))
+    end)
+
     it('validates input', function()
       local function assert(err_pat, input)
         local function update_input()
@@ -1774,6 +1978,29 @@ describe('vim.pack', function()
       local basic_data = make_basic_data(true, true)
       eq({ { defbranch_data, basic_data }, { basic_data } }, exec_lua('return _G.get_log'))
     end)
+
+    it('works with out of sync lockfile', function()
+      exec_lua(function()
+        vim.pack.add({ repos_src.basic, repos_src.defbranch })
+      end)
+      eq(2, vim.tbl_count(get_lock_tbl().plugins))
+      local basic_lua_file = vim.fs.joinpath(pack_get_plug_path('basic'), 'lua', 'basic.lua')
+
+      -- Should first autoinstall missing plugin (with confirmation)
+      vim.fs.rm(pack_get_plug_path('basic'), { force = true, recursive = true })
+      n.clear()
+      mock_confirm(1)
+      eq(2, exec_lua('return #vim.pack.get()'))
+
+      eq(1, exec_lua('return #_G.confirm_log'))
+      eq('return "basic main"', fn.readblob(basic_lua_file))
+
+      -- Should regenerate absent lockfile (from present plugins)
+      vim.fs.rm(get_lock_path())
+      n.clear()
+      eq(2, exec_lua('return #vim.pack.get()'))
+      eq(2, vim.tbl_count(get_lock_tbl().plugins))
+    end)
   end)
 
   describe('del()', function()
@@ -1829,6 +2056,31 @@ describe('vim.pack', function()
       eq({ plugins = {} }, get_lock_tbl())
     end)
 
+    it('works with out of sync lockfile', function()
+      exec_lua(function()
+        vim.pack.add({ repos_src.basic, repos_src.defbranch, repos_src.plugindirs })
+      end)
+      eq(3, vim.tbl_count(get_lock_tbl().plugins))
+
+      -- Should first autoinstall missing plugin (with confirmation)
+      vim.fs.rm(pack_get_plug_path('basic'), { force = true, recursive = true })
+      n.clear()
+      mock_confirm(1)
+      exec_lua('vim.pack.del({ "defbranch" })')
+
+      eq(1, exec_lua('return #_G.confirm_log'))
+      eq(true, pack_exists('basic'))
+      eq(false, pack_exists('defbranch'))
+      eq(true, pack_exists('plugindirs'))
+
+      -- Should regenerate absent lockfile (from present plugins)
+      vim.fs.rm(get_lock_path())
+      n.clear()
+      exec_lua('vim.pack.del({ "basic" })')
+      eq(1, exec_lua('return #vim.pack.get()'))
+      eq({ 'plugindirs' }, vim.tbl_keys(get_lock_tbl().plugins))
+    end)
+
     it('validates input', function()
       local function assert(err_pat, input)
         local function del_input()