Merge pull request #31011 from lewis6991/refactor/loader

runtime/doc/lua.txt

@@ -2519,8 +2519,8 @@ vim.loader.find({modname}, {opts})                         *vim.loader.find()*
         (`table[]`) A list of objects with the following fields:
         • {modpath} (`string`) Path of the module
         • {modname} (`string`) Name of the module
-        • {stat}? (`uv.uv_fs_t`) The fs_stat of the module path. Won't be
+        • {stat}? (`uv.fs_stat.result`) The fs_stat of the module path. Won't
-          returned for `modname="*"`
+          be returned for `modname="*"`
 
 vim.loader.reset({path})                                  *vim.loader.reset()*
     WARNING: This feature is experimental/unstable.

runtime/lua/vim/loader.lua

@@ -4,11 +4,14 @@ local uri_encode = vim.uri_encode --- @type function
 
 --- @type (fun(modename: string): fun()|string)[]
 local loaders = package.loaders
+local _loadfile = loadfile
+
+local VERSION = 4
 
 local M = {}
 
----@alias CacheHash {mtime: {nsec: integer, sec: integer}, size: integer, type?: string}
+--- @alias vim.loader.CacheHash {mtime: {nsec: integer, sec: integer}, size: integer, type?: string}
----@alias CacheEntry {hash:CacheHash, chunk:string}
+--- @alias vim.loader.CacheEntry {hash:vim.loader.CacheHash, chunk:string}
 
 --- @class vim.loader.find.Opts
 --- @inlinedoc
@@ -40,107 +43,97 @@ local M = {}
 --- @field modname string
 ---
 --- The fs_stat of the module path. Won't be returned for `modname="*"`
---- @field stat? uv.uv_fs_t
+--- @field stat? uv.fs_stat.result
 
----@alias LoaderStats table<string, {total:number, time:number, [string]:number?}?>
+--- @alias vim.loader.Stats table<string, {total:number, time:number, [string]:number?}?>
 
----@nodoc
+--- @private
 M.path = vim.fn.stdpath('cache') .. '/luac'
 
----@nodoc
+--- @private
 M.enabled = false
 
----@class (private) Loader
+--- @type vim.loader.Stats
----@field private _rtp string[]
+local stats = { find = { total = 0, time = 0, not_found = 0 } }
----@field private _rtp_pure string[]
+
----@field private _rtp_key string
+--- @type table<string, uv.fs_stat.result>?
----@field private _hashes? table<string, CacheHash>
+local fs_stat_cache
-local Loader = {
+
-  VERSION = 4,
+--- @type table<string, table<string,vim.loader.ModuleInfo>>
-  ---@type table<string, table<string,vim.loader.ModuleInfo>>
+local indexed = {}
-  _indexed = {},
-  ---@type table<string, string[]>
-  _topmods = {},
-  _loadfile = loadfile,
-  ---@type LoaderStats
-  _stats = {
-    find = { total = 0, time = 0, not_found = 0 },
-  },
-}
 
 --- @param path string
---- @return CacheHash
+--- @return uv.fs_stat.result?
---- @private
+local function fs_stat_cached(path)
-function Loader.get_hash(path)
+  if not fs_stat_cache then
-  if not Loader._hashes then
+    return uv.fs_stat(path)
-    return uv.fs_stat(path) --[[@as CacheHash]]
   end
 
-  if not Loader._hashes[path] then
+  if not fs_stat_cache[path] then
     -- Note we must never save a stat for a non-existent path.
     -- For non-existent paths fs_stat() will return nil.
-    Loader._hashes[path] = uv.fs_stat(path)
+    fs_stat_cache[path] = uv.fs_stat(path)
   end
-  return Loader._hashes[path]
+  return fs_stat_cache[path]
 end
 
 local function normalize(path)
   return fs.normalize(path, { expand_env = false, _fast = true })
 end
 
+local rtp_cached = {} --- @type string[]
+local rtp_cache_key --- @type  string?
+
 --- Gets the rtp excluding after directories.
 --- The result is cached, and will be updated if the runtime path changes.
 --- When called from a fast event, the cached value will be returned.
 --- @return string[] rtp, boolean updated
----@private
+local function get_rtp()
-function Loader.get_rtp()
   if vim.in_fast_event() then
-    return (Loader._rtp or {}), false
+    return (rtp_cached or {}), false
   end
   local updated = false
   local key = vim.go.rtp
-  if key ~= Loader._rtp_key then
+  if key ~= rtp_cache_key then
-    Loader._rtp = {}
+    rtp_cached = {}
     for _, path in ipairs(vim.api.nvim_get_runtime_file('', true)) do
       path = normalize(path)
       -- skip after directories
       if
         path:sub(-6, -1) ~= '/after'
-        and not (Loader._indexed[path] and vim.tbl_isempty(Loader._indexed[path]))
+        and not (indexed[path] and vim.tbl_isempty(indexed[path]))
       then
-        Loader._rtp[#Loader._rtp + 1] = path
+        rtp_cached[#rtp_cached + 1] = path
       end
     end
     updated = true
-    Loader._rtp_key = key
+    rtp_cache_key = key
   end
-  return Loader._rtp, updated
+  return rtp_cached, updated
 end
 
 --- Returns the cache file name
----@param name string can be a module name, or a file name
+--- @param name string can be a module name, or a file name
----@return string file_name
+--- @return string file_name
----@private
+local function cache_filename(name)
-function Loader.cache_file(name)
   local ret = ('%s/%s'):format(M.path, uri_encode(name, 'rfc2396'))
   return ret:sub(-4) == '.lua' and (ret .. 'c') or (ret .. '.luac')
 end
 
 --- Saves the cache entry for a given module or file
----@param name string module name or filename
+--- @param cname string cache filename
----@param entry CacheEntry
+--- @param hash vim.loader.CacheHash
----@private
+--- @param chunk function
-function Loader.write(name, entry)
+local function write_cachefile(cname, hash, chunk)
-  local cname = Loader.cache_file(name)
   local f = assert(uv.fs_open(cname, 'w', 438))
   local header = {
-    Loader.VERSION,
+    VERSION,
-    entry.hash.size,
+    hash.size,
-    entry.hash.mtime.sec,
+    hash.mtime.sec,
-    entry.hash.mtime.nsec,
+    hash.mtime.nsec,
   }
   uv.fs_write(f, table.concat(header, ',') .. '\0')
-  uv.fs_write(f, entry.chunk)
+  uv.fs_write(f, string.dump(chunk))
   uv.fs_close(f)
 end
 
@@ -150,151 +143,159 @@ end
 local function readfile(path, mode)
   local f = uv.fs_open(path, 'r', mode)
   if f then
-    local hash = assert(uv.fs_fstat(f))
+    local size = assert(uv.fs_fstat(f)).size
-    local data = uv.fs_read(f, hash.size, 0) --[[@as string?]]
+    local data = uv.fs_read(f, size, 0)
     uv.fs_close(f)
     return data
   end
 end
 
 --- Loads the cache entry for a given module or file
----@param name string module name or filename
+--- @param cname string cache filename
----@return CacheEntry?
+--- @return vim.loader.CacheHash? hash
----@private
+--- @return string? chunk
-function Loader.read(name)
+local function read_cachefile(cname)
-  local cname = Loader.cache_file(name)
   local data = readfile(cname, 438)
-  if data then
+  if not data then
-    local zero = data:find('\0', 1, true)
+    return
-    if not zero then
-      return
-    end
-
-    ---@type integer[]|{[0]:integer}
-    local header = vim.split(data:sub(1, zero - 1), ',')
-    if tonumber(header[1]) ~= Loader.VERSION then
-      return
-    end
-    return {
-      hash = {
-        size = tonumber(header[2]),
-        mtime = { sec = tonumber(header[3]), nsec = tonumber(header[4]) },
-      },
-      chunk = data:sub(zero + 1),
-    }
   end
+
+  local zero = data:find('\0', 1, true)
+  if not zero then
+    return
+  end
+
+  --- @type integer[]|{[0]:integer}
+  local header = vim.split(data:sub(1, zero - 1), ',')
+  if tonumber(header[1]) ~= VERSION then
+    return
+  end
+
+  local hash = {
+    size = tonumber(header[2]),
+    mtime = { sec = tonumber(header[3]), nsec = tonumber(header[4]) },
+  }
+
+  local chunk = data:sub(zero + 1)
+
+  return hash, chunk
 end
 
 --- The `package.loaders` loader for Lua files using the cache.
----@param modname string module name
+--- @param modname string module name
----@return string|function
+--- @return string|function
----@private
+local function loader_cached(modname)
-function Loader.loader(modname)
+  fs_stat_cache = {}
-  Loader._hashes = {}
   local ret = M.find(modname)[1]
   if ret then
     -- Make sure to call the global loadfile so we respect any augmentations done elsewhere.
     -- E.g. profiling
     local chunk, err = loadfile(ret.modpath)
-    Loader._hashes = nil
+    fs_stat_cache = nil
     return chunk or error(err)
   end
-  Loader._hashes = nil
+  fs_stat_cache = nil
   return ("\n\tcache_loader: module '%s' not found"):format(modname)
 end
 
---- The `package.loaders` loader for libs
+local is_win = vim.fn.has('win32') == 1
----@param modname string module name
----@return string|function
----@private
-function Loader.loader_lib(modname)
-  local is_win = vim.fn.has('win32') == 1
-  local ret = M.find(modname, { patterns = is_win and { '.dll' } or { '.so' } })[1]
-  if ret then
-    -- Making function name in Lua 5.1 (see src/loadlib.c:mkfuncname) is
-    -- a) strip prefix up to and including the first dash, if any
-    -- b) replace all dots by underscores
-    -- c) prepend "luaopen_"
-    -- So "foo-bar.baz" should result in "luaopen_bar_baz"
-    local dash = modname:find('-', 1, true)
-    local funcname = dash and modname:sub(dash + 1) or modname
-    local chunk, err = package.loadlib(ret.modpath, 'luaopen_' .. funcname:gsub('%.', '_'))
-    return chunk or error(err)
-  end
-  return ("\n\tcache_loader_lib: module '%s' not found"):format(modname)
-end
 
---- `loadfile` using the cache
+--- The `package.loaders` loader for libs
---- Note this has the mode and env arguments which is supported by LuaJIT and is 5.1 compatible.
+--- @param modname string module name
----@param filename? string
+--- @return string|function
----@param _mode? "b"|"t"|"bt"
+local function loader_lib_cached(modname)
----@param env? table
+  local ret = M.find(modname, { patterns = { is_win and '.dll' or '.so' } })[1]
----@return function?, string?  error_message
+  if not ret then
----@private
+    return ("\n\tcache_loader_lib: module '%s' not found"):format(modname)
-function Loader.loadfile(filename, _mode, env)
+  end
-  -- ignore mode, since we byte-compile the Lua source files
+
-  return Loader.load(normalize(filename), { env = env })
+  -- Making function name in Lua 5.1 (see src/loadlib.c:mkfuncname) is
+  -- a) strip prefix up to and including the first dash, if any
+  -- b) replace all dots by underscores
+  -- c) prepend "luaopen_"
+  -- So "foo-bar.baz" should result in "luaopen_bar_baz"
+  local dash = modname:find('-', 1, true)
+  local funcname = dash and modname:sub(dash + 1) or modname
+  local chunk, err = package.loadlib(ret.modpath, 'luaopen_' .. funcname:gsub('%.', '_'))
+  return chunk or error(err)
 end
 
 --- Checks whether two cache hashes are the same based on:
 --- * file size
 --- * mtime in seconds
 --- * mtime in nanoseconds
----@param h1 CacheHash
+--- @param a? vim.loader.CacheHash
----@param h2 CacheHash
+--- @param b? vim.loader.CacheHash
----@private
+local function hash_eq(a, b)
-function Loader.eq(h1, h2)
+  return a
-  return h1
+    and b
-    and h2
+    and a.size == b.size
-    and h1.size == h2.size
+    and a.mtime.sec == b.mtime.sec
-    and h1.mtime.sec == h2.mtime.sec
+    and a.mtime.nsec == b.mtime.nsec
-    and h1.mtime.nsec == h2.mtime.nsec
 end
 
---- Loads the given module path using the cache
+--- `loadfile` using the cache
----@param modpath string
+--- Note this has the mode and env arguments which is supported by LuaJIT and is 5.1 compatible.
----@param opts? {mode?: "b"|"t"|"bt", env?:table} (table|nil) Options for loading the module:
+--- @param filename? string
----    - mode: (string) the mode to load the module with. "b"|"t"|"bt" (defaults to `nil`)
+--- @param mode? "b"|"t"|"bt"
----    - env: (table) the environment to load the module in. (defaults to `nil`)
+--- @param env? table
----@see |luaL_loadfile()|
+--- @return function?, string?  error_message
----@return function?, string? error_message
+local function loadfile_cached(filename, mode, env)
----@private
+  local modpath = normalize(filename)
-function Loader.load(modpath, opts)
+  local stat = fs_stat_cached(modpath)
-  opts = opts or {}
+  local cname = cache_filename(modpath)
-  local hash = Loader.get_hash(modpath)
+  if stat then
-  ---@type function?, string?
+    local e_hash, e_chunk = read_cachefile(cname)
-  local chunk, err
+    if hash_eq(e_hash, stat) and e_chunk then
-
+      -- found in cache and up to date
-  if not hash then
+      local chunk, err = load(e_chunk, '@' .. modpath, mode, env)
-    -- trigger correct error
+      if not (err and err:find('cannot load incompatible bytecode', 1, true)) then
-    return Loader._loadfile(modpath, opts.mode, opts.env)
+        return chunk, err
-  end
+      end
-
-  local entry = Loader.read(modpath)
-  if entry and Loader.eq(entry.hash, hash) then
-    -- found in cache and up to date
-    chunk, err = load(entry.chunk --[[@as string]], '@' .. modpath, opts.mode, opts.env)
-    if not (err and err:find('cannot load incompatible bytecode', 1, true)) then
-      return chunk, err
     end
   end
-  entry = { hash = hash, modpath = modpath }
 
-  chunk, err = Loader._loadfile(modpath, opts.mode, opts.env)
+  local chunk, err = _loadfile(modpath, mode, env)
-  if chunk then
+  if chunk and stat then
-    entry.chunk = string.dump(chunk)
+    write_cachefile(cname, stat, chunk)
-    Loader.write(modpath, entry)
   end
   return chunk, err
 end
 
+--- Return the top-level \`/lua/*` modules for this path
+--- @param path string path to check for top-level Lua modules
+local function lsmod(path)
+  if not indexed[path] then
+    indexed[path] = {}
+    for name, t in fs.dir(path .. '/lua') do
+      local modpath = path .. '/lua/' .. name
+      -- HACK: type is not always returned due to a bug in luv
+      t = t or fs_stat_cached(modpath).type
+      --- @type string
+      local topname
+      local ext = name:sub(-4)
+      if ext == '.lua' or ext == '.dll' then
+        topname = name:sub(1, -5)
+      elseif name:sub(-3) == '.so' then
+        topname = name:sub(1, -4)
+      elseif t == 'link' or t == 'directory' then
+        topname = name
+      end
+      if topname then
+        indexed[path][topname] = { modpath = modpath, modname = topname }
+      end
+    end
+  end
+  return indexed[path]
+end
+
 --- Finds Lua modules for the given module name.
 ---
 --- @since 0
 ---
----@param modname string Module name, or `"*"` to find the top-level modules instead
+--- @param modname string Module name, or `"*"` to find the top-level modules instead
----@param opts? vim.loader.find.Opts Options for finding a module:
+--- @param opts? vim.loader.find.Opts Options for finding a module:
----@return vim.loader.ModuleInfo[]
+--- @return vim.loader.ModuleInfo[]
 function M.find(modname, opts)
   opts = opts or {}
 
@@ -320,7 +321,7 @@ function M.find(modname, opts)
     patterns[p] = '/lua/' .. basename .. pattern
   end
 
-  ---@type vim.loader.ModuleInfo[]
+  --- @type vim.loader.ModuleInfo[]
   local results = {}
 
   -- Only continue if we haven't found anything yet or we want to find all
@@ -330,23 +331,23 @@ function M.find(modname, opts)
 
   -- Checks if the given paths contain the top-level module.
   -- If so, it tries to find the module path for the given module name.
-  ---@param paths string[]
+  --- @param paths string[]
   local function _find(paths)
     for _, path in ipairs(paths) do
       if topmod == '*' then
-        for _, r in pairs(Loader.lsmod(path)) do
+        for _, r in pairs(lsmod(path)) do
           results[#results + 1] = r
           if not continue() then
             return
           end
         end
-      elseif Loader.lsmod(path)[topmod] then
+      elseif lsmod(path)[topmod] then
         for _, pattern in ipairs(patterns) do
           local modpath = path .. pattern
-          Loader._stats.find.stat = (Loader._stats.find.stat or 0) + 1
+          stats.find.stat = (stats.find.stat or 0) + 1
-          local hash = Loader.get_hash(modpath)
+          local stat = fs_stat_cached(modpath)
-          if hash then
+          if stat then
-            results[#results + 1] = { modpath = modpath, stat = hash, modname = modname }
+            results[#results + 1] = { modpath = modpath, stat = stat, modname = modname }
             if not continue() then
               return
             end
@@ -358,9 +359,9 @@ function M.find(modname, opts)
 
   -- always check the rtp first
   if opts.rtp ~= false then
-    _find(Loader._rtp or {})
+    _find(rtp_cached or {})
     if continue() then
-      local rtp, updated = Loader.get_rtp()
+      local rtp, updated = get_rtp()
       if updated then
         _find(rtp)
       end
@@ -374,7 +375,7 @@ function M.find(modname, opts)
 
   if #results == 0 then
     -- module not found
-    Loader._stats.find.not_found = Loader._stats.find.not_found + 1
+    stats.find.not_found = stats.find.not_found + 1
   end
 
   return results
@@ -384,17 +385,17 @@ end
 ---
 --- @since 0
 ---
----@param path string? path to reset
+--- @param path string? path to reset
 function M.reset(path)
   if path then
-    Loader._indexed[normalize(path)] = nil
+    indexed[normalize(path)] = nil
   else
-    Loader._indexed = {}
+    indexed = {}
   end
 
   -- Path could be a directory so just clear all the hashes.
-  if Loader._hashes then
+  if fs_stat_cache then
-    Loader._hashes = {}
+    fs_stat_cache = {}
   end
 end
 
@@ -411,11 +412,11 @@ function M.enable()
   end
   M.enabled = true
   vim.fn.mkdir(vim.fn.fnamemodify(M.path, ':p'), 'p')
-  _G.loadfile = Loader.loadfile
+  _G.loadfile = loadfile_cached
   -- add Lua loader
-  table.insert(loaders, 2, Loader.loader)
+  table.insert(loaders, 2, loader_cached)
   -- add libs loader
-  table.insert(loaders, 3, Loader.loader_lib)
+  table.insert(loaders, 3, loader_lib_cached)
   -- remove Nvim loader
   for l, loader in ipairs(loaders) do
     if loader == vim._load_package then
@@ -435,111 +436,75 @@ function M.disable()
     return
   end
   M.enabled = false
-  _G.loadfile = Loader._loadfile
+  _G.loadfile = _loadfile
   for l, loader in ipairs(loaders) do
-    if loader == Loader.loader or loader == Loader.loader_lib then
+    if loader == loader_cached or loader == loader_lib_cached then
       table.remove(loaders, l)
     end
   end
   table.insert(loaders, 2, vim._load_package)
 end
 
---- Return the top-level \`/lua/*` modules for this path
----@param path string path to check for top-level Lua modules
----@private
-function Loader.lsmod(path)
-  if not Loader._indexed[path] then
-    Loader._indexed[path] = {}
-    for name, t in fs.dir(path .. '/lua') do
-      local modpath = path .. '/lua/' .. name
-      -- HACK: type is not always returned due to a bug in luv
-      t = t or Loader.get_hash(modpath).type
-      ---@type string
-      local topname
-      local ext = name:sub(-4)
-      if ext == '.lua' or ext == '.dll' then
-        topname = name:sub(1, -5)
-      elseif name:sub(-3) == '.so' then
-        topname = name:sub(1, -4)
-      elseif t == 'link' or t == 'directory' then
-        topname = name
-      end
-      if topname then
-        Loader._indexed[path][topname] = { modpath = modpath, modname = topname }
-        Loader._topmods[topname] = Loader._topmods[topname] or {}
-        if not vim.list_contains(Loader._topmods[topname], path) then
-          table.insert(Loader._topmods[topname], path)
-        end
-      end
-    end
-  end
-  return Loader._indexed[path]
-end
-
 --- Tracks the time spent in a function
 --- @generic F: function
 --- @param f F
 --- @return F
---- @private
+local function track(stat, f)
-function Loader.track(stat, f)
   return function(...)
     local start = vim.uv.hrtime()
     local r = { f(...) }
-    Loader._stats[stat] = Loader._stats[stat] or { total = 0, time = 0 }
+    stats[stat] = stats[stat] or { total = 0, time = 0 }
-    Loader._stats[stat].total = Loader._stats[stat].total + 1
+    stats[stat].total = stats[stat].total + 1
-    Loader._stats[stat].time = Loader._stats[stat].time + uv.hrtime() - start
+    stats[stat].time = stats[stat].time + uv.hrtime() - start
     return unpack(r, 1, table.maxn(r))
   end
 end
 
----@class (private) vim.loader._profile.Opts
+--- @class (private) vim.loader._profile.Opts
----@field loaders? boolean Add profiling to the loaders
+--- @field loaders? boolean Add profiling to the loaders
 
 --- Debug function that wraps all loaders and tracks stats
----@private
+--- Must be called before vim.loader.enable()
----@param opts vim.loader._profile.Opts?
+--- @private
+--- @param opts vim.loader._profile.Opts?
 function M._profile(opts)
-  Loader.get_rtp = Loader.track('get_rtp', Loader.get_rtp)
+  get_rtp = track('get_rtp', get_rtp)
-  Loader.read = Loader.track('read', Loader.read)
+  read_cachefile = track('read', read_cachefile)
-  Loader.loader = Loader.track('loader', Loader.loader)
+  loader_cached = track('loader', loader_cached)
-  Loader.loader_lib = Loader.track('loader_lib', Loader.loader_lib)
+  loader_lib_cached = track('loader_lib', loader_lib_cached)
-  Loader.loadfile = Loader.track('loadfile', Loader.loadfile)
+  loadfile_cached = track('loadfile', loadfile_cached)
-  Loader.load = Loader.track('load', Loader.load)
+  M.find = track('find', M.find)
-  M.find = Loader.track('find', M.find)
+  lsmod = track('lsmod', lsmod)
-  Loader.lsmod = Loader.track('lsmod', Loader.lsmod)
 
   if opts and opts.loaders then
     for l, loader in pairs(loaders) do
       local loc = debug.getinfo(loader, 'Sn').source:sub(2)
-      loaders[l] = Loader.track('loader ' .. l .. ': ' .. loc, loader)
+      loaders[l] = track('loader ' .. l .. ': ' .. loc, loader)
     end
   end
 end
 
 --- Prints all cache stats
----@param opts? {print?:boolean}
+--- @param opts? {print?:boolean}
----@return LoaderStats
+--- @return vim.loader.Stats
----@private
+--- @private
 function M._inspect(opts)
   if opts and opts.print then
     local function ms(nsec)
       return math.floor(nsec / 1e6 * 1000 + 0.5) / 1000 .. 'ms'
     end
-    local chunks = {} ---@type string[][]
+    local chunks = {} --- @type string[][]
-    ---@type string[]
+    for _, stat in vim.spairs(stats) do
-    local stats = vim.tbl_keys(Loader._stats)
-    table.sort(stats)
-    for _, stat in ipairs(stats) do
       vim.list_extend(chunks, {
         { '\n' .. stat .. '\n', 'Title' },
         { '* total:    ' },
-        { tostring(Loader._stats[stat].total) .. '\n', 'Number' },
+        { tostring(stat.total) .. '\n', 'Number' },
         { '* time:     ' },
-        { ms(Loader._stats[stat].time) .. '\n', 'Bold' },
+        { ms(stat.time) .. '\n', 'Bold' },
         { '* avg time: ' },
-        { ms(Loader._stats[stat].time / Loader._stats[stat].total) .. '\n', 'Bold' },
+        { ms(stat.time / stat.total) .. '\n', 'Bold' },
       })
-      for k, v in pairs(Loader._stats[stat]) do
+      for k, v in pairs(stat) do
         if not vim.list_contains({ 'time', 'total' }, k) then
           chunks[#chunks + 1] = { '* ' .. k .. ':' .. string.rep(' ', 9 - #k) }
           chunks[#chunks + 1] = { tostring(v) .. '\n', 'Number' }
@@ -548,7 +513,7 @@ function M._inspect(opts)
     end
     vim.api.nvim_echo(chunks, true, {})
   end
-  return Loader._stats
+  return stats
 end
 
 return M