refactor(lua): reorganize builtin modules, phase 1

runtime/lua/vim/_load_package.lua

@@ -47,3 +47,6 @@ end
 
 -- Insert vim._load_package after the preloader at position 2
 table.insert(package.loaders, 2, vim._load_package)
+
+-- should always be available
+vim.inspect = require'vim.inspect'

src/nvim/CMakeLists.txt

@@ -332,14 +332,14 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E env
       "LUAC_PRG=${LUAC_PRG}"
       ${LUA_PRG} ${CHAR_BLOB_GENERATOR} -c ${VIM_MODULE_FILE}
-      ${LUA_VIM_MODULE_SOURCE} vim_module
+      ${LUA_LOAD_PACKAGE_MODULE_SOURCE} "vim._load_package"
-      ${LUA_SHARED_MODULE_SOURCE} shared_module
+      ${LUA_INSPECT_MODULE_SOURCE} "vim.inspect"
-      ${LUA_INSPECT_MODULE_SOURCE} inspect_module
+      ${LUA_VIM_MODULE_SOURCE} "vim"
-      ${LUA_F_MODULE_SOURCE} lua_F_module
+      ${LUA_SHARED_MODULE_SOURCE} "vim.shared"
-      ${LUA_META_MODULE_SOURCE} lua_meta_module
+      ${LUA_F_MODULE_SOURCE} "vim.F"
-      ${LUA_FILETYPE_MODULE_SOURCE} lua_filetype_module
+      ${LUA_META_MODULE_SOURCE} "vim._meta"
-      ${LUA_LOAD_PACKAGE_MODULE_SOURCE} lua_load_package_module
+      ${LUA_FILETYPE_MODULE_SOURCE} "vim.filetype"
-      ${LUA_KEYMAP_MODULE_SOURCE} lua_keymap_module
+      ${LUA_KEYMAP_MODULE_SOURCE} "vim.keymap"
   DEPENDS
     ${CHAR_BLOB_GENERATOR}
     ${LUA_VIM_MODULE_SOURCE}

src/nvim/generators/gen_char_blob.lua

@@ -28,16 +28,19 @@ local target = io.open(target_file, 'w')
 
 target:write('#include <stdint.h>\n\n')
 
+local index_items = {}
+
 local warn_on_missing_compiler = true
-local varnames = {}
+local modnames = {}
 for argi = 2, #arg, 2 do
   local source_file = arg[argi]
-  local varname = arg[argi + 1]
+  local modname = arg[argi + 1]
-  if varnames[varname] then
+  if modnames[modname] then
-    error(string.format("varname %q is already specified for file %q", varname, varnames[varname]))
+    error(string.format("modname %q is already specified for file %q", modname, modnames[modname]))
   end
-  varnames[varname] = source_file
+  modnames[modname] = source_file
 
+  local varname = string.gsub(modname,'%.','_dot_').."_module"
   target:write(('static const uint8_t %s[] = {\n'):format(varname))
 
   local output
@@ -78,6 +81,13 @@ for argi = 2, #arg, 2 do
   end
 
   target:write('  0};\n')
+  if modname ~= "_" then
+    table.insert(index_items, '  { "'..modname..'", '..varname..', sizeof '..varname..' },\n\n')
+  end
 end
 
+target:write('static ModuleDef builtin_modules[] = {\n')
+target:write(table.concat(index_items))
+target:write('};\n')
+
 target:close()

src/nvim/lua/executor.c

@@ -53,6 +53,12 @@ typedef struct {
   String lua_err_str;
 } LuaError;
 
+typedef struct {
+  char *name;
+  const uint8_t *data;
+  size_t size;
+} ModuleDef;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/executor.c.generated.h"
 # include "lua/vim_module.generated.h"
@@ -519,60 +525,70 @@ static void nlua_common_vim_init(lua_State *lstate, bool is_thread)
   lua_pop(lstate, 3);
 }
 
-static void nlua_common_package_init(lua_State *lstate)
+static void nlua_preload_modules(lua_State *lstate)
+{
+  lua_getglobal(lstate, "package");  // [package]
+  lua_getfield(lstate, -1, "preload");  // [package, preload]
+  for (size_t i = 0; i < ARRAY_SIZE(builtin_modules); i++) {
+    ModuleDef def = builtin_modules[i];
+    lua_pushinteger(lstate, (long)i);  // [package, preload, i]
+    lua_pushcclosure(lstate, nlua_module_preloader, 1);  // [package, preload, cclosure]
+    lua_setfield(lstate, -2, def.name);  // [package, preload]
+
+    if (nlua_disable_preload && strequal(def.name, "vim")) {
+      break;
+    }
+  }
+
+  lua_pop(lstate, 2);  // []
+}
+
+static int nlua_module_preloader(lua_State *lstate)
+{
+  size_t i = (size_t)lua_tointeger(lstate, lua_upvalueindex(1));
+  ModuleDef def = builtin_modules[i];
+  char name[256];
+  name[0] = '@';
+  size_t off = xstrlcpy(name+1, def.name, (sizeof name) - 2);
+  strchrsub(name+1, '.', '/');
+  xstrlcpy(name+1+off, ".lua", (sizeof name)-2-off);
+
+  if (luaL_loadbuffer(lstate, (const char *)def.data, def.size - 1, name)) {
+    return lua_error(lstate);
+  }
+
+  lua_call(lstate, 0, 1);  // propagates error to caller
+  return 1;
+}
+
+static bool nlua_common_package_init(lua_State *lstate)
   FUNC_ATTR_NONNULL_ALL
 {
-  {
+  nlua_preload_modules(lstate);
-    const char *code = (char *)&shared_module[0];
+
-    if (luaL_loadbuffer(lstate, code, sizeof(shared_module) - 1, "@vim/shared.lua")
+  lua_getglobal(lstate, "require");
-        || nlua_pcall(lstate, 0, 0)) {
+  lua_pushstring(lstate, "vim._load_package");
+  if (nlua_pcall(lstate, 1, 0)) {
+      nlua_error(lstate, _("E5106: Error while creating _load_package module: %.*s\n"));
+      return false;
+  }
+
+  // TODO(bfredl): ideally all initialization should be done as a single require
+  // call.
+  lua_getglobal(lstate, "require");
+  lua_pushstring(lstate, "vim.shared");
+  if (nlua_pcall(lstate, 1, 0)) {
       nlua_error(lstate, _("E5106: Error while creating shared module: %.*s\n"));
-      return;
+      return false;
-    }
   }
 
-  {
+  return true;
-    const char *code = (char *)&lua_load_package_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(lua_load_package_module) - 1, "@vim/_load_package.lua")
-        || lua_pcall(lstate, 0, 0, 0)) {
-      nlua_error(lstate, _("E5106: Error while creating _load_package module: %.*s"));
-      return;
-    }
-  }
-
-  {
-    lua_getglobal(lstate, "package");  // [package]
-    lua_getfield(lstate, -1, "loaded");  // [package, loaded]
-
-    const char *code = (char *)&inspect_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(inspect_module) - 1, "@vim/inspect.lua")
-        || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating inspect module: %.*s\n"));
-      return;
-    }
-
-    // [package, loaded, inspect]
-    lua_setfield(lstate, -2, "vim.inspect");  // [package, loaded]
-  }
-
-  {
-    const char *code = (char *)&lua_F_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(lua_F_module) - 1, "@vim/F.lua")
-        || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim.F module: %.*s\n"));
-      return;
-    }
-    // [package, loaded, module]
-    lua_setfield(lstate, -2, "vim.F");  // [package, loaded]
-
-    lua_pop(lstate, 2);  // []
-  }
 }
 
 /// Initialize lua interpreter state
 ///
 /// Called by lua interpreter itself to initialize state.
-static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
+static bool nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 {
   // print
   lua_pushcfunction(lstate, &nlua_print);
@@ -638,59 +654,18 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 
   lua_setglobal(lstate, "vim");
 
-  nlua_common_package_init(lstate);
+  if (!nlua_common_package_init(lstate)) {
-
+    return false;
-  {
-    lua_getglobal(lstate, "package");  // [package]
-    lua_getfield(lstate, -1, "loaded");  // [package, loaded]
-
-    char *code = (char *)&lua_filetype_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(lua_filetype_module) - 1, "@vim/filetype.lua")
-        || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim.filetype module: %.*s"));
-      return 1;
-    }
-    // [package, loaded, module]
-    lua_setfield(lstate, -2, "vim.filetype");  // [package, loaded]
-
-    code = (char *)&lua_keymap_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(lua_keymap_module) - 1, "@vim/keymap.lua")
-        || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim.keymap module: %.*s"));
-      return 1;
-    }
-    // [package, loaded, module]
-    lua_setfield(lstate, -2, "vim.keymap");  // [package, loaded]
-
-    lua_pop(lstate, 2);  // []
   }
 
-  {
+  lua_getglobal(lstate, "require");
-    const char *code = (char *)&vim_module[0];
+  lua_pushstring(lstate, "vim");
-    if (luaL_loadbuffer(lstate, code, sizeof(vim_module) - 1, "@vim.lua")
+  if (nlua_pcall(lstate, 1, 0)) {
-        || nlua_pcall(lstate, 0, 0)) {
+    nlua_error(lstate, _("E5106: Error while creating vim module: %.*s\n"));
-      nlua_error(lstate, _("E5106: Error while creating vim module: %.*s\n"));
+    return false;
-      return 1;
-    }
   }
 
-  {
+  return true;
-    lua_getglobal(lstate, "package");  // [package]
-    lua_getfield(lstate, -1, "loaded");  // [package, loaded]
-
-    const char *code = (char *)&lua_meta_module[0];
-    if (luaL_loadbuffer(lstate, code, sizeof(lua_meta_module) - 1, "@vim/_meta.lua")
-        || nlua_pcall(lstate, 0, 1)) {
-      nlua_error(lstate, _("E5106: Error while creating vim._meta module: %.*s\n"));
-      return 1;
-    }
-    // [package, loaded, module]
-    lua_setfield(lstate, -2, "vim._meta");  // [package, loaded]
-
-    lua_pop(lstate, 2);  // []
-  }
-
-  return 0;
 }
 
 /// Initialize global lua interpreter
@@ -707,11 +682,14 @@ void nlua_init(void)
 
   lua_State *lstate = luaL_newstate();
   if (lstate == NULL) {
-    emsg(_("E970: Failed to initialize lua interpreter"));
+    mch_errmsg(_("E970: Failed to initialize lua interpreter\n"));
-    preserve_exit();
+    os_exit(1);
   }
   luaL_openlibs(lstate);
-  nlua_state_init(lstate);
+  if (!nlua_state_init(lstate)) {
+    mch_errmsg(_("E970: Failed to initialize builtin lua modules\n"));
+    os_exit(1);
+  }
 
   luv_set_thread_cb(nlua_thread_acquire_vm, nlua_common_free_all_mem);
 
@@ -747,23 +725,20 @@ static lua_State *nlua_thread_acquire_vm(void)
 
   nlua_state_add_stdlib(lstate, true);
 
-  lua_setglobal(lstate, "vim");
-
-  nlua_common_package_init(lstate);
-
-  lua_getglobal(lstate, "vim");
-  lua_getglobal(lstate, "package");
-  lua_getfield(lstate, -1, "loaded");
-  lua_getfield(lstate, -1, "vim.inspect");
-  lua_setfield(lstate, -4, "inspect");
-  lua_pop(lstate, 3);
-
-  lua_getglobal(lstate, "vim");
   lua_createtable(lstate, 0, 0);
   lua_pushcfunction(lstate, nlua_thr_api_nvim__get_runtime);
   lua_setfield(lstate, -2, "nvim__get_runtime");
   lua_setfield(lstate, -2, "api");
-  lua_pop(lstate, 1);
+
+  lua_setglobal(lstate, "vim");
+
+  nlua_common_package_init(lstate);
+
+  lua_getglobal(lstate, "package");
+  lua_getfield(lstate, -1, "loaded");
+  lua_getglobal(lstate, "vim");
+  lua_setfield(lstate, -2, "vim");
+  lua_pop(lstate, 2);
 
   return lstate;
 }

src/nvim/lua/executor.h

@@ -38,5 +38,6 @@ typedef struct {
 #endif
 
 EXTERN nlua_ref_state_t *nlua_global_refs INIT(= NULL);
+EXTERN bool nlua_disable_preload INIT(= false);
 
 #endif  // NVIM_LUA_EXECUTOR_H

src/nvim/lua/vim.lua

@@ -36,16 +36,8 @@
 
 local vim = vim
 assert(vim)
-
-vim.inspect = package.loaded['vim.inspect']
 assert(vim.inspect)
 
-vim.filetype = package.loaded['vim.filetype']
-assert(vim.filetype)
-
-vim.keymap = package.loaded['vim.keymap']
-assert(vim.keymap)
-
 -- These are for loading runtime modules lazily since they aren't available in
 -- the nvim binary as specified in executor.c
 setmetatable(vim, {
@@ -53,6 +45,9 @@ setmetatable(vim, {
     if key == 'treesitter' then
       t.treesitter = require('vim.treesitter')
       return t.treesitter
+    elseif key == 'filetype' then
+      t.filetype = require('vim.filetype')
+      return t.filetype
     elseif key == 'F' then
       t.F = require('vim.F')
       return t.F
@@ -69,6 +64,9 @@ setmetatable(vim, {
     elseif key == 'diagnostic' then
       t.diagnostic = require('vim.diagnostic')
       return t.diagnostic
+    elseif key == 'keymap' then
+      t.keymap = require('vim.keymap')
+      return t.keymap
     elseif key == 'ui' then
       t.ui = require('vim.ui')
       return t.ui
@@ -662,4 +660,7 @@ function vim.pretty_print(...)
   return ...
 end
 
-return module
+
+require('vim._meta')
+
+return vim

src/nvim/main.c

@@ -253,12 +253,12 @@ int main(int argc, char **argv)
   // Check if we have an interactive window.
   check_and_set_isatty(&params);
 
-  nlua_init();
-
   // Process the command line arguments.  File names are put in the global
   // argument list "global_alist".
   command_line_scan(&params);
 
+  nlua_init();
+
   if (embedded_mode) {
     const char *err;
     if (!channel_from_stdio(true, CALLBACK_READER_INIT, &err)) {
@@ -918,6 +918,8 @@ static void command_line_scan(mparm_T *parmp)
           parmp->use_vimrc = "NONE";
           parmp->clean = true;
           set_option_value("shadafile", 0L, "NONE", 0);
+        } else if (STRNICMP(argv[0] + argv_idx, "luamod-dev", 9) == 0) {
+          nlua_disable_preload = true;
         } else {
           if (argv[0][argv_idx]) {
             mainerr(err_opt_unknown, argv[0]);
@@ -1990,6 +1992,8 @@ static void mainerr(const char *errstr, const char *str)
 /// Prints version information for "nvim -v" or "nvim --version".
 static void version(void)
 {
+  // TODO(bfred): not like this?
+  nlua_init();
   info_message = true;  // use mch_msg(), not mch_errmsg()
   list_version();
   msg_putchar('\n');

test/functional/lua/thread_spec.lua

@@ -102,6 +102,28 @@ describe('thread', function()
         print in thread                                   |
       ]])
     end)
+
+    it('vim.inspect', function()
+      exec_lua [[
+        local thread = vim.loop.new_thread(function()
+          print(vim.inspect({1,2}))
+        end)
+        vim.loop.thread_join(thread)
+      ]]
+
+      screen:expect([[
+        ^                                                  |
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        { 1, 2 }                                          |
+      ]])
+    end)
   end)
 
   describe('vim.*', function()

test/functional/lua/vim_spec.lua

@@ -23,9 +23,9 @@ local mkdir_p = helpers.mkdir_p
 local rmdir = helpers.rmdir
 local write_file = helpers.write_file
 
-before_each(clear)
 
 describe('lua stdlib', function()
+  before_each(clear)
   -- İ: `tolower("İ")` is `i` which has length 1 while `İ` itself has
   --    length 2 (in bytes).
   -- Ⱥ: `tolower("Ⱥ")` is `ⱥ` which has length 2 while `Ⱥ` itself has
@@ -2534,9 +2534,39 @@ describe('lua stdlib', function()
   end)
 end)
 
+describe('lua: builtin modules', function()
+  local function do_tests()
+    eq(2, exec_lua[[return vim.tbl_count {x=1,y=2}]])
+    eq('{ 10, "spam" }', exec_lua[[return vim.inspect {10, 'spam'}]])
+  end
+
+  it('works', function()
+    clear()
+    do_tests()
+  end)
+
+  it('works when disabled', function()
+    clear('--luamod-dev')
+    do_tests()
+  end)
+
+  it('works without runtime', function()
+    clear{env={VIMRUNTIME='fixtures/a'}}
+    do_tests()
+  end)
+
+
+  it('does not work when disabled without runtime', function()
+    clear{args={'--luamod-dev'}, env={VIMRUNTIME='fixtures/a'}}
+    -- error checking could be better here. just check that --luamod-dev
+    -- does anything at all by breaking with missing runtime..
+    eq(nil, exec_lua[[return vim.tbl_count {x=1,y=2}]])
+  end)
+end)
+
 describe('lua: require("mod") from packages', function()
   before_each(function()
-    command('set rtp+=test/functional/fixtures pp+=test/functional/fixtures')
+    clear('--cmd', 'set rtp+=test/functional/fixtures pp+=test/functional/fixtures')
   end)
 
   it('propagates syntax error', function()
@@ -2559,6 +2589,8 @@ describe('lua: require("mod") from packages', function()
 end)
 
 describe('vim.keymap', function()
+  before_each(clear)
+
   it('can make a mapping', function()
     eq(0, exec_lua [[
       GlobalCount = 0