feat: filetype.lua (#16600)

Adds a new vim.filetype module that provides support for filetype detection in
Lua.

runtime/doc/filetype.txt

@@ -24,12 +24,21 @@ Each time a new or existing file is edited, Vim will try to recognize the type
 of the file and set the 'filetype' option.  This will trigger the FileType
 event, which can be used to set the syntax highlighting, set options, etc.
 
-Detail: The ":filetype on" command will load this file:
+Detail: The ":filetype on" command will load these files:
+		$VIMRUNTIME/filetype.lua
 		$VIMRUNTIME/filetype.vim
-	This file is a Vim script that defines autocommands for the
+	filetype.lua creates an autocommand that fires for all BufNewFile and
-	BufNewFile and BufRead events.  If the file type is not found by the
+	BufRead events. It tries to detect the filetype based off of the
-	name, the file $VIMRUNTIME/scripts.vim is used to detect it from the
+	file's extension or name.
-	contents of the file.
+
+	filetype.vim is a Vim script that defines autocommands for the
+	BufNewFile and BufRead events. In contrast to filetype.lua, this
+	file creates separate BufNewFile and BufRead events for each filetype
+	pattern.
+
+	If the file type is not found by the name, the file
+	$VIMRUNTIME/scripts.vim is used to detect it from the contents of the
+	file.
 	When the GUI is running or will start soon, the |menu.vim| script is
 	also sourced.  See |'go-M'| about avoiding that.
 
@@ -149,9 +158,10 @@ is used.  The default value is set like this: >
 This means that the contents of compressed files are not inspected.
 
 							*new-filetype*
-If a file type that you want to use is not detected yet, there are four ways
+If a file type that you want to use is not detected yet, there are a few ways
-to add it.  In any way, it's better not to modify the $VIMRUNTIME/filetype.vim
+to add it.  In any way, it's better not to modify the $VIMRUNTIME/filetype.lua
-file.  It will be overwritten when installing a new version of Vim.
+or $VIMRUNTIME/filetype.vim files.  They will be overwritten when installing a
+new version of Nvim.
 
 A. If you want to overrule all default file type checks.
    This works by writing one file for each filetype.  The disadvantage is that
@@ -191,7 +201,7 @@ B. If you want to detect your file after the default file type checks.
 	au BufRead,BufNewFile *		if &ft == 'pascal' | set ft=mypascal
 								       | endif
 
-C. If your file type can be detected by the file name.
+C. If your file type can be detected by the file name or extension.
    1. Create your user runtime directory.  You would normally use the first
       item of the 'runtimepath' option.  Example for Unix: >
 	:!mkdir -p ~/.config/nvim
@@ -206,9 +216,38 @@ C. If your file type can be detected by the file name.
 	  au! BufRead,BufNewFile *.mine		setfiletype mine
 	  au! BufRead,BufNewFile *.xyz		setfiletype drawing
 	augroup END
-<     Write this file as "filetype.vim" in your user runtime directory.  For
+<
+      Write this file as "filetype.vim" in your user runtime directory.  For
       example, for Unix: >
 	:w ~/.config/nvim/filetype.vim
+<
+      Alternatively, create a file called "filetype.lua" that adds new
+      filetypes.
+      Example: >
+	vim.filetype.add({
+	  extension = {
+	    foo = "fooscript",
+	  },
+	  filename = {
+	    [".foorc"] = "foorc",
+	  },
+	  pattern = {
+	    [".*/etc/foo/.*%.conf"] = "foorc",
+	  },
+	})
+<
+      See |vim.filetype.add()|.
+						     *g:do_filetype_lua*
+      For now, Lua filetype detection is opt-in. You can enable it by adding
+      the following to your |init.vim|: >
+        let g:do_filetype_lua = 1
+<						     *g:did_load_filetypes*
+      In either case, the builtin filetype detection provided by Nvim can be
+      disabled by setting the did_load_filetypes global variable. If this
+      variable exists, $VIMRUNTIME/filetype.vim will not run.
+      Example: >
+	" Disable filetype.vim
+	let g:did_load_filetypes = 1
 
 <  3. To use the new filetype detection you must restart Vim.
 
@@ -245,9 +284,9 @@ D. If your filetype can only be detected by inspecting the contents of the
    $VIMRUNTIME/scripts.vim.
 
 						*remove-filetype*
-If a file type is detected that is wrong for you, install a filetype.vim or
+If a file type is detected that is wrong for you, install a filetype.lua,
-scripts.vim to catch it (see above).  You can set 'filetype' to a non-existing
+filetype.vim or scripts.vim to catch it (see above).  You can set 'filetype' to
-name to avoid that it will be set later anyway: >
+a non-existing name to avoid that it will be set later anyway: >
 	:set filetype=ignored
 
 If you are setting up a system with many users, and you don't want each user

runtime/doc/lua.txt

@@ -1780,4 +1780,71 @@ select({items}, {opts}, {on_choice})                         *vim.ui.select()*
                                  1-based index of `item` within `item` . `nil`
                                  if the user aborted the dialog.
 
+
+==============================================================================
+Lua module: filetype                                            *lua-filetype*
+
+add({filetypes})                                          *vim.filetype.add()*
+                Add new filetype mappings.
+
+                Filetype mappings can be added either by extension or by
+                filename (either the "tail" or the full file path). The full
+                file path is checked first, followed by the file name. If a
+                match is not found using the filename, then the filename is
+                matched against the list of patterns (sorted by priority)
+                until a match is found. Lastly, if pattern matching does not
+                find a filetype, then the file extension is used.
+
+                The filetype can be either a string (in which case it is used
+                as the filetype directly) or a function. If a function, it
+                takes the full path and buffer number of the file as arguments
+                (along with captures from the matched pattern, if any) and
+                should return a string that will be used as the buffer's
+                filetype.
+
+                Filename patterns can specify an optional priority to resolve
+                cases when a file path matches multiple patterns. Higher
+                priorities are matched first. When omitted, the priority
+                defaults to 0.
+
+                See $VIMRUNTIME/lua/vim/filetype.lua for more examples.
+
+                Note that Lua filetype detection is only enabled when
+                |g:do_filetype_lua| is set to 1.
+
+                Example: >
+
+                  vim.filetype.add({
+                    extension = {
+                      foo = "fooscript",
+                      bar = function(path, bufnr)
+                        if some_condition() then
+                          return "barscript"
+                        end
+                        return "bar"
+                      end,
+                    },
+                    filename = {
+                      [".foorc"] = "toml",
+                      ["/etc/foo/config"] = "toml",
+                    },
+                    pattern = {
+                      [".*‍/etc/foo/.*"] = "fooscript",
+                      -- Using an optional priority
+                      [".*‍/etc/foo/.*%.conf"] = { "dosini", { priority = 10 } },
+                      ["README.(%a+)$"] = function(path, bufnr, ext)
+                        if ext == "md" then
+                          return "markdown"
+                        elseif ext == "rst" then
+                          return "rst"
+                        end
+                      end,
+                    },
+                  })
+<
+
+                Parameters: ~
+                    {filetypes}  table A table containing new filetype maps
+                                 (see example).
+
  vim:tw=78:ts=8:ft=help:norl:

runtime/filetype.lua

@@ -0,0 +1,22 @@
+if vim.g.did_load_filetypes and vim.g.did_load_filetypes ~= 0 then
+  return
+end
+
+-- For now, make this opt-in with a global variable
+if vim.g.do_filetype_lua ~= 1 then
+  return
+end
+
+vim.cmd [[
+augroup filetypedetect
+au BufRead,BufNewFile * call v:lua.vim.filetype.match(str2nr(expand('<abuf>')))
+
+" These *must* be sourced after the autocommand above is created
+runtime! ftdetect/*.vim
+runtime! ftdetect/*.lua
+
+" Set a marker so that the ftdetect scripts are not sourced a second time by filetype.vim
+let g:did_load_ftdetect = 1
+
+augroup END
+]]

runtime/filetype.vim

@@ -2407,10 +2407,12 @@ au BufNewFile,BufRead *.txt
         \|   setf text
         \| endif       
 
-" Use the filetype detect plugins.  They may overrule any of the previously
+if !exists('g:did_load_ftdetect')
-" detected filetypes.
+  " Use the filetype detect plugins.  They may overrule any of the previously
-runtime! ftdetect/*.vim
+  " detected filetypes.
-runtime! ftdetect/*.lua
+  runtime! ftdetect/*.vim
+  runtime! ftdetect/*.lua
+endif
 
 " NOTE: The above command could have ended the filetypedetect autocmd group
 " and started another one. Let's make sure it has ended to get to a consistent

scripts/gen_filetype.lua

@@ -0,0 +1,201 @@
+local do_not_run = true
+if do_not_run then
+  print([[
+    This script was used to bootstrap the filetype patterns in runtime/lua/vim/filetype.lua. It
+    should no longer be used except for testing purposes. New filetypes, or changes to existing
+    filetypes, should be ported manually as part of the vim-patch process.
+  ]])
+  return
+end
+
+local filetype_vim = "runtime/filetype.vim"
+local filetype_lua = "runtime/lua/vim/filetype.lua"
+
+local keywords = {
+  ["for"] = true,
+  ["or"] = true,
+  ["and"] = true,
+  ["end"] = true,
+  ["do"] = true,
+  ["if"] = true,
+  ["while"] = true,
+  ["repeat"] = true,
+}
+
+local sections = {
+  extension = { str = {}, func = {} },
+  filename = { str = {}, func = {} },
+  pattern = { str = {}, func = {} },
+}
+
+local specialchars = "%*%?\\%$%[%]%{%}"
+
+local function add_pattern(pat, ft)
+  local ok = true
+
+  -- Patterns that start or end with { or } confuse splitting on commas and make parsing harder, so just skip those
+  if not string.find(pat, "^%{") and not string.find(pat, "%}$") then
+    for part in string.gmatch(pat, "[^,]+") do
+      if not string.find(part, "[" .. specialchars .. "]") then
+        if type(ft) == "string" then
+          sections.filename.str[part] = ft
+        else
+          sections.filename.func[part] = ft
+        end
+      elseif string.match(part, "^%*%.[^%./" .. specialchars .. "]+$") then
+        if type(ft) == "string" then
+          sections.extension.str[part:sub(3)] = ft
+        else
+          sections.extension.func[part:sub(3)] = ft
+        end
+      else
+        if string.match(part, "^%*/[^" .. specialchars .. "]+$") then
+          -- For patterns matching */some/pattern we want to easily match files
+          -- with path /some/pattern, so include those in filename detection
+          if type(ft) == "string" then
+            sections.filename.str[part:sub(2)] = ft
+          else
+            sections.filename.func[part:sub(2)] = ft
+          end
+        end
+
+        if string.find(part, "^[%w-_.*?%[%]/]+$") then
+          local p = part:gsub("%.", "%%."):gsub("%*", ".*"):gsub("%?", ".")
+          -- Insert into array to maintain order rather than setting
+          -- key-value directly
+          if type(ft) == "string" then
+            sections.pattern.str[p] = ft
+          else
+            sections.pattern.func[p] = ft
+          end
+        else
+          ok = false
+        end
+      end
+    end
+  end
+
+  return ok
+end
+
+local function parse_line(line)
+  local pat, ft
+  pat, ft = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+setf%s+(%S+)")
+  if pat then
+    return add_pattern(pat, ft)
+  else
+    local func
+    pat, func = line:match("^%s*au%a* Buf[%a,]+%s+(%S+)%s+call%s+(%S+)")
+    if pat then
+      return add_pattern(pat, function() return func end)
+    end
+  end
+end
+
+local unparsed = {}
+local full_line
+for line in io.lines(filetype_vim) do
+  local cont = string.match(line, "^%s*\\%s*(.*)$")
+  if cont then
+    full_line = full_line .. " " .. cont
+  else
+    if full_line then
+      if not parse_line(full_line) and string.find(full_line, "^%s*au%a* Buf") then
+        table.insert(unparsed, full_line)
+      end
+    end
+    full_line = line
+  end
+end
+
+if #unparsed > 0 then
+  print("Failed to parse the following patterns:")
+  for _, v in ipairs(unparsed) do
+    print(v)
+  end
+end
+
+local function add_item(indent, key, ft)
+  if type(ft) == "string" then
+    if string.find(key, "%A") or keywords[key] then
+      key = string.format("[\"%s\"]", key)
+    end
+    return string.format([[%s%s = "%s",]], indent, key, ft)
+  elseif type(ft) == "function" then
+    local func = ft()
+    if string.find(key, "%A") or keywords[key] then
+      key = string.format("[\"%s\"]", key)
+    end
+    -- Right now only a single argument is supported, which covers
+    -- everything in filetype.vim as of this writing
+    local arg = string.match(func, "%((.*)%)$")
+    func = string.gsub(func, "%(.*$", "")
+    if arg == "" then
+      -- Function with no arguments, call the function directly
+      return string.format([[%s%s = function() vim.fn["%s"]() end,]], indent, key, func)
+    elseif string.match(arg, [[^(["']).*%1$]]) then
+      -- String argument
+      if func == "s:StarSetf" then
+        return string.format([[%s%s = starsetf(%s),]], indent, key, arg)
+      else
+        return string.format([[%s%s = function() vim.fn["%s"](%s) end,]], indent, key, func, arg)
+      end
+    elseif string.find(arg, "%(") then
+      -- Function argument
+      return string.format([[%s%s = function() vim.fn["%s"](vim.fn.%s) end,]], indent, key, func, arg)
+    else
+      assert(false, arg)
+    end
+  end
+end
+
+do
+  local lines = {}
+  local start = false
+  for line in io.lines(filetype_lua) do
+    if line:match("^%s+-- END [A-Z]+$") then
+      start = false
+    end
+
+    if not start then
+      table.insert(lines, line)
+    end
+
+    local indent, section = line:match("^(%s+)-- BEGIN ([A-Z]+)$")
+    if section then
+      start = true
+      local t = sections[string.lower(section)]
+
+      local sorted = {}
+      for k, v in pairs(t.str) do
+        table.insert(sorted, {[k] = v})
+      end
+
+      table.sort(sorted, function(a, b)
+        return a[next(a)] < b[next(b)]
+      end)
+
+      for _, v in ipairs(sorted) do
+        local k = next(v)
+        table.insert(lines, add_item(indent, k, v[k]))
+      end
+
+      sorted = {}
+      for k, v in pairs(t.func) do
+        table.insert(sorted, {[k] = v})
+      end
+
+      table.sort(sorted, function(a, b)
+        return next(a) < next(b)
+      end)
+
+      for _, v in ipairs(sorted) do
+        local k = next(v)
+        table.insert(lines, add_item(indent, k, v[k]))
+      end
+    end
+  end
+  local f = io.open(filetype_lua, "w")
+  f:write(table.concat(lines, "\n") .. "\n")
+  f:close()
+end

scripts/gen_vimdoc.py

@@ -128,12 +128,14 @@ CONFIG = {
             'shared.lua',
             'uri.lua',
             'ui.lua',
+            'filetype.lua',
         ],
         'files': ' '.join([
             os.path.join(base_dir, 'src/nvim/lua/vim.lua'),
             os.path.join(base_dir, 'runtime/lua/vim/shared.lua'),
             os.path.join(base_dir, 'runtime/lua/vim/uri.lua'),
             os.path.join(base_dir, 'runtime/lua/vim/ui.lua'),
+            os.path.join(base_dir, 'runtime/lua/vim/filetype.lua'),
         ]),
         'file_patterns': '*.lua',
         'fn_name_prefix': '',
@@ -148,6 +150,7 @@ CONFIG = {
             'shared': 'vim',
             'uri': 'vim',
             'ui': 'vim.ui',
+            'filetype': 'vim.filetype',
         },
         'append_only': [
             'shared.lua',

src/nvim/CMakeLists.txt

@@ -62,6 +62,7 @@ set(LUA_SHARED_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/shared.lua)
 set(LUA_INSPECT_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/inspect.lua)
 set(LUA_F_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/F.lua)
 set(LUA_META_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/_meta.lua)
+set(LUA_FILETYPE_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/runtime/lua/vim/filetype.lua)
 set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
 set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json)
 set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint")
@@ -334,6 +335,7 @@ add_custom_command(
       ${LUA_INSPECT_MODULE_SOURCE} inspect_module
       ${LUA_F_MODULE_SOURCE} lua_F_module
       ${LUA_META_MODULE_SOURCE} lua_meta_module
+      ${LUA_FILETYPE_MODULE_SOURCE} lua_filetype_module
   DEPENDS
     ${CHAR_BLOB_GENERATOR}
     ${LUA_VIM_MODULE_SOURCE}
@@ -341,6 +343,7 @@ add_custom_command(
     ${LUA_INSPECT_MODULE_SOURCE}
     ${LUA_F_MODULE_SOURCE}
     ${LUA_META_MODULE_SOURCE}
+    ${LUA_FILETYPE_MODULE_SOURCE}
   VERBATIM
 )
 

src/nvim/ex_docmd.c

@@ -9551,15 +9551,18 @@ static void ex_filetype(exarg_T *eap)
 void filetype_maybe_enable(void)
 {
   if (filetype_detect == kNone) {
-    source_runtime(FILETYPE_FILE, true);
+    // Normally .vim files are sourced before .lua files when both are
+    // supported, but we reverse the order here because we want the Lua
+    // autocommand to be defined first so that it runs first
+    source_runtime(FILETYPE_FILE, DIP_ALL);
     filetype_detect = kTrue;
   }
   if (filetype_plugin == kNone) {
-    source_runtime(FTPLUGIN_FILE, true);
+    source_runtime(FTPLUGIN_FILE, DIP_ALL);
     filetype_plugin = kTrue;
   }
   if (filetype_indent == kNone) {
-    source_runtime(INDENT_FILE, true);
+    source_runtime(INDENT_FILE, DIP_ALL);
     filetype_indent = kTrue;
   }
 }

src/nvim/globals.h

@@ -28,7 +28,7 @@
 #endif
 
 #ifndef FILETYPE_FILE
-# define FILETYPE_FILE  "filetype.vim"
+# define FILETYPE_FILE "filetype.lua filetype.vim"
 #endif
 
 #ifndef FTPLUGIN_FILE

src/nvim/lua/executor.c

@@ -434,6 +434,15 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
     // [package, loaded, module]
     lua_setfield(lstate, -2, "vim.F");  // [package, loaded]
 
+    code = (char *)&lua_filetype_module[0];
+    if (luaL_loadbuffer(lstate, code, sizeof(lua_filetype_module) - 1, "@vim/filetype.lua")
+        || nlua_pcall(lstate, 0, 1)) {
+      nlua_error(lstate, _("E5106: Error while creating vim.filetype module: %.*s"));
+      return 1;
+    }
+    // [package, loaded, module]
+    lua_setfield(lstate, -2, "vim.filetype");  // [package, loaded]
+
     lua_pop(lstate, 2);  // []
   }
 

src/nvim/lua/vim.lua

@@ -40,6 +40,9 @@ assert(vim)
 vim.inspect = package.loaded['vim.inspect']
 assert(vim.inspect)
 
+vim.filetype = package.loaded['vim.filetype']
+assert(vim.filetype)
+
 local pathtrails = {}
 vim._so_trails = {}
 for s in  (package.cpath..';'):gmatch('[^;]*;') do

test/functional/lua/filetype_spec.lua

@@ -0,0 +1,100 @@
+local helpers = require('test.functional.helpers')(after_each)
+local exec_lua = helpers.exec_lua
+local eq = helpers.eq
+local clear = helpers.clear
+
+describe('vim.filetype', function()
+  before_each(function()
+    clear()
+
+    exec_lua [[
+      local bufnr = vim.api.nvim_create_buf(true, false)
+      vim.api.nvim_set_current_buf(bufnr)
+
+    ]]
+  end)
+
+  it('works with extensions', function()
+    eq('radicalscript', exec_lua [[
+      vim.filetype.add({
+        extension = {
+          rs = 'radicalscript',
+        },
+      })
+      vim.api.nvim_buf_set_name(0, '/home/user/src/main.rs')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+  end)
+
+  it('prioritizes filenames over extensions', function()
+    eq('somethingelse', exec_lua [[
+      vim.filetype.add({
+        extension = {
+          rs = 'radicalscript',
+        },
+        filename = {
+          ['main.rs'] = 'somethingelse',
+        },
+      })
+      vim.api.nvim_buf_set_name(0, '/home/usr/src/main.rs')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+  end)
+
+  it('works with filenames', function()
+    eq('nim', exec_lua [[
+      vim.filetype.add({
+        filename = {
+          ['s_O_m_e_F_i_l_e'] = 'nim',
+        },
+      })
+      vim.api.nvim_buf_set_name(0, '/home/user/src/s_O_m_e_F_i_l_e')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+
+    eq('dosini', exec_lua [[
+      vim.filetype.add({
+        filename = {
+          ['config'] = 'toml',
+          ['~/.config/fun/config'] = 'dosini',
+        },
+      })
+      vim.api.nvim_buf_set_name(0, '~/.config/fun/config')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+  end)
+
+  it('works with patterns', function()
+    eq('markdown', exec_lua [[
+      vim.filetype.add({
+        pattern = {
+          ['~/blog/.*%.txt'] = 'markdown',
+        }
+      })
+      vim.api.nvim_buf_set_name(0, '~/blog/why_neovim_is_awesome.txt')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+  end)
+
+  it('works with functions', function()
+    eq('foss', exec_lua [[
+      vim.filetype.add({
+        pattern = {
+          ["relevant_to_(%a+)"] = function(path, bufnr, capture)
+            if capture == "me" then
+              return "foss"
+            end
+          end,
+        }
+      })
+      vim.api.nvim_buf_set_name(0, 'relevant_to_me')
+      vim.filetype.match(0)
+      return vim.bo.filetype
+    ]])
+  end)
+end)