treesitter: add standard &rtp/parser/ search path for parsers

ci/build.ps1

@@ -123,22 +123,6 @@ if (-not $NoTests) {
   npm.cmd install -g neovim
   Get-Command -CommandType Application neovim-node-host.cmd
   npm.cmd link neovim
-
-
-  $env:TREE_SITTER_DIR = $env:USERPROFILE + "\tree-sitter-build"
-  mkdir "$env:TREE_SITTER_DIR\bin"
-
-  $xbits = if ($bits -eq '32') {'x86'} else {'x64'}
-  Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/tree-sitter/tree-sitter/releases/download/0.15.9/tree-sitter-windows-$xbits.gz" -OutFile tree-sitter.exe.gz
-  C:\msys64\usr\bin\gzip -d tree-sitter.exe.gz
-
-  Invoke-WebRequest -UseBasicParsing -Uri "https://codeload.github.com/tree-sitter/tree-sitter-c/zip/v0.15.2" -OutFile tree_sitter_c.zip
-  Expand-Archive .\tree_sitter_c.zip -DestinationPath .
-  cd tree-sitter-c-0.15.2
-  ..\tree-sitter.exe test
-  if (-Not (Test-Path -PathType Leaf "$env:TREE_SITTER_DIR\bin\c.dll")) {
-    exit 1
-  }
 }
 
 if ($compiler -eq 'MSVC') {

ci/install.sh

@@ -27,29 +27,5 @@ nvm use 10
 npm install -g neovim
 npm link neovim
 
-echo "Install tree-sitter npm package"
-
-# FIXME
-# https://github.com/tree-sitter/tree-sitter/commit/e14e285a1087264a8c74a7c62fcaecc49db9d904
-# If queries added to tree-sitter-c, we can use latest tree-sitter-cli
-npm install -g tree-sitter-cli@v0.15.9
-
-echo "Install tree-sitter c parser"
-curl "https://codeload.github.com/tree-sitter/tree-sitter-c/tar.gz/v0.15.2" -o tree_sitter_c.tar.gz
-tar xf tree_sitter_c.tar.gz
-cd tree-sitter-c-0.15.2
-export TREE_SITTER_DIR=$HOME/tree-sitter-build/
-mkdir -p "$TREE_SITTER_DIR/bin"
-
-if [[ "$BUILD_32BIT" != "ON" ]]; then
-  # builds c parser in $HOME/tree-sitter-build/bin/c.(so|dylib)
-  tree-sitter test
-else
-  # no tree-sitter binary for 32bit linux, so fake it (no tree-sitter unit tests)
-  cd src/
-  gcc -m32 -o "$TREE_SITTER_DIR/bin/c.so" -shared parser.c -I.
-fi
-test -f "$TREE_SITTER_DIR/bin/c.so"
-
 sudo cpanm -n Neovim::Ext || cat "$HOME/.cpanm/build.log"
 perl -W -e 'use Neovim::Ext; print $Neovim::Ext::VERSION'

ci/run_tests.sh

@@ -19,7 +19,7 @@ exit_suite --continue
 
 source ~/.nvm/nvm.sh
 nvm use 10
-export TREE_SITTER_DIR=$HOME/tree-sitter-build/
+
 
 enter_suite tests
 

runtime/doc/lua.txt

@@ -494,10 +494,12 @@ VIM.TREESITTER						*lua-treesitter*
 Nvim integrates the tree-sitter library for incremental parsing of buffers.
 
 Currently Nvim does not provide the tree-sitter parsers, instead these must
-be built separately, for instance using the tree-sitter utility.
+be built separately, for instance using the tree-sitter utility. The only
-The parser is loaded into nvim using >
+exception is a C parser being included in official builds for testing
+purposes. Parsers are searched for as `parser/{lang}.*` in any 'runtimepath'
+directory. A parser can also be loaded manually using a full path: >
 
-    vim.treesitter.add_language("/path/to/c_parser.so", "c")
+    vim.treesitter.require_language("python", "/path/to/python.so")
 
 <Create a parser for a buffer and a given language (if another plugin uses the
 same buffer/language combination, it will be safely reused). Use >

runtime/lua/vim/treesitter.lua

@@ -31,8 +31,6 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_
 end
 
 local M = {
-  add_language=vim._ts_add_language,
-  inspect_language=vim._ts_inspect_language,
   parse_query = vim._ts_parse_query,
 }
 
@@ -45,12 +43,34 @@ setmetatable(M, {
    end
  })
 
-function M.create_parser(bufnr, ft, id)
+function M.require_language(lang, path)
+  if vim._ts_has_language(lang) then
+    return true
+  end
+  if path == nil then
+    local fname = 'parser/' .. lang .. '.*'
+    local paths = a.nvim_get_runtime_file(fname, false)
+    if #paths == 0 then
+      -- TODO(bfredl): help tag?
+      error("no parser for '"..lang.."' language")
+    end
+    path = paths[1]
+  end
+  vim._ts_add_language(path, lang)
+end
+
+function M.inspect_language(lang)
+  M.require_language(lang)
+  return vim._ts_inspect_language(lang)
+end
+
+function M.create_parser(bufnr, lang, id)
+  M.require_language(lang)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser)
+  local self = setmetatable({bufnr=bufnr, lang=lang, valid=false}, Parser)
-  self._parser = vim._create_ts_parser(ft)
+  self._parser = vim._create_ts_parser(lang)
   self.change_cbs = {}
   self:parse()
     -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
@@ -94,6 +114,7 @@ local Query = {}
 Query.__index = Query
 
 function M.parse_query(lang, query)
+  M.require_language(lang)
   local self = setmetatable({}, Query)
   self.query = vim._ts_parse_query(lang, query)
   self.info = self.query:inspect()

src/nvim/lua/executor.c

@@ -1025,9 +1025,12 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
   lua_pushcfunction(lstate, create_tslua_parser);
   lua_setfield(lstate, -2, "_create_ts_parser");
 
-  lua_pushcfunction(lstate, tslua_register_lang);
+  lua_pushcfunction(lstate, tslua_add_language);
   lua_setfield(lstate, -2, "_ts_add_language");
 
+  lua_pushcfunction(lstate, tslua_has_language);
+  lua_setfield(lstate, -2, "_ts_has_language");
+
   lua_pushcfunction(lstate, tslua_inspect_lang);
   lua_setfield(lstate, -2, "_ts_inspect_language");
 

src/nvim/lua/treesitter.c

@@ -119,7 +119,14 @@ void tslua_init(lua_State *L)
   build_meta(L, "treesitter_querycursor", querycursor_meta);
 }
 
-int tslua_register_lang(lua_State *L)
+int tslua_has_language(lua_State *L)
+{
+  const char *lang_name = luaL_checkstring(L, 1);
+  lua_pushboolean(L, pmap_has(cstr_t)(langs, lang_name));
+  return 1;
+}
+
+int tslua_add_language(lua_State *L)
 {
   if (lua_gettop(L) < 2 || !lua_isstring(L, 1) || !lua_isstring(L, 2)) {
     return luaL_error(L, "string expected");

test/functional/lua/treesitter_spec.lua

@@ -6,7 +6,6 @@ local clear = helpers.clear
 local eq = helpers.eq
 local insert = helpers.insert
 local exec_lua = helpers.exec_lua
-local iswin = helpers.iswin
 local feed = helpers.feed
 local pcall_err = helpers.pcall_err
 local matches = helpers.matches
@@ -16,37 +15,35 @@ before_each(clear)
 describe('treesitter API', function()
   -- error tests not requiring a parser library
   it('handles missing language', function()
-    eq('Error executing lua: .../treesitter.lua: no such language: borklang',
+    eq("Error executing lua: .../treesitter.lua: no parser for 'borklang' language",
        pcall_err(exec_lua, "parser = vim.treesitter.create_parser(0, 'borklang')"))
 
     -- actual message depends on platform
-    matches('Error executing lua: Failed to load parser: uv_dlopen: .+',
+    matches("Error executing lua: Failed to load parser: uv_dlopen: .+",
-       pcall_err(exec_lua, "parser = vim.treesitter.add_language('borkbork.so', 'borklang')"))
+       pcall_err(exec_lua, "parser = vim.treesitter.require_language('borklang', 'borkbork.so')"))
 
-    eq('Error executing lua: [string "<nvim>"]:1: no such language: borklang',
+    eq("Error executing lua: .../treesitter.lua: no parser for 'borklang' language",
        pcall_err(exec_lua, "parser = vim.treesitter.inspect_language('borklang')"))
   end)
 
 end)
 
 describe('treesitter API with C parser', function()
-  local ts_path = os.getenv("TREE_SITTER_DIR")
+  local function check_parser()
-
+    local status, msg = unpack(exec_lua([[ return {pcall(vim.treesitter.require_language, 'c')} ]]))
-  -- The tests after this requires an actual parser
+    if not status then
-  if ts_path == nil then
+      if helpers.isCI() then
-    it("works", function() pending("TREE_SITTER_PATH not set, skipping treesitter parser tests") end)
+        error("treesitter C parser not found, required on CI: " .. msg)
-    return
+      else
+        pending('no C parser, skipping')
+      end
+    end
+    return status
   end
 
-  before_each(function()
-    local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so')
-    exec_lua([[
-      local path = ...
-      vim.treesitter.add_language(path,'c')
-    ]], path)
-  end)
-
   it('parses buffer', function()
+    if not check_parser() then return end
+
     insert([[
       int main() {
         int x = 3;
@@ -138,6 +135,8 @@ void ui_refresh(void)
   ]]
 
   it('support query and iter by capture', function()
+    if not check_parser() then return end
+
     insert(test_text)
 
     local res = exec_lua([[
@@ -167,6 +166,8 @@ void ui_refresh(void)
   end)
 
   it('support query and iter by match', function()
+    if not check_parser() then return end
+
     insert(test_text)
 
     local res = exec_lua([[
@@ -198,6 +199,8 @@ void ui_refresh(void)
   end)
 
   it('supports highlighting', function()
+    if not check_parser() then return end
+
     local hl_text = [[
 /// Schedule Lua callback on main loop's event queue
 static int nlua_schedule(lua_State *const lstate)
@@ -357,41 +360,43 @@ static int nlua_schedule(lua_State *const lstate)
   end)
 
   it('inspects language', function()
-      local keys, fields, symbols = unpack(exec_lua([[
+    if not check_parser() then return end
-        local lang = vim.treesitter.inspect_language('c')
-        local keys, symbols = {}, {}
-        for k,_ in pairs(lang) do
-          keys[k] = true
-        end
 
-        -- symbols array can have "holes" and is thus not a valid msgpack array
+    local keys, fields, symbols = unpack(exec_lua([[
-        -- but we don't care about the numbers here (checked in the parser test)
+      local lang = vim.treesitter.inspect_language('c')
-        for _, v in pairs(lang.symbols) do
+      local keys, symbols = {}, {}
-          table.insert(symbols, v)
+      for k,_ in pairs(lang) do
-        end
+        keys[k] = true
-        return {keys, lang.fields, symbols}
-      ]]))
-
-      eq({fields=true, symbols=true}, keys)
-
-      local fset = {}
-      for _,f in pairs(fields) do
-        eq("string", type(f))
-        fset[f] = true
       end
-      eq(true, fset["directive"])
-      eq(true, fset["initializer"])
 
-      local has_named, has_anonymous
+      -- symbols array can have "holes" and is thus not a valid msgpack array
-      for _,s in pairs(symbols) do
+      -- but we don't care about the numbers here (checked in the parser test)
-        eq("string", type(s[1]))
+      for _, v in pairs(lang.symbols) do
-        eq("boolean", type(s[2]))
+        table.insert(symbols, v)
-        if s[1] == "for_statement" and s[2] == true then
-          has_named = true
-        elseif s[1] == "|=" and s[2] == false then
-          has_anonymous = true
-        end
       end
-      eq({true,true}, {has_named,has_anonymous})
+      return {keys, lang.fields, symbols}
+    ]]))
+
+    eq({fields=true, symbols=true}, keys)
+
+    local fset = {}
+    for _,f in pairs(fields) do
+      eq("string", type(f))
+      fset[f] = true
+    end
+    eq(true, fset["directive"])
+    eq(true, fset["initializer"])
+
+    local has_named, has_anonymous
+    for _,s in pairs(symbols) do
+      eq("string", type(s[1]))
+      eq("boolean", type(s[2]))
+      if s[1] == "for_statement" and s[2] == true then
+        has_named = true
+      elseif s[1] == "|=" and s[2] == false then
+        has_anonymous = true
+      end
+    end
+    eq({true,true}, {has_named,has_anonymous})
   end)
 end)