From 08b47b196493694b48010fc5b14a2fb3be0ffdd3 Mon Sep 17 00:00:00 2001 From: Ryan Patterson Date: Fri, 19 Jun 2026 17:51:30 +0800 Subject: [PATCH] test: wasm tree-sitter #40304 Problem: No CI coverage for wasm builds. Solution: Add a basic workflow that builds with ENABLE_WASMTIME and runs wasm-specific tests. --- .github/workflows/test_wasm.yml | 53 ++++++++++++ cmake.deps/cmake/BuildTreesitterParsers.cmake | 19 +++++ cmake.deps/deps.txt | 2 + scripts/bump_deps.lua | 38 ++++++++- test/functional/treesitter/wasm_spec.lua | 85 +++++++++++++++++++ 5 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test_wasm.yml create mode 100644 test/functional/treesitter/wasm_spec.lua diff --git a/.github/workflows/test_wasm.yml b/.github/workflows/test_wasm.yml new file mode 100644 index 0000000000..a4c7cfacd5 --- /dev/null +++ b/.github/workflows/test_wasm.yml @@ -0,0 +1,53 @@ +name: wasm +on: + push: + branches: + - 'master' + - 'release-[0-9]+.[0-9]+' + pull_request: + branches: + - 'master' + - 'release-[0-9]+.[0-9]+' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + ASAN_OPTIONS: detect_leaks=1:check_initialization_order=1:log_path=${{ github.workspace }}/build/log/asan:intercept_tls_get_addr=0 + BIN_DIR: ${{ github.workspace }}/bin + BUILD_DIR: ${{ github.workspace }}/build + INSTALL_PREFIX: ${{ github.workspace }}/nvim-install + LOG_DIR: ${{ github.workspace }}/build/log + NVIM_LOG_FILE: ${{ github.workspace }}/build/nvim.log + TSAN_OPTIONS: log_path=${{ github.workspace }}/build/log/tsan + VALGRIND_LOG: ${{ github.workspace }}/build/log/valgrind-%p.log + # TEST_FILE: test/functional/core/startup_spec.lua + # TEST_FILTER: foo + +jobs: + wasmtime: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + - uses: ./.github/actions/setup + with: + install_flags: "--test" + - name: Build + run: | + cmake -S cmake.deps --preset ci -D ENABLE_WASMTIME=ON -D USE_BUNDLED_TS_PARSERS=ON + cmake --build .deps + cmake --preset ci -D ENABLE_WASMTIME=ON + cmake --build build + - name: functionaltest + timeout-minutes: 10 + env: + TEST_FILE: test/functional/treesitter/wasm_spec.lua + run: cmake --build build --target functionaltest diff --git a/cmake.deps/cmake/BuildTreesitterParsers.cmake b/cmake.deps/cmake/BuildTreesitterParsers.cmake index 060447e6fe..f71f1335ab 100644 --- a/cmake.deps/cmake/BuildTreesitterParsers.cmake +++ b/cmake.deps/cmake/BuildTreesitterParsers.cmake @@ -32,3 +32,22 @@ foreach(lang c lua vim vimdoc query) BuildTSParser(LANG ${lang}) endforeach() BuildTSParser(LANG markdown CMAKE_FILE MarkdownParserCMakeLists.txt) + +if(USE_BUNDLED_TS_PARSERS AND ENABLE_WASMTIME) + if(DEPS_IGNORE_SHA) + set(_lua_wasm_hash "") + else() + set(_lua_wasm_hash "URL_HASH;SHA256=${TREESITTER_LUA_WASM_SHA256}") + endif() + ExternalProject_Add(treesitter_lua_wasm + DOWNLOAD_NO_PROGRESS TRUE + DOWNLOAD_NO_EXTRACT TRUE + URL ${TREESITTER_LUA_WASM_URL} + ${_lua_wasm_hash} + DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/treesitter_lua_wasm + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy + ${DEPS_DOWNLOAD_DIR}/treesitter_lua_wasm/tree-sitter-lua.wasm + ${DEPS_INSTALL_DIR}/lib/nvim/parser/lua.wasm) +endif() diff --git a/cmake.deps/deps.txt b/cmake.deps/deps.txt index 0e98843303..57d2f76acf 100644 --- a/cmake.deps/deps.txt +++ b/cmake.deps/deps.txt @@ -35,6 +35,8 @@ TREESITTER_C_URL https://github.com/tree-sitter/tree-sitter-c/archive/v0.24.2.ta TREESITTER_C_SHA256 2eeb4db31f8fa0865e45488503d13403923bcb485a1bdb637abff8c42dd97364 TREESITTER_LUA_URL https://github.com/tree-sitter-grammars/tree-sitter-lua/archive/v0.5.0.tar.gz TREESITTER_LUA_SHA256 cf01b93f4b61b96a6d27942cf28eeda4cbce7d503c3bef773a8930b3d778a2d9 +TREESITTER_LUA_WASM_URL https://github.com/tree-sitter-grammars/tree-sitter-lua/releases/download/v0.5.0/tree-sitter-lua.wasm +TREESITTER_LUA_WASM_SHA256 df08a1704e504c70b8dba4a3e6f8e0c99a4fb94e1b1693d2969f53141d09f0d4 TREESITTER_VIM_URL https://github.com/tree-sitter-grammars/tree-sitter-vim/archive/v0.8.1.tar.gz TREESITTER_VIM_SHA256 93cafb9a0269420362454ace725a118ff1c3e08dcdfdc228aa86334b54d53c2a TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v4.1.0.tar.gz diff --git a/scripts/bump_deps.lua b/scripts/bump_deps.lua index 3927492dcc..df655406c3 100755 --- a/scripts/bump_deps.lua +++ b/scripts/bump_deps.lua @@ -78,8 +78,9 @@ local function get_archive_info(repo, ref) local temp_dir = os.getenv('TMPDIR') or os.getenv('TEMP') or '/tmp' local archive_name = ref .. '.tar.gz' - local archive_path = temp_dir .. '/' .. archive_name - local archive_url = 'https://github.com/' .. repo .. '/archive/' .. archive_name + local archive_path = vim.fs.joinpath(temp_dir, archive_name) + local archive_url = ('https://github.com/%s/archive/%s'):format(repo, archive_name) + print(vim.inspect(archive_path), vim.inspect(archive_url)) run_die( { 'curl', '-sfL', archive_url, '-o', archive_path }, @@ -90,10 +91,32 @@ local function get_archive_info(repo, ref) vim.fn.executable('sha256sum') == 1 and { 'sha256sum', archive_path } or { 'shasum', '-a', '256', archive_path } ) - local archive_sha = run(shacmd):gmatch('%w+')() + local archive_sha = run_die(shacmd):gmatch('%w+')() return { url = archive_url, sha = archive_sha } end +--- @param repo string +--- @param ref string +--- @param filename string +local function get_release_artifact_info(repo, ref, filename) + local temp_dir = os.getenv('TMPDIR') or os.getenv('TEMP') or '/tmp' + + local artifact_path = vim.fs.joinpath(temp_dir, filename) + local artifact_url = ('https://github.com/%s/releases/download/%s/%s'):format(repo, ref, filename) + + run_die( + { 'curl', '-sfL', artifact_url, '-o', artifact_path }, + 'Failed to download release artifact from GitHub' + ) + + local shacmd = ( + vim.fn.executable('sha256sum') == 1 and { 'sha256sum', artifact_path } + or { 'shasum', '-a', '256', artifact_path } + ) + local artifact_sha = run_die(shacmd):gmatch('%w+')() + return { url = artifact_url, sha = artifact_sha } +end + local function get_gh_commit_sha(repo, ref) local full_repo = string.format('https://github.com/%s.git', repo) local tag_exists = run_die({ 'git', 'ls-remote', full_repo, 'refs/tags/' .. ref }) ~= '' @@ -151,6 +174,15 @@ local function ref(name, _ref) print('Updating ' .. name .. ' to ' .. archive.url .. '\n') update_deps_file(symbol_upper, 'URL', archive.url:gsub('/', '\\/')) update_deps_file(symbol_upper, 'SHA256', archive.sha) + + -- Keep wasm version in sync with native version + if name == 'tree-sitter-lua' then + local wasm = get_release_artifact_info(repo, _ref, 'tree-sitter-lua.wasm') + print('Updating ' .. name .. ' WASM to ' .. wasm.url .. '\n') + update_deps_file(symbol_upper .. '_WASM', 'URL', wasm.url:gsub('/', '\\/')) + update_deps_file(symbol_upper .. '_WASM', 'SHA256', wasm.sha) + end + run_die({ 'git', 'add', deps_file }) local zig = zig_mode[symbol] diff --git a/test/functional/treesitter/wasm_spec.lua b/test/functional/treesitter/wasm_spec.lua new file mode 100644 index 0000000000..a3014133a9 --- /dev/null +++ b/test/functional/treesitter/wasm_spec.lua @@ -0,0 +1,85 @@ +local t = require('test.testutil') +local n = require('test.functional.testnvim')() + +local clear = n.clear +local command = n.command +local eq = t.eq +local exec_lua = n.exec_lua +local insert = n.insert +local skip = t.skip + +local lua_text = [[ +local M = {} + +-- returns the sum of two numbers +local function add(a, b) + return a + b +end + +M.add = add +return M +]] + +describe('treesitter WASM parser', function() + before_each(clear) + + it('lua WASM parser produces same highlight captures as native parser', function() + skip( + exec_lua('return vim._ts_add_language_from_wasm == nil'), + 'N/A ENABLE_WASMTIME not enabled' + ) + + local wasm_files = exec_lua(function() + return vim.api.nvim_get_runtime_file('parser/lua.wasm', true) + end) + assert( + #wasm_files > 0, + 'lua.wasm not found in runtimepath (rebuild with ENABLE_WASMTIME=ON USE_BUNDLED_TS_PARSERS=ON)' + ) + + local wasm_path = wasm_files[1] + + local query_str = exec_lua(function() + local files = vim.api.nvim_get_runtime_file('queries/lua/highlights.scm', false) + assert(#files > 0, 'queries/lua/highlights.scm not found') + local f = assert(io.open(files[1])) + local s = f:read('*a') + f:close() + return s + end) + + -- Buffer 1: parse with the native lua parser. + insert(lua_text) + local native_captures = exec_lua(function(qstr) + local query = vim.treesitter.query.parse('lua', qstr) + local parser = vim.treesitter.get_parser(0, 'lua') + local tree = parser:parse()[1] + local result = {} + for id, node in query:iter_captures(tree:root(), 0) do + local r = { node:range() } + table.insert(result, { query.captures[id], r[1], r[2], r[3], r[4] }) + end + return result + end, query_str) + + -- Buffer 2: same content, but the lua language loaded from WASM. + command('new') + insert(lua_text) + local wasm_captures = exec_lua(function(path, qstr) + -- Remove the native registration so the WASM one can take its place. + vim._ts_remove_language('lua') + vim._ts_add_language_from_wasm(path, 'lua') + local query = vim.treesitter.query.parse('lua', qstr) + local parser = vim.treesitter.get_parser(0, 'lua') + local tree = parser:parse()[1] + local result = {} + for id, node in query:iter_captures(tree:root(), 0) do + local r = { node:range() } + table.insert(result, { query.captures[id], r[1], r[2], r[3], r[4] }) + end + return result + end, wasm_path, query_str) + + eq(native_captures, wasm_captures) + end) +end)