refactor: rewrite python provider in lua

runtime/autoload/provider/python3.vim

@@ -1,45 +1,15 @@
-" The Python3 provider uses a Python3 host to emulate an environment for running
-" python3 plugins. :help provider
-"
-" Associating the plugin with the Python3 host is the first step because
-" plugins will be passed as command-line arguments
-
 if exists('g:loaded_python3_provider')
   finish
 endif
-let [s:prog, s:err] = provider#pythonx#Detect(3)
-let g:loaded_python3_provider = empty(s:prog) ? 1 : 2
-
-function! provider#python3#Prog() abort
-  return s:prog
-endfunction
-
-function! provider#python3#Error() abort
-  return s:err
-endfunction
-
-" The Python3 provider plugin will run in a separate instance of the Python3
-" host.
-call remote#host#RegisterClone('legacy-python3-provider', 'python3')
-call remote#host#RegisterPlugin('legacy-python3-provider', 'script_host.py', [])
 
 function! provider#python3#Call(method, args) abort
-  if s:err != ''
+  return v:lua.require'vim.provider.python'.call(a:method, a:args)
-    return
-  endif
-  if !exists('s:host')
-    let s:rpcrequest = function('rpcrequest')
-
-    " Ensure that we can load the Python3 host before bootstrapping
-    try
-      let s:host = remote#host#Require('legacy-python3-provider')
-    catch
-      let s:err = v:exception
-      echohl WarningMsg
-      echomsg v:exception
-      echohl None
-      return
-    endtry
-  endif
-  return call(s:rpcrequest, insert(insert(a:args, 'python_'.a:method), s:host))
 endfunction
+
+function! provider#python3#Require(host) abort
+  return v:lua.require'vim.provider.python'.require(a:host)
+endfunction
+
+let s:prog = v:lua.require'vim.provider.python'.detect_by_module('neovim')
+let g:loaded_python3_provider = empty(s:prog) ? 1 : 2
+call v:lua.require'vim.provider.python'.start()

runtime/autoload/provider/pythonx.vim

@@ -1,112 +0,0 @@
-" The Python provider helper
-if exists('s:loaded_pythonx_provider')
-  finish
-endif
-
-let s:loaded_pythonx_provider = 1
-
-function! provider#pythonx#Require(host) abort
-  " Python host arguments
-  let prog = provider#python3#Prog()
-  let args = [prog, '-c', 'import sys; sys.path = [p for p in sys.path if p != ""]; import neovim; neovim.start_host()']
-
-
-  " Collect registered Python plugins into args
-  let python_plugins = remote#host#PluginsForHost(a:host.name)
-  for plugin in python_plugins
-    call add(args, plugin.path)
-  endfor
-
-  return provider#Poll(args, a:host.orig_name, '$NVIM_PYTHON_LOG_FILE', {'overlapped': v:true})
-endfunction
-
-function! s:get_python_executable_from_host_var(major_version) abort
-  return expand(get(g:, 'python'.(a:major_version == 3 ? '3' : execute("throw 'unsupported'")).'_host_prog', ''), v:true)
-endfunction
-
-function! s:get_python_candidates(major_version) abort
-  return {
-        \ 3: ['python3', 'python3.12', 'python3.11', 'python3.10', 'python3.9', 'python3.8', 'python3.7', 'python']
-        \ }[a:major_version]
-endfunction
-
-" Returns [path_to_python_executable, error_message]
-function! provider#pythonx#Detect(major_version) abort
-  return provider#pythonx#DetectByModule('neovim', a:major_version)
-endfunction
-
-" Returns [path_to_python_executable, error_message]
-function! provider#pythonx#DetectByModule(module, major_version) abort
-  let python_exe = s:get_python_executable_from_host_var(a:major_version)
-
-  if !empty(python_exe)
-    return [exepath(expand(python_exe, v:true)), '']
-  endif
-
-  let candidates = s:get_python_candidates(a:major_version)
-  let errors = []
-
-  for exe in candidates
-    let [result, error] = provider#pythonx#CheckForModule(exe, a:module, a:major_version)
-    if result
-      return [exe, error]
-    endif
-    " Accumulate errors in case we don't find any suitable Python executable.
-    call add(errors, error)
-  endfor
-
-  " No suitable Python executable found.
-  return ['', 'Could not load Python '.a:major_version.":\n".join(errors, "\n")]
-endfunction
-
-" Returns array: [prog_exitcode, prog_version]
-function! s:import_module(prog, module) abort
-  let prog_version = system([a:prog, '-W', 'ignore', '-c', printf(
-        \ 'import sys, importlib.util; ' .
-        \ 'sys.path = [p for p in sys.path if p != ""]; ' .
-        \ 'sys.stdout.write(str(sys.version_info[0]) + "." + str(sys.version_info[1])); ' .
-        \ 'sys.exit(2 * int(importlib.util.find_spec("%s") is None))',
-        \ a:module)])
-  return [v:shell_error, prog_version]
-endfunction
-
-" Returns array: [was_success, error_message]
-function! provider#pythonx#CheckForModule(prog, module, major_version) abort
-  let prog_path = exepath(a:prog)
-  if prog_path ==# ''
-    return [0, a:prog . ' not found in search path or not executable.']
-  endif
-
-  let min_version = '3.7'
-
-  " Try to load module, and output Python version.
-  " Exit codes:
-  "   0  module can be loaded.
-  "   2  module cannot be loaded.
-  "   Otherwise something else went wrong (e.g. 1 or 127).
-  let [prog_exitcode, prog_version] = s:import_module(a:prog, a:module)
-
-  if prog_exitcode == 2 || prog_exitcode == 0
-    " Check version only for expected return codes.
-    if prog_version !~ '^' . a:major_version
-      return [0, prog_path . ' is Python ' . prog_version . ' and cannot provide Python '
-            \ . a:major_version . '.']
-    elseif prog_version =~ '^' . a:major_version && str2nr(prog_version[2:]) < str2nr(min_version[2:])
-      return [0, prog_path . ' is Python ' . prog_version . ' and cannot provide Python >= '
-            \ . min_version . '.']
-    endif
-  endif
-
-  if prog_exitcode == 2
-    return [0, prog_path.' does not have the "' . a:module . '" module.']
-  elseif prog_exitcode == 127
-    " This can happen with pyenv's shims.
-    return [0, prog_path . ' does not exist: ' . prog_version]
-  elseif prog_exitcode
-    return [0, 'Checking ' . prog_path . ' caused an unknown error. '
-          \ . '(' . prog_exitcode . ', output: ' . prog_version . ')'
-          \ . ' Report this at https://github.com/neovim/neovim']
-  endif
-
-  return [1, '']
-endfunction

runtime/autoload/remote/host.vim

@@ -190,11 +190,9 @@ endfunction
 
 " Registration of standard hosts
 
-" Python/Python3
+" Python3
-call remote#host#Register('python', '*',
-      \ function('provider#pythonx#Require'))
 call remote#host#Register('python3', '*',
-      \ function('provider#pythonx#Require'))
+      \ function('provider#python3#Require'))
 
 " Ruby
 call remote#host#Register('ruby', '*.rb',

runtime/lua/nvim/health.lua

@@ -183,10 +183,6 @@ local function check_rplugin_manifest()
   health.start('Remote Plugins')
 
   local existing_rplugins = {}
-  for _, item in ipairs(vim.fn['remote#host#PluginsForHost']('python')) do
-    existing_rplugins[item.path] = 'python'
-  end
-
   for _, item in ipairs(vim.fn['remote#host#PluginsForHost']('python3')) do
     existing_rplugins[item.path] = 'python3'
   end

runtime/lua/provider/python/health.lua

@@ -217,7 +217,7 @@ end
 function M.check()
   health.start('Python 3 provider (optional)')
 
-  local pyname = 'python3'
+  local pyname = 'python3' ---@type string?
   local python_exe = ''
   local virtual_env = os.getenv('VIRTUAL_ENV')
   local venv = virtual_env and vim.fn.resolve(virtual_env) or ''
@@ -237,11 +237,10 @@ function M.check()
     health.info(message)
   end
 
-  local python_table = vim.fn['provider#pythonx#Detect'](3)
+  local pythonx_warnings
-  pyname = python_table[1]
+  pyname, pythonx_warnings = require('vim.provider.python').detect_by_module('neovim')
-  local pythonx_warnings = python_table[2]
 
-  if pyname == '' then
+  if not pyname then
     health.warn(
       'No Python executable found that can `import neovim`. '
         .. 'Using the first available executable for diagnostics.'
@@ -251,7 +250,7 @@ function M.check()
   end
 
   -- No Python executable could `import neovim`, or host_prog_var was used.
-  if pythonx_warnings ~= '' then
+  if pythonx_warnings then
     health.warn(pythonx_warnings, {
       'See :help provider-python for more information.',
       'You may disable this provider (and warning) by adding `let g:loaded_python3_provider = 0` to your init.vim',
@@ -364,9 +363,8 @@ function M.check()
     -- can import 'pynvim'. If so, that Python failed to import 'neovim' as
     -- well, which is most probably due to a failed pip upgrade:
     -- https://github.com/neovim/neovim/wiki/Following-HEAD#20181118
-    local pynvim_table = vim.fn['provider#pythonx#DetectByModule']('pynvim', 3)
+    local pynvim_exe = require('vim.provider.python').detect_by_module('pynvim')
-    local pynvim_exe = pynvim_table[1]
+    if pynvim_exe then
-    if pynvim_exe ~= '' then
       local message = 'Detected pip upgrade failure: Python executable can import "pynvim" but not "neovim": '
         .. pynvim_exe
       local advice = {

runtime/lua/vim/provider/python.lua

@@ -0,0 +1,150 @@
+local M = {}
+local min_version = '3.7'
+local s_err ---@type string?
+local s_host ---@type string?
+
+local python_candidates = {
+  'python3',
+  'python3.12',
+  'python3.11',
+  'python3.10',
+  'python3.9',
+  'python3.8',
+  'python3.7',
+  'python',
+}
+
+--- @param prog string
+--- @param module string
+--- @return integer, string
+local function import_module(prog, module)
+  local program = [[
+import sys, importlib.util;
+sys.path = [p for p in sys.path if p != ""];
+sys.stdout.write(str(sys.version_info[0]) + "." + str(sys.version_info[1]));]]
+
+  program = program
+    .. string.format('sys.exit(2 * int(importlib.util.find_spec("%s") is None))', module)
+
+  local out = vim.system({ prog, '-W', 'ignore', '-c', program }):wait()
+  return out.code, assert(out.stdout)
+end
+
+--- @param prog string
+--- @param module string
+--- @return string?
+local function check_for_module(prog, module)
+  local prog_path = vim.fn.exepath(prog)
+  if prog_path == '' then
+    return prog .. ' not found in search path or not executable.'
+  end
+
+  --   Try to load module, and output Python version.
+  --   Exit codes:
+  --     0  module can be loaded.
+  --     2  module cannot be loaded.
+  --     Otherwise something else went wrong (e.g. 1 or 127).
+  local prog_exitcode, prog_version = import_module(prog, module)
+  if prog_exitcode == 2 or prog_exitcode == 0 then
+    -- Check version only for expected return codes.
+    if vim.version.lt(prog_version, min_version) then
+      return string.format(
+        '%s is Python %s and cannot provide Python >= %s.',
+        prog_path,
+        prog_version,
+        min_version
+      )
+    end
+  end
+
+  if prog_exitcode == 2 then
+    return string.format('%s does not have the "%s" module.', prog_path, module)
+  elseif prog_exitcode == 127 then
+    -- This can happen with pyenv's shims.
+    return string.format('%s does not exist: %s', prog_path, prog_version)
+  elseif prog_exitcode ~= 0 then
+    return string.format(
+      'Checking %s caused an unknown error. (%s, output: %s) Report this at https://github.com/neovim/neovim',
+      prog_path,
+      prog_exitcode,
+      prog_version
+    )
+  end
+
+  return nil
+end
+
+--- @param module string
+--- @return string? path to detected python, if any; nil if not found
+--- @return string? error message if python can't be detected by {module}; nil if success
+function M.detect_by_module(module)
+  local python_exe = vim.fn.expand(vim.g.python3_host_prog or '', true)
+
+  if python_exe ~= '' then
+    return vim.fn.exepath(vim.fn.expand(python_exe, true)), nil
+  end
+
+  local errors = {}
+  for _, exe in ipairs(python_candidates) do
+    local error = check_for_module(exe, module)
+    if not error then
+      return exe, error
+    end
+    -- Accumulate errors in case we don't find any suitable Python executable.
+    table.insert(errors, error)
+  end
+
+  -- No suitable Python executable found.
+  return nil, 'Could not load Python :\n' .. table.concat(errors, '\n')
+end
+
+function M.require(host)
+  -- Python host arguments
+  local prog = M.detect_by_module('neovim')
+  local args = {
+    prog,
+    '-c',
+    'import sys; sys.path = [p for p in sys.path if p != ""]; import neovim; neovim.start_host()',
+  }
+
+  -- Collect registered Python plugins into args
+  local python_plugins = vim.fn['remote#host#PluginsForHost'](host.name) ---@type any
+  ---@param plugin any
+  for _, plugin in ipairs(python_plugins) do
+    table.insert(args, plugin.path)
+  end
+
+  return vim.fn['provider#Poll'](
+    args,
+    host.orig_name,
+    '$NVIM_PYTHON_LOG_FILE',
+    { ['overlapped'] = true }
+  )
+end
+
+function M.call(method, args)
+  if s_err then
+    return
+  end
+
+  if not s_host then
+    -- Ensure that we can load the Python3 host before bootstrapping
+    local ok, result = pcall(vim.fn['remote#host#Require'], 'legacy-python3-provider') ---@type any, any
+    if not ok then
+      s_err = result
+      vim.api.nvim_echo({ result, 'WarningMsg' }, true, {})
+      return
+    end
+    s_host = result
+  end
+
+  return vim.fn.rpcrequest(s_host, 'python_' .. method, unpack(args))
+end
+
+function M.start()
+  -- The Python3 provider plugin will run in a separate instance of the Python3 host.
+  vim.fn['remote#host#RegisterClone']('legacy-python3-provider', 'python3')
+  vim.fn['remote#host#RegisterPlugin']('legacy-python3-provider', 'script_host.py', {})
+end
+
+return M

test/functional/ex_cmds/script_spec.lua

@@ -96,7 +96,7 @@ describe('script_get-based command', function()
 
   -- Provider-based scripts
   test_garbage_exec('ruby', not missing_provider('ruby'))
-  test_garbage_exec('python3', not missing_provider('python3'))
+  test_garbage_exec('python3', not missing_provider('python'))
 
   -- Missing scripts
   test_garbage_exec('python', false)

test/functional/helpers.lua

@@ -932,17 +932,14 @@ function module.new_pipename()
 end
 
 --- @param provider string
---- @return string|false?
+--- @return string|boolean?
 function module.missing_provider(provider)
   if provider == 'ruby' or provider == 'node' or provider == 'perl' then
     --- @type string?
     local e = module.fn['provider#' .. provider .. '#Detect']()[2]
     return e ~= '' and e or false
-  elseif provider == 'python' or provider == 'python3' then
+  elseif provider == 'python' then
-    local py_major_version = (provider == 'python3' and 3 or 2)
+    return module.exec_lua([[return {require('vim.provider.python').detect_by_module('neovim')}]])[2]
-    --- @type string?
-    local e = module.fn['provider#pythonx#Detect'](py_major_version)[2]
-    return e ~= '' and e or false
   end
   assert(false, 'Unknown provider: ' .. provider)
 end

test/functional/provider/python3_spec.lua

@@ -13,7 +13,7 @@ local dedent = helpers.dedent
 
 do
   clear()
-  local reason = missing_provider('python3')
+  local reason = missing_provider('python')
   if reason then
     it(':python3 reports E319 if provider is missing', function()
       local expected = [[Vim%(py3.*%):E319: No "python3" provider found.*]]

test/old/testdir/test_python3.vim

@@ -169,7 +169,7 @@ func Test_Catch_Exception_Message()
   try
     py3 raise RuntimeError( 'TEST' )
   catch /.*/
-    call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception )
+    call assert_match('^Vim(.*):.*RuntimeError: TEST.*$', v:exception )
   endtry
 endfunc
 

test/old/testdir/test_pyx3.vim

@@ -76,7 +76,7 @@ func Test_Catch_Exception_Message()
   try
     pyx raise RuntimeError( 'TEST' )
   catch /.*/
-    call assert_match('^Vim(.*):.*RuntimeError: TEST$', v:exception )
+    call assert_match('^Vim(.*):.*RuntimeError: TEST.*$', v:exception )
   endtry
 endfunc