runtime: Reimplement python/clipboard providers in vimscript

Clipboard is implemented with platform-specific shell commands, and python is
implemented with the external plugin facility (rpc#* functions). The
script_host.py file(legacy python-vim emulation plugin) was moved/adapted from
the python client repository.

runtime/autoload/provider/clipboard.vim

@@ -0,0 +1,40 @@
+" The clipboard provider uses shell commands to communicate with the clipboard.
+" The provider function will only be registered if one of the supported
+" commands are available.
+let s:copy = ''
+let s:paste = ''
+
+if executable('pbcopy')
+  let s:copy = 'pbcopy'
+  let s:paste = 'pbpaste'
+elseif executable('xsel')
+  let s:copy = 'xsel -i -b'
+  let s:paste = 'xsel -o -b'
+elseif executable('xclip')
+  let s:copy = 'xclip -i -selection clipboard'
+  let s:paste = 'xclip -o -selection clipboard'
+endif
+
+if s:copy == ''
+  echom 'No shell command for communicating with the clipboard found.'
+  finish
+endif
+
+let s:methods = {}
+
+function! s:ClipboardGet(...)
+  return systemlist(s:paste)
+endfunction
+
+function! s:ClipboardSet(...)
+  call systemlist(s:copy, a:1)
+endfunction
+
+let s:methods = {
+      \ 'get': function('s:ClipboardGet'),
+      \ 'set': function('s:ClipboardSet')
+      \ }
+
+function! provider#clipboard#Call(method, args)
+  return s:methods[a:method](a:args)
+endfunction

runtime/autoload/provider/python.vim

@@ -0,0 +1,28 @@
+" The python provider uses a python host to emulate an environment for running
+" python-vim plugins(:h python-vim). See :h nvim-providers for more
+" information.
+"
+" Associating the plugin with the python host is the first step because plugins
+" will be passed as command-line arguments
+if exists('s:loaded_python_provider') || &cp
+  finish
+endif
+let s:loaded_python_provider = 1
+let s:plugin_path = expand('<sfile>:p:h').'/script_host.py'
+" The python provider plugin will run in a separate instance of the python
+" host.
+call rpc#host#RegisterClone('legacy-python-provider', 'python')
+call rpc#host#RegisterPlugin('legacy-python-provider', s:plugin_path, [])
+" Ensure that we can load the python host before bootstrapping
+try
+  let s:host = rpc#host#Require('legacy-python-provider')
+catch
+  echomsg v:exception
+  finish
+endtry
+
+let s:rpcrequest = function('rpcrequest')
+
+function! provider#python#Call(method, args)
+  return call(s:rpcrequest, insert(insert(a:args, 'python_'.a:method), s:host))
+endfunction

runtime/autoload/provider/script_host.py

@@ -0,0 +1,243 @@
+"""Legacy python-vim emulation."""
+import imp
+import logging
+import os
+import sys
+
+import neovim
+
+__all__ = ('ScriptHost',)
+
+
+logger = logging.getLogger(__name__)
+debug, info, warn = (logger.debug, logger.info, logger.warn,)
+
+IS_PYTHON3 = sys.version_info >= (3, 0)
+
+if IS_PYTHON3:
+    basestring = str
+
+
+@neovim.plugin
+class ScriptHost(object):
+
+    """Provides an environment for running python plugins created for Vim."""
+
+    def __init__(self, nvim):
+        """Initialize the legacy python-vim environment."""
+        self.setup(nvim)
+        # context where all code will run
+        self.module = imp.new_module('__main__')
+        nvim.script_context = self.module
+        # it seems some plugins assume 'sys' is already imported, so do it now
+        exec('import sys', self.module.__dict__)
+        self.legacy_vim = nvim.with_hook(LegacyEvalHook())
+        if IS_PYTHON3:
+            self.legacy_vim = self.legacy_vim.with_hook(
+                neovim.DecodeHook(
+                    encoding=nvim.options['encoding'].decode('ascii')))
+        sys.modules['vim'] = self.legacy_vim
+
+    def setup(self, nvim):
+        """Setup import hooks and global streams.
+
+        This will add import hooks for importing modules from runtime
+        directories and patch the sys module so 'print' calls will be
+        forwarded to Nvim.
+        """
+        self.nvim = nvim
+        info('install import hook/path')
+        self.hook = path_hook(nvim)
+        sys.path_hooks.append(self.hook)
+        nvim.VIM_SPECIAL_PATH = '_vim_path_'
+        sys.path.append(nvim.VIM_SPECIAL_PATH)
+        info('redirect sys.stdout and sys.stderr')
+        self.saved_stdout = sys.stdout
+        self.saved_stderr = sys.stderr
+        sys.stdout = RedirectStream(lambda data: nvim.out_write(data))
+        sys.stderr = RedirectStream(lambda data: nvim.err_write(data))
+
+    def teardown(self):
+        """Restore state modified from the `setup` call."""
+        for plugin in self.installed_plugins:
+            if hasattr(plugin, 'on_teardown'):
+                plugin.teardown()
+        nvim = self.nvim
+        info('uninstall import hook/path')
+        sys.path.remove(nvim.VIM_SPECIAL_PATH)
+        sys.path_hooks.remove(self.hook)
+        info('restore sys.stdout and sys.stderr')
+        sys.stdout = self.saved_stdout
+        sys.stderr = self.saved_stderr
+
+    @neovim.rpc_export('python_execute', sync=True)
+    def python_execute(self, script, range_start, range_stop):
+        """Handle the `python` ex command."""
+        self._set_current_range(range_start, range_stop)
+        exec(script, self.module.__dict__)
+
+    @neovim.rpc_export('python_execute_file', sync=True)
+    def python_execute_file(self, file_path, range_start, range_stop):
+        """Handle the `pyfile` ex command."""
+        self._set_current_range(range_start, range_stop)
+        with open(file_path) as f:
+            script = compile(f.read(), file_path, 'exec')
+            exec(script, self.module.__dict__)
+
+    @neovim.rpc_export('python_do_range', sync=True)
+    def python_do_range(self, start, stop, code):
+        """Handle the `pydo` ex command."""
+        self._set_current_range(start, stop)
+        nvim = self.nvim
+        start -= 1
+        stop -= 1
+        fname = '_vim_pydo'
+
+        # Python3 code (exec) must be a string, mixing bytes with
+        # function_def would use bytes.__repr__ instead
+        if isinstance and isinstance(code, bytes):
+            code = code.decode(nvim.options['encoding'].decode('ascii'))
+        # define the function
+        function_def = 'def %s(line, linenr):\n %s' % (fname, code,)
+        exec(function_def, self.module.__dict__)
+        # get the function
+        function = self.module.__dict__[fname]
+        while start <= stop:
+            # Process batches of 5000 to avoid the overhead of making multiple
+            # API calls for every line. Assuming an average line length of 100
+            # bytes, approximately 488 kilobytes will be transferred per batch,
+            # which can be done very quickly in a single API call.
+            sstart = start
+            sstop = min(start + 5000, stop)
+            lines = nvim.current.buffer.get_line_slice(sstart, sstop, True,
+                                                       True)
+
+            exception = None
+            newlines = []
+            linenr = sstart + 1
+            for i, line in enumerate(lines):
+                result = function(line, linenr)
+                if result is None:
+                    # Update earlier lines, and skip to the next
+                    if newlines:
+                        end = sstart + len(newlines) - 1
+                        nvim.current.buffer.set_line_slice(sstart, end,
+                                                           True, True,
+                                                           newlines)
+                    sstart += len(newlines) + 1
+                    newlines = []
+                    pass
+                elif isinstance(result, basestring):
+                    newlines.append(result)
+                else:
+                    exception = TypeError('pydo should return a string ' +
+                                          'or None, found %s instead'
+                                          % result.__class__.__name__)
+                    break
+                linenr += 1
+
+            start = sstop + 1
+            if newlines:
+                end = sstart + len(newlines) - 1
+                nvim.current.buffer.set_line_slice(sstart, end, True, True,
+                                                   newlines)
+            if exception:
+                raise exception
+        # delete the function
+        del self.module.__dict__[fname]
+
+    @neovim.rpc_export('python_eval', sync=True)
+    def python_eval(self, expr):
+        """Handle the `pyeval` vim function."""
+        return eval(expr, self.module.__dict__)
+
+    def _set_current_range(self, start, stop):
+        current = self.legacy_vim.current
+        current.range = current.buffer.range(start, stop)
+
+
+class RedirectStream(object):
+    def __init__(self, redirect_handler):
+        self.redirect_handler = redirect_handler
+
+    def write(self, data):
+        self.redirect_handler(data)
+
+    def writelines(self, seq):
+        self.redirect_handler('\n'.join(seq))
+
+
+class LegacyEvalHook(neovim.SessionHook):
+
+    """Injects legacy `vim.eval` behavior to a Nvim instance."""
+
+    def __init__(self):
+        super(LegacyEvalHook, self).__init__(from_nvim=self._string_eval)
+
+    def _string_eval(self, obj, session, method, kind):
+        if method == 'vim_eval' and isinstance(obj, (int, long, float)):
+            return str(obj)
+        return obj
+
+
+# This was copied/adapted from nvim-python help
+def path_hook(nvim):
+    def _get_paths():
+        return discover_runtime_directories(nvim)
+
+    def _find_module(fullname, oldtail, path):
+        idx = oldtail.find('.')
+        if idx > 0:
+            name = oldtail[:idx]
+            tail = oldtail[idx+1:]
+            fmr = imp.find_module(name, path)
+            module = imp.load_module(fullname[:-len(oldtail)] + name, *fmr)
+            return _find_module(fullname, tail, module.__path__)
+        else:
+            fmr = imp.find_module(fullname, path)
+            return imp.load_module(fullname, *fmr)
+
+    class VimModuleLoader(object):
+        def __init__(self, module):
+            self.module = module
+
+        def load_module(self, fullname, path=None):
+            return self.module
+
+    class VimPathFinder(object):
+        @classmethod
+        def find_module(cls, fullname, path=None):
+            try:
+                return VimModuleLoader(
+                    _find_module(fullname, fullname, path or _get_paths()))
+            except ImportError:
+                return None
+
+        @classmethod
+        def load_module(cls, fullname, path=None):
+            return _find_module(fullname, fullname, path or _get_paths())
+
+    def hook(path):
+        if path == nvim.VIM_SPECIAL_PATH:
+            return VimPathFinder
+        else:
+            raise ImportError
+
+    return hook
+
+
+def discover_runtime_directories(nvim):
+    rv = []
+    for path in nvim.list_runtime_paths():
+        if not os.path.exists(path):
+            continue
+        path1 = os.path.join(path, b'pythonx')
+        if IS_PYTHON3:
+            path2 = os.path.join(path, b'python3')
+        else:
+            path2 = os.path.join(path, b'python2')
+        if os.path.exists(path1):
+            rv.append(path1)
+        if os.path.exists(path2):
+            rv.append(path2)
+    return rv