fix(vim.ui)!: change open() to return pcall-like values #28502

Problem:
`vim.ui.open` unnecessarily invents a different success/failure
convention. Its return type was changed in 57adf8c6e0, so we might as
well change it to have a more conventional form.

Solution:
Change the signature to use the `pcall` convention of `status, result`.

runtime/lua/vim/ui.lua

@@ -118,16 +118,16 @@ end
 --- vim.ui.open("https://neovim.io/")
 --- vim.ui.open("~/path/to/file")
 --- -- Synchronous (wait until the process exits).
---- local cmd, err = vim.ui.open("$VIMRUNTIME")
---- if cmd then
+--- local ok, cmd = vim.ui.open("$VIMRUNTIME")
+--- if ok then
 ---   cmd:wait()
 --- end
 --- ```
 ---
 ---@param path string Path or URL to open
 ---
----@return vim.SystemObj|nil # Command object, or nil if not found.
----@return string|nil # Error message on failure
+---@return boolean # false if command not found, else true.
+---@return vim.SystemObj|string # Command object, or error message on failure
 ---
 ---@see |vim.system()|
 function M.open(path)
@@ -147,7 +147,7 @@ function M.open(path)
     if vim.fn.executable('rundll32') == 1 then
       cmd = { 'rundll32', 'url.dll,FileProtocolHandler', path }
     else
-      return nil, 'vim.ui.open: rundll32 not found'
+      return false, 'vim.ui.open: rundll32 not found'
     end
   elseif vim.fn.executable('wslview') == 1 then
     cmd = { 'wslview', path }
@@ -156,10 +156,10 @@ function M.open(path)
   elseif vim.fn.executable('xdg-open') == 1 then
     cmd = { 'xdg-open', path }
   else
-    return nil, 'vim.ui.open: no handler found (tried: wslview, explorer.exe, xdg-open)'
+    return false, 'vim.ui.open: no handler found (tried: wslview, explorer.exe, xdg-open)'
   end
 
-  return vim.system(cmd, { text = true, detach = true }), nil
+  return true, vim.system(cmd, { text = true, detach = true })
 end
 
 return M