fix(lsp): handle out-of-bounds character positions #30288

Problem:
str_byteindex_enc could return an error if the index was longer than the
lline length. This was handled in each of the calls to it individually

Solution:
* Fix the call at the source level so that if the index is higher than
  the line length, line length is returned as per LSP specification
* Remove pcalls on str_byteindex_enc calls. No longer needed now that
  str_byteindex_enc has a bounds check.

runtime/lua/vim/lsp/inlay_hint.lua

@@ -77,12 +77,7 @@ function M.on_inlayhint(err, result, ctx, _)
     local col = position.character
     if col > 0 then
       local line = lines[position.line + 1] or ''
-      local ok, convert_result
-      ok, convert_result = pcall(util._str_byteindex_enc, line, col, client.offset_encoding)
-      if ok then
-        return convert_result
-      end
-      return math.min(#line, col)
+      return util._str_byteindex_enc(line, col, client.offset_encoding)
     end
     return col
   end

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -140,12 +140,7 @@ local function tokens_to_ranges(data, bufnr, client, request)
     local function _get_byte_pos(col)
       if col > 0 then
         local buf_line = lines[line + 1] or ''
-        local ok, result
-        ok, result = pcall(util._str_byteindex_enc, buf_line, col, client.offset_encoding)
-        if ok then
-          return result
-        end
-        return math.min(#buf_line, col)
+        return util._str_byteindex_enc(buf_line, col, client.offset_encoding)
       end
       return col
     end

runtime/lua/vim/lsp/util.lua

@@ -148,6 +148,12 @@ end
 ---@param encoding string utf-8|utf-16|utf-32| defaults to utf-16
 ---@return integer byte (utf-8) index of `encoding` index `index` in `line`
 function M._str_byteindex_enc(line, index, encoding)
+  local len = vim.fn.strlen(line)
+  if index > len then
+    -- LSP spec: if character > line length, default to the line length.
+    -- https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#position
+    return len
+  end
   if not encoding then
     encoding = 'utf-16'
   end
@@ -167,7 +173,6 @@ function M._str_byteindex_enc(line, index, encoding)
 end
 
 local _str_utfindex_enc = M._str_utfindex_enc
-local _str_byteindex_enc = M._str_byteindex_enc
 
 --- Replaces text in a range with new text.
 ---
@@ -333,12 +338,7 @@ local function get_line_byte_from_position(bufnr, position, offset_encoding)
   -- character
   if col > 0 then
     local line = get_line(bufnr, position.line) or ''
-    local ok, result
-    ok, result = pcall(_str_byteindex_enc, line, col, offset_encoding)
-    if ok then
-      return result
-    end
-    return math.min(#line, col)
+    return M._str_byteindex_enc(line, col, offset_encoding or 'utf-16')
   end
   return col
 end
@@ -495,14 +495,15 @@ function M.apply_text_edits(text_edits, bufnr, offset_encoding)
         e.end_col = last_line_len
         has_eol_text_edit = true
       else
-        -- If the replacement is over the end of a line (i.e. e.end_col is out of bounds and the
+        -- If the replacement is over the end of a line (i.e. e.end_col is equal to the line length and the
         -- replacement text ends with a newline We can likely assume that the replacement is assumed
         -- to be meant to replace the newline with another newline and we need to make sure this
         -- doesn't add an extra empty line. E.g. when the last line to be replaced contains a '\r'
         -- in the file some servers (clangd on windows) will include that character in the line
         -- while nvim_buf_set_text doesn't count it as part of the line.
         if
-          e.end_col > last_line_len
+          e.end_col >= last_line_len
+          and text_edit.range['end'].character > e.end_col
           and #text_edit.newText > 0
           and string.sub(text_edit.newText, -1) == '\n'
         then