refactor(lsp): drop str_byteindex/str_utfindex wrappers #30915

* deprecate old signatures
* move to new str_byteindex/str_utfindex signature
* use single-underscore name (double-underscore is reserved for Lua itself)

runtime/lua/vim/lsp/completion.lua

@@ -315,7 +315,7 @@ local function adjust_start_col(lnum, line, items, encoding)
     end
   end
   if min_start_char then
-    return lsp.util._str_byteindex_enc(line, min_start_char, encoding)
+    return vim.str_byteindex(line, encoding, min_start_char, false)
   else
     return nil
   end

runtime/lua/vim/lsp/diagnostic.lua

@@ -33,25 +33,6 @@ local function severity_vim_to_lsp(severity)
   return severity
 end
 
----@param lines string[]?
----@param lnum integer
----@param col integer
----@param offset_encoding string
----@return integer
-local function line_byte_from_position(lines, lnum, col, offset_encoding)
-  if not lines or offset_encoding == 'utf-8' then
-    return col
-  end
-
-  local line = lines[lnum + 1]
-  local ok, result = pcall(vim.str_byteindex, line, col, offset_encoding == 'utf-16')
-  if ok then
-    return result --- @type integer
-  end
-
-  return col
-end
-
 ---@param bufnr integer
 ---@return string[]?
 local function get_buf_lines(bufnr)
@@ -118,12 +99,13 @@ local function diagnostic_lsp_to_vim(diagnostics, bufnr, client_id)
       )
       message = diagnostic.message.value
     end
+    local line = buf_lines and buf_lines[start.line + 1] or ''
     --- @type vim.Diagnostic
     return {
       lnum = start.line,
-      col = line_byte_from_position(buf_lines, start.line, start.character, offset_encoding),
+      col = vim.str_byteindex(line, offset_encoding, start.character, false),
       end_lnum = _end.line,
-      end_col = line_byte_from_position(buf_lines, _end.line, _end.character, offset_encoding),
+      end_col = vim.str_byteindex(line, offset_encoding, _end.character, false),
       severity = severity_lsp_to_vim(diagnostic.severity),
       message = message,
       source = diagnostic.source,

runtime/lua/vim/lsp/inlay_hint.lua

@@ -70,20 +70,12 @@ function M.on_inlayhint(err, result, ctx, _)
   end
 
   local lines = api.nvim_buf_get_lines(bufnr, 0, -1, false)
-  ---@param position lsp.Position
-  ---@return integer
-  local function pos_to_byte(position)
-    local col = position.character
-    if col > 0 then
-      local line = lines[position.line + 1] or ''
-      return util._str_byteindex_enc(line, col, client.offset_encoding)
-    end
-    return col
-  end
 
   for _, hint in ipairs(result) do
     local lnum = hint.position.line
-    hint.position.character = pos_to_byte(hint.position)
+    local line = lines and lines[lnum + 1] or ''
+    hint.position.character =
+      vim.str_byteindex(line, client.offset_encoding, hint.position.character, false)
     table.insert(new_lnum_hints[lnum], hint)
   end
 

runtime/lua/vim/lsp/semantic_tokens.lua

@@ -137,16 +137,10 @@ local function tokens_to_ranges(data, bufnr, client, request)
     local token_type = token_types[data[i + 3] + 1]
     local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
 
-    local function _get_byte_pos(col)
-      if col > 0 then
-        local buf_line = lines[line + 1] or ''
-        return util._str_byteindex_enc(buf_line, col, client.offset_encoding)
-      end
-      return col
-    end
-
-    local start_col = _get_byte_pos(start_char)
-    local end_col = _get_byte_pos(start_char + data[i + 2])
+    local end_char = start_char + data[i + 2]
+    local buf_line = lines and lines[line + 1] or ''
+    local start_col = vim.str_byteindex(buf_line, client.offset_encoding, start_char, false)
+    local end_col = vim.str_byteindex(buf_line, client.offset_encoding, end_char, false)
 
     if token_type then
       ranges[#ranges + 1] = {

runtime/lua/vim/lsp/sync.lua

@@ -48,45 +48,6 @@ local str_utfindex = vim.str_utfindex
 local str_utf_start = vim.str_utf_start
 local str_utf_end = vim.str_utf_end
 
--- Given a line, byte idx, and offset_encoding convert to the
--- utf-8, utf-16, or utf-32 index.
----@param line string the line to index into
----@param byte integer the byte idx
----@param offset_encoding string utf-8|utf-16|utf-32|nil (default: utf-8)
----@return integer utf_idx for the given encoding
-local function byte_to_utf(line, byte, offset_encoding)
-  -- convert to 0 based indexing for str_utfindex
-  byte = byte - 1
-
-  local utf_idx, _ --- @type integer, integer
-  -- Convert the byte range to utf-{8,16,32} and convert 1-based (lua) indexing to 0-based
-  if offset_encoding == 'utf-16' then
-    _, utf_idx = str_utfindex(line, byte)
-  elseif offset_encoding == 'utf-32' then
-    utf_idx, _ = str_utfindex(line, byte)
-  else
-    utf_idx = byte
-  end
-
-  -- convert to 1 based indexing
-  return utf_idx + 1
-end
-
----@param line string
----@param offset_encoding string
----@return integer
-local function compute_line_length(line, offset_encoding)
-  local length, _ --- @type integer, integer
-  if offset_encoding == 'utf-16' then
-    _, length = str_utfindex(line)
-  elseif offset_encoding == 'utf-32' then
-    length, _ = str_utfindex(line)
-  else
-    length = #line
-  end
-  return length
-end
-
 -- Given a line, byte idx, alignment, and offset_encoding convert to the aligned
 -- utf-8 index and either the utf-16, or utf-32 index.
 ---@param line string the line to index into
@@ -101,7 +62,7 @@ local function align_end_position(line, byte, offset_encoding)
     char = byte
     -- Called in the case of extending an empty line "" -> "a"
   elseif byte == #line + 1 then
-    char = compute_line_length(line, offset_encoding) + 1
+    char = str_utfindex(line, offset_encoding) + 1
   else
     -- Modifying line, find the nearest utf codepoint
     local offset = str_utf_start(line, byte)
@@ -111,9 +72,10 @@ local function align_end_position(line, byte, offset_encoding)
       byte = byte + str_utf_end(line, byte) + 1
     end
     if byte <= #line then
-      char = byte_to_utf(line, byte, offset_encoding)
+      --- Convert to 0 based for input, and from 0 based for output
+      char = str_utfindex(line, offset_encoding, byte - 1) + 1
     else
-      char = compute_line_length(line, offset_encoding) + 1
+      char = str_utfindex(line, offset_encoding) + 1
     end
     -- Extending line, find the nearest utf codepoint for the last valid character
   end
@@ -153,7 +115,7 @@ local function compute_start_range(
     if line then
       line_idx = firstline - 1
       byte_idx = #line + 1
-      char_idx = compute_line_length(line, offset_encoding) + 1
+      char_idx = str_utfindex(line, offset_encoding) + 1
     else
       line_idx = firstline
       byte_idx = 1
@@ -190,10 +152,11 @@ local function compute_start_range(
     char_idx = 1
   elseif start_byte_idx == #prev_line + 1 then
     byte_idx = start_byte_idx
-    char_idx = compute_line_length(prev_line, offset_encoding) + 1
+    char_idx = str_utfindex(prev_line, offset_encoding) + 1
   else
     byte_idx = start_byte_idx + str_utf_start(prev_line, start_byte_idx)
-    char_idx = byte_to_utf(prev_line, byte_idx, offset_encoding)
+    --- Convert to 0 based for input, and from 0 based for output
+    char_idx = vim.str_utfindex(prev_line, offset_encoding, byte_idx - 1) + 1
   end
 
   -- Return the start difference (shared for new and prev lines)
@@ -230,7 +193,7 @@ local function compute_end_range(
     return {
       line_idx = lastline - 1,
       byte_idx = #prev_line + 1,
-      char_idx = compute_line_length(prev_line, offset_encoding) + 1,
+      char_idx = str_utfindex(prev_line, offset_encoding) + 1,
     }, { line_idx = 1, byte_idx = 1, char_idx = 1 }
   end
   -- If firstline == new_lastline, the first change occurred on a line that was deleted.
@@ -376,7 +339,7 @@ local function compute_range_length(lines, start_range, end_range, offset_encodi
   local start_line = lines[start_range.line_idx]
   local range_length --- @type integer
   if start_line and #start_line > 0 then
-    range_length = compute_line_length(start_line, offset_encoding)
+    range_length = str_utfindex(start_line, offset_encoding)
       - start_range.char_idx
       + 1
       + line_ending_length
@@ -389,7 +352,7 @@ local function compute_range_length(lines, start_range, end_range, offset_encodi
   for idx = start_range.line_idx + 1, end_range.line_idx - 1 do
     -- Length full line plus newline character
     if #lines[idx] > 0 then
-      range_length = range_length + compute_line_length(lines[idx], offset_encoding) + #line_ending
+      range_length = range_length + str_utfindex(lines[idx], offset_encoding) + #line_ending
     else
       range_length = range_length + line_ending_length
     end

runtime/lua/vim/lsp/util.lua

@@ -116,71 +116,6 @@ local function create_window_without_focus()
   return new
 end
 
---- Convert byte index to `encoding` index.
---- Convenience wrapper around vim.str_utfindex
----@param line string line to be indexed
----@param index integer? byte index (utf-8), or `nil` for length
----@param encoding 'utf-8'|'utf-16'|'utf-32'? defaults to utf-16
----@return integer `encoding` index of `index` in `line`
-function M._str_utfindex_enc(line, index, encoding)
-  local len32, len16 = vim.str_utfindex(line)
-  if not encoding then
-    encoding = 'utf-16'
-  end
-  if encoding == 'utf-8' then
-    if index then
-      return index
-    else
-      return #line
-    end
-  elseif encoding == 'utf-16' then
-    if not index or index > len16 then
-      return len16
-    end
-    local _, col16 = vim.str_utfindex(line, index)
-    return col16
-  elseif encoding == 'utf-32' then
-    if not index or index > len32 then
-      return len32
-    end
-    local col32, _ = vim.str_utfindex(line, index)
-    return col32
-  else
-    error('Invalid encoding: ' .. vim.inspect(encoding))
-  end
-end
-
---- Convert UTF index to `encoding` index.
---- Convenience wrapper around vim.str_byteindex
----Alternative to vim.str_byteindex that takes an encoding.
----@param line string line to be indexed
----@param index integer UTF index
----@param encoding string utf-8|utf-16|utf-32| defaults to utf-16
----@return integer byte (utf-8) index of `encoding` index `index` in `line`
-function M._str_byteindex_enc(line, index, encoding)
-  -- LSP spec: if character > line length, default to the line length.
-  -- https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#position
-  local len8 = #line
-  if not encoding then
-    encoding = 'utf-16'
-  end
-  if encoding == 'utf-8' then
-    if index and index <= len8 then
-      return index
-    else
-      return len8
-    end
-  end
-  local len32, len16 = vim.str_utfindex(line)
-  if encoding == 'utf-16' then
-    return index <= len16 and vim.str_byteindex(line, index, true) or len8
-  elseif encoding == 'utf-32' then
-    return index <= len32 and vim.str_byteindex(line, index) or len8
-  else
-    error('Invalid encoding: ' .. vim.inspect(encoding))
-  end
-end
-
 --- Replaces text in a range with new text.
 ---
 --- CAUTION: Changes in-place!
@@ -352,7 +287,7 @@ local function get_line_byte_from_position(bufnr, position, offset_encoding)
   -- character
   if col > 0 then
     local line = get_line(bufnr, position.line) or ''
-    return M._str_byteindex_enc(line, col, offset_encoding)
+    return vim.str_byteindex(line, offset_encoding, col, false)
   end
   return col
 end
@@ -1787,8 +1722,8 @@ function M.locations_to_items(locations, offset_encoding)
       local end_row = end_pos.line
       local line = lines[row] or ''
       local end_line = lines[end_row] or ''
-      local col = M._str_byteindex_enc(line, pos.character, offset_encoding)
-      local end_col = M._str_byteindex_enc(end_line, end_pos.character, offset_encoding)
+      local col = vim.str_byteindex(line, offset_encoding, pos.character, false)
+      local end_col = vim.str_byteindex(end_line, offset_encoding, end_pos.character, false)
 
       items[#items + 1] = {
         filename = filename,
@@ -1911,7 +1846,7 @@ local function make_position_param(window, offset_encoding)
     return { line = 0, character = 0 }
   end
 
-  col = M._str_utfindex_enc(line, col, offset_encoding)
+  col = vim.str_utfindex(line, offset_encoding, col, false)
 
   return { line = row, character = col }
 end
@@ -2092,7 +2027,7 @@ function M.character_offset(buf, row, col, offset_encoding)
     )
     offset_encoding = vim.lsp.get_clients({ bufnr = buf })[1].offset_encoding
   end
-  return M._str_utfindex_enc(line, col, offset_encoding)
+  return vim.str_utfindex(line, offset_encoding, col, false)
 end
 
 --- Helper function to return nested values in language server settings