backport: fix(mpack): boundary values for negative integer encoding (#38411)

fix(mpack): boundary values for negative integer encoding

Problem:
libmpack encodes boundary values -129 and -32769 with wrong integer
sizes:
- -129 as int8 instead of int16
- -32769 as int16 instead of int32
because the boundary checks compare against the wrong values (e.g., lo
< 0xffffff7f instead of lo < 0xffffff80). This caused data corruption:
-129 would decode as 127.

Solution:
Fix off-by-one errors in the two's complement boundary constants:
0xffffff80 (-128, min int8) and 0xffff8000 (-32768, min int16).

Co-authored-by: benarcher2691 <ben.archer2691@gmail.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

src/mpack/conv.c

@@ -122,8 +122,8 @@ MPACK_API mpack_token_t mpack_pack_number(double v)
     if (!tok.data.value.lo) tok.data.value.hi++;
     if (tok.data.value.lo == 0 && tok.data.value.hi == 0) tok.length = 1;
     else if (tok.data.value.lo < 0x80000000) tok.length = 8;
-    else if (tok.data.value.lo < 0xffff7fff) tok.length = 4;
+    else if (tok.data.value.lo < 0xffff8000) tok.length = 4;
-    else if (tok.data.value.lo < 0xffffff7f) tok.length = 2;
+    else if (tok.data.value.lo < 0xffffff80) tok.length = 2;
     else tok.length = 1;
   } else {
     tok.type = MPACK_TOKEN_UINT;

src/mpack/mpack_core.c

@@ -408,11 +408,11 @@ static int mpack_wnint(char **buf, size_t *buflen, mpack_value_t val)
     return mpack_w1(buf, buflen, 0xd3) ||
            mpack_w4(buf, buflen, hi)   ||
            mpack_w4(buf, buflen, lo);
-  } else if (lo < 0xffff7fff) {
+  } else if (lo < 0xffff8000) {
     /* int 32 */
     return mpack_w1(buf, buflen, 0xd2) ||
            mpack_w4(buf, buflen, lo);
-  } else if (lo < 0xffffff7f) {
+  } else if (lo < 0xffffff80) {
     /* int 16 */
     return mpack_w1(buf, buflen, 0xd1) ||
            mpack_w2(buf, buflen, lo);

test/functional/lua/mpack_spec.lua

@@ -27,4 +27,43 @@ describe('lua vim.mpack', function()
       end)
     )
   end)
+
+  it('encodes negative integers at type boundaries correctly #37202', function()
+    -- Test boundary values between int8/int16/int32
+    -- int8 range: -128 to -33 (fixint handles -32 to -1)
+    -- int16 range: -32768 to -129
+    -- int32 range: -2147483648 to -32769
+    local result = exec_lua(function()
+      local tests = {
+        { -128, -128 }, -- int8 boundary (minimum int8)
+        { -129, -129 }, -- int16 boundary (one past int8)
+        { -32768, -32768 }, -- int16 boundary (minimum int16)
+        { -32769, -32769 }, -- int32 boundary (one past int16)
+      }
+      local results = {}
+      for _, test in ipairs(tests) do
+        local input, expected = test[1], test[2]
+        local decoded = vim.mpack.decode(vim.mpack.encode(input))
+        table.insert(results, { input = input, decoded = decoded, ok = decoded == expected })
+      end
+      return results
+    end)
+    for _, r in ipairs(result) do
+      eq(true, r.ok, string.format('encode/decode %d returned %d', r.input, r.decoded))
+    end
+  end)
+
+  it('encodes dict keys of length 20-31 as fixstr #32784', function()
+    -- MessagePack fixstr format: 0xa0 | length (for lengths 0-31)
+    -- Before #36737, strings 20-31 bytes were incorrectly encoded as str8 (0xd9, len)
+    for len = 20, 31 do
+      local expected_header = string.char(0xa0 + len) -- fixstr header
+      local result = exec_lua(function(keylen)
+        local key = string.rep('x', keylen)
+        return vim.mpack.encode({ [key] = 1 })
+      end, len)
+      -- Byte 1 is fixmap header (0x81), byte 2 should be fixstr header for the key
+      eq(expected_header, result:sub(2, 2), 'dict key length ' .. len .. ' should use fixstr')
+    end
+  end)
 end)

test/unit/msgpack_spec.lua

@@ -1,7 +1,8 @@
 local t = require('test.unit.testutil')
 local cimport = t.cimport
 local itp = t.gen_itp(it)
-local lib = cimport('./src/nvim/msgpack_rpc/unpacker.h', './src/nvim/memory.h')
+local lib =
+  cimport('./src/nvim/msgpack_rpc/unpacker.h', './src/nvim/memory.h', './src/mpack/conv.h')
 local ffi = t.ffi
 local eq = t.eq
 local to_cstr = t.to_cstr
@@ -77,3 +78,31 @@ describe('msgpack', function()
     end)
   end)
 end)
+
+describe('libmpack', function()
+  describe('mpack_pack_number', function()
+    itp('token length at negative integer boundaries #37202', function()
+      -- tok.length is the byte count for the integer value (not including the msgpack type prefix byte).
+
+      -- int8 min: -128 → 1 byte
+      local tok = lib.mpack_pack_number(-128)
+      eq(lib.MPACK_TOKEN_SINT, tok.type)
+      eq(1, tok.length)
+
+      -- one past int8: -129 → 2 bytes (int16)
+      tok = lib.mpack_pack_number(-129)
+      eq(lib.MPACK_TOKEN_SINT, tok.type)
+      eq(2, tok.length)
+
+      -- int16 min: -32768 → 2 bytes
+      tok = lib.mpack_pack_number(-32768)
+      eq(lib.MPACK_TOKEN_SINT, tok.type)
+      eq(2, tok.length)
+
+      -- one past int16: -32769 → 4 bytes (int32)
+      tok = lib.mpack_pack_number(-32769)
+      eq(lib.MPACK_TOKEN_SINT, tok.type)
+      eq(4, tok.length)
+    end)
+  end)
+end)