feat(stdlib): vim.json.decode() can allow comments #37795

Problem:
`vim.json.decode()` could not parse JSONC (JSON with Comments)
extension, which is commonly used in configuration files.

Solution:
Introduce an `skip_comments` option, which is disabled by default. When
enabled, allows JavaScript-style comments within JSON data.

runtime/doc/lua.txt

@@ -3433,6 +3433,11 @@ vim.json.decode({str}, {opts})                             *vim.json.decode()*
                 • {luanil}? (`{ object?: boolean, array?: boolean }`, default:
                   `nil`) Convert `null` in JSON objects and/or arrays to Lua
                   `nil` instead of |vim.NIL|.
+                • {skip_comments}? (`boolean`, default: `false`) Allows
+                  JavaScript-style comments within JSON data. Comments are
+                  treated as whitespace and may appear anywhere whitespace is
+                  valid in JSON. Supports single-line comments beginning with
+                  '//' and block comments enclosed with '/' and '/'.
 
     Return: ~
         (`any`)

runtime/doc/news.txt

@@ -325,6 +325,7 @@ LUA
 • |vim.json.encode()| has an `indent` option for pretty-formatting.
 • |vim.json.encode()| has an `sort_keys` option.
 • |Range:is_empty()| to check if a |vim.Range| is empty.
+• |vim.json.decode()| has an `skip_comments` option.
 
 OPTIONS
 

runtime/lua/vim/_meta/json.lua

@@ -11,15 +11,20 @@ vim.json = {}
 --- Convert `null` in JSON objects and/or arrays to Lua `nil` instead of |vim.NIL|.
 --- (default: `nil`)
 --- @field luanil? { object?: boolean, array?: boolean }
+---
+--- Allows JavaScript-style comments within JSON data. Comments are treated as whitespace and may
+--- appear anywhere whitespace is valid in JSON. Supports single-line comments beginning with '//'
+--- and block comments enclosed with '/*' and '*/'.
+--- (default: `false`)
+--- @field skip_comments? boolean
 
 --- @class vim.json.encode.Opts
 --- @inlinedoc
 ---
----  Escape slash characters "/" in string values.
+--- Escape slash characters "/" in string values.
 --- (default: `false`)
 --- @field escape_slash? boolean
 ---
----
 --- If non-empty, the returned JSON is formatted with newlines and whitespace, where `indent`
 --- defines the whitespace at each nesting level.
 --- (default: `""`)

src/cjson/lua_cjson.c

@@ -85,6 +85,7 @@
 #define DEFAULT_ENCODE_NUMBER_PRECISION 16
 #define DEFAULT_ENCODE_EMPTY_TABLE_AS_OBJECT 0
 #define DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT 0
+#define DEFAULT_DECODE_SKIP_COMMENTS 0
 #define DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH 1
 #define DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES 0
 #define DEFAULT_ENCODE_INDENT NULL
@@ -206,6 +207,7 @@ typedef struct {
     int decode_invalid_numbers;
     int decode_max_depth;
     int decode_array_with_array_mt;
+    int decode_skip_comments;
     int encode_skip_unsupported_value_types;
 } json_config_t;
 
@@ -230,6 +232,7 @@ typedef struct {
     bool luanil_object;
     /* convert null in json arrays to lua nil instead of vim.NIL */
     bool luanil_array;
+    bool skip_comments;
 } json_options_t;
 
 typedef struct {
@@ -455,6 +458,18 @@ static int json_cfg_decode_array_with_array_mt(lua_State *l)
 }
 */
 
+/* Configures whether decoder should skip comments */
+/*
+static int json_cfg_decode_skip_comments(lua_State *l)
+{
+    json_config_t *cfg = json_arg_init(l, 1);
+
+    json_enum_option(l, 1, &cfg->decode_skip_comments, NULL, 1);
+
+    return 1;
+}
+*/
+
 /* Configure how to treat invalid types */
 /*
 static int json_cfg_encode_skip_unsupported_value_types(lua_State *l)
@@ -610,6 +625,7 @@ static void json_create_config(lua_State *l)
     cfg->encode_number_precision = DEFAULT_ENCODE_NUMBER_PRECISION;
     cfg->encode_empty_table_as_object = DEFAULT_ENCODE_EMPTY_TABLE_AS_OBJECT;
     cfg->decode_array_with_array_mt = DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT;
+    cfg->decode_skip_comments = DEFAULT_DECODE_SKIP_COMMENTS;
     cfg->encode_escape_forward_slash = DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH;
     cfg->encode_skip_unsupported_value_types = DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES;
     cfg->encode_indent = DEFAULT_ENCODE_INDENT;
@@ -1569,13 +1585,46 @@ static void json_next_token(json_parse_t *json, json_token_t *token)
     const json_token_type_t *ch2token = json->cfg->ch2token;
     int ch;
 
-    /* Eat whitespace. */
     while (1) {
-        ch = (unsigned char)*(json->ptr);
-        token->type = ch2token[ch];
-        if (token->type != T_WHITESPACE)
+        /* Eat whitespace. */
+        while (1) {
+            ch = (unsigned char)*(json->ptr);
+            token->type = ch2token[ch];
+            if (token->type != T_WHITESPACE)
+                break;
+            json->ptr++;
+        }
+
+        if (!json->options->skip_comments)
             break;
-        json->ptr++;
+
+        /* Eat comments. */
+        if ((unsigned char)json->ptr[0] != '/' ||
+            ((unsigned char)json->ptr[1] != '/' &&
+            (unsigned char)json->ptr[1] != '*')) {
+            break;
+        }
+
+        if (json->ptr[1] == '/') {
+            /* Handle single-line comment */
+            json->ptr += 2;
+            while (*json->ptr != '\0' && *json->ptr != '\n')
+                json->ptr++;
+        } else {
+            /* Handle multi-line comment */
+            json->ptr += 2;
+            while (1) {
+                if (*json->ptr == '\0') {
+                    json_set_token_error(token, json, "unclosed multi-line comment");
+                    return;
+                }
+                if (json->ptr[0] == '*' && json->ptr[1] == '/') {
+                    json->ptr += 2;
+                    break;
+                }
+                json->ptr++;
+            }
+        }
     }
 
     /* Store location of new token. Required when throwing errors
@@ -1821,8 +1870,7 @@ static int json_decode(lua_State *l)
 {
     json_parse_t json;
     json_token_t token;
-    json_options_t options = { .luanil_object = false, .luanil_array = false };
-
+    json_options_t options = { .luanil_object = false, .luanil_array = false, .skip_comments = false };
 
     size_t json_len;
 
@@ -1831,9 +1879,12 @@ static int json_decode(lua_State *l)
         break;
     case 2:
         luaL_checktype(l, 2, LUA_TTABLE);
-        lua_getfield(l, 2, "luanil");
 
-        /* We only handle the luanil option for now */
+        lua_getfield(l, 2, "skip_comments");
+        options.skip_comments = lua_toboolean(l, -1);
+        lua_pop(l, 1);
+
+        lua_getfield(l, 2, "luanil");
         if (lua_isnil(l, -1)) {
             lua_pop(l, 1);
             break;
@@ -1951,6 +2002,7 @@ int lua_cjson_new(lua_State *l)
         /*
         { "encode_empty_table_as_object", json_cfg_encode_empty_table_as_object },
         { "decode_array_with_array_mt", json_cfg_decode_array_with_array_mt },
+        { "decode_skip_comments", json_cfg_decode_skip_comments },
         { "encode_sparse_array", json_cfg_encode_sparse_array },
         { "encode_max_depth", json_cfg_encode_max_depth },
         { "decode_max_depth", json_cfg_decode_max_depth },

test/functional/lua/json_spec.lua

@@ -146,6 +146,38 @@ describe('vim.json.decode()', function()
     local str = ('%s{%s"key"%s:%s[%s"val"%s,%s"val2"%s]%s,%s"key2"%s:%s1%s}%s'):gsub('%%s', s)
     eq({ key = { 'val', 'val2' }, key2 = 1 }, exec_lua([[return vim.json.decode(...)]], str))
   end)
+
+  it('skip_comments', function()
+    eq({}, exec_lua([[return vim.json.decode('{//comment\n}', { skip_comments = true })]]))
+    eq({}, exec_lua([[return vim.json.decode('{//comment\r\n}', { skip_comments = true })]]))
+    eq(
+      'test // /* */ string',
+      exec_lua(
+        [[return vim.json.decode('"test // /* */ string"//comment', { skip_comments = true })]]
+      )
+    )
+    eq(
+      {},
+      exec_lua([[return vim.json.decode('{/* A multi-line\ncomment*/}', { skip_comments = true })]])
+    )
+    eq(
+      { a = 1 },
+      exec_lua([[return vim.json.decode('{"a" /* Comment */: 1}', { skip_comments = true })]])
+    )
+    eq(
+      { a = 1 },
+      exec_lua([[return vim.json.decode('{"a": /* Comment */ 1}', { skip_comments = true })]])
+    )
+    eq({}, exec_lua([[return vim.json.decode('/*first*//*second*/{}', { skip_comments = true })]]))
+    eq(
+      'Expected the end but found unclosed multi-line comment at character 13',
+      pcall_err(exec_lua, [[return vim.json.decode('{}/*Unclosed', { skip_comments = true })]])
+    )
+    eq(
+      'Expected comma or object end but found T_INTEGER at character 12',
+      pcall_err(exec_lua, [[return vim.json.decode('{"a":1/*x*/0}', { skip_comments = true })]])
+    )
+  end)
 end)
 
 describe('vim.json.encode()', function()