api/msgpack-rpc: Parse type information from api/private/defs.h

Enhance msgpack-gen.lua to extract custom api type codes from the ObjectType
enum in api/private/defs.h. The type information is made available from the api
metadata and clients can use to correctly serialize/deserialize these types
using msgpack EXT type.

scripts/msgpack-gen.lua

@@ -38,9 +38,28 @@ assert(#arg >= 1)
 -- api metadata
 api = {
   functions = {},
-  -- Helpers for object-oriented languages
-  classes = {'Buffer', 'Window', 'Tabpage'}
+  types = {}
 }
+
+-- Extract type codes from api/private/defs.h. The codes are values between
+-- comment markers in the ObjectType enum
+local typedefs_header = arg[1]
+local input = io.open(typedefs_header, 'rb')
+local reading_types = false
+while true do
+  local line = input:read('*l'):gsub("^%s*(.-)%s*$", "%1")
+  if reading_types then
+    if line == '// end custom types' then
+      break
+    end
+    local type_name = line:gsub("^kObjectType(.-),$", "%1")
+    api.types[#api.types + 1] = type_name
+  else
+    reading_types = line == '// start custom types'
+  end
+end
+input:close()
+
 -- names of all headers relative to the source root(for inclusion in the
 -- generated file)
 headers = {}
@@ -48,7 +67,7 @@ headers = {}
 outputf = arg[#arg]
 
 -- read each input file, parse and append to the api metadata
-for i = 1, #arg - 1 do
+for i = 2, #arg - 1 do
   local full_path = arg[i]
   local parts = {}
   for part in string.gmatch(full_path, '[^/]+') do

src/nvim/CMakeLists.txt

@@ -3,6 +3,7 @@ include(CheckLibraryExists)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(DISPATCH_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/msgpack-gen.lua)
 file(GLOB API_HEADERS api/*.h)
+file(GLOB API_DEFS api/private/defs.h)
 set(MSGPACK_RPC_HEADER ${PROJECT_SOURCE_DIR}/src/nvim/os/msgpack_rpc.h)
 set(MSGPACK_DISPATCH ${GENERATED_DIR}/msgpack_dispatch.c)
 set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
@@ -123,9 +124,10 @@ foreach(sfile ${NEOVIM_SOURCES}
 endforeach()
 
 add_custom_command(OUTPUT ${MSGPACK_DISPATCH}
-  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_HEADERS} ${MSGPACK_DISPATCH}
+  COMMAND ${LUA_PRG} ${DISPATCH_GENERATOR} ${API_DEFS} ${API_HEADERS} ${MSGPACK_DISPATCH}
   DEPENDS
     ${API_HEADERS}
+    ${API_DEFS}
     ${MSGPACK_RPC_HEADER}
     ${DISPATCH_GENERATOR}
 )

src/nvim/api/private/defs.h

@@ -49,32 +49,36 @@ typedef struct {
 } Dictionary;
 
 typedef enum {
+// The following comments are markers that msgpack-gen.lua uses to extract
+// types, don't remove!
+// start custom types
+  kObjectTypePosition,
+  kObjectTypeBuffer,
+  kObjectTypeWindow,
+  kObjectTypeTabpage,
+// end custom types
   kObjectTypeNil,
   kObjectTypeBoolean,
   kObjectTypeInteger,
   kObjectTypeFloat,
   kObjectTypeString,
-  kObjectTypeBuffer,
-  kObjectTypeWindow,
-  kObjectTypeTabpage,
   kObjectTypeArray,
   kObjectTypeDictionary,
-  kObjectTypePosition,
 } ObjectType;
 
 struct object {
   ObjectType type;
   union {
+    Position position;
+    Buffer buffer;
+    Window window;
+    Tabpage tabpage;
     Boolean boolean;
     Integer integer;
     Float floating;
     String string;
-    Buffer buffer;
-    Window window;
-    Tabpage tabpage;
     Array array;
     Dictionary dictionary;
-    Position position;
   } data;
 };