generators: separate source generators from scripts

src/nvim/CMakeLists.txt

@@ -11,13 +11,14 @@ endif()
 endif()
 
 set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches)
+set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
-set(MSGPACK_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genmsgpack.lua)
-set(API_UI_EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_api_ui_events.lua)
+set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
+set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
 set(API_METADATA ${PROJECT_BINARY_DIR}/api_metadata.mpack)
 set(FUNCS_DATA ${PROJECT_BINARY_DIR}/funcs_data.mpack)
 set(MSGPACK_LUA_C_BINDINGS ${GENERATED_DIR}/msgpack_lua_c_bindings.generated.c)
-set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
+set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua)
 set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include)
 set(GENERATED_API_DISPATCH ${GENERATED_DIR}/api/private/dispatch_wrappers.generated.h)
 set(GENERATED_FUNCS_METADATA ${GENERATED_DIR}/api/private/funcs_metadata.generated.h)
@@ -31,16 +32,16 @@ set(GENERATED_FUNCS ${GENERATED_DIR}/funcs.generated.h)
 set(GENERATED_EVENTS_ENUM ${GENERATED_INCLUDES_DIR}/auevents_enum.generated.h)
 set(GENERATED_EVENTS_NAMES_MAP ${GENERATED_DIR}/auevents_name_map.generated.h)
 set(GENERATED_OPTIONS ${GENERATED_DIR}/options.generated.h)
-set(EX_CMDS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genex_cmds.lua)
-set(FUNCS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/geneval.lua)
-set(EVENTS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gen_events.lua)
-set(OPTIONS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genoptions.lua)
-set(UNICODE_TABLES_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genunicodetables.lua)
+set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
+set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua)
+set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
+set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua)
+set(UNICODE_TABLES_GENERATOR ${GENERATOR_DIR}/gen_unicode_tables.lua)
 set(UNICODE_DIR ${PROJECT_SOURCE_DIR}/unicode)
 set(GENERATED_UNICODE_TABLES ${GENERATED_DIR}/unicode_tables.generated.h)
 set(VIM_MODULE_FILE ${GENERATED_DIR}/lua/vim_module.generated.h)
 set(VIM_MODULE_SOURCE ${PROJECT_SOURCE_DIR}/src/nvim/lua/vim.lua)
-set(CHAR_BLOB_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gencharblob.lua)
+set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
 set(LINT_SUPPRESS_FILE ${PROJECT_BINARY_DIR}/errors.json)
 set(LINT_SUPPRESS_URL_BASE "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint")
 set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json")
@@ -238,7 +239,7 @@ add_custom_command(OUTPUT ${GENERATED_UNICODE_TABLES}
 add_custom_command(
   OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_FUNCS_METADATA}
          ${API_METADATA} ${MSGPACK_LUA_C_BINDINGS}
-  COMMAND ${LUA_PRG} ${MSGPACK_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
+  COMMAND ${LUA_PRG} ${API_DISPATCH_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
                      ${GENERATED_API_DISPATCH}
                      ${GENERATED_FUNCS_METADATA} ${API_METADATA}
                      ${MSGPACK_LUA_C_BINDINGS}
@@ -246,7 +247,7 @@ add_custom_command(
   DEPENDS
     ${API_HEADERS}
     ${MSGPACK_RPC_HEADERS}
-    ${MSGPACK_GENERATOR}
+    ${API_DISPATCH_GENERATOR}
     ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
 )
 
@@ -263,10 +264,11 @@ list(APPEND NVIM_GENERATED_SOURCES
   "${MSGPACK_LUA_C_BINDINGS}"
 )
 
-add_custom_command(OUTPUT ${GENERATED_UI_EVENTS}
-                          ${GENERATED_UI_EVENTS_CALL}
-                          ${GENERATED_UI_EVENTS_REMOTE}
-                          ${GENERATED_UI_EVENTS_BRIDGE}
+add_custom_command(
+  OUTPUT ${GENERATED_UI_EVENTS}
+         ${GENERATED_UI_EVENTS_CALL}
+         ${GENERATED_UI_EVENTS_REMOTE}
+         ${GENERATED_UI_EVENTS_BRIDGE}
   COMMAND ${LUA_PRG} ${API_UI_EVENTS_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}
                      ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h
                      ${GENERATED_UI_EVENTS}

src/nvim/generators/c_grammar.lua

@@ -0,0 +1,51 @@
+lpeg = require('lpeg')
+
+-- lpeg grammar for building api metadata from a set of header files. It
+-- ignores comments and preprocessor commands and parses a very small subset
+-- of C prototypes with a limited set of types
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local num = R('09')
+local alpha = letter + num
+local nl = P('\r\n') + P('\n')
+local not_nl = any - nl
+local ws = S(' \t') + nl
+local fill = ws ^ 0
+local c_comment = P('//') * (not_nl ^ 0)
+local c_preproc = P('#') * (not_nl ^ 0)
+local typed_container =
+  (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')')
+local c_id = (
+  typed_container +
+  (letter * (alpha ^ 0))
+)
+local c_void = P('void')
+local c_param_type = (
+  ((P('Error') * fill * P('*') * fill) * Cc('error')) +
+  (C(c_id) * (ws ^ 1))
+  )
+local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
+local c_param = Ct(c_param_type * C(c_id))
+local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
+local c_params = Ct(c_void + c_param_list)
+local c_proto = Ct(
+  Cg(c_type, 'return_type') * Cg(c_id, 'name') *
+  fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') *
+  Cg(Cc(false), 'async') *
+  (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) *
+  (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'),
+              'deprecated_since') ^ -1) *
+  (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) *
+  (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
+  (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
+  (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
+  fill * P(';')
+  )
+
+local grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
+return {grammar=grammar, typed_container=typed_container}

src/nvim/generators/dump_bin_array.lua

@@ -0,0 +1,17 @@
+local function dump_bin_array(output, name, data)
+  output:write([[
+  static const uint8_t ]]..name..[[[] = {
+]])
+
+  for i = 1, #data do
+    output:write(string.byte(data, i)..', ')
+    if i % 10 == 0 then
+      output:write('\n  ')
+    end
+  end
+  output:write([[
+};
+]])
+end
+
+return dump_bin_array

src/nvim/generators/gen_api_dispatch.lua

@@ -1,51 +1,5 @@
-lpeg = require('lpeg')
 mpack = require('mpack')
 
--- lpeg grammar for building api metadata from a set of header files. It
--- ignores comments and preprocessor commands and parses a very small subset
--- of C prototypes with a limited set of types
-P, R, S = lpeg.P, lpeg.R, lpeg.S
-C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
-
-any = P(1) -- (consume one character)
-letter = R('az', 'AZ') + S('_$')
-num = R('09')
-alpha = letter + num
-nl = P('\r\n') + P('\n')
-not_nl = any - nl
-ws = S(' \t') + nl
-fill = ws ^ 0
-c_comment = P('//') * (not_nl ^ 0)
-c_preproc = P('#') * (not_nl ^ 0)
-typed_container =
-  (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')')
-c_id = (
-  typed_container +
-  (letter * (alpha ^ 0))
-)
-c_void = P('void')
-c_param_type = (
-  ((P('Error') * fill * P('*') * fill) * Cc('error')) +
-  (C(c_id) * (ws ^ 1))
-  )
-c_type = (C(c_void) * (ws ^ 1)) + c_param_type
-c_param = Ct(c_param_type * C(c_id))
-c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
-c_params = Ct(c_void + c_param_list)
-c_proto = Ct(
-  Cg(c_type, 'return_type') * Cg(c_id, 'name') *
-  fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') *
-  Cg(Cc(false), 'async') *
-  (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) *
-  (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'),
-              'deprecated_since') ^ -1) *
-  (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) *
-  (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
-  (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) *
-  fill * P(';')
-  )
-grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
-
 -- we need at least 4 arguments since the last two are output files
 if arg[1] == '--help' then
   print('Usage: genmsgpack.lua args')
@@ -59,8 +13,8 @@ end
 assert(#arg >= 4)
 functions = {}
 
-local nvimsrcdir = arg[1]
-package.path = nvimsrcdir .. '/?.lua;' .. package.path
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
 
 -- names of all headers relative to the source root (for inclusion in the
 -- generated file)
@@ -77,6 +31,8 @@ lua_c_bindings_outputf = arg[5]
 -- set of function names, used to detect duplicates
 function_names = {}
 
+c_grammar = require('generators.c_grammar')
+
 -- read each input file, parse and append to the api metadata
 for i = 6, #arg do
   local full_path = arg[i]
@@ -88,7 +44,7 @@ for i = 6, #arg do
 
   local input = io.open(full_path, 'rb')
 
-  local tmp = grammar:match(input:read('*all'))
+  local tmp = c_grammar.grammar:match(input:read('*all'))
   for i = 1, #tmp do
     local fn = tmp[i]
     if not fn.noexport then
@@ -193,23 +149,12 @@ for _,f in ipairs(functions) do
 end
 
 
-funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb')
-funcs_metadata_output:write([[
-static const uint8_t funcs_metadata[] = {
-]])
-
 -- serialize the API metadata using msgpack and embed into the resulting
 -- binary for easy querying by clients
-packed_exported_functions = mpack.pack(exported_functions)
-for i = 1, #packed_exported_functions do
-  funcs_metadata_output:write(string.byte(packed_exported_functions, i)..', ')
-  if i % 10 == 0 then
-    funcs_metadata_output:write('\n  ')
-  end
-end
-funcs_metadata_output:write([[
-};
-]])
+funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb')
+packed = mpack.pack(exported_functions)
+dump_bin_array = require("generators.dump_bin_array")
+dump_bin_array(funcs_metadata_output, 'funcs_metadata', packed)
 funcs_metadata_output:close()
 
 -- start building the dispatch wrapper output
@@ -217,7 +162,7 @@ output = io.open(dispatch_outputf, 'wb')
 
 local function real_type(type)
   local rv = type
-  if typed_container:match(rv) then
+  if c_grammar.typed_container:match(rv) then
     if rv:match('Array') then
       rv = 'Array'
     else

src/nvim/generators/gen_api_ui_events.lua

@@ -1,52 +1,8 @@
-lpeg = require('lpeg')
 mpack = require('mpack')
 
--- TODO: reduce copying
--- lpeg grammar for building api metadata from a set of header files. It
--- ignores comments and preprocessor commands and parses a very small subset
--- of C prototypes with a limited set of types
-P, R, S = lpeg.P, lpeg.R, lpeg.S
-C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
 
-any = P(1) -- (consume one character)
-letter = R('az', 'AZ') + S('_$')
-num = R('09')
-alpha = letter + num
-nl = P('\r\n') + P('\n')
-not_nl = any - nl
-ws = S(' \t') + nl
-fill = ws ^ 0
-c_comment = P('//') * (not_nl ^ 0)
-c_preproc = P('#') * (not_nl ^ 0)
-typed_container =
-  (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')')
-c_id = (
-  typed_container +
-  (letter * (alpha ^ 0))
-)
-c_void = P('void')
-c_param_type = (
-  ((P('Error') * fill * P('*') * fill) * Cc('error')) +
-  (C(c_id) * (ws ^ 1))
-  )
-c_type = (C(c_void) * (ws ^ 1)) + c_param_type
-c_param = Ct(c_param_type * C(c_id))
-c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
-c_params = Ct(c_void + c_param_list)
-c_proto = Ct(
-  Cg(c_type, 'return_type') * Cg(c_id, 'name') *
-  fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') *
-  Cg(Cc(false), 'async') *
-  (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) *
-  (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
-  (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
-  (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
-  fill * P(';')
-  )
-grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
-
-
--- we need at least 4 arguments since the last two are output files
 assert(#arg == 6)
 input = io.open(arg[2], 'rb')
 proto_output = io.open(arg[3], 'wb')
@@ -54,9 +10,8 @@ call_output = io.open(arg[4], 'wb')
 remote_output = io.open(arg[5], 'wb')
 bridge_output = io.open(arg[6], 'wb')
 
-functions = {}
-
-local events = grammar:match(input:read('*all'))
+c_grammar = require('generators.c_grammar')
+local events = c_grammar.grammar:match(input:read('*all'))
 
 function write_signature(output, ev, prefix, notype)
   output:write('('..prefix)
@@ -176,8 +131,6 @@ for i = 1, #events do
 
 end
 
-
-
 proto_output:close()
 call_output:close()
 remote_output:close()