generators: separate source generators from scripts

src/nvim/generators/c_grammar.lua

@@ -0,0 +1,51 @@
+lpeg = require('lpeg')
+
+-- lpeg grammar for building api metadata from a set of header files. It
+-- ignores comments and preprocessor commands and parses a very small subset
+-- of C prototypes with a limited set of types
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+local any = P(1) -- (consume one character)
+local letter = R('az', 'AZ') + S('_$')
+local num = R('09')
+local alpha = letter + num
+local nl = P('\r\n') + P('\n')
+local not_nl = any - nl
+local ws = S(' \t') + nl
+local fill = ws ^ 0
+local c_comment = P('//') * (not_nl ^ 0)
+local c_preproc = P('#') * (not_nl ^ 0)
+local typed_container =
+  (P('ArrayOf(') + P('DictionaryOf(')) * ((any - P(')')) ^ 1) * P(')')
+local c_id = (
+  typed_container +
+  (letter * (alpha ^ 0))
+)
+local c_void = P('void')
+local c_param_type = (
+  ((P('Error') * fill * P('*') * fill) * Cc('error')) +
+  (C(c_id) * (ws ^ 1))
+  )
+local c_type = (C(c_void) * (ws ^ 1)) + c_param_type
+local c_param = Ct(c_param_type * C(c_id))
+local c_param_list = c_param * (fill * (P(',') * fill * c_param) ^ 0)
+local c_params = Ct(c_void + c_param_list)
+local c_proto = Ct(
+  Cg(c_type, 'return_type') * Cg(c_id, 'name') *
+  fill * P('(') * fill * Cg(c_params, 'parameters') * fill * P(')') *
+  Cg(Cc(false), 'async') *
+  (fill * Cg((P('FUNC_API_SINCE(') * C(num ^ 1)) * P(')'), 'since') ^ -1) *
+  (fill * Cg((P('FUNC_API_DEPRECATED_SINCE(') * C(num ^ 1)) * P(')'),
+              'deprecated_since') ^ -1) *
+  (fill * Cg((P('FUNC_API_ASYNC') * Cc(true)), 'async') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
+  (fill * Cg((P('FUNC_API_NOEVAL') * Cc(true)), 'noeval') ^ -1) *
+  (fill * Cg((P('REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
+  (fill * Cg((P('REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
+  (fill * Cg((P('BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
+  fill * P(';')
+  )
+
+local grammar = Ct((c_proto + c_comment + c_preproc + ws) ^ 1)
+return {grammar=grammar, typed_container=typed_container}

src/nvim/generators/dump_bin_array.lua

@@ -0,0 +1,17 @@
+local function dump_bin_array(output, name, data)
+  output:write([[
+  static const uint8_t ]]..name..[[[] = {
+]])
+
+  for i = 1, #data do
+    output:write(string.byte(data, i)..', ')
+    if i % 10 == 0 then
+      output:write('\n  ')
+    end
+  end
+  output:write([[
+};
+]])
+end
+
+return dump_bin_array

src/nvim/generators/gen_api_dispatch.lua

@@ -0,0 +1,458 @@
+mpack = require('mpack')
+
+-- we need at least 4 arguments since the last two are output files
+if arg[1] == '--help' then
+  print('Usage: genmsgpack.lua args')
+  print('Args: 1: source directory')
+  print('      2: dispatch output file (dispatch_wrappers.generated.h)')
+  print('      3: functions metadata output file (funcs_metadata.generated.h)')
+  print('      4: API metadata output file (api_metadata.mpack)')
+  print('      5: lua C bindings output file (msgpack_lua_c_bindings.generated.c)')
+  print('      rest: C files where API functions are defined')
+end
+assert(#arg >= 4)
+functions = {}
+
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
+
+-- names of all headers relative to the source root (for inclusion in the
+-- generated file)
+headers = {}
+
+-- output h file with generated dispatch functions
+dispatch_outputf = arg[2]
+-- output h file with packed metadata
+funcs_metadata_outputf = arg[3]
+-- output metadata mpack file, for use by other build scripts
+mpack_outputf = arg[4]
+lua_c_bindings_outputf = arg[5]
+
+-- set of function names, used to detect duplicates
+function_names = {}
+
+c_grammar = require('generators.c_grammar')
+
+-- read each input file, parse and append to the api metadata
+for i = 6, #arg do
+  local full_path = arg[i]
+  local parts = {}
+  for part in string.gmatch(full_path, '[^/]+') do
+    parts[#parts + 1] = part
+  end
+  headers[#headers + 1] = parts[#parts - 1]..'/'..parts[#parts]
+
+  local input = io.open(full_path, 'rb')
+
+  local tmp = c_grammar.grammar:match(input:read('*all'))
+  for i = 1, #tmp do
+    local fn = tmp[i]
+    if not fn.noexport then
+      functions[#functions + 1] = tmp[i]
+      function_names[fn.name] = true
+      if #fn.parameters ~= 0 and fn.parameters[1][2] == 'channel_id' then
+        -- this function should receive the channel id
+        fn.receives_channel_id = true
+        -- remove the parameter since it won't be passed by the api client
+        table.remove(fn.parameters, 1)
+      end
+      if #fn.parameters ~= 0 and fn.parameters[#fn.parameters][1] == 'error' then
+        -- function can fail if the last parameter type is 'Error'
+        fn.can_fail = true
+        -- remove the error parameter, msgpack has it's own special field
+        -- for specifying errors
+        fn.parameters[#fn.parameters] = nil
+      end
+    end
+  end
+  input:close()
+end
+
+local function shallowcopy(orig)
+  local copy = {}
+  for orig_key, orig_value in pairs(orig) do
+    copy[orig_key] = orig_value
+  end
+  return copy
+end
+
+local function startswith(String,Start)
+  return string.sub(String,1,string.len(Start))==Start
+end
+
+-- Export functions under older deprecated names.
+-- These will be removed eventually.
+local deprecated_aliases = require("api.dispatch_deprecated")
+for i,f in ipairs(shallowcopy(functions)) do
+  local ismethod = false
+  if startswith(f.name, "nvim_") then
+    if startswith(f.name, "nvim__") then
+      f.since = -1
+    elseif f.since == nil then
+      print("Function "..f.name.." lacks since field.\n")
+      os.exit(1)
+    end
+    f.since = tonumber(f.since)
+    if f.deprecated_since ~= nil then
+      f.deprecated_since = tonumber(f.deprecated_since)
+    end
+
+    if startswith(f.name, "nvim_buf_") then
+      ismethod = true
+    elseif startswith(f.name, "nvim_win_") then
+      ismethod = true
+    elseif startswith(f.name, "nvim_tabpage_") then
+      ismethod = true
+    end
+  else
+    f.noeval = true
+    f.since = 0
+    f.deprecated_since = 1
+  end
+  f.method = ismethod
+  local newname = deprecated_aliases[f.name]
+  if newname ~= nil then
+    if function_names[newname] then
+      -- duplicate
+      print("Function "..f.name.." has deprecated alias\n"
+            ..newname.." which has a separate implementation.\n"..
+            "Please remove it from src/nvim/api/dispatch_deprecated.lua")
+      os.exit(1)
+    end
+    local newf = shallowcopy(f)
+    newf.name = newname
+    if newname == "ui_try_resize" then
+      -- The return type was incorrectly set to Object in 0.1.5.
+      -- Keep it that way for clients that rely on this.
+      newf.return_type = "Object"
+    end
+    newf.impl_name = f.name
+    newf.noeval = true
+    newf.since = 0
+    newf.deprecated_since = 1
+    functions[#functions+1] = newf
+  end
+end
+
+-- don't expose internal attributes like "impl_name" in public metadata
+exported_attributes = {'name', 'parameters', 'return_type', 'method',
+                       'since', 'deprecated_since'}
+exported_functions = {}
+for _,f in ipairs(functions) do
+  if not startswith(f.name, "nvim__") then
+    local f_exported = {}
+    for _,attr in ipairs(exported_attributes) do
+      f_exported[attr] = f[attr]
+    end
+    exported_functions[#exported_functions+1] = f_exported
+  end
+end
+
+
+-- serialize the API metadata using msgpack and embed into the resulting
+-- binary for easy querying by clients
+funcs_metadata_output = io.open(funcs_metadata_outputf, 'wb')
+packed = mpack.pack(exported_functions)
+dump_bin_array = require("generators.dump_bin_array")
+dump_bin_array(funcs_metadata_output, 'funcs_metadata', packed)
+funcs_metadata_output:close()
+
+-- start building the dispatch wrapper output
+output = io.open(dispatch_outputf, 'wb')
+
+local function real_type(type)
+  local rv = type
+  if c_grammar.typed_container:match(rv) then
+    if rv:match('Array') then
+      rv = 'Array'
+    else
+      rv = 'Dictionary'
+    end
+  end
+  return rv
+end
+
+local function attr_name(rt)
+  if rt == 'Float' then
+    return 'floating'
+  else
+    return rt:lower()
+  end
+end
+
+-- start the handler functions. Visit each function metadata to build the
+-- handler function with code generated for validating arguments and calling to
+-- the real API.
+for i = 1, #functions do
+  local fn = functions[i]
+  if fn.impl_name == nil then
+    local args = {}
+
+    output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)')
+    output:write('\n{')
+    output:write('\n  Object ret = NIL;')
+    -- Declare/initialize variables that will hold converted arguments
+    for j = 1, #fn.parameters do
+      local param = fn.parameters[j]
+      local converted = 'arg_'..j
+      output:write('\n  '..param[1]..' '..converted..';')
+    end
+    output:write('\n')
+    output:write('\n  if (args.size != '..#fn.parameters..') {')
+    output:write('\n    api_set_error(error, kErrorTypeException, "Wrong number of arguments: expecting '..#fn.parameters..' but got %zu", args.size);')
+    output:write('\n    goto cleanup;')
+    output:write('\n  }\n')
+
+    -- Validation/conversion for each argument
+    for j = 1, #fn.parameters do
+      local converted, convert_arg, param, arg
+      param = fn.parameters[j]
+      converted = 'arg_'..j
+      local rt = real_type(param[1])
+      if rt ~= 'Object' then
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then
+          -- Buffer, Window, and Tabpage have a specific type, but are stored in integer
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {')
+          output:write('\n    '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;')
+        else
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {')
+          output:write('\n    '..converted..' = args.items['..(j - 1)..'].data.'..attr_name(rt)..';')
+        end
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then
+          -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages
+          output:write('\n  } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {')
+          output:write('\n    '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;')
+        end
+        output:write('\n  } else {')
+        output:write('\n    api_set_error(error, kErrorTypeException, "Wrong type for argument '..j..', expecting '..param[1]..'");')
+        output:write('\n    goto cleanup;')
+        output:write('\n  }\n')
+      else
+        output:write('\n  '..converted..' = args.items['..(j - 1)..'];\n')
+      end
+
+      args[#args + 1] = converted
+    end
+
+    -- function call
+    local call_args = table.concat(args, ', ')
+    output:write('\n  ')
+    if fn.return_type ~= 'void' then
+      -- has a return value, prefix the call with a declaration
+      output:write(fn.return_type..' rv = ')
+    end
+
+    -- write the function name and the opening parenthesis
+    output:write(fn.name..'(')
+
+    if fn.receives_channel_id then
+      -- if the function receives the channel id, pass it as first argument
+      if #args > 0 or fn.can_fail then
+        output:write('channel_id, '..call_args)
+      else
+        output:write('channel_id')
+      end
+    else
+      output:write(call_args)
+    end
+
+    if fn.can_fail then
+      -- if the function can fail, also pass a pointer to the local error object
+      if #args > 0 then
+        output:write(', error);\n')
+      else
+        output:write('error);\n')
+      end
+      -- and check for the error
+      output:write('\n  if (ERROR_SET(error)) {')
+      output:write('\n    goto cleanup;')
+      output:write('\n  }\n')
+    else
+      output:write(');\n')
+    end
+
+    if fn.return_type ~= 'void' then
+      output:write('\n  ret = '..string.upper(real_type(fn.return_type))..'_OBJ(rv);')
+    end
+    output:write('\n\ncleanup:');
+
+    output:write('\n  return ret;\n}\n\n');
+  end
+end
+
+-- Generate a function that initializes method names with handler functions
+output:write([[
+void msgpack_rpc_init_method_table(void)
+{
+  methods = map_new(String, MsgpackRpcRequestHandler)();
+
+]])
+
+for i = 1, #functions do
+  local fn = functions[i]
+  output:write('  msgpack_rpc_add_method_handler('..
+               '(String) {.data = "'..fn.name..'", '..
+               '.size = sizeof("'..fn.name..'") - 1}, '..
+               '(MsgpackRpcRequestHandler) {.fn = handle_'..  (fn.impl_name or fn.name)..
+               ', .async = '..tostring(fn.async)..'});\n')
+
+end
+
+output:write('\n}\n\n')
+output:close()
+
+mpack_output = io.open(mpack_outputf, 'wb')
+mpack_output:write(mpack.pack(functions))
+mpack_output:close()
+
+local function include_headers(output, headers)
+  for i = 1, #headers do
+    if headers[i]:sub(-12) ~= '.generated.h' then
+      output:write('\n#include "nvim/'..headers[i]..'"')
+    end
+  end
+end
+
+local function write_shifted_output(output, str)
+  str = str:gsub('\n  ', '\n')
+  str = str:gsub('^  ', '')
+  str = str:gsub(' +$', '')
+  output:write(str)
+end
+
+-- start building lua output
+output = io.open(lua_c_bindings_outputf, 'wb')
+
+output:write([[
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+
+#include "nvim/func_attr.h"
+#include "nvim/api/private/defs.h"
+#include "nvim/api/private/helpers.h"
+#include "nvim/lua/converter.h"
+]])
+include_headers(output, headers)
+output:write('\n')
+
+lua_c_functions = {}
+
+local function process_function(fn)
+  lua_c_function_name = ('nlua_msgpack_%s'):format(fn.name)
+  write_shifted_output(output, string.format([[
+
+  static int %s(lua_State *lstate)
+  {
+    Error err = ERROR_INIT;
+    if (lua_gettop(lstate) != %i) {
+      api_set_error(&err, kErrorTypeValidation, "Expected %i argument%s");
+      goto exit_0;
+    }
+  ]], lua_c_function_name, #fn.parameters, #fn.parameters,
+      (#fn.parameters == 1) and '' or 's'))
+  lua_c_functions[#lua_c_functions + 1] = {
+    binding=lua_c_function_name,
+    api=fn.name
+  }
+  local cparams = ''
+  local free_code = {}
+  for j = #fn.parameters,1,-1 do
+    param = fn.parameters[j]
+    cparam = string.format('arg%u', j)
+    param_type = real_type(param[1])
+    lc_param_type = param_type:lower()
+    write_shifted_output(output, string.format([[
+    const %s %s = nlua_pop_%s(lstate, &err);
+
+    if (ERROR_SET(&err)) {
+      goto exit_%u;
+    }
+    ]], param[1], cparam, param_type, #fn.parameters - j))
+    free_code[#free_code + 1] = ('api_free_%s(%s);'):format(
+      lc_param_type, cparam)
+    cparams = cparam .. ', ' .. cparams
+  end
+  if fn.receives_channel_id then
+    cparams = 'LUA_INTERNAL_CALL, ' .. cparams
+  end
+  if fn.can_fail then
+    cparams = cparams .. '&err'
+  else
+    cparams = cparams:gsub(', $', '')
+  end
+  local free_at_exit_code = ''
+  for i = 1, #free_code do
+    local rev_i = #free_code - i + 1
+    local code = free_code[rev_i]
+    if i == 1 then
+      free_at_exit_code = free_at_exit_code .. ('\n    %s'):format(code)
+    else
+      free_at_exit_code = free_at_exit_code .. ('\n  exit_%u:\n    %s'):format(
+        rev_i, code)
+    end
+  end
+  local err_throw_code = [[
+
+  exit_0:
+    if (ERROR_SET(&err)) {
+      luaL_where(lstate, 1);
+      lua_pushstring(lstate, err.msg);
+      api_clear_error(&err);
+      lua_concat(lstate, 2);
+      return lua_error(lstate);
+    }
+  ]]
+  if fn.return_type ~= 'void' then
+    if fn.return_type:match('^ArrayOf') then
+      return_type = 'Array'
+    else
+      return_type = fn.return_type
+    end
+    write_shifted_output(output, string.format([[
+    const %s ret = %s(%s);
+    nlua_push_%s(lstate, ret);
+    api_free_%s(ret);
+  %s
+  %s
+    return 1;
+    ]], fn.return_type, fn.name, cparams, return_type, return_type:lower(),
+        free_at_exit_code, err_throw_code))
+  else
+    write_shifted_output(output, string.format([[
+    %s(%s);
+  %s
+  %s
+    return 0;
+    ]], fn.name, cparams, free_at_exit_code, err_throw_code))
+  end
+  write_shifted_output(output, [[
+  }
+  ]])
+end
+
+for _, fn in ipairs(functions) do
+  if not fn.noeval or fn.name:sub(1, 4) == '_vim' then
+    process_function(fn)
+  end
+end
+
+output:write(string.format([[
+void nlua_add_api_functions(lua_State *lstate)
+  FUNC_ATTR_NONNULL_ALL
+{
+  lua_createtable(lstate, 0, %u);
+]], #lua_c_functions))
+for _, func in ipairs(lua_c_functions) do
+  output:write(string.format([[
+
+  lua_pushcfunction(lstate, &%s);
+  lua_setfield(lstate, -2, "%s");]], func.binding, func.api))
+end
+output:write([[
+
+  lua_setfield(lstate, -2, "api");
+}
+]])
+
+output:close()

src/nvim/generators/gen_api_ui_events.lua

@@ -0,0 +1,136 @@
+mpack = require('mpack')
+
+local nvimdir = arg[1]
+package.path = nvimdir .. '/?.lua;' .. package.path
+
+assert(#arg == 6)
+input = io.open(arg[2], 'rb')
+proto_output = io.open(arg[3], 'wb')
+call_output = io.open(arg[4], 'wb')
+remote_output = io.open(arg[5], 'wb')
+bridge_output = io.open(arg[6], 'wb')
+
+c_grammar = require('generators.c_grammar')
+local events = c_grammar.grammar:match(input:read('*all'))
+
+function write_signature(output, ev, prefix, notype)
+  output:write('('..prefix)
+  if prefix == "" and #ev.parameters == 0 then
+    output:write('void')
+  end
+  for j = 1, #ev.parameters do
+    if j > 1 or prefix ~= '' then
+      output:write(', ')
+    end
+    local param = ev.parameters[j]
+    if not notype then
+      output:write(param[1]..' ')
+    end
+    output:write(param[2])
+  end
+  output:write(')')
+end
+
+function write_arglist(output, ev, need_copy)
+  output:write('  Array args = ARRAY_DICT_INIT;\n')
+  for j = 1, #ev.parameters do
+    local param = ev.parameters[j]
+    local kind = string.upper(param[1])
+    local do_copy = need_copy and (kind == "ARRAY" or kind == "DICTIONARY" or kind == "STRING")
+    output:write('  ADD(args, ')
+    if do_copy then
+      output:write('copy_object(')
+    end
+    output:write(kind..'_OBJ('..param[2]..')')
+    if do_copy then
+      output:write(')')
+    end
+    output:write(');\n')
+  end
+end
+
+for i = 1, #events do
+  ev = events[i]
+  assert(ev.return_type == 'void')
+
+  if not ev.remote_only then
+    proto_output:write('  void (*'..ev.name..')')
+    write_signature(proto_output, ev, 'UI *ui')
+    proto_output:write(';\n')
+
+    if not ev.remote_impl then
+      remote_output:write('static void remote_ui_'..ev.name)
+      write_signature(remote_output, ev, 'UI *ui')
+      remote_output:write('\n{\n')
+      write_arglist(remote_output, ev, true)
+      remote_output:write('  push_call(ui, "'..ev.name..'", args);\n')
+      remote_output:write('}\n\n')
+    end
+
+    if not ev.bridge_impl then
+
+      send, argv, recv, recv_argv, recv_cleanup = '', '', '', '', ''
+      argc = 1
+      for j = 1, #ev.parameters do
+        local param = ev.parameters[j]
+        copy = 'copy_'..param[2]
+        if param[1] == 'String' then
+          send = send..'  String copy_'..param[2]..' = copy_string('..param[2]..');\n'
+          argv = argv..', '..copy..'.data, INT2PTR('..copy..'.size)'
+          recv = (recv..'  String '..param[2]..
+                          ' = (String){.data = argv['..argc..'],'..
+                          '.size = (size_t)argv['..(argc+1)..']};\n')
+          recv_argv = recv_argv..', '..param[2]
+          recv_cleanup = recv_cleanup..'  api_free_string('..param[2]..');\n'
+          argc = argc+2
+        elseif param[1] == 'Array' then
+          send = send..'  Array copy_'..param[2]..' = copy_array('..param[2]..');\n'
+          argv = argv..', '..copy..'.items, INT2PTR('..copy..'.size)'
+          recv = (recv..'  Array '..param[2]..
+                          ' = (Array){.items = argv['..argc..'],'..
+                          '.size = (size_t)argv['..(argc+1)..']};\n')
+          recv_argv = recv_argv..', '..param[2]
+          recv_cleanup = recv_cleanup..'  api_free_array('..param[2]..');\n'
+          argc = argc+2
+        elseif param[1] == 'Integer' or param[1] == 'Boolean' then
+          argv = argv..', INT2PTR('..param[2]..')'
+          recv_argv = recv_argv..', PTR2INT(argv['..argc..'])'
+          argc = argc+1
+        else
+          assert(false)
+        end
+      end
+      bridge_output:write('static void ui_bridge_'..ev.name..
+                          '_event(void **argv)\n{\n')
+      bridge_output:write('  UI *ui = UI(argv[0]);\n')
+      bridge_output:write(recv)
+      bridge_output:write('  ui->'..ev.name..'(ui'..recv_argv..');\n')
+      bridge_output:write(recv_cleanup)
+      bridge_output:write('}\n\n')
+
+      bridge_output:write('static void ui_bridge_'..ev.name)
+      write_signature(bridge_output, ev, 'UI *ui')
+      bridge_output:write('\n{\n')
+      bridge_output:write(send)
+      bridge_output:write('  UI_CALL(ui, '..ev.name..', '..argc..', ui'..argv..');\n}\n')
+    end
+  end
+
+  call_output:write('void ui_call_'..ev.name)
+  write_signature(call_output, ev, '')
+  call_output:write('\n{\n')
+  if ev.remote_only then
+    write_arglist(call_output, ev, false)
+    call_output:write('  ui_event("'..ev.name..'", args);\n')
+  else
+    call_output:write('  UI_CALL')
+    write_signature(call_output, ev, ev.name, true)
+    call_output:write(";\n")
+  end
+  call_output:write("}\n\n")
+
+end
+
+proto_output:close()
+call_output:close()
+remote_output:close()

src/nvim/generators/gen_char_blob.lua

@@ -0,0 +1,48 @@
+if arg[1] == '--help' then
+  print('Usage:')
+  print('  gencharblob.lua source target varname')
+  print('')
+  print('Generates C file with big uint8_t blob.')
+  print('Blob will be stored in a static const array named varname.')
+  os.exit()
+end
+
+assert(#arg == 3)
+
+local source_file = arg[1]
+local target_file = arg[2]
+local varname = arg[3]
+
+source = io.open(source_file, 'r')
+target = io.open(target_file, 'w')
+
+target:write('#include <stdint.h>\n\n')
+target:write(('static const uint8_t %s[] = {\n'):format(varname))
+
+num_bytes = 0
+MAX_NUM_BYTES = 15  -- 78 / 5: maximum number of bytes on one line
+target:write(' ')
+
+increase_num_bytes = function()
+  num_bytes = num_bytes + 1
+  if num_bytes == MAX_NUM_BYTES then
+    num_bytes = 0
+    target:write('\n ')
+  end
+end
+
+for line in source:lines() do
+  for i = 1,string.len(line) do
+    byte = string.byte(line, i)
+    assert(byte ~= 0)
+    target:write(string.format(' %3u,', byte))
+    increase_num_bytes()
+  end
+  target:write(string.format(' %3u,', string.byte('\n', 1)))
+  increase_num_bytes()
+end
+
+target:write('   0};\n')
+
+source:close()
+target:close()

src/nvim/generators/gen_declarations.lua

@@ -0,0 +1,287 @@
+#!/usr/bin/lua
+
+local fname = arg[1]
+local static_fname = arg[2]
+local non_static_fname = arg[3]
+local preproc_fname = arg[4]
+
+
+local lpeg = require('lpeg')
+
+local fold = function (func, ...)
+  local result = nil
+  for i, v in ipairs({...}) do
+    if result == nil then
+      result = v
+    else
+      result = func(result, v)
+    end
+  end
+  return result
+end
+
+local folder = function (func)
+  return function (...)
+    return fold(func, ...)
+  end
+end
+
+local lit = lpeg.P
+local set = function(...)
+  return lpeg.S(fold(function (a, b) return a .. b end, ...))
+end
+local any_character = lpeg.P(1)
+local rng = function(s, e) return lpeg.R(s .. e) end
+local concat = folder(function (a, b) return a * b end)
+local branch = folder(function (a, b) return a + b end)
+local one_or_more = function(v) return v ^ 1 end
+local two_or_more = function(v) return v ^ 2 end
+local any_amount = function(v) return v ^ 0 end
+local one_or_no = function(v) return v ^ -1 end
+local look_behind = lpeg.B
+local look_ahead = function(v) return #v end
+local neg_look_ahead = function(v) return -v end
+local neg_look_behind = function(v) return -look_behind(v) end
+
+local w = branch(
+  rng('a', 'z'),
+  rng('A', 'Z'),
+  lit('_')
+)
+local aw = branch(
+  w,
+  rng('0', '9')
+)
+local s = set(' ', '\n', '\t')
+local raw_word = concat(w, any_amount(aw))
+local right_word = concat(
+  raw_word,
+  neg_look_ahead(aw)
+)
+local word = branch(
+  concat(
+    branch(lit('ArrayOf('), lit('DictionaryOf(')), -- typed container macro
+    one_or_more(any_character - lit(')')),
+    lit(')')
+  ),
+  concat(
+    neg_look_behind(aw),
+    right_word
+  )
+)
+local inline_comment = concat(
+  lit('/*'),
+  any_amount(concat(
+    neg_look_ahead(lit('*/')),
+    any_character
+  )),
+  lit('*/')
+)
+local spaces = any_amount(branch(
+  s,
+  -- Comments are really handled by preprocessor, so the following is not needed
+  inline_comment,
+  concat(
+    lit('//'),
+    any_amount(concat(
+      neg_look_ahead(lit('\n')),
+      any_character
+    )),
+    lit('\n')
+  ),
+  -- Linemarker inserted by preprocessor
+  concat(
+    lit('# '),
+    any_amount(concat(
+      neg_look_ahead(lit('\n')),
+      any_character
+    )),
+    lit('\n')
+  )
+))
+local typ_part = concat(
+  word,
+  any_amount(concat(
+    spaces,
+    lit('*')
+  )),
+  spaces
+)
+local typ = one_or_more(typ_part)
+local typ_id = two_or_more(typ_part)
+local arg = typ_id         -- argument name is swallowed by typ
+local pattern = concat(
+  any_amount(branch(set(' ', '\t'), inline_comment)),
+  typ_id,                  -- return type with function name
+  spaces,
+  lit('('),
+  spaces,
+  one_or_no(branch(        -- function arguments
+    concat(
+      arg,                 -- first argument, does not require comma
+      any_amount(concat(   -- following arguments, start with a comma
+        spaces,
+        lit(','),
+        spaces,
+        arg,
+        any_amount(concat(
+          lit('['),
+          spaces,
+          any_amount(aw),
+          spaces,
+          lit(']')
+        ))
+      )),
+      one_or_no(concat(
+        spaces,
+        lit(','),
+        spaces,
+        lit('...')
+      ))
+    ),
+    lit('void')            -- also accepts just void
+  )),
+  spaces,
+  lit(')'),
+  any_amount(concat(       -- optional attributes
+    spaces,
+    lit('FUNC_'),
+    any_amount(aw),
+    one_or_no(concat(      -- attribute argument
+      spaces,
+      lit('('),
+      any_amount(concat(
+        neg_look_ahead(lit(')')),
+        any_character
+      )),
+      lit(')')
+    ))
+  )),
+  look_ahead(concat(       -- definition must be followed by "{"
+    spaces,
+    lit('{')
+  ))
+)
+
+if fname == '--help' then
+  print'Usage:'
+  print()
+  print'  gendeclarations.lua definitions.c static.h non-static.h preprocessor.i'
+  os.exit()
+end
+
+local preproc_f = io.open(preproc_fname)
+local text = preproc_f:read("*all")
+preproc_f:close()
+
+
+local header = [[
+#ifndef DEFINE_FUNC_ATTRIBUTES
+# define DEFINE_FUNC_ATTRIBUTES
+#endif
+#include "nvim/func_attr.h"
+#undef DEFINE_FUNC_ATTRIBUTES
+]]
+
+local footer = [[
+#include "nvim/func_attr.h"
+]]
+
+local non_static = header
+local static = header
+
+local filepattern = '^#%a* (%d+) "([^"]-)/?([^"/]+)"'
+local curfile
+
+local init = 0
+local curfile = nil
+local neededfile = fname:match('[^/]+$')
+local declline = 0
+local declendpos = 0
+local curdir = nil
+local is_needed_file = false
+while init ~= nil do
+  init = text:find('[\n;}]', init)
+  if init == nil then
+    break
+  end
+  local init_is_nl = text:sub(init, init) == '\n'
+  init = init + 1
+  if init_is_nl and is_needed_file then
+    declline = declline + 1
+  end
+  if init_is_nl and text:sub(init, init) == '#' then
+    local line, dir, file = text:match(filepattern, init)
+    if file ~= nil then
+      curfile = file
+      is_needed_file = (curfile == neededfile)
+      declline = tonumber(line) - 1
+      local curdir_start = dir:find('src/nvim/')
+      if curdir_start ~= nil then
+        curdir = dir:sub(curdir_start + #('src/nvim/'))
+      else
+        curdir = dir
+      end
+    else
+      declline = declline - 1
+    end
+  elseif init < declendpos then
+    -- Skipping over declaration
+  elseif is_needed_file then
+    s = init
+    e = pattern:match(text, init)
+    if e ~= nil then
+      local declaration = text:sub(s, e - 1)
+      -- Comments are really handled by preprocessor, so the following is not 
+      -- needed
+      declaration = declaration:gsub('/%*.-%*/', '')
+      declaration = declaration:gsub('//.-\n', '\n')
+
+      declaration = declaration:gsub('# .-\n', '')
+
+      declaration = declaration:gsub('\n', ' ')
+      declaration = declaration:gsub('%s+', ' ')
+      declaration = declaration:gsub(' ?%( ?', '(')
+      -- declaration = declaration:gsub(' ?%) ?', ')')
+      declaration = declaration:gsub(' ?, ?', ', ')
+      declaration = declaration:gsub(' ?(%*+) ?', ' %1')
+      declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1')
+      declaration = declaration:gsub(' $', '')
+      declaration = declaration:gsub('^ ', '')
+      declaration = declaration .. ';'
+      declaration = declaration .. ('  // %s/%s:%u'):format(
+          curdir, curfile, declline)
+      declaration = declaration .. '\n'
+      if declaration:sub(1, 6) == 'static' then
+        static = static .. declaration
+      else
+        non_static = non_static .. declaration
+      end
+      declendpos = e
+    end
+  end
+end
+
+non_static = non_static .. footer
+static = static .. footer
+
+local F
+F = io.open(static_fname, 'w')
+F:write(static)
+F:close()
+
+-- Before generating the non-static headers, check if the current file(if
+-- exists) is different from the new one. If they are the same, we won't touch
+-- the current version to avoid triggering an unnecessary rebuilds of modules
+-- that depend on this one
+F = io.open(non_static_fname, 'r')
+if F ~= nil then
+  if F:read('*a') == non_static then
+    os.exit(0)
+  end
+  io.close(F)
+end
+
+F = io.open(non_static_fname, 'w')
+F:write(non_static)
+F:close()

src/nvim/generators/gen_eval.lua

@@ -0,0 +1,67 @@
+mpack = require('mpack')
+
+local nvimsrcdir = arg[1]
+local autodir = arg[2]
+local metadata_file = arg[3]
+local funcs_file = arg[4]
+
+if nvimsrcdir == '--help' then
+  print([[
+Usage:
+  lua geneval.lua src/nvim build/src/nvim/auto
+
+Will generate build/src/nvim/auto/funcs.generated.h with definition of functions 
+static const array.
+]])
+  os.exit(0)
+end
+
+package.path = nvimsrcdir .. '/?.lua;' .. package.path
+
+local funcsfname = autodir .. '/funcs.generated.h'
+
+local gperfpipe = io.open(funcsfname .. '.gperf', 'wb')
+
+local funcs = require('eval').funcs
+local metadata = mpack.unpack(io.open(arg[3], 'rb'):read("*all"))
+for i,fun in ipairs(metadata) do
+  if not fun.noeval then
+    funcs[fun.name] = {
+      args=#fun.parameters,
+      func='api_wrapper',
+      data='&handle_'..fun.name,
+    }
+  end
+end
+
+local funcsdata = io.open(funcs_file, 'w')
+funcsdata:write(mpack.pack(funcs))
+funcsdata:close()
+
+gperfpipe:write([[
+%language=ANSI-C
+%global-table
+%readonly-tables
+%define initializer-suffix ,0,0,NULL,NULL
+%define word-array-name functions
+%define hash-function-name hash_internal_func_gperf
+%define lookup-function-name find_internal_func_gperf
+%omit-struct-type
+%struct-type
+VimLFuncDef;
+%%
+]])
+
+for name, def in pairs(funcs) do
+  args = def.args or 0
+  if type(args) == 'number' then
+    args = {args, args}
+  elseif #args == 1 then
+    args[2] = 'MAX_FUNC_ARGS'
+  end
+  func = def.func or ('f_' .. name)
+  data = def.data or "NULL"
+  gperfpipe:write(('%s,  %s, %s, &%s, (FunPtr)%s\n')
+                  :format(name, args[1], args[2], func, data))
+end
+gperfpipe:close()

src/nvim/generators/gen_events.lua

@@ -0,0 +1,65 @@
+if arg[1] == '--help' then
+  print('Usage: gen_events.lua src/nvim enum_file event_names_file')
+  os.exit(0)
+end
+
+local nvimsrcdir = arg[1]
+local fileio_enum_file = arg[2]
+local names_file = arg[3]
+
+package.path = nvimsrcdir .. '/?.lua;' .. package.path
+
+local auevents = require('auevents')
+local events = auevents.events
+local aliases = auevents.aliases
+
+enum_tgt = io.open(fileio_enum_file, 'w')
+names_tgt = io.open(names_file, 'w')
+
+enum_tgt:write('typedef enum auto_event {')
+names_tgt:write([[
+static const struct event_name {
+  size_t len;
+  char *name;
+  event_T event;
+} event_names[] = {]])
+
+for i, event in ipairs(events) do
+  if i > 1 then
+    comma = ',\n'
+  else
+    comma = '\n'
+  end
+  enum_tgt:write(('%s  EVENT_%s = %u'):format(comma, event:upper(), i - 1))
+  names_tgt:write(('%s  {%u, "%s", EVENT_%s}'):format(comma, #event, event, event:upper()))
+end
+
+for alias, event in pairs(aliases) do
+  names_tgt:write((',\n {%u, "%s", EVENT_%s}'):format(#alias, alias, event:upper()))
+end
+
+names_tgt:write(',\n  {0, NULL, (event_T)0}')
+
+enum_tgt:write('\n} event_T;\n')
+names_tgt:write('\n};\n')
+
+enum_tgt:write(('\n#define NUM_EVENTS %u\n'):format(#events))
+names_tgt:write('\nstatic AutoPat *first_autopat[NUM_EVENTS] = {\n ')
+line_len = 1
+for i = 1,((#events) - 1) do
+  line_len = line_len + #(' NULL,')
+  if line_len > 80 then
+    names_tgt:write('\n ')
+    line_len = 1 + #(' NULL,')
+  end
+  names_tgt:write(' NULL,')
+end
+if line_len + #(' NULL') > 80 then
+  names_tgt:write('\n  NULL')
+else
+  names_tgt:write(' NULL')
+end
+names_tgt:write('\n};\n')
+
+enum_tgt:close()
+names_tgt:close()

src/nvim/generators/gen_ex_cmds.lua

@@ -0,0 +1,88 @@
+local nvimsrcdir = arg[1]
+local includedir = arg[2]
+local autodir = arg[3]
+
+if nvimsrcdir == '--help' then
+  print ([[
+Usage:
+  lua genex_cmds.lua src/nvim build/include build/src/nvim/auto
+
+Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T 
+enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands 
+definitions.
+]])
+  os.exit(0)
+end
+
+package.path = nvimsrcdir .. '/?.lua;' .. package.path
+
+local enumfname = includedir .. '/ex_cmds_enum.generated.h'
+local defsfname = autodir .. '/ex_cmds_defs.generated.h'
+
+local enumfile = io.open(enumfname, 'w')
+local defsfile = io.open(defsfname, 'w')
+
+local defs = require('ex_cmds')
+local lastchar = nil
+
+local i
+local cmd
+local first = true
+local prevfirstchar = nil
+
+local byte_a = string.byte('a')
+local byte_z = string.byte('z')
+
+local cmdidxs = string.format([[
+static const cmdidx_T cmdidxs[%u] = {
+]], byte_z - byte_a + 2)
+
+enumfile:write([[
+typedef enum CMD_index {
+]])
+defsfile:write(string.format([[
+static CommandDefinition cmdnames[%u] = {
+]], #defs))
+for i, cmd in ipairs(defs) do
+  local enumname = cmd.enum or ('CMD_' .. cmd.command)
+  firstchar = string.byte(cmd.command)
+  if firstchar ~= prevfirstchar then
+    if (not prevfirstchar
+        or (byte_a <= firstchar      and firstchar     <= byte_z)
+        or (byte_a <= prevfirstchar  and prevfirstchar <= byte_z)) then
+      if not first then
+        cmdidxs = cmdidxs .. ',\n'
+      end
+      cmdidxs = cmdidxs .. '  ' .. enumname
+    end
+    prevfirstchar = firstchar
+  end
+  if first then
+    first = false
+  else
+    defsfile:write(',\n')
+  end
+  enumfile:write('  ' .. enumname .. ',\n')
+  defsfile:write(string.format([[
+  [%s] = {
+    .cmd_name = (char_u *) "%s",
+    .cmd_func = (ex_func_T)&%s,
+    .cmd_argt = %uL,
+    .cmd_addr_type = %i
+  }]], enumname, cmd.command, cmd.func, cmd.flags, cmd.addr_type))
+end
+defsfile:write([[
+
+};
+]])
+enumfile:write([[
+  CMD_SIZE,
+  CMD_USER = -1,
+  CMD_USER_BUF = -2
+} cmdidx_T;
+]])
+cmdidxs = cmdidxs .. [[
+
+};
+]]
+defsfile:write(cmdidxs)

src/nvim/generators/gen_options.lua

@@ -0,0 +1,190 @@
+if arg[1] == '--help' then
+  print('Usage: genoptions.lua src/nvim options_file')
+  os.exit(0)
+end
+
+local nvimsrcdir = arg[1]
+local options_file = arg[2]
+
+package.path = nvimsrcdir .. '/?.lua;' .. package.path
+
+local opt_fd = io.open(options_file, 'w')
+
+local w = function(s)
+  if s:match('^    %.') then
+    opt_fd:write(s .. ',\n')
+  else
+    opt_fd:write(s .. '\n')
+  end
+end
+
+local options = require('options')
+
+cstr = options.cstr
+
+local type_flags={
+  bool='P_BOOL',
+  number='P_NUM',
+  string='P_STRING',
+}
+
+local redraw_flags={
+  statuslines='P_RSTAT',
+  current_window='P_RWIN',
+  current_window_only='P_RWINONLY',
+  current_buffer='P_RBUF',
+  all_windows='P_RALL',
+  everything='P_RCLR',
+  curswant='P_CURSWANT',
+}
+
+local list_flags={
+  comma='P_COMMA',
+  onecomma='P_ONECOMMA',
+  flags='P_FLAGLIST',
+  flagscomma='P_COMMA|P_FLAGLIST',
+}
+
+local get_flags = function(o)
+  local ret = {type_flags[o.type]}
+  local add_flag = function(f)
+    ret[1] = ret[1] .. '|' .. f
+  end
+  if o.list then
+    add_flag(list_flags[o.list])
+  end
+  if o.redraw then
+    for _, r_flag in ipairs(o.redraw) do
+      add_flag(redraw_flags[r_flag])
+    end
+  end
+  if o.expand then
+    add_flag('P_EXPAND')
+    if o.expand == 'nodefault' then
+      add_flag('P_NO_DEF_EXP')
+    end
+  end
+  for _, flag_desc in ipairs({
+    {'alloced'},
+    {'nodefault'},
+    {'no_mkrc'},
+    {'vi_def'},
+    {'vim'},
+    {'secure'},
+    {'gettext'},
+    {'noglob'},
+    {'normal_fname_chars', 'P_NFNAME'},
+    {'pri_mkrc'},
+    {'deny_in_modelines', 'P_NO_ML'},
+    {'deny_duplicates', 'P_NODUP'},
+  }) do
+    local key_name = flag_desc[1]
+    local def_name = flag_desc[2] or ('P_' .. key_name:upper())
+    if o[key_name] then
+      add_flag(def_name)
+    end
+  end
+  return ret[1]
+end
+
+local get_cond
+get_cond = function(c, base_string)
+  local cond_string = base_string or '#if '
+  if type(c) == 'table' then
+    cond_string = cond_string .. get_cond(c[1], '')
+    for i, subc in ipairs(c) do
+      if i > 1 then
+        cond_string = cond_string .. ' && ' .. get_cond(subc, '')
+      end
+    end
+  elseif c:sub(1, 1) == '!' then
+    cond_string = cond_string .. '!defined(' .. c:sub(2) .. ')'
+  else
+    cond_string = cond_string .. 'defined(' .. c .. ')'
+  end
+  return cond_string
+end
+
+value_dumpers = {
+  ['function']=function(v) return v() end,
+  string=cstr,
+  boolean=function(v) return v and 'true' or 'false' end,
+  number=function(v) return ('%iL'):format(v) end,
+  ['nil']=function(v) return '0L' end,
+}
+
+local get_value = function(v)
+  return '(char_u *) ' .. value_dumpers[type(v)](v)
+end
+
+local get_defaults = function(d)
+  return '{' .. get_value(d.vi) .. ', ' .. get_value(d.vim) .. '}'
+end
+
+local defines = {}
+
+local dump_option = function(i, o)
+  w('  [' .. ('%u'):format(i - 1) .. ']={')
+  w('    .fullname=' .. cstr(o.full_name))
+  if o.abbreviation then
+    w('    .shortname=' .. cstr(o.abbreviation))
+  end
+  w('    .flags=' .. get_flags(o))
+  if o.enable_if then
+    w(get_cond(o.enable_if))
+  end
+  if o.varname then
+    w('    .var=(char_u *)&' .. o.varname)
+  elseif #o.scope == 1 and o.scope[1] == 'window' then
+    w('    .var=VAR_WIN')
+  end
+  if o.enable_if then
+    w('#endif')
+  end
+  if #o.scope == 1 and o.scope[1] == 'global' then
+    w('    .indir=PV_NONE')
+  else
+    assert (#o.scope == 1 or #o.scope == 2)
+    assert (#o.scope == 1 or o.scope[1] == 'global')
+    local min_scope = o.scope[#o.scope]
+    local varname = o.pv_name or o.varname or (
+      'p_' .. (o.abbreviation or o.full_name))
+    local pv_name = (
+      'OPT_' .. min_scope:sub(1, 3):upper() .. '(' .. (
+        min_scope:sub(1, 1):upper() .. 'V_' .. varname:sub(3):upper()
+      ) .. ')'
+    )
+    if #o.scope == 2 then
+      pv_name = 'OPT_BOTH(' .. pv_name .. ')'
+    end
+    defines['PV_' .. varname:sub(3):upper()] = pv_name
+    w('    .indir=' .. pv_name)
+  end
+  if o.defaults then
+    if o.defaults.condition then
+      w(get_cond(o.defaults.condition))
+    end
+    w('    .def_val=' .. get_defaults(o.defaults.if_true))
+    if o.defaults.condition then
+      if o.defaults.if_false then
+        w('#else')
+        w('    .def_val=' .. get_defaults(o.defaults.if_false))
+      end
+      w('#endif')
+    end
+  end
+  w('  },')
+end
+
+w('static vimoption_T options[] = {')
+for i, o in ipairs(options.options) do
+  dump_option(i, o)
+end
+w('  [' .. ('%u'):format(#options.options) .. ']={.fullname=NULL}')
+w('};')
+w('')
+
+for k, v in pairs(defines) do
+  w('#define ' .. k .. ' ' .. v)
+end
+opt_fd:close()

src/nvim/generators/gen_unicode_tables.lua

@@ -0,0 +1,327 @@
+-- Script creates the following tables in unicode_tables.generated.h:
+--
+-- 1. doublewidth and ambiguous tables: sorted list of non-overlapping closed 
+--    intervals. Codepoints in these intervals have double (W or F) or ambiguous 
+--    (A) east asian width respectively.
+-- 2. combining table: same as the above, but characters inside are combining 
+--    characters (i.e. have general categories equal to Mn, Mc or Me).
+-- 3. foldCase, toLower and toUpper tables used to convert characters to 
+--    folded/lower/upper variants. In these tables first two values are 
+--    character ranges: like in previous tables they are sorted and must be 
+--    non-overlapping. Third value means step inside the range: e.g. if it is 
+--    2 then interval applies only to first, third, fifth, … character in range. 
+--    Fourth value is number that should be added to the codepoint to yield 
+--    folded/lower/upper codepoint.
+-- 4. emoji_width and emoji_all tables: sorted lists of non-overlapping closed
+--    intervals of Emoji characters.  emoji_width contains all the characters
+--    which don't have ambiguous or double width, and emoji_all has all Emojis.
+if arg[1] == '--help' then
+  print('Usage:')
+  print('  genunicodetables.lua unicode/ unicode_tables.generated.h')
+  os.exit(0)
+end
+
+local basedir = arg[1]
+local pathsep = package.config:sub(1, 1)
+local get_path = function(fname)
+  return basedir .. pathsep .. fname
+end
+
+local unicodedata_fname = get_path('UnicodeData.txt')
+local casefolding_fname = get_path('CaseFolding.txt')
+local eastasianwidth_fname = get_path('EastAsianWidth.txt')
+local emoji_fname = get_path('emoji-data.txt')
+
+local utf_tables_fname = arg[2]
+
+local split_on_semicolons = function(s)
+  local ret = {}
+  local idx = 1
+  while idx <= #s + 1 do
+    item = s:match('^[^;]*', idx)
+    idx = idx + #item + 1
+    if idx <= #s + 1 then
+      assert(s:sub(idx - 1, idx - 1) == ';')
+    end
+    item = item:gsub('^%s*', '')
+    item = item:gsub('%s*$', '')
+    table.insert(ret, item)
+  end
+  return ret
+end
+
+local fp_lines_to_lists = function(fp, n, has_comments)
+  local ret = {}
+  local line
+  local i = 0
+  while true do
+    i = i + 1
+    line = fp:read('*l')
+    if not line then
+      break
+    end
+    if (not has_comments
+        or (line:sub(1, 1) ~= '#' and not line:match('^%s*$'))) then
+      local l = split_on_semicolons(line)
+      if #l ~= n then
+        io.stderr:write(('Found %s items in line %u, expected %u\n'):format(
+          #l, i, n))
+        io.stderr:write('Line: ' .. line .. '\n')
+        return nil
+      end
+      table.insert(ret, l)
+    end
+  end
+  return ret
+end
+
+local parse_data_to_props = function(ud_fp)
+  return fp_lines_to_lists(ud_fp, 15, false)
+end
+
+local parse_fold_props = function(cf_fp)
+  return fp_lines_to_lists(cf_fp, 4, true)
+end
+
+local parse_width_props = function(eaw_fp)
+  return fp_lines_to_lists(eaw_fp, 2, true)
+end
+
+local parse_emoji_props = function(emoji_fp)
+  return fp_lines_to_lists(emoji_fp, 2, true)
+end
+
+local make_range = function(start, end_, step, add)
+  if step and add then
+    return ('  {0x%x, 0x%x, %d, %d},\n'):format(
+      start, end_, step == 0 and -1 or step, add)
+  else
+    return ('  {0x%04x, 0x%04x},\n'):format(start, end_)
+  end
+end
+
+local build_convert_table = function(ut_fp, props, cond_func, nl_index,
+                                     table_name)
+  ut_fp:write('static const convertStruct ' .. table_name .. '[] = {\n')
+  local start = -1
+  local end_ = -1
+  local step = 0
+  local add = -1
+  for _, p in ipairs(props) do
+    if cond_func(p) then
+      local n = tonumber(p[1], 16)
+      local nl = tonumber(p[nl_index], 16)
+      if start >= 0 and add == (nl - n) and (step == 0 or n - end_ == step) then
+        -- Continue with the same range.
+        step = n - end_
+        end_ = n
+      else
+        if start >= 0 then
+          -- Produce previous range.
+          ut_fp:write(make_range(start, end_, step, add))
+        end
+        start = n
+        end_ = n
+        step = 0
+        add = nl - n
+      end
+    end
+  end
+  if start >= 0 then
+    ut_fp:write(make_range(start, end_, step, add))
+  end
+  ut_fp:write('};\n')
+end
+
+local build_case_table = function(ut_fp, dataprops, table_name, index)
+  local cond_func = function(p)
+    return p[index] ~= ''
+  end
+  return build_convert_table(ut_fp, dataprops, cond_func, index,
+                             'to' .. table_name)
+end
+
+local build_fold_table = function(ut_fp, foldprops)
+  local cond_func = function(p)
+    return (p[2] == 'C' or p[2] == 'S')
+  end
+  return build_convert_table(ut_fp, foldprops, cond_func, 3, 'foldCase')
+end
+
+local build_combining_table = function(ut_fp, dataprops)
+  ut_fp:write('static const struct interval combining[] = {\n')
+  local start = -1
+  local end_ = -1
+  for _, p in ipairs(dataprops) do
+    if (({Mn=true, Mc=true, Me=true})[p[3]]) then
+      local n = tonumber(p[1], 16)
+      if start >= 0 and end_ + 1 == n then
+        -- Continue with the same range.
+        end_ = n
+      else
+        if start >= 0 then
+          -- Produce previous range.
+          ut_fp:write(make_range(start, end_))
+        end
+        start = n
+        end_ = n
+      end
+    end
+  end
+  if start >= 0 then
+    ut_fp:write(make_range(start, end_))
+  end
+  ut_fp:write('};\n')
+end
+
+local build_width_table = function(ut_fp, dataprops, widthprops, widths,
+                                   table_name)
+  ut_fp:write('static const struct interval ' .. table_name .. '[] = {\n')
+  local start = -1
+  local end_ = -1
+  local dataidx = 1
+  local ret = {}
+  for _, p in ipairs(widthprops) do
+    if widths[p[2]:sub(1, 1)] then
+      local rng_start, rng_end = p[1]:find('%.%.')
+      local n, n_last
+      if rng_start then
+        -- It is a range. We don’t check for composing char then.
+        n = tonumber(p[1]:sub(1, rng_start - 1), 16)
+        n_last = tonumber(p[1]:sub(rng_end + 1), 16)
+      else
+        n = tonumber(p[1], 16)
+        n_last = n
+      end
+      local dn
+      while true do
+        dn = tonumber(dataprops[dataidx][1], 16)
+        if dn >= n then
+          break
+        end
+        dataidx = dataidx + 1
+      end
+      if dn ~= n and n_last == n then
+        io.stderr:write('Cannot find character ' .. n .. ' in data table.\n')
+      end
+      -- Only use the char when it’s not a composing char.
+      -- But use all chars from a range.
+      local dp = dataprops[dataidx]
+      if (n_last > n) or (not (({Mn=true, Mc=true, Me=true})[dp[3]])) then
+        if start >= 0 and end_ + 1 == n then
+          -- Continue with the same range.
+        else
+          if start >= 0 then
+            ut_fp:write(make_range(start, end_))
+            table.insert(ret, {start, end_})
+          end
+          start = n
+        end
+        end_ = n_last
+      end
+    end
+  end
+  if start >= 0 then
+    ut_fp:write(make_range(start, end_))
+    table.insert(ret, {start, end_})
+  end
+  ut_fp:write('};\n')
+  return ret
+end
+
+local build_emoji_table = function(ut_fp, emojiprops, doublewidth, ambiwidth)
+  local emojiwidth = {}
+  local emoji = {}
+  for _, p in ipairs(emojiprops) do
+    if p[2]:match('Emoji%s+#') then
+      local rng_start, rng_end = p[1]:find('%.%.')
+      if rng_start then
+        n = tonumber(p[1]:sub(1, rng_start - 1), 16)
+        n_last = tonumber(p[1]:sub(rng_end + 1), 16)
+      else
+        n = tonumber(p[1], 16)
+        n_last = n
+      end
+      if #emoji > 0 and n - 1 == emoji[#emoji][2] then
+        emoji[#emoji][2] = n_last
+      else
+        table.insert(emoji, { n, n_last })
+      end
+
+      -- Characters below 1F000 may be considered single width traditionally,
+      -- making them double width causes problems.
+      if n >= 0x1f000 then
+        -- exclude characters that are in the ambiguous/doublewidth table
+        for _, ambi in ipairs(ambiwidth) do
+          if n >= ambi[1] and n <= ambi[2] then
+            n = ambi[2] + 1
+          end
+          if n_last >= ambi[1] and n_last <= ambi[2] then
+            n_last = ambi[1] - 1
+          end
+        end
+        for _, double in ipairs(doublewidth) do
+          if n >= double[1] and n <= double[2] then
+            n = double[2] + 1
+          end
+          if n_last >= double[1] and n_last <= double[2] then
+            n_last = double[1] - 1
+          end
+        end
+
+        if n <= n_last then
+          if #emojiwidth > 0 and n - 1 == emojiwidth[#emojiwidth][2] then
+            emojiwidth[#emojiwidth][2] = n_last
+          else
+            table.insert(emojiwidth, { n, n_last })
+          end
+        end
+      end
+    end
+  end
+
+  ut_fp:write('static const struct interval emoji_all[] = {\n')
+  for _, p in ipairs(emoji) do
+    ut_fp:write(make_range(p[1], p[2]))
+  end
+  ut_fp:write('};\n')
+
+  ut_fp:write('static const struct interval emoji_width[] = {\n')
+  for _, p in ipairs(emojiwidth) do
+    ut_fp:write(make_range(p[1], p[2]))
+  end
+  ut_fp:write('};\n')
+end
+
+local ud_fp = io.open(unicodedata_fname, 'r')
+local dataprops = parse_data_to_props(ud_fp)
+ud_fp:close()
+
+local ut_fp = io.open(utf_tables_fname, 'w')
+
+build_case_table(ut_fp, dataprops, 'Lower', 14)
+build_case_table(ut_fp, dataprops, 'Upper', 13)
+build_combining_table(ut_fp, dataprops)
+
+local cf_fp = io.open(casefolding_fname, 'r')
+local foldprops = parse_fold_props(cf_fp)
+cf_fp:close()
+
+build_fold_table(ut_fp, foldprops)
+
+local eaw_fp = io.open(eastasianwidth_fname, 'r')
+local widthprops = parse_width_props(eaw_fp)
+eaw_fp:close()
+
+local doublewidth = build_width_table(ut_fp, dataprops, widthprops,
+                                      {W=true, F=true}, 'doublewidth')
+local ambiwidth = build_width_table(ut_fp, dataprops, widthprops,
+                                    {A=true}, 'ambiguous')
+
+local emoji_fp = io.open(emoji_fname, 'r')
+local emojiprops = parse_emoji_props(emoji_fp)
+emoji_fp:close()
+
+build_emoji_table(ut_fp, emojiprops, doublewidth, ambiwidth)
+
+ut_fp:close()