Merge #5864 from ZyX-I/fix-5857

unittest: Allow multiple indirect includes

test/functional/helpers.lua

@@ -21,6 +21,9 @@ local nvim_argv = {nvim_prog, '-u', 'NONE', '-i', 'NONE', '-N',
 
 local mpack = require('mpack')
 
+local tmpname = global_helpers.tmpname
+local uname = global_helpers.uname
+
 -- Formulate a path to the directory containing nvim.  We use this to
 -- help run test executables.  It helps to keep the tests working, even
 -- when the build is not in the default location.
@@ -334,44 +337,6 @@ local function write_file(name, text, dont_dedent)
   file:close()
 end
 
--- Tries to get platform name from $SYSTEM_NAME, uname; fallback is "Windows".
-local uname = (function()
-  local platform = nil
-  return (function()
-    if platform then
-      return platform
-    end
-
-    platform = os.getenv("SYSTEM_NAME")
-    if platform then
-      return platform
-    end
-
-    local status, f = pcall(io.popen, "uname -s")
-    if status then
-      platform = f:read("*l")
-    else
-      platform = 'Windows'
-    end
-    return platform
-  end)
-end)()
-
-local function tmpname()
-  local fname = os.tmpname()
-  if uname() == 'Windows' and fname:sub(1, 2) == '\\s' then
-    -- In Windows tmpname() returns a filename starting with
-    -- special sequence \s, prepend $TEMP path
-    local tmpdir = os.getenv('TEMP')
-    return tmpdir..fname
-  elseif fname:match('^/tmp') and uname() == 'Darwin' then
-    -- In OS X /tmp links to /private/tmp
-    return '/private'..fname
-  else
-    return fname
-  end
-end
-
 local function source(code)
   local fname = tmpname()
   write_file(fname, code)

test/helpers.lua

@@ -52,9 +52,49 @@ local function check_logs()
   assert(0 == runtime_errors)
 end
 
+-- Tries to get platform name from $SYSTEM_NAME, uname; fallback is "Windows".
+local uname = (function()
+  local platform = nil
+  return (function()
+    if platform then
+      return platform
+    end
+
+    platform = os.getenv("SYSTEM_NAME")
+    if platform then
+      return platform
+    end
+
+    local status, f = pcall(io.popen, "uname -s")
+    if status then
+      platform = f:read("*l")
+    else
+      platform = 'Windows'
+    end
+    return platform
+  end)
+end)()
+
+local function tmpname()
+  local fname = os.tmpname()
+  if uname() == 'Windows' and fname:sub(1, 2) == '\\s' then
+    -- In Windows tmpname() returns a filename starting with
+    -- special sequence \s, prepend $TEMP path
+    local tmpdir = os.getenv('TEMP')
+    return tmpdir..fname
+  elseif fname:match('^/tmp') and uname() == 'Darwin' then
+    -- In OS X /tmp links to /private/tmp
+    return '/private'..fname
+  else
+    return fname
+  end
+end
+
 return {
   eq = eq,
   neq = neq,
   ok = ok,
   check_logs = check_logs,
+  uname = uname,
+  tmpname = tmpname,
 }

test/unit/formatc.lua

@@ -219,13 +219,7 @@ local function standalone(...)  -- luacheck: ignore
   Preprocess.add_to_include_path('./../../build/include')
   Preprocess.add_to_include_path('./../../.deps/usr/include')
 
-  local input = Preprocess.preprocess_stream(arg[1])
+  local raw = Preprocess.preprocess('', arg[1])
-  local raw = input:read('*all')
-  input:close()
-
-  if raw == nil then
-    print("ERROR: Preprocess.preprocess_stream():read() returned empty")
-  end
 
   local formatted
   if #arg == 2 and arg[2] == 'no' then

test/unit/helpers.lua

@@ -45,6 +45,8 @@ local function filter_complex_blocks(body)
   return table.concat(result, "\n")
 end
 
+local previous_defines = ''
+
 -- use this helper to import C files, you can pass multiple paths at once,
 -- this helper will return the C namespace of the nvim library.
 local function cimport(...)
@@ -66,17 +68,8 @@ local function cimport(...)
     return libnvim
   end
 
-  local body = nil
+  local body
-  for _ = 1, 10 do
+  body, previous_defines = Preprocess.preprocess(previous_defines, unpack(paths))
-    local stream = Preprocess.preprocess_stream(unpack(paths))
-    body = stream:read("*a")
-    stream:close()
-    if body ~= nil then break end
-  end
-
-  if body == nil then
-    print("ERROR: helpers.lua: Preprocess.preprocess_stream():read() returned empty")
-  end
 
   -- format it (so that the lines are "unique" statements), also filter out
   -- Objective-C blocks

test/unit/preprocess.lua

@@ -7,22 +7,22 @@ local ccs = {}
 
 local env_cc = os.getenv("CC")
 if env_cc then
-  table.insert(ccs, {path = "/usr/bin/env " .. tostring(env_cc), type = "gcc"})
+  table.insert(ccs, {path = {"/usr/bin/env", env_cc}, type = "gcc"})
 end
 
 if ffi.os == "Windows" then
-  table.insert(ccs, {path = "cl", type = "msvc"})
+  table.insert(ccs, {path = {"cl"}, type = "msvc"})
 end
 
-table.insert(ccs, {path = "/usr/bin/env cc", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "cc"}, type = "gcc"})
-table.insert(ccs, {path = "/usr/bin/env gcc", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "gcc"}, type = "gcc"})
-table.insert(ccs, {path = "/usr/bin/env gcc-4.9", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.9"}, type = "gcc"})
-table.insert(ccs, {path = "/usr/bin/env gcc-4.8", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.8"}, type = "gcc"})
-table.insert(ccs, {path = "/usr/bin/env gcc-4.7", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "gcc-4.7"}, type = "gcc"})
-table.insert(ccs, {path = "/usr/bin/env clang", type = "clang"})
+table.insert(ccs, {path = {"/usr/bin/env", "clang"}, type = "clang"})
-table.insert(ccs, {path = "/usr/bin/env icc", type = "gcc"})
+table.insert(ccs, {path = {"/usr/bin/env", "icc"}, type = "gcc"})
 
-local quote_me = '[^%w%+%-%=%@%_%/]' -- complement (needn't quote)
+local quote_me = '[^.%w%+%-%@%_%/]' -- complement (needn't quote)
 local function shell_quote(str)
   if string.find(str, quote_me) or str == '' then
     return "'" .. string.gsub(str, "'", [['"'"']]) .. "'"
@@ -61,12 +61,12 @@ end
 -- will produce a string that represents a meta C header file that includes
 -- all the passed in headers. I.e.:
 --
--- headerize({"stdio.h", "math.h", true}
+-- headerize({"stdio.h", "math.h"}, true)
 -- produces:
 -- #include <stdio.h>
 -- #include <math.h>
 --
--- headerize({"vim.h", "memory.h", false}
+-- headerize({"vim.h", "memory.h"}, false)
 -- produces:
 -- #include "vim.h"
 -- #include "memory.h"
@@ -79,8 +79,7 @@ local function headerize(headers, global)
   end
 
   local formatted = {}
-  for i = 1, #headers do
+  for _, hdr in ipairs(headers) do
-    local hdr = headers[i]
     formatted[#formatted + 1] = "#include " ..
                                 tostring(pre) ..
                                 tostring(hdr) ..
@@ -91,49 +90,77 @@ local function headerize(headers, global)
 end
 
 local Gcc = {
+  preprocessor_extra_flags = {},
+  get_defines_extra_flags = {'-std=c99', '-dM', '-E'},
+  get_declarations_extra_flags = {'-std=c99', '-P', '-E'},
+}
+
+function Gcc:define(name, args, val)
+  local define = '-D' .. name
+  if args ~= nil then
+    define = define .. '(' .. table.concat(args, ',') .. ')'
+  end
+  if val ~= nil then
+    define = define .. '=' .. val
+  end
+  self.preprocessor_extra_flags[#self.preprocessor_extra_flags + 1] = define
+end
+
+function Gcc:undefine(name)
+  self.preprocessor_extra_flags[#self.preprocessor_extra_flags + 1] = (
+      '-U' .. name)
+end
+
+function Gcc:init_defines()
   -- preprocessor flags that will hopefully make the compiler produce C
   -- declarations that the LuaJIT ffi understands.
-  preprocessor_extra_flags = {
+  self:define('aligned', {'ARGS'}, '')
-   '-D "aligned(ARGS)="',
+  self:define('__attribute__', {'ARGS'}, '')
-   '-D "__attribute__(ARGS)="',
+  self:define('__asm', {'ARGS'}, '')
-   '-D "__asm(ARGS)="',
+  self:define('__asm__', {'ARGS'}, '')
-   '-D "__asm__(ARGS)="',
+  self:define('__inline__', nil, '')
-   '-D "__inline__="',
+  self:define('EXTERN', nil, 'extern')
-   '-D "EXTERN=extern"',
+  self:define('INIT', {'...'}, '')
-   '-D "INIT(...)="',
+  self:define('_GNU_SOURCE')
-   '-D_GNU_SOURCE',
+  self:define('INCLUDE_GENERATED_DECLARATIONS')
-   '-DINCLUDE_GENERATED_DECLARATIONS',
+  -- Needed for FreeBSD
-
+  self:define('_Thread_local', nil, '')
-   -- Needed for FreeBSD
+  -- Needed for macOS Sierra
-   '-D "_Thread_local="',
+  self:define('_Nullable', nil, '')
-
+  self:define('_Nonnull', nil, '')
-   -- Needed for macOS Sierra
+  self:undefine('__BLOCKS__')
-   '-D "_Nullable="',
+end
-   '-D "_Nonnull="',
-   '-U__BLOCKS__',
-  }
-}
 
 function Gcc:new(obj)
   obj = obj or {}
   setmetatable(obj, self)
   self.__index = self
+  self:init_defines()
   return obj
 end
 
 function Gcc:add_to_include_path(...)
-  local paths = {...}
+  for i = 1, select('#', ...) do
-  for i = 1, #paths do
+    local path = select(i, ...)
-    local path = paths[i]
-    local directive = '-I ' .. '"' .. path .. '"'
     local ef = self.preprocessor_extra_flags
-    ef[#ef + 1] = directive
+    ef[#ef + 1] = '-I' .. path
   end
 end
 
+local function argss_to_cmd(...)
+  local cmd = ''
+  for i = 1, select('#', ...) do
+    for _, arg in ipairs(select(i, ...)) do
+      cmd = cmd .. ' ' .. shell_quote(arg)
+    end
+  end
+  return cmd
+end
+
 -- returns a list of the headers files upon which this file relies
 function Gcc:dependencies(hdr)
-  local out = io.popen(tostring(self.path) .. " -M " .. tostring(hdr) .. " 2>&1")
+  local cmd = argss_to_cmd(self.path, {'-M', hdr}) .. ' 2>&1'
+  local out = io.popen(cmd)
   local deps = out:read("*a")
   out:close()
   if deps then
@@ -143,23 +170,51 @@ function Gcc:dependencies(hdr)
   end
 end
 
+local function repeated_call(...)
+  local cmd = argss_to_cmd(...)
+  for _ = 1, 10 do
+    local stream = io.popen(cmd)
+    local ret = stream:read('*a')
+    stream:close()
+    if ret then
+      return ret
+    end
+  end
+  print('ERROR: preprocess.lua: Failed to execute ' .. cmd .. ': nil return after 10 attempts')
+  return nil
+end
+
 -- returns a stream representing a preprocessed form of the passed-in headers.
 -- Don't forget to close the stream by calling the close() method on it.
-function Gcc:preprocess_stream(...)
+function Gcc:preprocess(previous_defines, ...)
   -- create pseudo-header
   local pseudoheader = headerize({...}, false)
-  local defines = table.concat(self.preprocessor_extra_flags, ' ')
+  local pseudoheader_fname = 'tmp_pseudoheader.h'
-  local cmd = ("echo $hdr | " ..
+  local pseudoheader_file = io.open(pseudoheader_fname, 'w')
-               tostring(self.path) ..
+  pseudoheader_file:write(previous_defines)
-               " " ..
+  pseudoheader_file:write("\n")
-               tostring(defines) ..
+  pseudoheader_file:write(pseudoheader)
-               " -std=c99 -P -E -"):gsub('$hdr', shell_quote(pseudoheader))
+  pseudoheader_file:flush()
+  pseudoheader_file:close()
+
+  local defines = repeated_call(self.path, self.preprocessor_extra_flags,
+                                self.get_defines_extra_flags,
+                                {pseudoheader_fname})
+
   -- lfs = require("lfs")
   -- print("CWD: #{lfs.currentdir!}")
   -- print("CMD: #{cmd}")
   -- io.stderr\write("CWD: #{lfs.currentdir!}\n")
   -- io.stderr\write("CMD: #{cmd}\n")
-  return io.popen(cmd)
+
+  local declarations = repeated_call(self.path, self.preprocessor_extra_flags,
+                                     self.get_declarations_extra_flags,
+                                     {pseudoheader_fname})
+
+  os.remove(pseudoheader_fname)
+
+  assert(declarations and defines)
+  return declarations, defines
 end
 
 local Clang = Gcc:new()
@@ -197,8 +252,8 @@ return {
   includes = function(hdr)
     return cc:dependencies(hdr)
   end,
-  preprocess_stream = function(...)
+  preprocess = function(...)
-    return cc:preprocess_stream(...)
+    return cc:preprocess(...)
   end,
   add_to_include_path = function(...)
     return cc:add_to_include_path(...)