refactor(test): dedup benchmark utils #40478

test/benchmark/keycodes_spec.lua

@@ -1,4 +1,5 @@
 local n = require('test.functional.testnvim')()
+local t = require('test.testutil')
 local clear = n.clear
 local api = n.api
 local fn = n.fn
@@ -9,7 +10,6 @@ describe('nvim_replace_termcodes performance', function()
   it('200 calls with a key repeated 5000 times', function()
     clear()
     local stats = {}
-    local sum = 0
     local ms = 1 / 1000000
 
     for _, keycode in ipairs(keycodes.names) do
@@ -23,24 +23,12 @@ describe('nvim_replace_termcodes performance', function()
       local elapsed = vim.uv.hrtime() - start
 
       table.insert(stats, elapsed)
-      sum = sum + elapsed
       io.stdout:write(('\n%-20s%14.6f ms'):format(notation, elapsed * ms))
       io.stdout:flush()
     end
     io.stdout:write('\n')
 
-    table.sort(stats)
-    print(('%18s'):rep(6):format('avg', 'min', '25%', 'median', '75%', 'max'))
-    print(
-      (' %14.6f ms'):rep(6):format(
-        sum / #stats * ms,
-        stats[1] * ms,
-        stats[1 + math.floor(#stats * 0.25)] * ms,
-        stats[1 + math.floor(#stats * 0.5)] * ms,
-        stats[1 + math.floor(#stats * 0.75)] * ms,
-        stats[#stats] * ms
-      )
-    )
+    t.bench_report(stats, { unit = 'ms' })
   end)
 end)
 
@@ -48,7 +36,6 @@ describe('keytrans() performance', function()
   it('200 calls with a key repeated 5000 times', function()
     clear()
     local stats = {}
-    local sum = 0
     local ms = 1 / 1000000
 
     for _, keycode in ipairs(keycodes.names) do
@@ -62,23 +49,11 @@ describe('keytrans() performance', function()
       local elapsed = vim.uv.hrtime() - start
 
       table.insert(stats, elapsed)
-      sum = sum + elapsed
       io.stdout:write(('\n%-20s%14.6f ms'):format(notation, elapsed * ms))
       io.stdout:flush()
     end
     io.stdout:write('\n')
 
-    table.sort(stats)
-    print((' %17s'):rep(6):format('avg', 'min', '25%', 'median', '75%', 'max'))
-    print(
-      (' %14.6f ms'):rep(6):format(
-        sum / #stats * ms,
-        stats[1] * ms,
-        stats[1 + math.floor(#stats * 0.25)] * ms,
-        stats[1 + math.floor(#stats * 0.5)] * ms,
-        stats[1 + math.floor(#stats * 0.75)] * ms,
-        stats[#stats] * ms
-      )
-    )
+    t.bench_report(stats, { unit = 'ms' })
   end)
 end)

test/benchmark/option_spec.lua

@@ -1,101 +1,54 @@
 local n = require('test.functional.testnvim')()
 
 local clear = n.clear
-local exec_lua = n.exec_lua
+local bench = n.bench
 
 describe('option set perf', function()
   before_each(clear)
 
-  --- Runs `fn` (a string of Lua) `iters` times, prints min/median/max, and returns nothing.
-  --- @param name string
-  --- @param iters integer
-  --- @param body string  Lua statement(s) executed each iteration; may use the loop var `i`.
-  local function bench(name, iters, body)
-    local stats = exec_lua(
-      [[
-      local iters, body = ...
-      local fn = assert(loadstring('local i = ...\n' .. body))
-      -- Warmup (JIT, option metadata cache, etc.).
-      for i = 1, 100 do
-        fn(i)
-      end
-      local samples = {}
-      for i = 1, iters do
-        local t0 = vim.uv.hrtime()
-        fn(i)
-        samples[i] = vim.uv.hrtime() - t0
-      end
-      table.sort(samples)
-      return samples
-    ]],
-      iters,
-      body
-    )
-
-    local ms = 1 / 1000000
-    print(
-      ('\n%-44s min %0.4fms  median %0.4fms  max %0.4fms'):format(
-        name,
-        stats[1] * ms,
-        stats[1 + math.floor(#stats * 0.5)] * ms,
-        stats[#stats] * ms
-      )
-    )
-  end
-
-  local ITERS = 20000
+  local trials = 20000
 
   it('vim.o scalar (number)', function()
-    bench('vim.o.scrolloff = i % 10', ITERS, [[vim.o.scrolloff = i % 10]])
+    bench([[vim.o.scrolloff = i % 10]], { n = trials, label = 'vim.o.scrolloff = i % 10' })
   end)
 
   it('vim.o scalar (boolean)', function()
-    bench('vim.o.wrap = (i % 2 == 0)', ITERS, [[vim.o.wrap = (i % 2 == 0)]])
+    bench([[vim.o.wrap = (i % 2 == 0)]], { n = trials, label = 'vim.o.wrap = (i % 2 == 0)' })
   end)
 
   it('vim.o scalar (string)', function()
     -- makeprg has a trivial did_set handler, so this isolates the set path itself.
-    bench("vim.o.makeprg = 'make'", ITERS, [[vim.o.makeprg = 'make']])
+    bench([[vim.o.makeprg = 'make']], { n = trials, label = "vim.o.makeprg = 'make'" })
   end)
 
   it('vim.opt array via table', function()
-    bench(
-      "vim.opt.wildignore = {'*.o','*.a','*.so'}",
-      ITERS,
-      [[
-      vim.opt.wildignore = { '*.o', '*.a', '*.so' }
-    ]]
-    )
+    bench([[vim.opt.wildignore = { '*.o', '*.a', '*.so' }]], {
+      n = trials,
+      label = "vim.opt.wildignore = {'*.o','*.a','*.so'}",
+    })
   end)
 
   it('vim.opt map via table', function()
-    bench(
-      "vim.opt.listchars = { eol='~', space='.', tab='> ' }",
-      ITERS,
-      [[
-      vim.opt.listchars = { eol = '~', space = '.', tab = '> ' }
-    ]]
-    )
+    bench([[vim.opt.listchars = { eol = '~', space = '.', tab = '> ' }]], {
+      n = trials,
+      label = "vim.opt.listchars = { eol='~', space='.', tab='> ' }",
+    })
   end)
 
   it('vim.opt:append (operation)', function()
     bench(
-      'vim.opt.wildignore:append(...)',
-      ITERS,
       [[
       vim.o.wildignore = ''
       vim.opt.wildignore:append({ '*.tmp', '*.bak' })
-    ]]
+    ]],
+      { n = trials, label = 'vim.opt.wildignore:append(...)' }
     )
   end)
 
   it('nvim_set_option_value with table (direct API)', function()
-    bench(
-      'nvim_set_option_value(listchars, {..})',
-      ITERS,
-      [[
-      vim.api.nvim_set_option_value('listchars', { eol = '~', space = '.' }, {})
-    ]]
-    )
+    bench([[vim.api.nvim_set_option_value('listchars', { eol = '~', space = '.' }, {})]], {
+      n = trials,
+      label = 'nvim_set_option_value(listchars, {..})',
+    })
   end)
 end)

test/benchmark/text_spec.lua

@@ -1,52 +1,17 @@
+local t = require('test.testutil')
+
 describe('vim.text', function()
-  --- @param t number[]
-  local function mean(t)
-    assert(#t > 0)
-    local sum = 0
-    for _, v in ipairs(t) do
-      sum = sum + v
-    end
-    return sum / #t
-  end
-
-  --- @param t number[]
-  local function median(t)
-    local len = #t
-    if len % 2 == 0 then
-      return t[len / 2]
-    end
-    return t[(len + 1) / 2]
-  end
-
-  --- @param f fun(t: number[]): table<number, number|string|table>
-  local function measure(f, input, N)
-    local stats = {} ---@type number[]
-    for _ = 1, N do
-      local tic = vim.uv.hrtime()
-      f(input)
-      local toc = vim.uv.hrtime()
-      stats[#stats + 1] = (toc - tic) / 1000000
-    end
-    table.sort(stats)
-    print(
-      string.format(
-        '\nN: %d, Min: %0.6f ms, Max: %0.6f ms, Median: %0.6f ms, Mean: %0.6f ms',
-        N,
-        math.min(unpack(stats)),
-        math.max(unpack(stats)),
-        median(stats),
-        mean(stats)
-      )
-    )
-  end
-
   local input, output = string.rep('😂', 2 ^ 16), string.rep('F09F9882', 2 ^ 16)
 
   it('hexencode', function()
-    measure(vim.text.hexencode, input, 100)
+    t.bench(function()
+      vim.text.hexencode(input)
+    end, { n = 100, label = 'hexencode' })
   end)
 
   it('hexdecode', function()
-    measure(vim.text.hexdecode, output, 100)
+    t.bench(function()
+      vim.text.hexdecode(output)
+    end, { n = 100, label = 'hexdecode' })
   end)
 end)

test/benchmark/trim_spec.lua

@@ -1,45 +1,6 @@
+local t = require('test.testutil')
+
 describe('vim.trim()', function()
-  --- @param t number[]
-  local function mean(t)
-    assert(#t > 0)
-    local sum = 0
-    for _, v in ipairs(t) do
-      sum = sum + v
-    end
-    return sum / #t
-  end
-
-  --- @param t number[]
-  local function median(t)
-    local len = #t
-    if len % 2 == 0 then
-      return t[len / 2]
-    end
-    return t[(len + 1) / 2]
-  end
-
-  --- @param f fun(t: number[]): table<number, number|string|table>
-  local function measure(f, input, N)
-    local stats = {} ---@type number[]
-    for _ = 1, N do
-      local tic = vim.uv.hrtime()
-      f(input)
-      local toc = vim.uv.hrtime()
-      stats[#stats + 1] = (toc - tic) / 1000000
-    end
-    table.sort(stats)
-    print(
-      string.format(
-        '\nN: %d, Min: %0.6f ms, Max: %0.6f ms, Median: %0.6f ms, Mean: %0.6f ms',
-        N,
-        math.min(unpack(stats)),
-        math.max(unpack(stats)),
-        median(stats),
-        mean(stats)
-      )
-    )
-  end
-
   local strings = {
     ['10000 whitespace characters'] = string.rep(' ', 10000),
     ['10000 whitespace characters and one non-whitespace at the end'] = string.rep(' ', 10000)
@@ -64,7 +25,9 @@ describe('vim.trim()', function()
 
   for _, name in ipairs(string_names) do
     it(name, function()
-      measure(vim.trim, strings[name], 100)
+      t.bench(function()
+        vim.trim(strings[name])
+      end, { n = 100, label = name })
     end)
   end
 end)

test/functional/testnvim.lua

@@ -957,6 +957,36 @@ function M.exec_lua(code, ...)
   return require('test.functional.testnvim.exec_lua')(session, 2, code, ...)
 end
 
+--- Benchmarks `fn` in the Nvim-under-test: runs it `opts.n` times, timing each run in-session, then reports.
+---
+--- Note: see `testutil.bench()` to run in the test-runner process.
+---
+--- @param body string  Lua source executed each iteration; the iteration index is available as `i`.
+--- @param opts { n: integer, label?: string, unit?: 'ms'|'us', warmup?: integer }
+--- @return table stats  See `testutil.bench_report()`.
+function M.bench(body, opts)
+  local samples = M.exec_lua(
+    [[
+    local body, n, warmup = ...
+    local fn = assert(loadstring('local i = ...\n' .. body))
+    for i = 1, warmup do
+      fn(i)
+    end
+    local samples = {}
+    for i = 1, n do
+      local t0 = vim.uv.hrtime()
+      fn(i)
+      samples[i] = vim.uv.hrtime() - t0
+    end
+    return samples
+  ]],
+    body,
+    opts.n,
+    opts.warmup or 100
+  )
+  return t.bench_report(samples, opts)
+end
+
 function M.get_pathsep()
   return is_os('win') and '\\' or '/'
 end

test/testutil.lua

@@ -897,4 +897,74 @@ function M.translations_enabled()
   return M.paths.translations_enabled
 end
 
+--- Sorts timing `samples` and prints + returns min/q25/median/q75/max/mean.
+---
+--- Shared reporter for `test/benchmark/`. Pair it with `M.bench()` (in-process) or `n.bench()`
+--- (Nvim-under-test), or call it directly with samples you collected yourself.
+---
+--- @param samples number[]  Per-iteration durations, in nanoseconds. Sorted in place.
+--- @param opts? { label?: string, unit?: 'ms'|'us' }
+--- @return { n: integer, min: number, q25: number, median: number, q75: number, max: number, mean: number }
+function M.bench_report(samples, opts)
+  opts = opts or {}
+  local n = #samples
+  assert(n > 0, 'bench_report: no samples')
+  local unit = opts.unit or 'ms'
+  local scale = unit == 'us' and 1e3 or 1e6
+
+  table.sort(samples)
+  local sum = 0
+  for _, v in ipairs(samples) do
+    sum = sum + v
+  end
+  --- @param p number percentile in [0,1]
+  local function pct(p)
+    return samples[math.max(1, math.min(n, 1 + math.floor(n * p)))] / scale
+  end
+  local stats = {
+    n = n,
+    min = samples[1] / scale,
+    q25 = pct(0.25),
+    median = pct(0.5),
+    q75 = pct(0.75),
+    max = samples[n] / scale,
+    mean = (sum / n) / scale,
+  }
+
+  print(
+    ('\n%smin %.4f  q25 %.4f  median %.4f  q75 %.4f  max %.4f  mean %.4f  (%s, n=%d)'):format(
+      opts.label and (opts.label .. '\n  ') or '',
+      stats.min,
+      stats.q25,
+      stats.median,
+      stats.q75,
+      stats.max,
+      stats.mean,
+      unit,
+      n
+    )
+  )
+  return stats
+end
+
+--- Benchmarks `fn` in the test-runner process: runs it `opts.n` times, timing each run, then reports.
+---
+--- Note: see `testnvim.bench()` to run in the Nvim-under-test.
+---
+--- @param fn fun()
+--- @param opts { n: integer, label?: string, unit?: 'ms'|'us', warmup?: integer }
+--- @return table stats  See `M.bench_report()`.
+function M.bench(fn, opts)
+  for _ = 1, (opts.warmup or 0) do
+    fn()
+  end
+  local samples = {} --- @type number[]
+  for i = 1, opts.n do
+    local t0 = uv.hrtime()
+    fn()
+    samples[i] = uv.hrtime() - t0
+  end
+  return M.bench_report(samples, opts)
+end
+
 return M