From 8ca36b9783d84e29bf6cb085c38e717d832b5594 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 25 Aug 2026 15:01:28 +0200 Subject: [PATCH 1/3] =?UTF-8?q?test(harness):=20support=20`{retries=3D?= =?UTF-8?q?=E2=80=A6}`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Cannot retry a test with its full `after_each`/`before_each` lifecycle. Solution: - Overload `it()` to accept an `opts` param: ``` it('flaky', { retries = 2 }, function(ctx) end) -- 3 attempts. ``` - Pass `ctx` to test functions. - Fix a bug in `t.read_file_list()`. --- test/functional/harness/harness_spec.lua | 61 ++++++++++++ test/harness.lua | 114 ++++++++++++++--------- test/testutil.lua | 4 +- 3 files changed, 135 insertions(+), 44 deletions(-) diff --git a/test/functional/harness/harness_spec.lua b/test/functional/harness/harness_spec.lua index 8ef99f5db6..86fedaecae 100644 --- a/test/functional/harness/harness_spec.lua +++ b/test/functional/harness/harness_spec.lua @@ -1015,6 +1015,67 @@ describe('test harness', function() not_matches('missing value for --repeat', output, true) end) + it('it() rejects a non-table, non-function second argument', function() + eq( + '.../harness.lua:0: it() arg 2 must be an opts table or a function', + t.pcall_err(it, 'nope', 'not opts', function() end) + ) + end) + + it('{retries=n} reruns before_each/after_each/finally, only for the retried test', function() + local log = t.tmpname(false) + local suite_dir = write_suite({ + ['one_spec.lua'] = ([[ + local function log(s) + vim.fn.writefile({ s }, '%s', 'a') + end + describe('retry', function() + before_each(function() log('d-before') end) + after_each(function() log('d-after') end) + it('succeeds on the second attempt', { retries = 2 }, function(ctx) + finally(function() log('t1-finally' .. ctx.retry) end) + eq(true, ctx.retry >= 1) + end) + it('runs once', function(ctx) + log('t2-next' .. ctx.retry) + end) + end) + ]]):format(log), + }) + + local code, output = run_harness(suite_dir) + + eq(0, code) + matches('PASSED ', output, true) + not_matches('FAILED ', output, true) + -- Attempt 2 succeeded so attempt 3 never ran, and the next test starts back at retry 0. + eq( + 'd-before t1-finally0 d-after d-before t1-finally1 d-after d-before t2-next0 d-after', + table.concat(vim.fn.readfile(log), ' ') + ) + end) + + it('{retries=n} reports one failure after all retries are exhausted', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('retry', function() + local attempts = 0 + before_each(function() attempts = attempts + 1 end) + it('never succeeds', { retries = 2 }, function() + error('boom on attempt ' .. attempts) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + matches('boom on attempt 3', output, true) + not_matches('boom on attempt 1', output, true) + matches('1 test, listed below', output, true) -- One failure, not one per attempt. + end) + it('reports test-body errors as failures', function() local suite_dir = write_suite({ ['one_spec.lua'] = [[ diff --git a/test/harness.lua b/test/harness.lua index 56cf89af5c..6ddabbc6a9 100644 --- a/test/harness.lua +++ b/test/harness.lua @@ -71,8 +71,13 @@ --- @field fn? fun() --- @field parent test.harness.Suite --- @field pending_message? string +--- @field retries? integer --- @field selected? boolean +--- Test context: state passed to the `it` callback (the test body). +--- @class test.harness.Context +--- @field retry integer Attempt index: 0 on the first run, 1 on the first retry. + --- Normalized result returned from running a test or hook. --- @class test.harness.Result --- @field status test.harness.ResultStatus @@ -399,11 +404,13 @@ end --- @param fn? fun() --- @param pending_message? string --- @return test.harness.Test -local function register_test(name, fn, pending_message) +local function register_test(name, fn, pending_message, opts) assert(type(name) == 'string' and name ~= '', 'test name must be a non-empty string') if fn ~= nil then assert(type(fn) == 'function', 'test body must be a function') end + local retries = opts and opts.retries or 0 + assert(type(retries) == 'number' and retries >= 0, 'retries must be a non-negative number') local suite = current_suite() local test = { @@ -413,6 +420,7 @@ local function register_test(name, fn, pending_message) parent = suite, trace = caller_trace(3), pending_message = pending_message, + retries = retries, } table.insert(suite.children, test) return test @@ -463,11 +471,21 @@ function M.describe(name, fn) end --- Define a test. ---- @param name string ---- @param fn? fun() +--- +--- @param name string Test description. +--- @param opts? table|fun() Options: +--- - `retries`: (default: 0) Retry the test, including setup/teardown +--- (`before_each`/`after_each`/`finally`), up to this many times. +--- Only the last attempt is reported. +--- @param fn? fun(ctx: test.harness.Context) Test body. +--- - `ctx.retry` is the index of the `retries` attempt, or 0 if this is not a retry. --- @return test.harness.Test -function M.it(name, fn) - return register_test(name, fn, nil) +function M.it(name, opts, fn) + if type(opts) == 'function' then + opts, fn = nil, opts + end + assert(opts == nil or type(opts) == 'table', 'it() arg 2 must be an opts table or a function') + return register_test(name, fn, nil, opts) end --- Mark the current test as pending or define a pending test. @@ -685,7 +703,7 @@ end --- @param callable test.harness.RegisteredCallback --- @param fallback_status? test.harness.ResultStatus --- @return test.harness.Result, test.harness.Trace? -local function run_callable(scope, callable, fallback_status) +local function run_callable(scope, callable, fallback_status, ctx) local previous_execution = state.current_execution --- @type test.harness.Execution local execution = { @@ -694,7 +712,10 @@ local function run_callable(scope, callable, fallback_status) } state.current_execution = execution - local ok, err = xpcall(callable.fn, exception_handler) + -- Note: xpcall variadic args not supported by Lua 5.1. + local ok, err = xpcall(function() + return callable.fn(ctx) + end, exception_handler) local finalizer_err local finalizer_trace for i = #execution.finalizers, 1, -1 do @@ -999,48 +1020,57 @@ local function run_test(test, reporter, summary, file_summary) message = test.pending_message, } else - result = { status = 'success' } + local attempt = 0 + repeat + attempt = attempt + 1 + --- @type test.harness.Context + local ctx = { retry = attempt - 1 } + result = { status = 'success' } - for _, hook in ipairs(gather_before_each(test.parent)) do - result, report_trace = run_callable('before_each', hook, 'failure') - if result.status ~= 'success' then - break + for _, hook in ipairs(gather_before_each(test.parent)) do + result, report_trace = run_callable('before_each', hook, 'failure') + if result.status ~= 'success' then + break + end end - end - reporter:test_start(name) + if attempt == 1 then + reporter:test_start(name) + end - if result.status == 'success' then - result, report_trace = run_callable('test', { fn = test.fn, trace = test.trace }, 'failure') - end - - for _, hook in ipairs(gather_after_each(test.parent)) do - local hook_result, hook_trace = run_callable('after_each', hook, 'failure') if result.status == 'success' then - result = hook_result - report_trace = hook_trace - elseif hook_result.status ~= 'success' then - local hook_report_trace = hook_trace or hook.trace - result.message = (result.message or '') - .. (result.message and result.message ~= '' and '\n\n' or '') - .. 'after_each: ' - .. hook_result.message - if not result.traceback then - result.traceback = hook_result.traceback - end - if not result.trace then - result.trace = hook_result.trace - end - if result.status == 'pending' then - result.status = 'error' - report_trace = hook_report_trace - elseif not report_trace then - report_trace = hook_report_trace + result, report_trace = + run_callable('test', { fn = test.fn, trace = test.trace }, 'failure', ctx) + end + + for _, hook in ipairs(gather_after_each(test.parent)) do + local hook_result, hook_trace = run_callable('after_each', hook, 'failure') + if result.status == 'success' then + result = hook_result + report_trace = hook_trace + elseif hook_result.status ~= 'success' then + local hook_report_trace = hook_trace or hook.trace + result.message = (result.message or '') + .. (result.message and result.message ~= '' and '\n\n' or '') + .. 'after_each: ' + .. hook_result.message + if not result.traceback then + result.traceback = hook_result.traceback + end + if not result.trace then + result.trace = hook_result.trace + end + if result.status == 'pending' then + result.status = 'error' + report_trace = hook_report_trace + elseif not report_trace then + report_trace = hook_report_trace + end end end - end - -- check for interrupts - vim.wait(0) + -- check for interrupts + vim.wait(0) + until result.status == 'success' or attempt > test.retries end test.duration = now_seconds() - start_time diff --git a/test/testutil.lua b/test/testutil.lua index 0f3ae9805c..8dba141487 100644 --- a/test/testutil.lua +++ b/test/testutil.lua @@ -786,9 +786,9 @@ function M.read_file_list(filename, start) local i = 1 local line = file:read('*l') while line ~= nil do - if i >= start then + if i >= lnum then table.insert(lines, line) - if #lines > maxlines then + if maxlines and #lines > maxlines then table.remove(lines, 1) end end From 03dfbe333c589939539eead20d81abc9e2034735 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 25 Aug 2026 15:42:53 +0200 Subject: [PATCH 2/3] =?UTF-8?q?test(channel):=20unreliable=20"chansend=20s?= =?UTF-8?q?ends=20lines=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/functional/terminal/channel_spec.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/functional/terminal/channel_spec.lua b/test/functional/terminal/channel_spec.lua index 8d6d8d7896..11bb07374f 100644 --- a/test/functional/terminal/channel_spec.lua +++ b/test/functional/terminal/channel_spec.lua @@ -126,9 +126,11 @@ it('chansend sends lines to terminal channel in proper order', function() local screen = Screen.new(100, 20) screen._default_attr_ids = nil local shells = is_os('win') and { 'cmd.exe', 'pwsh.exe -nop', 'powershell.exe -nop' } or { 'sh' } + -- Prompt which indicates the shell is ready to read. + local prompt = is_os('win') and '>' or '%$ ' for _, sh in ipairs(shells) do command([[let id = jobstart(']] .. sh .. [[', {'term':v:true})]]) - screen:sleep(50) -- Wait some time for the shell to start. + screen:expect({ any = prompt, attr_ids = {} }) command([[call chansend(id, ['echo "hello"', 'echo "world"', ''])]]) -- With PowerShell the command may be highlighted, so specify attr_ids = {}. screen:expect({ any = [[echo "hello".*echo "world"]], attr_ids = {} }) From 3aa323584ef61ed16bbc7a84a7eb4f7467199d87 Mon Sep 17 00:00:00 2001 From: "Justin M. Keyes" Date: Tue, 25 Aug 2026 16:37:05 +0200 Subject: [PATCH 3/3] =?UTF-8?q?test(shell):=20unreliable=20"throttles=20sh?= =?UTF-8?q?ell-command=20output=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/functional/ui/output_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/ui/output_spec.lua b/test/functional/ui/output_spec.lua index 896e773897..c95c51b192 100644 --- a/test/functional/ui/output_spec.lua +++ b/test/functional/ui/output_spec.lua @@ -64,7 +64,7 @@ describe('shell command :!', function() ]]) end) - it('throttles shell-command output greater than ~10KB', function() + it('throttles shell-command output greater than ~10KB', { retries = 2 }, function() skip(is_os('openbsd'), 'FIXME #10804') skip(is_os('win')) tt.feed_data((':!%s REP 150001 foo\n'):format(testprg('shell-test')))