mirrors/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-10-03 08:28:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

vim-patch:partial:9.1.1110: Vim tests are slow and flaky

Browse Source 
Problem:  Vim tests are slow and flaky at the same time due to reliance
          on timeouts which are unreliable.
Solution: improve Vim test performance and reduce flakiness
          (Yee Cheng Chin)

A lot of Vim tests currently rely on waiting a specific amount of time
before asserting a condition. This is bad because 1) it is slow, as the
timeout is hardcoded, 2) it's unreliable as a resource-starved runner
may overshoot the timeout. Also, there are a lot of builtin sleep
commands in commonly used utilities like VerifyScreenDump and WaitFor()
which leads to a lot of unnecessary idle time.

Fix these issues by doing the following:
1. Make utilities like VerifyScreenDump and WaitFor use the lowest wait
   time possible (1 ms). This essentially turns it into a spin wait. On
   fast machines, these will finish very quickly. For existing tests
   that had an implicit reliance on the old timeouts (e.g.
   VerifyScreenDump had a 50ms wait before), fix the tests to wait that
   specific amount explicitly.
2. Fix tests that sleep or wait for long amounts of time to instead
   explicitly use a callback mechanism to be notified when a child
   terminal job has finished. This allows the test to only take as much
   time as possible instead of having to hard code an unreliable
   timeout.

With these fixes, tests should 1) completely quickly on fast machines,
and 2) on slow machines they will still run to completion albeit slowly.
Note that previoulsy both were not true. The hardcoded timeouts meant
that on fast machines the tests were mostly idling wasting time, whereas
on slow machines, the timeouts often were not generous enough to allow
them to run to completion.

closes: vim/vim#16615

e70587dbdb

Part of shared.vim and test_crash.vim changes only.

Co-authored-by: Yee Cheng Chin <ychin.git@gmail.com>
This commit is contained in:
zeertzjq
2025-10-02 12:11:03 +08:00
parent 9f33b69cfd
commit dfd3ac6f48
2 changed files with 38 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/old/testdir/shared.vim
View File

@@ -211,11 +211,11 @@ func s:WaitForCommon(expr, assert, timeout)
call remove(v:errors, -1) call remove(v:errors, -1)
endif endif
sleep 10m sleep 1m
if exists('*reltimefloat') if exists('*reltimefloat')
let slept = float2nr(reltimefloat(reltime(start)) * 1000) let slept = float2nr(reltimefloat(reltime(start)) * 1000)
else else
let slept += 10 let slept += 1
endif endif
endwhile endwhile

85
test/old/testdir/test_crash.vim
View File

@@ -4,11 +4,15 @@ source screendump.vim
CheckScreendump CheckScreendump
" Run the command in terminal and wait for it to complete via notification
func s:RunCommandAndWait(buf, cmd)
call term_sendkeys(a:buf, a:cmd .. "; printf '" .. TermNotifyParentCmd(v:false) .. "'\<cr>")
call WaitForChildNotification()
endfunc
func Test_crash1() func Test_crash1()
CheckNotBSD CheckNotBSD
CheckExecutable dash CheckExecutable dash
" Test 7 fails on Mac ...
CheckNotMac
" The following used to crash Vim " The following used to crash Vim
let opts = #{cmd: 'sh'} let opts = #{cmd: 'sh'}
@@ -19,70 +23,59 @@ func Test_crash1()
let file = 'crash/poc_huaf1' let file = 'crash/poc_huaf1'
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 1: [OK]" > X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 1: [OK]" > X_crash1_result.txt')
call TermWait(buf, 50)
let file = 'crash/poc_huaf2' let file = 'crash/poc_huaf2'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 2: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 2: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 50)
let file = 'crash/poc_huaf3' let file = 'crash/poc_huaf3'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 3: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 3: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 100)
let file = 'crash/bt_quickfix_poc' let file = 'crash/bt_quickfix_poc'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 4: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 4: [OK]" >> X_crash1_result.txt')
" clean up " clean up
call delete('Xerr') call delete('Xerr')
" This test takes a bit longer
call TermWait(buf, 1000)
let file = 'crash/poc_tagfunc.vim' let file = 'crash/poc_tagfunc.vim'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
" using || because this poc causes vim to exit with exitstatus != 0 " using || because this poc causes vim to exit with exitstatus != 0
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' || echo "crash 5: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' || echo "crash 5: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 100)
let file = 'crash/bt_quickfix1_poc' let file = 'crash/bt_quickfix1_poc'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 6: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 6: [OK]" >> X_crash1_result.txt')
" clean up " clean up
call delete('X') call delete('X')
call TermWait(buf, 3000)
let file = 'crash/vim_regsub_both_poc' let file = 'crash/vim_regsub_both_poc'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 7: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 7: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 3000)
let file = 'crash/vim_msg_trunc_poc' let file = 'crash/vim_msg_trunc_poc'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' || echo "crash 8: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' || echo "crash 8: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 3000)
let file = 'crash/crash_scrollbar' let file = 'crash/crash_scrollbar'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 9: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' && echo "crash 9: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 1000)
let file = 'crash/editing_arg_idx_POC_1' let file = 'crash/editing_arg_idx_POC_1'
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' || echo "crash 10: [OK]" >> X_crash1_result.txt' .. "\<cr>") \ ' || echo "crash 10: [OK]" >> X_crash1_result.txt')
call TermWait(buf, 1000)
call delete('Xerr') call delete('Xerr')
call delete('@') call delete('@')
@@ -113,7 +106,6 @@ endfunc
func Test_crash1_2() func Test_crash1_2()
CheckNotBSD CheckNotBSD
CheckExecutable dash CheckExecutable dash
let g:test_is_flaky = 1
" The following used to crash Vim " The following used to crash Vim
let opts = #{cmd: 'sh'} let opts = #{cmd: 'sh'}
@@ -125,37 +117,32 @@ func Test_crash1_2()
let file = 'crash/poc1' let file = 'crash/poc1'
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 1: [OK]" > '.. result .. "\<cr>") \ ' && echo "crash 1: [OK]" > '.. result)
call TermWait(buf, 150)
let file = 'crash/poc_win_enter_ext' let file = 'crash/poc_win_enter_ext'
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 2: [OK]" >> '.. result .. "\<cr>") \ ' && echo "crash 2: [OK]" >> '.. result)
call TermWait(buf, 350)
let file = 'crash/poc_suggest_trie_walk' let file = 'crash/poc_suggest_trie_walk'
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' && echo "crash 3: [OK]" >> '.. result .. "\<cr>") \ ' && echo "crash 3: [OK]" >> '.. result)
call TermWait(buf, 150)
let file = 'crash/poc_did_set_langmap' let file = 'crash/poc_did_set_langmap'
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' ; echo "crash 4: [OK]" >> '.. result .. "\<cr>") \ ' ; echo "crash 4: [OK]" >> '.. result)
call TermWait(buf, 150)
let file = 'crash/reverse_text_overflow' let file = 'crash/reverse_text_overflow'
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'" let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
let args = printf(cmn_args, vim, file) let args = printf(cmn_args, vim, file)
call term_sendkeys(buf, args .. call s:RunCommandAndWait(buf, args ..
\ ' ; echo "crash 5: [OK]" >> '.. result .. "\<cr>") \ ' ; echo "crash 5: [OK]" >> '.. result)
call TermWait(buf, 150)
" clean up " clean up
exe buf .. "bw!" exe buf .. "bw!"