improve nimsuggest/tester, minor improvements to koch.nim (#17879)

* improve nimsuggest/tester

* koch improvements

koch.nim

@@ -154,7 +154,7 @@ proc bundleNimbleExe(latest: bool, args: string) =
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
-                 options = "-d:release -d:danger " & args)
+                 options = "-d:danger " & args)
 
 proc buildVccTool(args: string) =
   let input = "tools/vccexe/vccexe.nim"
@@ -558,28 +558,27 @@ proc runCI(cmd: string) =
     ## run tests
     execFold("Test nimscript", "nim e tests/test_nimscript.nims")
     when defined(windows):
-      # note: will be over-written below
-      execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
+      execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
 
     # main bottleneck here
     # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`.
     # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343
-    execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
+    execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
 
-    block CT_FFI:
+    block: # nimHasLibFFI:
       when defined(posix): # windows can be handled in future PR's
         execFold("nimble install -y libffi", "nimble install -y libffi")
-        const nimFFI = "./bin/nim.ctffi"
+        const nimFFI = "bin/nim.ctffi"
         # no need to bootstrap with koch boot (would be slower)
         let backend = if doUseCpp(): "cpp" else: "c"
         execFold("build with -d:nimHasLibFFI", "nim $1 -d:release -d:nimHasLibFFI -o:$2 compiler/nim.nim" % [backend, nimFFI])
         execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend])
 
-    execFold("Run nimdoc tests", "nim c -r nimdoc/tester")
-    execFold("Run rst2html tests", "nim c -r nimdoc/rsttester")
-    execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim")
+    execFold("Run nimdoc tests", "nim r nimdoc/tester")
+    execFold("Run rst2html tests", "nim r nimdoc/rsttester")
+    execFold("Run nimpretty tests", "nim r nimpretty/tester.nim")
     when defined(posix):
-      execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester")
+      execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
 
 proc testUnixInstall(cmdLineRest: string) =
   csource("-d:danger" & cmdLineRest)
@@ -605,7 +604,7 @@ proc testUnixInstall(cmdLineRest: string) =
       execCleanPath("./koch tools")
       # check the tests work:
       putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
-      execCleanPath("./koch tests --nim:./bin/nim cat megatest", destDir / "bin")
+      execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin")
     else:
       echo "Version check: failure"
   finally:

nimdoc/testproject/testproject.nim

@@ -341,7 +341,7 @@ when true: # issue #14473
   template doit(): untyped =
     ## doit
     ## return output only
-    toSeq([1,2])
+    toSeq(["D20210427T172228"]) # make it searcheable at least until we figure out a way to avoid echo
   echo doit() # using doAssert or similar to avoid echo would "hide" the original bug
 
 when true: # issue #14846

nimpretty/tester.nim

@@ -3,7 +3,7 @@
 import strutils, os, sequtils
 
 const
-  dir = "nimpretty/tests/"
+  dir = "nimpretty/tests"
   outputdir = dir / "outputdir"
 
 var

nimsuggest/tester.nim

@@ -2,6 +2,8 @@
 # Every test file can have a #[!]# comment that is deleted from the input
 # before 'nimsuggest' is invoked to ensure this token doesn't make a
 # crucial difference for Nim's parser.
+# When debugging, to run a single test, use for e.g.:
+# `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim`
 
 import os, osproc, strutils, streams, re, sexp, net
 
@@ -13,16 +15,16 @@ type
     disabled: bool
 
 const
-  curDir = when defined(windows): "" else: ""
   DummyEof = "!EOF!"
-
-template tpath(): untyped = getAppDir() / "tests"
+  tpath = "nimsuggest/tests"
+  # we could also use `stdtest/specialpaths`
 
 import std/compilesettings
 
 proc parseTest(filename: string; epcMode=false): Test =
   const cursorMarker = "#[!]#"
-  let nimsug = curDir & addFileExt("nimsuggest", ExeExt)
+  let nimsug = "bin" / addFileExt("nimsuggest", ExeExt)
+  doAssert nimsug.fileExists, nimsug
   const libpath = querySetting(libPath)
   result.filename = filename
   result.dest = getTempDir() / extractFilename(filename)
@@ -63,7 +65,7 @@ proc parseTest(filename: string; epcMode=false): Test =
       elif x.startsWith(">"):
         # since 'markers' here are not complete yet, we do the $substitutions
         # afterwards
-        result.script.add((x.substr(1).replaceWord("$path", tpath()), ""))
+        result.script.add((x.substr(1).replaceWord("$path", tpath), ""))
       elif x.len > 0:
         # expected output line:
         let x = x % ["file", filename, "lib", libpath]
@@ -104,7 +106,7 @@ proc parseCmd(c: string): seq[string] =
 proc edit(tmpfile: string; x: seq[string]) =
   if x.len != 3 and x.len != 4:
     quit "!edit takes two or three arguments"
-  let f = if x.len >= 4: tpath() / x[3] else: tmpfile
+  let f = if x.len >= 4: tpath / x[3] else: tmpfile
   try:
     let content = readFile(f)
     let newcontent = content.replace(x[1], x[2])
@@ -121,12 +123,12 @@ proc exec(x: seq[string]) =
 
 proc copy(x: seq[string]) =
   if x.len != 3: quit "!copy takes two arguments"
-  let rel = tpath()
+  let rel = tpath
   copyFile(rel / x[1], rel / x[2])
 
 proc del(x: seq[string]) =
   if x.len != 2: quit "!del takes one argument"
-  removeFile(tpath() / x[1])
+  removeFile(tpath / x[1])
 
 proc runCmd(cmd, dest: string): bool =
   result = cmd[0] == '!'
@@ -317,7 +319,7 @@ proc runTest(filename: string): int =
     try:
       inp.writeLine("quit")
       inp.flush()
-    except:
+    except IOError, OSError:
       # assume it's SIGPIPE, ie, the child already died
       discard
     close(p)
@@ -334,7 +336,7 @@ proc main() =
     failures += runTest(xx)
     failures += runEpcTest(xx)
   else:
-    for x in walkFiles(tpath() / "t*.nim"):
+    for x in walkFiles(tpath / "t*.nim"):
       echo "Test ", x
       let xx = expandFilename x
       when not defined(windows):