fixes #10039 : CI now runs buildTools (eg, nimfind wasn't being compiled before); refactoring (#10242)

koch.nim

@@ -26,7 +26,6 @@ import
 import tools / kochdocs
 
 const VersionAsString = system.NimVersion
-const env_NIM_COMPILE_TO_CPP = "NIM_COMPILE_TO_CPP"
 
 const
   HelpText = """
@@ -47,6 +46,8 @@ Possible Commands:
   boot [options]           bootstraps with given command line options
   distrohelper [bindir]    helper for distro packagers
   tools                    builds Nim related tools
+  toolsNoNimble            builds Nim related tools (except nimble)
+                           doesn't require network connectivity
   nimble                   builds the Nimble tool
 Boot options:
   -d:release               produce a release version of the compiler
@@ -73,6 +74,11 @@ Web options:
                            build the official docs, use UA-48159761-1
 """
 
+let kochExe* = os.getAppFilename()
+
+proc kochExec*(cmd: string) =
+  exec kochExe.quoteShell & " " & cmd
+
 template withDir(dir, body) =
   let old = getCurrentDir()
   try:
@@ -123,9 +129,6 @@ proc bundleNimbleExe(latest: bool) =
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim", options = "-d:release --nilseqs:on")
 
-proc buildNimfind() =
-  nimCompile("tools/nimfind.nim", options = "-d:release")
-
 proc buildNimble(latest: bool) =
   # old installations created nim/nimblepkg/*.nim files. We remove these
   # here so that it cannot cause problems (nimble bug #306):
@@ -199,13 +202,12 @@ proc buildTool(toolname, args: string) =
   nimexec("cc $# $#" % [args, toolname])
   copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe)
 
-proc buildTools(latest: bool) =
+proc buildTools() =
   bundleNimsuggest()
   nimCompile("tools/nimgrep.nim", options = "-d:release")
   when defined(windows): buildVccTool()
   nimCompile("nimpretty/nimpretty.nim", options = "-d:release")
-  buildNimble(latest)
-  buildNimfind()
+  nimCompile("tools/nimfind.nim", options = "-d:release")
 
 proc nsis(latest: bool; args: string) =
   bundleNimbleExe(latest)
@@ -274,7 +276,7 @@ proc boot(args: string) =
   var output = "compiler" / "nim".exe
   var finalDest = "bin" / "nim".exe
   # default to use the 'c' command:
-  let defaultCommand = if getEnv(env_NIM_COMPILE_TO_CPP, "false") == "true": "cpp" else: "c"
+  let defaultCommand = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": "cpp" else: "c"
   let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: ""
   echo "boot: defaultCommand: ", defaultCommand, " bootOptions: ", bootOptions
   let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") &
@@ -440,32 +442,38 @@ proc runCI(cmd: string) =
     # todo: implement `execWithEnv`
     exec("env NIM_COMPILE_TO_CPP=false $1 boot" % kochExe.quoteShell)
   kochExec "boot -d:release"
+
+  ## build nimble early on to enable remainder to depend on it if needed
   kochExec "nimble"
-  exec "nim e tests/test_nimscript.nims"
 
   when false:
     for pkg in "zip opengl sdl1 jester@#head niminst".split:
       exec "nimble install -y" & pkg
 
+  buildTools() # altenatively, kochExec "tools --toolsNoNimble"
+
+  ## run tests
+  exec "nim e tests/test_nimscript.nims"
   when defined(windows):
     # note: will be over-written below
     exec "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester"
-    when false:
-      kochExec "csource"
-      kochExec "zip"
 
-  # main bottleneck: runs all main tests
+  # main bottleneck here
   exec "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines"
+
   exec "nim c -r nimdoc/tester"
-
-  nimCompile "nimpretty/nimpretty.nim"
   exec "nim c -r nimpretty/tester.nim"
+  when defined(posix):
+    exec "nim c -r nimsuggest/tester"
 
+  ## remaining actions
   when defined(posix):
     kochExec "docs --git.commit:devel"
     kochExec "csource"
-    kochExec "nimsuggest"
-    exec "nim c -r nimsuggest/tester"
+  elif defined(windows):
+    when false:
+      kochExec "csource"
+      kochExec "zip"
 
 proc pushCsources() =
   if not dirExists("../csources/.git"):
@@ -550,6 +558,10 @@ when isMainModule:
   var op = initOptParser()
   var latest = false
   var stable = false
+  template isLatest(): bool =
+    if stable: false
+    else:
+      existsDir(".git") or latest
   while true:
     op.next()
     case op.kind
@@ -580,13 +592,13 @@ when isMainModule:
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools()
-      of "nimble":
-        if stable: buildNimble(false)
-        else: buildNimble(existsDir(".git") or latest)
+      of "nimble": buildNimble(isLatest())
       of "nimsuggest": bundleNimsuggest()
+      of "toolsnonimble":
+        buildTools()
       of "tools":
-        if stable: buildTools(false)
-        else: buildTools(existsDir(".git") or latest)
+        buildTools()
+        buildNimble(isLatest())
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
       else: showHelp()

tools/kochdocs.nim

@@ -51,12 +51,6 @@ proc execCleanPath*(cmd: string,
   if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
   putEnv("PATH", prevPath)
 
-let kochExe* = os.getAppFilename()
-  # note: assumes `kochdocs` is only used by koch.nim
-
-proc kochExec*(cmd: string) =
-  exec kochExe.quoteShell & " " & cmd
-
 proc nimexec*(cmd: string) =
   # Consider using `nimCompile` instead
   exec findNim() & " " & cmd