fix hintProcessing dots interference with static:echo and hintCC; add tests for nim secret, add tests for hintProcessing, misc other bug fixes (#16495)

* fix dots interfering with static:echo
* add tests
* fix hintProcessing dots for hintCC
* improve trunner tests
* fix bug: readLineFromStdin now writes prompt to stdout, consistent with linenoise and rdstdin
* disable a failing test for windows

compiler/extccomp.nim

@@ -848,7 +848,10 @@ proc callCCompiler*(conf: ConfigRef) =
   var cmds: TStringSeq
   var prettyCmds: TStringSeq
   let prettyCb = proc (idx: int) =
-    if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+    if prettyCmds[idx].len > 0:
+      flushDot(conf)
+      # xxx should probably use stderr like other compiler messages, not stdout
+      echo prettyCmds[idx]
 
   for idx, it in conf.toCompile:
     # call the C compiler for the .c file:

compiler/llstream.nim

@@ -75,10 +75,10 @@ proc llStreamClose*(s: PLLStream) =
 when not declared(readLineFromStdin):
   # fallback implementation:
   proc readLineFromStdin(prompt: string, line: var string): bool =
-    stderr.write(prompt)
+    stdout.write(prompt)
     result = readLine(stdin, line)
     if not result:
-      stderr.write("\n")
+      stdout.write("\n")
       quit(0)
 
 proc endsWith*(x: string, s: set[char]): bool =

compiler/main.nim

@@ -138,7 +138,7 @@ proc commandInteractive(graph: ModuleGraph) =
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
     var idgen = IdGenerator(module: m.itemId.module, item: m.itemId.item)
-    let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config, stderr))
+    let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
     processModule(graph, m, idgen, s)
 
 proc commandScan(cache: IdentCache, config: ConfigRef) =

compiler/msgs.nim

@@ -19,8 +19,10 @@ template instLoc(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
 template toStdOrrKind(stdOrr): untyped =
   if stdOrr == stdout: stdOrrStdout else: stdOrrStderr
 
-template flushDot*(conf, stdOrr) =
+proc flushDot*(conf: ConfigRef) =
   ## safe to call multiple times
+  # xxx one edge case not yet handled is when `printf` is called at CT with `compiletimeFFI`.
+  let stdOrr = if optStdout in conf.globalOptions: stdout else: stderr
   let stdOrrKind = toStdOrrKind(stdOrr)
   if stdOrrKind in conf.lastMsgWasDot:
     conf.lastMsgWasDot.excl stdOrrKind
@@ -311,12 +313,12 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
     conf.writelnHook(s)
   elif optStdout in conf.globalOptions or msgStdout in flags:
     if eStdOut in conf.m.errorOutputs:
-      flushDot(conf, stdout)
+      flushDot(conf)
       writeLine(stdout, s)
       flushFile(stdout)
   else:
     if eStdErr in conf.m.errorOutputs:
-      flushDot(conf, stderr)
+      flushDot(conf)
       writeLine(stderr, s)
       # On Windows stderr is fully-buffered when piped, regardless of C std.
       when defined(windows):
@@ -368,11 +370,11 @@ template styledMsgWriteln*(args: varargs[typed]) =
     callIgnoringStyle(callWritelnHook, nil, args)
   elif optStdout in conf.globalOptions:
     if eStdOut in conf.m.errorOutputs:
-      flushDot(conf, stdout)
+      flushDot(conf)
       callIgnoringStyle(writeLine, stdout, args)
       flushFile(stdout)
   elif eStdErr in conf.m.errorOutputs:
-    flushDot(conf, stderr)
+    flushDot(conf)
     if optUseColors in conf.globalOptions:
       callStyledWriteLineStderr(args)
     else:

compiler/options.nim

@@ -368,8 +368,11 @@ proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) =
     if enabled: incl(conf.notes, note) else: excl(conf.notes, note)
 
 proc hasHint*(conf: ConfigRef, note: TNoteKind): bool =
+  # ternary states instead of binary states would simplify logic
   if optHints notin conf.options: false
-  elif note in {hintConf}: # could add here other special notes like hintSource
+  elif note in {hintConf, hintProcessing}:
+    # could add here other special notes like hintSource
+    # these notes apply globally.
     note in conf.mainPackageNotes
   else: note in conf.notes
 

compiler/vm.nim

@@ -1153,14 +1153,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, "node is not a proc symbol")
     of opcEcho:
       let rb = instr.regB
-      if rb == 1:
-        msgWriteln(c.config, regs[ra].node.strVal, {msgStdout})
+      template fn(s) = msgWriteln(c.config, s, {msgStdout})
+      if rb == 1: fn(regs[ra].node.strVal)
       else:
         var outp = ""
         for i in ra..ra+rb-1:
           #if regs[i].kind != rkNode: debug regs[i]
           outp.add(regs[i].node.strVal)
-        msgWriteln(c.config, outp, {msgStdout})
+        fn(outp)
     of opcContainsSet:
       decodeBC(rkInt)
       regs[ra].intVal = ord(inSet(regs[rb].node, regs[rc].regToNode))

tests/misc/trunner.nim

@@ -12,10 +12,18 @@ from std/sequtils import toSeq,mapIt
 from std/algorithm import sorted
 import stdtest/[specialpaths, unittest_light]
 from std/private/globs import nativeToUnixPath
-
+from strutils import startsWith, strip, removePrefix
+from std/sugar import dup
 import "$lib/../compiler/nimpaths"
 
+proc isDots(a: string): bool =
+  ## test for `hintProcessing` dots
+  a.startsWith(".") and a.strip(chars = {'.'}) == ""
+
 const
+  defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off"
+    # useful when you want to turn only some hints on, and some common ones off.
+    # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off`
   nim = getCurrentCompilerExe()
   mode =
     when defined(c): "c"
@@ -93,10 +101,9 @@ else: # don't run twice the same test
       check exitCode == 0
       let ret = toSeq(walkDirRec(htmldocsDir, relative=true)).mapIt(it.nativeToUnixPath).sorted.join("\n")
       let context = $(i, ret, cmd)
-      var expected = ""
       case i
       of 0,5:
-        let htmlFile = htmldocsDir/"mmain.html"
+        let htmlFile = htmldocsDir/mainFname
         check htmlFile in outp # sanity check for `hintSuccessX`
         assertEquals ret, fmt"""
 {dotdotMangle}/imp.html
@@ -106,7 +113,7 @@ imp.html
 imp.idx
 imp2.html
 imp2.idx
-mmain.html
+{mainFname}
 mmain.idx
 {nimdocOutCss}
 {theindexFname}""", context
@@ -119,21 +126,21 @@ tests/nimdoc/sub/imp.html
 tests/nimdoc/sub/imp.idx
 tests/nimdoc/sub/imp2.html
 tests/nimdoc/sub/imp2.idx
-tests/nimdoc/sub/mmain.html
+tests/nimdoc/sub/{mainFname}
 tests/nimdoc/sub/mmain.idx
 {theindexFname}"""
       of 2, 3: assertEquals ret, fmt"""
 {docHackJsFname}
-mmain.html
+{mainFname}
 mmain.idx
 {nimdocOutCss}""", context
       of 4: assertEquals ret, fmt"""
 {docHackJsFname}
 {nimdocOutCss}
-sub/mmain.html
+sub/{mainFname}
 sub/mmain.idx""", context
       of 6: assertEquals ret, fmt"""
-mmain.html
+{mainFname}
 {nimdocOutCss}""", context
       else: doAssert false
 
@@ -222,3 +229,44 @@ mmain.html
     check fmt"""{nim} {opt} --eval:"echo defined(nimscript)"""".execCmdEx == ("true\n", 0)
     check fmt"""{nim} r {opt} --eval:"echo defined(c)"""".execCmdEx == ("true\n", 0)
     check fmt"""{nim} r -b:js {opt} --eval:"echo defined(js)"""".execCmdEx == ("true\n", 0)
+
+  block: # `hintProcessing` dots should not interfere with `static: echo` + friends
+    let cmd = fmt"""{nim} r {defaultHintsOff} --hint:processing -f --eval:"static: echo 1+1""""
+    let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut})
+    template check3(cond) = doAssert cond, $(outp,)
+    doAssert exitCode == 0
+    let lines = outp.splitLines
+    check3 lines.len == 3
+    when not defined(windows): # xxx: on windows, dots not properly handled, gives: `....2\n\n`
+      check3 lines[0].isDots
+      check3 lines[1] == "2"
+      check3 lines[2] == ""
+    else:
+      check3 "2" in outp
+
+  block: # nim secret
+    let opt = fmt"{defaultHintsOff} --hint:processing"
+    template check3(cond) = doAssert cond, $(outp,)
+    for extra in ["", "--stdout"]:
+      let cmd = fmt"""{nim} secret {opt} {extra}"""
+      # xxx minor bug: `nim --hint:QuitCalled:off secret` ignores the hint cmdline flag
+      template run(input2): untyped =
+        execCmdEx(cmd, options = {poStdErrToStdOut}, input = input2)
+      block:
+        let (outp, exitCode) = run """echo 1+2; import strutils; echo strip(" ab "); quit()"""
+        let lines = outp.splitLines
+        when not defined(windows):
+          check3 lines.len == 5
+          check3 lines[0].isDots
+          check3 lines[1].dup(removePrefix(">>> ")) == "3" # prompt depends on `nimUseLinenoise`
+          check3 lines[2].isDots
+          check3 lines[3] == "ab"
+          check3 lines[4] == ""
+        else:
+          check3 "3" in outp
+          check3 "ab" in outp
+        doAssert exitCode == 0
+      block:
+        let (outp, exitCode) = run "echo 1+2; quit(2)"
+        check3 "3" in outp
+        doAssert exitCode == 2