Merge pull request #470 from gradha/pr_idetools_tests

Adds idetools tests

.gitignore

@@ -167,4 +167,10 @@ examples/cross_calculator/android/tags
 /run.json
 /testresults.html
 /testresults.json
+/tests/caas/idetools_api
+/tests/caas/imported
+/tests/caas/issue_416
+/tests/caas/issue_452
+/tests/caas/main
+/tests/caasdriver
 /tools/nimgrep

tests/caas/basic-recompile.txt

@@ -2,6 +2,9 @@ main.nim
 > c
 SuccessX
 > c
-! Processing
+# The "Processing" string will be found always in proc mode since each
+# compilation command will generate it. We need to test it only in Caas mode to
+# verify the server is not recompiling again the file.
+CaasRun ! Processing
 SuccessX
 

tests/caas/compile-suggest.txt

@@ -1,7 +1,8 @@
 main.nim
 > c
 SuccessX
-> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
-skField\tx
-skField\ty
+# Ugh, undocumented trackDirty, how is it supposed to work? Fails in proc mode.
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest
+CaasRun skField\tx
+CaasRun skField\ty
 

tests/caas/compile-then-def.txt

@@ -1,9 +1,9 @@
 main.nim
 > c
 SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+! SuccessX
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX

tests/caas/def-def-compile.txt

@@ -1,10 +1,10 @@
 main.nim
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+! SuccessX
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX
 > c
 SuccessX
 

tests/caas/def-then-compile.txt

@@ -1,7 +1,7 @@
 main.nim
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX
 > c
 SuccessX
 

tests/caas/idetools_api.nim

@@ -0,0 +1,23 @@
+import unicode, sequtils
+
+proc test_enums() =
+  var o: Tfile
+  if o.open("files " & "test.txt", fmWrite):
+    o.write("test")
+    o.close()
+
+proc test_iterators(filename = "tests.nim") =
+  let
+    input = readFile(filename)
+    letters = toSeq(runes(string(input)))
+  for letter in letters: echo int(letter)
+
+const SOME_SEQUENCE = @[1, 2]
+type
+  bad_string = distinct string
+  TPerson = object of TObject
+    name*: bad_string
+    age: int
+
+proc adder(a, b: int): int =
+  result = a + b

tests/caas/idetools_api.txt

@@ -0,0 +1,41 @@
+idetools_api.nim
+> c
+SuccessX
+> idetools --track:idetools_api.nim,4,11 --def
+def\tskType\tsystem.TFile\tTFile
+> idetools --track:idetools_api.nim,5,7 --def
+def\tskProc\tsystem.Open\tproc \(var TFile, string, TFileMode, int\): bool
+> idetools --track:idetools_api.nim,5,21 --def
+def\tskProc\tsystem.\&\tproc \(string, string\): string\{.noSideEffect.\}
+> idetools --track:idetools_api.nim,5,38 --def
+def\tskEnumField\tsystem.TFileMode.fmWrite\tTFileMode
+> idetools --track:idetools_api.nim,7,6 --def
+def\tskProc\tsystem.Close\tproc \(TFile\)
+> idetools --track:idetools_api.nim,12,23 --def
+def\tskIterator\tunicode.runes\titerator \(string\): TRune
+> idetools --track:idetools_api.nim,12,15 --def
+def\tskTemplate\tsequtils.toSeq\tproc \(expr\): expr
+> idetools --track:idetools_api.nim,15,7 --def
+
+# ProcRun mode will fail the next line, because the type is returned empty.
+def\tskConst\tidetools_api.SOME_SEQUENCE\tseq\[int\]\t
+> idetools --track:idetools_api.nim,15,23 --def
+def\tskProc\tsystem.@\tproc \(array\[IDX, T\]\): seq\[T\]\{.noSideEffect.\}
+> idetools --track:idetools_api.nim,17,3 --def
+
+# ProcRun mode will fail the next line, because the type is returned empty.
+def\tskType\tidetools_api.bad_string\tbad_string\t
+> idetools --track:idetools_api.nim,11,24 --def
+def\tskParam\tidetools_api.test_iterators.filename\tstring
+> idetools --track:idetools_api.nim,6,5 --def
+def\tskVar\tidetools_api.test_enums.o\tTFile
+> idetools --track:idetools_api.nim,12,34 --def
+def\tskLet\tidetools_api.test_iterators.input\tTaintedString
+> idetools --track:idetools_api.nim,13,35 --def
+def\tskForVar\tidetools_api.test_iterators.letter\tTRune
+> idetools --track:idetools_api.nim,23,3 --def
+def\tskResult\tidetools_api.adder.result\tint
+> idetools --track:idetools_api.nim,19,6 --def
+
+# ProcRun mode will fail the next line, because the type is returned empty.
+def\tskField\tidetools_api.TPerson.name\tbad_string\t

tests/caas/issue_416.nim

@@ -0,0 +1,17 @@
+import unicode, sequtils
+
+proc test() =
+  let input = readFile("weird.nim")
+  for letter in runes(string(input)):
+    echo int(letter)
+
+when 1 > 0:
+  proc failtest() =
+    let
+      input = readFile("weird.nim")
+      letters = toSeq(runes(string(input)))
+    for letter in letters:
+      echo int(letter)
+
+when isMainModule:
+  test()

tests/caas/issue_416.txt

@@ -0,0 +1,14 @@
+issue_416.nim
+> c
+SuccessX
+> idetools --track:issue_416.nim,12,28 --def
+def\tskType\tsystem.string\tstring
+> idetools --track:issue_416.nim,12,35 --def
+def\tskLet\tissue_416.failtest.input\tTaintedString
+
+# The following fail because they seem shifted one colum to the right.
+> idetools --track:issue_416.nim,12,16 --def
+def\tskTemplate\tsequtils.toSeq\tproc \(expr\): expr
+> idetools --track:issue_416.nim,12,22 --def
+def\tskIterator\tunicode.runes\titerator \(string\): TRune
+

tests/caas/issue_452.nim

@@ -0,0 +1,8 @@
+const
+  VERSION_STR1* = "0.5.0" ## Idetools shifts this one column.
+  VERSION_STR2 = "0.5.0" ## This one is ok.
+  VERSION_STR3* = "0.5.0" ## Bad.
+  VERSION_STR4 = "0.5.0" ## Ok.
+
+proc forward1*(): string = result = ""
+proc forward2(): string = result = ""

tests/caas/issue_452.txt

@@ -0,0 +1,11 @@
+issue_452.nim
+> c
+SuccessX
+> idetools --track:issue_452.nim,2,2 --def
+def\tskConst\tissue_452.VERSION_STR1\tstring
+> idetools --track:issue_452.nim,3,2 --def
+def\tskConst\tissue_452.VERSION_STR2\tstring
+> idetools --track:issue_452.nim,7,5 --def
+def\tskProc\tissue_452.forward1\tproc \(\): string\t
+> idetools --track:issue_452.nim,8,5 --def
+def\tskProc\tissue_452.forward2\tproc \(\): string\t

tests/caas/suggest-compile.txt

@@ -1,5 +1,6 @@
 main.nim
-> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
+# Ugh, undocumented trackDirty, how is it supposed to work? Fails in proc mode.
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest
 skField\tx
 skField\ty
 > c

tests/caasdriver.nim

@@ -1,26 +1,75 @@
 import osproc, streams, os, strutils, re
 
-type
-  TNimrodSession* = object
-    nim: PProcess
+## Compiler as a service tester.
+##
+## This test cases uses the txt files in the caas/ subdirectory.
+##
+## Each of the text files inside encodes a session with the compiler:
+##
+## The first line indicates the main project file.
+##
+## Lines starting with '>' indicate a command to be sent to the compiler and
+## the lines following a command include checks for expected or forbidden
+## output (! for forbidden).
+##
+## If a line starts with '#' it will be ignored completely, so you can use that
+## for comments.
+##
+## All the tests are run both in ProcRun (each command creates a separate
+## process) and CaasRun (first command starts up a server and it is reused for
+## the rest) modes. Since some cases are specific to either ProcRun or CaasRun
+## modes, you can prefix a line with the mode and the line will be processed
+## only in that mode.
+##
+## The rest of the line is treated as a regular expression, so be careful
+## escaping metacharacters like parenthesis.
+##
+## You can optionally pass parameters at the command line to modify the
+## behaviour of the test suite. By default only tests which fail will be echoed
+## to stdout. If you want to see all the output pass the word "verbose" as a
+## parameter.
+##
+## If you don't want to run all the test case files, you can pass any substring
+## as a parameter. Only files matching the passed substring will be run. The
+## filtering doesn't use any globbing metacharacters, it's a plain match.
+##
+## Example to run only "*-compile*.txt" tests in verbose mode:
+##
+##   ./caasdriver verbose -compile
 
-proc dirname(path: string): string = path.splitPath()[0]
+
+type
+  TRunMode = enum
+    ProcRun, CaasRun
+
+  TNimrodSession* = object
+    nim: PProcess # Holds the open process for CaasRun sessions, nil otherwise.
+    mode: TRunMode # Stores the type of run mode the session was started with.
+    lastOutput: string # Preserves the last output, needed for ProcRun mode.
+    filename: string # Appended to each command starting with '>'.
+
+const modes = [CaasRun, ProcRun]
 
 var
   TesterDir = getAppDir()
   NimrodBin = TesterDir / "../bin/nimrod"
 
-proc startNimrodSession*(project: string): TNimrodSession =
-  result.nim = startProcess(NimrodBin,
-    workingDir = project.dirname,
-    args = ["serve", "--server.type:stdin", project])
+proc startNimrodSession(project: string, mode: TRunMode): TNimrodSession =
+  let (dir, name) = project.SplitPath
+  result.mode = mode
+  result.lastOutput = ""
+  result.filename = name
+  if mode == CaasRun:
+    result.nim = startProcess(NimrodBin, workingDir = dir,
+      args = ["serve", "--server.type:stdin", name])
 
-proc doCommand*(session: var TNimrodSession, command: string): string =
+proc doCaasCommand(session: var TNimrodSession, command: string): string =
+  assert session.mode == CaasRun
   session.nim.inputStream.write(command & "\n")
   session.nim.inputStream.flush
-  
+
   result = ""
-  
+
   while true:
     var line = TaintedString("")
     if session.nim.outputStream.readLine(line):
@@ -30,30 +79,66 @@ proc doCommand*(session: var TNimrodSession, command: string): string =
       result = "FAILED TO EXECUTE: " & command & "\n" & result
       break
 
-proc close(session: var TNimrodSession) {.destructor.} =
-  session.nim.close
+proc doProcCommand(session: var TNimrodSession, command: string): string =
+  assert session.mode == ProcRun
+  except: result = "FAILED TO EXECUTE: " & command & "\n" & result
+  var
+    process = startProcess(NimrodBin, args = command.split)
+    stream = outputStream(process)
+    line = TaintedString("")
 
-proc doScenario(script: string, output: PStream): bool =
+  result = ""
+  while stream.readLine(line):
+    if result.len > 0: result &= "\n"
+    result &= line.string
+
+  process.close()
+
+proc doCommand(session: var TNimrodSession, command: string) =
+  if session.mode == CaasRun:
+    session.lastOutput = doCaasCommand(session,
+                                       command & " " & session.filename)
+  else:
+    session.lastOutput = doProcCommand(session,
+                                       command & " " & session.filename)
+
+proc close(session: var TNimrodSession) {.destructor.} =
+  if session.mode == CaasRun:
+    session.nim.close
+
+proc doScenario(script: string, output: PStream, mode: TRunMode): bool =
   result = true
 
   var f = open(script)
   var project = TaintedString("")
-  
+
   if f.readLine(project):
     var
-      s = startNimrodSession(script.dirname / project.string)
+      s = startNimrodSession(script.parentDir / project.string, mode)
       tline = TaintedString("")
-      lastOutput = ""
       ln = 1
 
     while f.readLine(tline):
       var line = tline.string
       inc ln
+
+      # Filter lines by run mode, removing the prefix if the mode is current.
+      for testMode in modes:
+        if line.startsWith($testMode):
+          if testMode != mode:
+            line = ""
+          else:
+            line = line[len($testMode)..len(line) - 1].strip
+          break
+
       if line.strip.len == 0: continue
 
-      if line.startsWith(">"):
-        lastOutput = s.doCommand(line.substr(1).strip)
-        output.writeln line, "\n", lastOutput
+      if line.startsWith("#"):
+        output.writeln line
+        continue
+      elif line.startsWith(">"):
+        s.doCommand(line.substr(1).strip)
+        output.writeln line, "\n", s.lastOutput
       else:
         var expectMatch = true
         var pattern = line
@@ -61,7 +146,8 @@ proc doScenario(script: string, output: PStream): bool =
           pattern = line.substr(1).strip
           expectMatch = false
 
-        var actualMatch = lastOutput.find(re(pattern)) != -1
+        let actualMatch =
+          s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
 
         if expectMatch == actualMatch:
           output.writeln "SUCCESS ", line
@@ -70,17 +156,33 @@ proc doScenario(script: string, output: PStream): bool =
           result = false
 
 iterator caasTestsRunner*(filter = ""): tuple[test, output: string,
-                                              status: bool] =
+                                              status: bool, mode: TRunMode] =
   for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
     if filter.len > 0 and find(scenario, filter) == -1: continue
-    var outStream = newStringStream()
-    let r = doScenario(scenario, outStream)
-    yield (scenario, outStream.data, r)
+    for mode in modes:
+      var outStream = newStringStream()
+      let r = doScenario(scenario, outStream, mode)
+      yield (scenario, outStream.data, r, mode)
 
 when isMainModule:
-  var filter = ""
-  if paramCount() > 0: filter = paramStr(1)
-  
-  for t, o, r in caasTestsRunner(filter):
-    echo t, "\n", o
-    
+  var
+    filter = ""
+    failures = 0
+    verbose = false
+
+  for i in 0..ParamCount() - 1:
+    let param = paramStr(i + 1)
+    case param
+    of "verbose": verbose = true
+    else: filter = param
+
+  if verbose and len(filter) > 0:
+    echo "Running only test cases matching filter '$1'" % [filter]
+
+  for test, output, result, mode in caasTestsRunner(filter):
+    if not result or verbose:
+      echo test, "\n", output, "-> ", $mode, ":", $result, "\n-----"
+    if not result:
+      failures += 1
+
+  quit(failures)