Print error summary in Travis CI (#10474)

.travis.yml

@@ -42,6 +42,7 @@ before_script:
   - cd ..
   - export PATH=$(pwd)/bin${PATH:+:$PATH}
   - echo PATH:${PATH}
+  - set +e   # prevents breaking after_failure
 
 script:
   - nim c koch
@@ -59,3 +60,6 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
+
+# Extract failed tests
+after_failure: nim c -r tools/ci_testresults.nim

tools/ci_testresults.nim

@@ -0,0 +1,24 @@
+## Print summary of failed tests for CI
+
+import os, json, sets, strformat
+
+const skip = toSet(["reDisabled", "reIgnored", "reSuccess", "reJoined"])
+
+when isMainModule:
+  for fn in walkFiles("testresults/*.json"):
+    let entries = fn.readFile().parseJson()
+    for j in entries:
+      let res = j["result"].getStr()
+      if skip.contains(res):
+        continue
+      echo fmt """
+Category: {j["category"].getStr()}
+Name: {j["name"].getStr()}
+Action: {j["action"].getStr()}
+Result: {res}
+-------- Expected -------
+{j["expected"].getStr()}
+--------- Given  --------
+{j["given"].getStr()}
+-------------------------
+"""