testament: add azure integration

(cherry picked from commit acebcd7899)

testament/azure.nim

@@ -0,0 +1,95 @@
+#
+#
+#              The Nim Tester
+#        (c) Copyright 2019 Leorize
+#
+#    Look at license.txt for more info.
+#    All rights reserved.
+
+import base64, json, httpclient, os, strutils
+import specs
+
+const
+  ApiRuns = "/_apis/test/runs"
+  ApiVersion = "?api-version=5.0"
+  ApiResults = ApiRuns & "/$1/results"
+
+var runId* = -1
+
+proc getAzureEnv(env: string): string =
+  # Conversion rule at:
+  # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#set-variables-in-pipeline
+  env.toUpperAscii().replace('.', '_').getEnv
+
+proc invokeRest(httpMethod: HttpMethod; api: string; body = ""): Response =
+  let http = newHttpClient()
+  defer: close http
+  result = http.request(getAzureEnv("System.TeamFoundationCollectionUri") &
+                        getAzureEnv("System.TeamProjectId") & api & ApiVersion,
+                        httpMethod,
+                        $body,
+                        newHttpHeaders {
+                          "Accept": "application/json",
+                          "Authorization": "Basic " & encode(':' & getAzureEnv("System.AccessToken")),
+                          "Content-Type": "application/json"
+                        })
+  if not result.code.is2xx:
+    raise newException(HttpRequestError, "Server returned: " & result.body)
+
+proc finish*() {.noconv.} =
+  if not isAzure or runId < 0:
+    return
+
+  try:
+    discard invokeRest(HttpPatch,
+                       ApiRuns & "/" & $runId,
+                       $ %* { "state": "Completed" })
+  except:
+    stderr.writeLine "##vso[task.logissue type=warning;]Unable to finalize Azure backend"
+    stderr.writeLine getCurrentExceptionMsg()
+
+  runId = -1
+
+# TODO: Only obtain a run id if tests are run
+# NOTE: We can't delete test runs with Azure's access token
+proc start*() =
+  if not isAzure:
+    return
+  try:
+    if runId < 0:
+      runId = invokeRest(HttpPost,
+                         ApiRuns,
+                         $ %* {
+                           "automated": true,
+                           "build": { "id": getAzureEnv("Build.BuildId") },
+                           "buildPlatform": hostCPU,
+                           "controller": "nim-testament",
+                           "name": getAzureEnv("Agent.JobName")
+                         }).body.parseJson["id"].getInt(-1)
+  except:
+    stderr.writeLine "##vso[task.logissue type=warning;]Unable to initialize Azure backend"
+    stderr.writeLine getCurrentExceptionMsg()
+
+proc addTestResult*(name, category: string; durationInMs: int; errorMsg: string;
+                    outcome: TResultEnum) =
+  if not isAzure or runId < 0:
+    return
+  let outcome = case outcome
+                of reSuccess: "Passed"
+                of reDisabled, reJoined: "NotExecuted"
+                else: "Failed"
+  try:
+    discard invokeRest(HttpPost,
+                       ApiResults % [$runId],
+                       $ %* [{
+                         "automatedTestName": name,
+                         "automatedTestStorage": category,
+                         "durationInMs": durationInMs,
+                         "errorMessage": errorMsg,
+                         "outcome": outcome,
+                         "testCaseTitle": name
+                       }])
+  except:
+    stderr.writeLine "##vso[task.logissue type=warning;]Unable to log test case: ",
+                     name, ", outcome: ", outcome
+    stderr.writeLine getCurrentExceptionMsg()

testament/specs.nim

@@ -13,6 +13,7 @@ var compilerPrefix* = findExe("nim")
 
 let isTravis* = existsEnv("TRAVIS")
 let isAppVeyor* = existsEnv("APPVEYOR")
+let isAzure* = existsEnv("TF_BUILD")
 
 var skips*: seq[string]
 
@@ -214,6 +215,8 @@ proc parseSpec*(filename: string): TSpec =
           if isTravis: result.err = reDisabled
         of "appveyor":
           if isAppVeyor: result.err = reDisabled
+        of "azure":
+          if isAzure: result.err = reDisabled
         of "32bit":
           if sizeof(int) == 4:
             result.err = reDisabled

testament/testament.nim

@@ -12,7 +12,7 @@
 import
   strutils, pegs, os, osproc, streams, json,
   backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, times, md5, sequtils
+  algorithm, times, md5, sequtils, azure
 
 include compiler/nodejs
 
@@ -284,7 +284,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       maybeStyledEcho styleBright, given, "\n"
 
 
-  if backendLogging and existsEnv("APPVEYOR"):
+  if backendLogging and (isAppVeyor or isAzure):
     let (outcome, msg) =
       case success
       of reSuccess:
@@ -295,14 +295,17 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
         ("Failed", "Failure: " & $success & "\n" & given)
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
-    var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
-                         "-Framework", "nim-testament", "-FileName",
-                         test.cat.string,
-                         "-Outcome", outcome, "-ErrorMessage", msg,
-                         "-Duration", $(duration*1000).int],
-                         options={poStdErrToStdOut, poUsePath, poParentStreams})
-    discard waitForExit(p)
-    close(p)
+    if isAzure:
+      azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success)
+    else:
+      var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
+                           "-Framework", "nim-testament", "-FileName",
+                           test.cat.string,
+                           "-Outcome", outcome, "-ErrorMessage", msg,
+                           "-Duration", $(duration*1000).int],
+                           options={poStdErrToStdOut, poUsePath, poParentStreams})
+      discard waitForExit(p)
+      close(p)
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
   if strip(expected.msg) notin strip(given.msg):
@@ -641,6 +644,8 @@ proc main() =
         quit Usage
     of "skipfrom":
       skipFrom = p.val.string
+    of "azurerunid":
+      runId = p.val.parseInt
     else:
       quit Usage
     p.next()
@@ -653,6 +658,7 @@ proc main() =
   of "all":
     #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
 
+    azure.start()
     var myself = quoteShell(findExe("testament" / "testament"))
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
@@ -661,6 +667,8 @@ proc main() =
 
     if skipFrom.len > 0:
       myself &= " " & quoteShell("--skipFrom:" & skipFrom)
+    if isAzure:
+      myself &= " " & quoteShell("--azureRunId:" & $runId)
 
     var cats: seq[string]
     let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
@@ -686,13 +694,16 @@ proc main() =
         progressStatus(i)
         processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
     else:
+      addQuitProc azure.finish
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
+    azure.start()
     skips = loadSkipFrom(skipFrom)
     var cat = Category(p.key)
     p.next
     processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
   of "pcat":
+    azure.start()
     skips = loadSkipFrom(skipFrom)
     # 'pcat' is used for running a category in parallel. Currently the only
     # difference is that we don't want to run joinable tests here as they
@@ -707,6 +718,7 @@ proc main() =
     p.next
     processPattern(r, pattern, p.cmdLineRest.string, simulate)
   of "r", "run":
+    azure.start()
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})
     # "stdlib/nre/captures.nim" -> "stdlib" + "nre/captures.nim"
@@ -722,6 +734,7 @@ proc main() =
     if action == "html": openDefaultBrowser(resultsFile)
     else: echo r, r.data
   backend.close()
+  if isMainProcess: azure.finish()
   var failed = r.total - r.passed - r.skipped
   if failed != 0:
     echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",

testament/testament.nim.cfg

@@ -1 +1,2 @@
 path = "$nim" # For compiler/nodejs
+-d:ssl # For azure