testament: no sqlite or re dependencies anymore

tests/testament/backend.nim

@@ -1,56 +1,16 @@
 #
 #
 #              The Nim Tester
-#        (c) Copyright 2015 Andreas Rumpf
+#        (c) Copyright 2017 Andreas Rumpf
 #
 #    Look at license.txt for more info.
 #    All rights reserved.
 
-import strutils, db_sqlite, os, osproc
-
-var db: DbConn
-
-proc createDb() =
-  db.exec(sql"""
-    create table if not exists Machine(
-      id integer primary key,
-      name varchar(100) not null,
-      os varchar(20) not null,
-      cpu varchar(20) not null
-    );""")
-
-  db.exec(sql"""
-    create table if not exists [Commit](
-      id integer primary key,
-      hash varchar(256) not null,
-      branch varchar(50) not null
-    );""")
-
-  db.exec(sql"""
-    create table if not exists TestResult(
-      id integer primary key,
-      name varchar(100) not null,
-      category varchar(100) not null,
-      target varchar(20) not null,
-      action varchar(10) not null,
-      result varchar(30) not null,
-      [commit] int not null,
-      machine int not null,
-      expected varchar(10000) not null,
-      given varchar(10000) not null,
-      created timestamp not null default (DATETIME('now')),
-
-      foreign key ([commit]) references [commit](id),
-      foreign key (machine) references machine(id)
-    );""")
-
-  #db.exec(sql"""
-  #  --create unique index if not exists TsstNameIx on TestResult(name);
-  #  """, [])
+import strutils, os, osproc, json
 
 type
-  MachineId* = distinct int64
-  CommitId = distinct int64
+  MachineId* = distinct string
+  CommitId = distinct string
 
 proc `$`*(id: MachineId): string {.borrow.}
 proc `$`(id: CommitId): string {.borrow.}
@@ -58,11 +18,12 @@ proc `$`(id: CommitId): string {.borrow.}
 var
   thisMachine: MachineId
   thisCommit: CommitId
+  thisBranch: string
 
 {.experimental.}
 proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
 
-proc getMachine*(db: DbConn): MachineId =
+proc getMachine*(): MachineId =
   var name = "hostname"()
   if name.len == 0:
     name = when defined(posix): getenv"HOSTNAME".string
@@ -70,54 +31,44 @@ proc getMachine*(db: DbConn): MachineId =
   if name.len == 0:
     quit "cannot determine the machine name"
 
-  let id = db.getValue(sql"select id from Machine where name = ?", name)
-  if id.len > 0:
-    result = id.parseInt.MachineId
-  else:
-    result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)",
-                         name, system.hostOS, system.hostCPU).MachineId
+  result = MachineId(name)
 
-proc getCommit(db: DbConn): CommitId =
+proc getCommit(): CommitId =
   const commLen = "commit ".len
   let hash = "git log -n 1"()[commLen..commLen+10]
-  let branch = "git symbolic-ref --short HEAD"()
-  if hash.len == 0 or branch.len == 0: quit "cannot determine git HEAD"
+  thisBranch = "git symbolic-ref --short HEAD"()
+  if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
+  result = CommitId(hash)
 
-  let id = db.getValue(sql"select id from [Commit] where hash = ? and branch = ?",
-                       hash, branch)
-  if id.len > 0:
-    result = id.parseInt.CommitId
-  else:
-    result = db.insertId(sql"insert into [Commit](hash, branch) values (?, ?)",
-                         hash, branch).CommitId
+var
+  results: File
+  currentCategory: string
+  entries: int
 
 proc writeTestResult*(name, category, target,
                       action, result, expected, given: string) =
-  let id = db.getValue(sql"""select id from TestResult
-                             where name = ? and category = ? and target = ? and
-                                machine = ? and [commit] = ?""",
-                                name, category, target,
-                                thisMachine, thisCommit)
-  if id.len > 0:
-    db.exec(sql"""update TestResult
-                  set action = ?, result = ?, expected = ?, given = ?
-                  where id = ?""", action, result, expected, given, id)
-  else:
-    db.exec(sql"""insert into TestResult(name, category, target,
-                                         action,
-                                         result, expected, given,
-                                         [commit], machine)
-                  values (?,?,?,?,?,?,?,?,?) """, name, category, target,
-                                        action,
-                                        result, expected, given,
-                                        thisCommit, thisMachine)
+  createDir("testresults")
+  if currentCategory != category:
+    if currentCategory.len > 0:
+      results.writeLine("]")
+      close(results)
+    currentCategory = category
+    results = open("testresults" / category.addFileExt"json", fmWrite)
+    results.writeLine("[")
+    entries = 0
+
+  let jentry = %*{"name": name, "category": category, "target": target,
+    "action": action, "result": result, "expected": expected, "given": given,
+    "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch}
+  if entries > 0:
+    results.writeLine(",")
+  results.write($jentry)
+  inc entries
 
 proc open*() =
-  let dbFile = if existsEnv("TRAVIS") or existsEnv("APPVEYOR"): ":memory:" else: "testament.db"
-  db = open(connection=dbFile, user="testament", password="",
-            database="testament")
-  createDb()
-  thisMachine = getMachine(db)
-  thisCommit = getCommit(db)
+  thisMachine = getMachine()
+  thisCommit = getCommit()
 
-proc close*() = close(db)
+proc close*() =
+  results.writeLine("]")
+  close(results)

tests/testament/htmlgen.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim Tester
-#        (c) Copyright 2015 Andreas Rumpf
+#        (c) Copyright 2017 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -9,20 +9,20 @@
 
 ## HTML generator for the tester.
 
-import db_sqlite, cgi, backend, strutils, json
+import cgi, backend, strutils, json, os
 
 import "testamenthtml.templ"
 
-proc generateTestRunTabListItemPartial(outfile: File, testRunRow: Row, firstRow = false) =
+proc generateTestRunTabListItemPartial(outfile: File, testRunRow: JsonNode, firstRow = false) =
   let
     # The first tab gets the bootstrap class for a selected tab
     firstTabActiveClass = if firstRow: "active"
                           else: ""
-    commitId = testRunRow[0]
-    hash = htmlQuote(testRunRow[1])
-    branch = htmlQuote(testRunRow[2])
-    machineId = testRunRow[3]
-    machineName = htmlQuote(testRunRow[4])
+    commitId = htmlQuote testRunRow["commit"].str
+    hash = htmlQuote(testRunRow["commit"].str)
+    branch = htmlQuote(testRunRow["branch"].str)
+    machineId = htmlQuote testRunRow["machine"].str
+    machineName = htmlQuote(testRunRow["machine"].str)
 
   outfile.generateHtmlTabListItem(
       firstTabActiveClass,
@@ -33,17 +33,17 @@ proc generateTestRunTabListItemPartial(outfile: File, testRunRow: Row, firstRow
       machineName
     )
 
-proc generateTestResultPanelPartial(outfile: File, testResultRow: Row, onlyFailing = false) =
+proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode, onlyFailing = false) =
   let
-    trId = testResultRow[0]
-    name = testResultRow[1].htmlQuote()
-    category = testResultRow[2].htmlQuote()
-    target = testResultRow[3].htmlQuote()
-    action = testResultRow[4].htmlQuote()
-    result = testResultRow[5]
-    expected = testResultRow[6]
-    gotten = testResultRow[7]
-    timestamp = testResultRow[8]
+    trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str)
+    name = testResultRow["name"].str.htmlQuote()
+    category = testResultRow["category"].str.htmlQuote()
+    target = testResultRow["target"].str.htmlQuote()
+    action = testResultRow["action"].str.htmlQuote()
+    result = htmlQuote testResultRow["result"].str
+    expected = htmlQuote testResultRow["expected"].str
+    gotten = htmlQuote testResultRow["given"].str
+    timestamp = "unknown"
   var panelCtxClass, textCtxClass, bgCtxClass, resultSign, resultDescription: string
   case result
   of "reSuccess":
@@ -71,8 +71,8 @@ proc generateTestResultPanelPartial(outfile: File, testResultRow: Row, onlyFaili
 
   outfile.generateHtmlTestresultPanelBegin(
     trId, name, target, category, action, resultDescription,
-    timestamp, 
-    result, resultSign, 
+    timestamp,
+    result, resultSign,
     panelCtxClass, textCtxClass, bgCtxClass
   )
   if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace():
@@ -84,92 +84,65 @@ proc generateTestResultPanelPartial(outfile: File, testResultRow: Row, onlyFaili
     )
   outfile.generateHtmlTestresultPanelEnd()
 
-proc generateTestResultsPanelGroupPartial(outfile: File, db: DbConn, commitid, machineid: string, onlyFailing = false) =
-  const testResultsSelect = sql"""
-SELECT [tr].[id]
-  , [tr].[name]
-  , [tr].[category]
-  , [tr].[target]
-  , [tr].[action]
-  , [tr].[result]
-  , [tr].[expected]
-  , [tr].[given]
-  , [tr].[created]
-FROM [TestResult] AS [tr]
-WHERE [tr].[commit] = ?
-  AND [tr].[machine] = ?"""
-  for testresultRow in db.rows(testResultsSelect, commitid, machineid):
+type
+  AllTests = object
+    data: JSonNode
+    totalCount, successCount, ignoredCount, failedCount: int
+    successPercentage, ignoredPercentage, failedPercentage: BiggestFloat
+
+proc allTestResults(): AllTests =
+  result.data = newJArray()
+  for file in os.walkFiles("testresults/*.json"):
+    let data = parseFile(file)
+    if data.kind != JArray:
+      echo "[ERROR] ignoring json file that is not an array: ", file
+    else:
+      for elem in data:
+        result.data.add elem
+        let state = elem["result"]
+        if state.contains("reSuccess"): inc result.successCount
+        elif state.contains("reIgnored"): inc result.ignoredCount
+
+  result.totalCount = result.data.len
+  result.successPercentage = 100 * (result.successCount.toBiggestFloat() / result.totalCount.toBiggestFloat())
+  result.ignoredPercentage = 100 * (result.ignoredCount.toBiggestFloat() / result.totalCount.toBiggestFloat())
+  result.failedCount = result.totalCount - result.successCount - result.ignoredCount
+  result.failedPercentage = 100 * (result.failedCount.toBiggestFloat() / result.totalCount.toBiggestFloat())
+
+
+proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode, onlyFailing = false) =
+  for testresultRow in allResults:
     generateTestResultPanelPartial(outfile, testresultRow, onlyFailing)
 
-proc generateTestRunTabContentPartial(outfile: File, db: DbConn, testRunRow: Row, onlyFailing = false, firstRow = false) =
+proc generateTestRunTabContentPartial(outfile: File, allResults: AllTests, testRunRow: JsonNode, onlyFailing = false, firstRow = false) =
   let
     # The first tab gets the bootstrap classes for a selected and displaying tab content
     firstTabActiveClass = if firstRow: " in active"
                           else: ""
-    commitId = testRunRow[0]
-    hash = htmlQuote(testRunRow[1])
-    branch = htmlQuote(testRunRow[2])
-    machineId = testRunRow[3]
-    machineName = htmlQuote(testRunRow[4])
-    os = htmlQuote(testRunRow[5])
-    cpu = htmlQuote(testRunRow[6])
-
-  const
-    totalClause = """
-SELECT COUNT(*)
-FROM [TestResult] AS [tr]
-WHERE [tr].[commit] = ?
-  AND [tr].[machine] = ?"""
-    successClause = totalClause & "\L" & """
-  AND [tr].[result] LIKE 'reSuccess'"""
-    ignoredClause = totalClause & "\L" & """
-  AND [tr].[result] LIKE 'reIgnored'"""
-  let
-    totalCount = db.getValue(sql(totalClause), commitId, machineId).parseBiggestInt()
-    successCount = db.getValue(sql(successClause), commitId, machineId).parseBiggestInt()
-    successPercentage = 100 * (successCount.toBiggestFloat() / totalCount.toBiggestFloat())
-    ignoredCount = db.getValue(sql(ignoredClause), commitId, machineId).parseBiggestInt()
-    ignoredPercentage = 100 * (ignoredCount.toBiggestFloat() / totalCount.toBiggestFloat())
-    failedCount = totalCount - successCount - ignoredCount
-    failedPercentage = 100 * (failedCount.toBiggestFloat() / totalCount.toBiggestFloat())
+    commitId = htmlQuote testRunRow["commit"].str
+    hash = htmlQuote(testRunRow["hash"].str)
+    branch = htmlQuote(testRunRow["branch"].str)
+    machineId = htmlQuote testRunRow["machine"].str
+    machineName = htmlQuote(testRunRow["machine"].str)
+    os = htmlQuote("unknown_os")
+    cpu = htmlQuote("unknown_cpu")
 
   outfile.generateHtmlTabPageBegin(
     firstTabActiveClass, commitId,
     machineId, branch, hash, machineName, os, cpu,
-    totalCount,
-    successCount, formatBiggestFloat(successPercentage, ffDecimal, 2) & "%",
-    ignoredCount, formatBiggestFloat(ignoredPercentage, ffDecimal, 2) & "%",
-    failedCount, formatBiggestFloat(failedPercentage, ffDecimal, 2) & "%"
+    allResults.totalCount,
+    allResults.successCount, formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%",
+    allResults.ignoredCount, formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%",
+    allResults.failedCount, formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%"
   )
-  generateTestResultsPanelGroupPartial(outfile, db, commitId, machineId, onlyFailing)
+  generateTestResultsPanelGroupPartial(outfile, allResults.data, onlyFailing)
   outfile.generateHtmlTabPageEnd()
 
-proc generateTestRunsHtmlPartial(outfile: File, db: DbConn, onlyFailing = false) =
-  # Select a cross-join of Commits and Machines ensuring that the selected combination
-  # contains testresults
-  const testrunSelect = sql"""
-SELECT [c].[id] AS [CommitId]
-  , [c].[hash] as [Hash]
-  , [c].[branch] As [Branch]
-  , [m].[id] AS [MachineId]
-  , [m].[name] AS [MachineName]
-  , [m].[os] AS [OS]
-  , [m].[cpu] AS [CPU]
-FROM [Commit] AS [c], [Machine] AS [m]
-WHERE (
-    SELECT COUNT(*)
-    FROM [TestResult] AS [tr]
-    WHERE [tr].[commit] = [c].[id]
-      AND [tr].[machine] = [m].[id]
-  ) > 0
-ORDER BY [c].[id] DESC
-"""
+proc generateTestRunsHtmlPartial(outfile: File, allResults: AllTests, onlyFailing = false) =
   # Iterating the results twice, get entire result set in one go
-  var testRunRowSeq = db.getAllRows(testrunSelect)
-
   outfile.generateHtmlTabListBegin()
   var firstRow = true
-  for testRunRow in testRunRowSeq:
+  for testRunRow in allResults.data:
     generateTestRunTabListItemPartial(outfile, testRunRow, firstRow)
     if firstRow:
       firstRow = false
@@ -177,96 +150,20 @@ ORDER BY [c].[id] DESC
 
   outfile.generateHtmlTabContentsBegin()
   firstRow = true
-  for testRunRow in testRunRowSeq:
-    generateTestRunTabContentPartial(outfile, db, testRunRow, onlyFailing, firstRow)
+  for testRunRow in allResults.data:
+    generateTestRunTabContentPartial(outfile, allResults, testRunRow, onlyFailing, firstRow)
     if firstRow:
       firstRow = false
   outfile.generateHtmlTabContentsEnd()
 
-proc generateHtml*(filename: string, commit: int; onlyFailing: bool) =
-  var db = open(connection="testament.db", user="testament", password="",
-                database="testament")
+proc generateHtml*(filename: string, onlyFailing: bool) =
   var outfile = open(filename, fmWrite)
 
   outfile.generateHtmlBegin()
 
-  generateTestRunsHtmlPartial(outfile, db, onlyFailing)
+  generateTestRunsHtmlPartial(outfile, allTestResults(), onlyFailing)
 
   outfile.generateHtmlEnd()
-  
+
   outfile.flushFile()
   close(outfile)
-  close(db)
-
-proc getCommit(db: DbConn, c: int): string =
-  var commit = c
-  for thisCommit in db.rows(sql"select id from [Commit] order by id desc"):
-    if commit == 0: result = thisCommit[0]
-    inc commit
-
-proc generateJson*(filename: string, commit: int) =
-  const
-    selRow = """select count(*),
-                           sum(result = 'reSuccess'),
-                           sum(result = 'reIgnored')
-                from TestResult
-                where [commit] = ? and machine = ?
-                order by category"""
-    selDiff = """select A.category || '/' || A.target || '/' || A.name,
-                        A.result,
-                        B.result
-                from TestResult A
-                inner join TestResult B
-                on A.name = B.name and A.category = B.category
-                where A.[commit] = ? and B.[commit] = ? and A.machine = ?
-                   and A.result != B.result"""
-    selResults = """select
-                      category || '/' || target || '/' || name,
-                      category, target, action, result, expected, given
-                    from TestResult
-                    where [commit] = ?"""
-  var db = open(connection="testament.db", user="testament", password="",
-                database="testament")
-  let lastCommit = db.getCommit(commit)
-  if lastCommit.isNil:
-    quit "cannot determine commit " & $commit
-
-  let previousCommit = db.getCommit(commit-1)
-
-  var outfile = open(filename, fmWrite)
-
-  let machine = $backend.getMachine(db)
-  let data = db.getRow(sql(selRow), lastCommit, machine)
-
-  outfile.writeLine("""{"total": $#, "passed": $#, "skipped": $#""" % data)
-
-  let results = newJArray()
-  for row in db.rows(sql(selResults), lastCommit):
-    var obj = newJObject()
-    obj["name"] = %row[0]
-    obj["category"] = %row[1]
-    obj["target"] = %row[2]
-    obj["action"] = %row[3]
-    obj["result"] = %row[4]
-    obj["expected"] = %row[5]
-    obj["given"] = %row[6]
-    results.add(obj)
-  outfile.writeLine(""", "results": """)
-  outfile.write(results.pretty)
-
-  if not previousCommit.isNil:
-    let diff = newJArray()
-
-    for row in db.rows(sql(selDiff), previousCommit, lastCommit, machine):
-      var obj = newJObject()
-      obj["name"] = %row[0]
-      obj["old"] = %row[1]
-      obj["new"] = %row[2]
-      diff.add obj
-    outfile.writeLine(""", "diff": """)
-    outfile.writeLine(diff.pretty)
-
-  outfile.writeLine "}"
-  close(db)
-  close(outfile)
-

tests/testament/tester.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim Tester
-#        (c) Copyright 2015 Andreas Rumpf
+#        (c) Copyright 2017 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -12,7 +12,7 @@
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
   marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, compiler/nodejs, re, times, sets
+  algorithm, compiler/nodejs, times, sets
 
 const
   resultsFile = "testresults.html"
@@ -24,9 +24,7 @@ Command:
   all                         run all tests
   c|category <category>       run all the tests of a certain category
   r|run <test>                run single test file
-  html [commit]               generate $1 from the database; uses the latest
-                              commit or a specific one (use -1 for the commit
-                              before latest etc)
+  html                        generate $1 from the database
 Arguments:
   arguments are passed to the compiler
 Options:
@@ -191,7 +189,12 @@ proc addResult(r: var TResults, test: TTest,
         ("Skipped", "")
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
-    var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, "-Duration", $(duration*1000).int], options={poStdErrToStdOut, poUsePath, poParentStreams})
+    var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
+                         "-Framework", "nim-testament", "-FileName",
+                         test.cat.string,
+                         "-Outcome", outcome, "-ErrorMessage", msg,
+                         "-Duration", $(duration*1000).int],
+                         options={poStdErrToStdOut, poUsePath, poParentStreams})
     discard waitForExit(p)
     close(p)
 
@@ -290,7 +293,7 @@ proc analyzeAndConsolidateOutput(s: string): string =
       result = substr(rows[i], pos) & "\n"
       for i in i+1 ..< rows.len:
         result.add rows[i] & "\n"
-        if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"):
+        if not (rows[i] =~ peg"['(']+ '(' \d+ ')' \s+"):
           return
     elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1):
       result = substr(rows[i], pos)
@@ -473,10 +476,7 @@ proc main() =
     var cat = Category(subdir)
     processSingleTest(r, cat, p.cmdLineRest.string, file)
   of "html":
-    var commit = 0
-    discard parseInt(p.cmdLineRest.string, commit)
-    generateHtml(resultsFile, commit, optFailing)
-    generateJson(jsonFile, commit)
+    generateHtml(resultsFile, optFailing)
   else:
     quit Usage