first version of the new memory tracking feature

compiler/ccgexprs.nim

@@ -355,6 +355,14 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   else: internalError("genAssignment: " & $ty.kind)
 
+  if optMemTracker in p.options and dest.s in {OnHeap, OnUnknown}:
+    #writeStackTrace()
+    #echo p.currLineInfo, " requesting"
+    linefmt(p, cpsStmts, "#memTrackerWrite((void*)$1, $2, $3, $4);$n",
+            addrLoc(dest), rope getSize(dest.t),
+            makeCString(p.currLineInfo.toFullPath),
+            rope p.currLineInfo.safeLineNm)
+
 proc genDeepCopy(p: BProc; dest, src: TLoc) =
   var ty = skipTypes(dest.t, abstractVarRange)
   case ty.kind
@@ -1946,6 +1954,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
       d.s = OnStatic
 
 proc expr(p: BProc, n: PNode, d: var TLoc) =
+  p.currLineInfo = n.info
   case n.kind
   of nkSym:
     var sym = n.sym

compiler/cgendata.nim

@@ -68,6 +68,7 @@ type
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
+    currLineInfo*: TLineInfo  # AST codegen will make this superfluous
     nestedTryStmts*: seq[PNode]   # in how many nested try statements we are
                                   # (the vars must be volatile then)
     inExceptBlock*: int       # are we currently inside an except block?

compiler/commands.nim

@@ -242,6 +242,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "linetrace": result = contains(gOptions, optLineTrace)
   of "debugger": result = contains(gOptions, optEndb)
   of "profiler": result = contains(gOptions, optProfiler)
+  of "memtracker": result = contains(gOptions, optMemTracker)
   of "checks", "x": result = gOptions * ChecksOptions == ChecksOptions
   of "floatchecks":
     result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
@@ -446,6 +447,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     processOnOffSwitch({optProfiler}, arg, pass, info)
     if optProfiler in gOptions: defineSymbol("profiler")
     else: undefSymbol("profiler")
+  of "memtracker":
+    processOnOffSwitch({optMemTracker}, arg, pass, info)
+    if optMemTracker in gOptions: defineSymbol("memtracker")
+    else: undefSymbol("memtracker")
   of "checks", "x": processOnOffSwitch(ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info)

compiler/options.nim

@@ -34,7 +34,8 @@ type                          # please make sure we have under 32 options
     optProfiler,              # profiler turned on
     optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
-    optPatterns               # en/disable pattern matching
+    optPatterns,              # en/disable pattern matching
+    optMemTracker
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**

compiler/pragmas.nim

@@ -323,7 +323,8 @@ proc processOption(c: PContext, n: PNode): bool =
     of wStacktrace: onOff(c, n, {optStackTrace})
     of wLinetrace: onOff(c, n, {optLineTrace})
     of wDebugger: onOff(c, n, {optEndb})
-    of wProfiler: onOff(c, n, {optProfiler})
+    of wProfiler: onOff(c, n, {optProfiler, optMemTracker})
+    of wMemTracker: onOff(c, n, {optMemTracker})
     of wByRef: onOff(c, n, {optByRef})
     of wDynlib: processDynLib(c, n, nil)
     of wOptimization:

compiler/wordrecg.nim

@@ -34,7 +34,7 @@ type
 
     wColon, wColonColon, wEquals, wDot, wDotDot,
     wStar, wMinus,
-    wMagic, wThread, wFinal, wProfiler, wObjChecks,
+    wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks,
     wIntDefine, wStrDefine,
 
     wDestroy,
@@ -121,7 +121,7 @@ const
 
     ":", "::", "=", ".", "..",
     "*", "-",
-    "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine",
+    "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine",
 
     "destroy",
 

doc/advopt.txt

@@ -61,6 +61,7 @@ Advanced options:
   --taintMode:on|off        turn taint mode on|off
   --implicitStatic:on|off   turn implicit compile time evaluation on|off
   --patterns:on|off         turn pattern matching on|off
+  --memTracker:on|off       turn memory tracker on|off
   --skipCfg                 do not read the general configuration file
   --skipUserCfg             do not read the user's configuration file
   --skipParentCfg           do not read the parent dirs' configuration files

lib/pure/nimtracker.nim

@@ -0,0 +1,71 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Memory tracking support for Nim.
+
+when isMainModule:
+  import db_sqlite
+  var db = open("memtrack.db", "", "", "")
+  db.exec sql"""
+  create table if not exists Tracking(
+    id integer primary key,
+    op varchar not null,
+    address integer not null,
+    size integer not null,
+    file varchar not null,
+    line integer not null
+  )"""
+  db.close()
+else:
+  when not defined(memTracker):
+    {.error: "Memory tracking support is turned off!".}
+
+  {.push memtracker: off.}
+  # we import the low level wrapper and are careful not to use Nim's
+  # memory manager for anything here.
+  import sqlite3
+
+  var
+    dbHandle: PSqlite3
+    insertStmt: Pstmt
+
+  template sbind(x: int; value) =
+    when value is cstring:
+      let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT)
+      if ret != SQLITE_OK:
+        quit "could not bind value"
+    else:
+      let ret = insertStmt.bindInt64(x, value)
+      if ret != SQLITE_OK:
+        quit "could not bind value"
+
+  proc logEntries(log: TrackLog) {.nimcall.} =
+    for i in 0..log.count-1:
+      var success = false
+      let e = log.data[i]
+      discard sqlite3.reset(insertStmt)
+      discard clearBindings(insertStmt)
+      sbind 1, e.op
+      sbind(2, cast[int](e.address))
+      sbind 3, e.size
+      sbind 4, e.file
+      sbind 5, e.line
+      if step(insertStmt) == SQLITE_DONE:
+        success = true
+      if not success:
+        quit "could not write to database!"
+
+  if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK:
+    const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)"
+    if prepare_v2(dbHandle, query,
+        query.len, insertStmt, nil) == SQLITE_OK:
+      setTrackLogger logEntries
+    else:
+      quit "could not prepare statement"
+  {.pop.}

lib/system.nim

@@ -1272,12 +1272,14 @@ const
 
   seqShallowFlag = low(int)
 
+{.push profiler: off.}
 when defined(nimKnowsNimvm):
   let nimvm* {.magic: "Nimvm".}: bool = false
     ## may be used only in "when" expression.
     ## It is true in Nim VM context and false otherwise
 else:
   const nimvm*: bool = false
+{.pop.}
 
 proc compileOption*(option: string): bool {.
   magic: "CompileOption", noSideEffect.}
@@ -2544,6 +2546,7 @@ when hostOS == "standalone":
   include "$projectpath/panicoverride"
 
 when not declared(sysFatal):
+  {.push profiler: off.}
   when hostOS == "standalone":
     proc sysFatal(exceptn: typedesc, message: string) {.inline.} =
       panic(message)
@@ -2563,6 +2566,7 @@ when not declared(sysFatal):
       new(e)
       e.msg = message & arg
       raise e
+  {.pop.}
 
 proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.}
   ## get type information for `x`. Ordinary code should not use this, but
@@ -2616,8 +2620,10 @@ when not defined(JS): #and not defined(nimscript):
       when declared(setStackBottom):
         setStackBottom(locals)
 
+    {.push profiler: off.}
     var
       strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic})
+    {.pop.}
 
 
   # ----------------- IO Part ------------------------------------------------
@@ -2950,6 +2956,8 @@ when not defined(JS): #and not defined(nimscript):
         ## lead to the ``raise`` statement. This only works for debug builds.
 
     {.push stack_trace: off, profiler:off.}
+    when defined(memtracker):
+      include "system/memtracker"
     when hostOS == "standalone":
       include "system/embedded"
     else:
@@ -2992,7 +3000,9 @@ when not defined(JS): #and not defined(nimscript):
       else:
         result = n.sons[n.len]
 
+    {.push profiler:off.}
     when hasAlloc: include "system/mmdisp"
+    {.pop.}
     {.push stack_trace: off, profiler:off.}
     when hasAlloc: include "system/sysstr"
     {.pop.}

lib/system/excpt.nim

@@ -339,6 +339,8 @@ when not defined(noSignalHandler):
           action("unknown signal\n")
 
     # print stack trace and quit
+    when defined(memtracker):
+      logPendingOps()
     when hasSomeStackTrace:
       GC_disable()
       var buf = newStringOfCap(2000)

lib/system/memtracker.nim

@@ -0,0 +1,62 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Memory tracking support for Nim.
+
+when not defined(memTracker):
+  {.error: "Memory tracking support is turned off! Enable memory tracking by passing `--memtracker:on` to the compiler (see the Nim Compiler User Guide for more options).".}
+
+when defined(noSignalHandler):
+  {.error: "Memory tracking works better with the default signal handler.".}
+
+# We don't want to memtrack the tracking code ...
+{.push memtracker: off.}
+
+type
+  LogEntry* = object
+    op*: cstring
+    address*: pointer
+    size*: int
+    file*: cstring
+    line*: int
+  TrackLog* = object
+    count*: int
+    data*: array[4000, LogEntry]
+  TrackLogger* = proc (log: TrackLog) {.nimcall.}
+
+var
+  gLog*: TrackLog
+  gLogger*: TrackLogger = proc (log: TrackLog) = discard
+
+proc setTrackLogger*(logger: TrackLogger) =
+  gLogger = logger
+
+proc addEntry(entry: LogEntry) =
+  if gLog.count > high(gLog.data):
+    gLogger(gLog)
+    gLog.count = 0
+  gLog.data[gLog.count] = entry
+  inc gLog.count
+
+proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.compilerProc.} =
+  addEntry LogEntry(op: "write", address: address,
+      size: size, file: file, line: line)
+
+proc memTrackerOp*(op: cstring; address: pointer; size: int) =
+  addEntry LogEntry(op: op, address: address, size: size,
+      file: "", line: 0)
+
+proc logPendingOps() {.noconv.} =
+  # forward declared and called from Nim's signal handler.
+  gLogger(gLog)
+  gLog.count = 0
+
+addQuitProc logPendingOps
+
+{.pop.}