From c67520a7c5d474be409bee98d9b258030115c259 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 28 Nov 2012 01:14:02 +0200 Subject: [PATCH] temporary debugging code for the memory leak investigation --- compiler/ast.nim | 2 +- compiler/magicsys.nim | 9 ++++++ compiler/main.nim | 70 +++++++++++++++++++++++++++++++++++++---- compiler/ropes.nim | 4 +++ compiler/semdata.nim | 2 +- compiler/semstmts.nim | 2 +- lib/system.nim | 6 ++-- lib/system/cellsets.nim | 2 +- lib/system/gc.nim | 52 +++++++++++++++++++++++------- lib/system/mmdisp.nim | 1 + 10 files changed, 125 insertions(+), 25 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 7e57d0e85b..6943add46f 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -517,7 +517,7 @@ type TNodeSeq* = seq[PNode] PType* = ref TType PSym* = ref TSym - TNode*{.acyclic, final.} = object # on a 32bit machine, this takes 32 bytes + TNode*{.final.} = object # on a 32bit machine, this takes 32 bytes typ*: PType comment*: string info*: TLineInfo diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 2681f93d6f..6ff714dc70 100755 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -77,6 +77,15 @@ proc getSysType(kind: TTypeKind): PType = var intTypeCache: array[-5..64, PType] +proc resetSysTypes* = + systemModule = nil + initStrTable(compilerprocs) + for i in low(gSysTypes)..high(gSysTypes): + gSysTypes[i] = nil + + for i in low(intTypeCache)..high(intTypeCache): + intTypeCache[i] = nil + proc getIntLitType*(literal: PNode): PType = # we cache some common integer literal types for performance: let value = literal.intVal diff --git a/compiler/main.nim b/compiler/main.nim index 2ce47217b5..b4a4617399 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -89,13 +89,23 @@ proc addDep(x: Psym, dep: int32) = growCache gMemCacheData, dep gMemCacheData[x.position].deps.safeAdd(dep) +proc ResetModule(fileIdx: int32) = + echo "HARD RESETTING ", fileIdx.toFilename + gMemCacheData[fileIdx].needsRecompile = Yes + gCompiledModules[fileIdx] = nil + cgendata.gModules[fileIdx] = nil + +proc ResetAllModules = + for i in 0..gCompiledModules.high: + if gCompiledModules[i] != nil: + ResetModule(i.int32) + + for m in cgenModules(): + echo "CGEN MODULE FOUND" + proc checkDepMem(fileIdx: int32): TNeedRecompile = template markDirty = - echo "HARD RESETTING ", fileIdx.toFilename - gMemCacheData[fileIdx].needsRecompile = Yes - gCompiledModules[fileIdx] = nil - cgendata.gModules[fileIdx] = nil - + ResetModule(fileIdx) return Yes if gMemCacheData[fileIdx].needsRecompile != Maybe: @@ -377,7 +387,13 @@ proc wantMainModule = if gProjectFull.len == 0: Fatal(gCmdLineInfo, errCommandExpectsFilename) gProjectMainIdx = addFileExt(gProjectFull, nimExt).fileInfoIdx - + +var oss: PGenericSeq +proc dbgseqimp(x: PGenericSeq) {.cdecl.} = + oss = x + +seqdbg = dbgseqimp + proc MainCommand = # In "nimrod serve" scenario, each command must reset the registered passes clearPasses() @@ -482,6 +498,48 @@ proc MainCommand = of "e": # XXX: temporary command for easier testing commandEval(mainCommandArg()) + of "reset": + ResetModule(gProjectMainIdx) + gcDebugging = true + GC_fullCollect() + + resetCompilationLists() + ccgutils.resetCaches() + ResetAllModules() + resetRopeCache() + resetSysTypes() + gGenericsCache = nil + gOwners = @[] + rangeDestructorProc = nil + + # XXX: clean these global vars + # ccgstmts.gBreakpoints + # ccgthreadvars.nimtv + # ccgthreadvars.nimtVDeps + # ccgthreadvars.nimtvDeclared + # cgendata + # cgmeth? + # condsyms? + # depends? + # lexer.gLinesCompiled + # msgs - error counts + # magicsys, when system.nim changes + # rodread.rodcompilerProcs + # rodread.gTypeTable + # rodread.gMods + + # !! ropes.cache + # !! semdata.gGenericsCache + # semthreads.computed? + # + # suggest.usageSym + # + # XXX: can we run out of IDs? + # XXX: detect config reloading (implement as error/require restart) + # XXX: options are appended (they will accumulate over time) + # vis = visimpl + GC_fullCollect() + echo GC_getStatistics() of "idetools": gCmd = cmdIdeTools if gEvalExpr != "": diff --git a/compiler/ropes.nim b/compiler/ropes.nim index d5472a53a8..18c10fabdb 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -114,6 +114,10 @@ proc freezeMutableRope*(r: PRope) {.inline.} = var cache: array[0..2048*2 -1, PRope] +proc resetRopeCache* = + for i in low(cache)..high(cache): + cache[i] = nil + proc RopeInvariant(r: PRope): bool = if r == nil: result = true diff --git a/compiler/semdata.nim b/compiler/semdata.nim index d3122bb1e6..54662111bc 100755 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -86,7 +86,7 @@ type # naming it multiple times var - gGenericsCache: PGenericsCache # save for modularity + gGenericsCache*: PGenericsCache # save for modularity proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair = result.genericSym = s diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 74ebae204f..f8860212b4 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -933,7 +933,7 @@ var destructorName = getIdent"destroy_" destructorParam = getIdent"this_" destructorPragma = newIdentNode(getIdent"destructor", UnknownLineInfo()) - rangeDestructorProc: PSym + rangeDestructorProc*: PSym proc destroyField(c: PContext, field: PSym, holder: PNode): PNode = if instantiateDestructor(c, field.typ): diff --git a/lib/system.nim b/lib/system.nim index 982c7a5f97..9fa5258ce0 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -179,9 +179,9 @@ when not defined(niminheritable): when not defined(EcmaScript) and not defined(NimrodVM): type - TGenericSeq {.compilerproc, pure, inheritable.} = object + TGenericSeq* {.compilerproc, pure, inheritable.} = object len, reserved: int - PGenericSeq {.exportc.} = ptr TGenericSeq + PGenericSeq* {.exportc.} = ptr TGenericSeq # len and space without counting the terminating zero: NimStringDesc {.compilerproc, final.} = object of TGenericSeq data: array[0..100_000_000, char] @@ -558,7 +558,7 @@ proc abs*(x: int64): int64 {.magic: "AbsI64", noSideEffect.} ## checking is turned on). type - IntMax32 = int|int8|int16|int32 + IntMax32 = bool|int|int8|int16|int32 proc `+%` *(x, y: IntMax32): IntMax32 {.magic: "AddU", noSideEffect.} proc `+%` *(x, y: Int64): Int64 {.magic: "AddU", noSideEffect.} diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index b860ef38cc..55004e8ec7 100755 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -13,7 +13,7 @@ type TCell {.pure.} = object refcount: int # the refcount and some flags typ: PNimType - when leakDetector: + when trackAllocationSource: filename: cstring line: int diff --git a/lib/system/gc.nim b/lib/system/gc.nim index ec656e0efc..42631411ed 100755 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -355,9 +355,10 @@ proc forAllChildren(cell: PCell, op: TWalkOp) = var d = cast[TAddress](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: + let baseAddr = d +% GenericSeqSize for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) + forAllChildrenAux(cast[pointer](baseAddr +% i *% cell.typ.base.size), + cell.typ.base, op) else: nil proc addNewObjToZCT(res: PCell, gch: var TGcHeap) {.inline.} = @@ -414,10 +415,12 @@ proc rawNewObj(typ: PNimType, size: int, gch: var TGcHeap): pointer = sysAssert((cast[TAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when trackAllocationSource and not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil and framePtr.prev.prev != nil: + res.filename = framePtr.prev.prev.filename + res.line = framePtr.prev.prev.line + else: + res.filename = "nofile" res.refcount = rcZct # refcount is zero, but mark it to be in the ZCT sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") # its refcount is zero, so add it to the ZCT: @@ -456,10 +459,12 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert((cast[TAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when trackAllocationSource and not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil and framePtr.prev.prev != nil: + res.filename = framePtr.prev.prev.filename + res.line = framePtr.prev.prev.line + else: + res.filename = "nofile" res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") when logGC: writeCell("new cell", res) @@ -597,6 +602,21 @@ proc collectCycles(gch: var TGcHeap) = Deinit(gch.cycleRoots) Init(gch.cycleRoots) +var gcDebugging* = false +var vis*: proc (a: pointer, b: PNimType) + +proc debugNode(n: ptr TNimNode) = + c_fprintf(c_stdout, "node %s\n", n.name) + for i in 0..n.len-1: + debugNode(n.sons[i]) + +proc debugTyp(x: PNimType) = + c_fprintf(c_stdout, "type %d\n", x.kind) + if x.node != nil: + debugNode(x.node) + +var seqdbg* : proc (s: PGenericSeq) {.cdecl.} + proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") @@ -607,8 +627,16 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} = var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) if objStart != nil: # mark the cell: - objStart.refcount = objStart.refcount +% rcIncrement - add(gch.decStack, objStart) + if gcDebugging: + c_fprintf(c_stdout, "object root found %d\nfile: %s\nline: %d\n", objStart.typ.kind, objStart.filename, objStart.line) + if objStart.typ.kind == tySequence: + let sq = cast[PGenericSeq](cellToUsr(objStart)) + c_fprintf(c_stdout, "seq len: %d\noffset: %ld\n", sq.len, cast[TAddress](p) - cast[TAddress](sq)) + seqdbg(sq) + + if not gcDebugging: + objStart.refcount = objStart.refcount +% rcIncrement + add(gch.decStack, objStart) when false: if isAllocatedPtr(gch.region, cell): sysAssert false, "allocated pointer but not interior?" diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 1abf3fbbf1..97a42ed32c 100755 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -21,6 +21,7 @@ const alwaysGC = false # collect after every memory allocation (for debugging) leakDetector = false overwriteFree = false + trackAllocationSource = true or leakDetector cycleGC = true # (de)activate the cycle GC reallyDealloc = true # for debugging purposes this can be set to false