diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index 069ed48df7..a1ad3ae047 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -16,7 +16,7 @@ ## implementation. template detectVersion(field, corename) = - if m.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcHooks}: + if m.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc, gcHooks}: result = 2 else: result = 1 diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 11fe701c18..98b9ab9a60 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -277,7 +277,7 @@ proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool = of ctStruct: let t = skipTypes(rettype, typedescInst) if rettype.isImportedCppType or t.isImportedCppType or - (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc}): + (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}): # prevents nrvo for cdecl procs; # bug #23401 result = false else: @@ -1692,7 +1692,7 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: echo "ayclic but has this =trace ", t, " ", theProc.ast else: when false: - if op == attachedTrace and m.config.selectedGC == gcOrc and + if op == attachedTrace and m.config.selectedGC in {gcOrc, gcYrc} and containsGarbageCollectedRef(t): # unfortunately this check is wrong for an object type that only contains # .cursor fields like 'Node' inside 'cycleleak'. diff --git a/compiler/commands.nim b/compiler/commands.nim index 9aa66f7887..3d2aabdc03 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -245,7 +245,7 @@ proc processCompile(conf: ConfigRef; filename: string) = extccomp.addExternalFileToCompile(conf, found) const - errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" + errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'yrc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found" errGuiConsoleOrLibExpectedButXFound = "'gui', 'console', 'lib' or 'staticlib' expected, but '$1' found" errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found" @@ -266,6 +266,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo of "markandsweep": result = conf.selectedGC == gcMarkAndSweep of "destructors", "arc": result = conf.selectedGC == gcArc of "orc": result = conf.selectedGC == gcOrc + of "yrc": result = conf.selectedGC == gcYrc of "hooks": result = conf.selectedGC == gcHooks of "go": result = conf.selectedGC == gcGo of "none": result = conf.selectedGC == gcNone @@ -570,6 +571,7 @@ proc unregisterArcOrc*(conf: ConfigRef) = undefSymbol(conf.symbols, "gcdestructors") undefSymbol(conf.symbols, "gcarc") undefSymbol(conf.symbols, "gcorc") + undefSymbol(conf.symbols, "gcyrc") undefSymbol(conf.symbols, "gcatomicarc") undefSymbol(conf.symbols, "nimSeqsV2") undefSymbol(conf.symbols, "nimV2") @@ -603,6 +605,10 @@ proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, conf.selectedGC = gcOrc defineSymbol(conf.symbols, "gcorc") registerArcOrc(pass, conf) + of "yrc": + conf.selectedGC = gcYrc + defineSymbol(conf.symbols, "gcyrc") + registerArcOrc(pass, conf) of "atomicarc": conf.selectedGC = gcAtomicArc defineSymbol(conf.symbols, "gcatomicarc") diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index e6ddf79a8a..f23e6ed04d 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -69,7 +69,7 @@ proc hasDestructor(c: Con; t: PType): bool {.inline.} = result = ast.hasDestructor(t) when toDebug.len > 0: # for more effective debugging - if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}: + if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}: assert(not containsGarbageCollectedRef(t)) proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode = @@ -165,7 +165,7 @@ proc isLastReadImpl(n: PNode; c: var Con; scope: var Scope): bool = template hasDestructorOrAsgn(c: var Con, typ: PType): bool = # bug #23354; an object type could have a non-trivial assignements when it is passed to a sink parameter - hasDestructor(c, typ) or (c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and + hasDestructor(c, typ) or (c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc} and typ.kind == tyObject and not isTrivial(getAttachedOp(c.graph, typ, attachedAsgn))) proc isLastRead(n: PNode; c: var Con; s: var Scope): bool = @@ -329,14 +329,14 @@ proc isCriticalLink(dest: PNode): bool {.inline.} = result = dest.kind != nkSym proc finishCopy(c: var Con; result, dest: PNode; flags: set[MoveOrCopyFlag]; isFromSink: bool) = - if c.graph.config.selectedGC == gcOrc and IsExplicitSink notin flags: + if c.graph.config.selectedGC in {gcOrc, gcYrc} and IsExplicitSink notin flags: # add cyclic flag, but not to sink calls, which IsExplicitSink generates let t = dest.typ.skipTypes(tyUserTypeClasses + {tyGenericInst, tyAlias, tySink, tyDistinct}) if cyclicType(c.graph, t): result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest)) proc genMarkCyclic(c: var Con; result, dest: PNode) = - if c.graph.config.selectedGC == gcOrc: + if c.graph.config.selectedGC in {gcOrc, gcYrc}: let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct}) if cyclicType(c.graph, t): if t.kind == tyRef: @@ -495,7 +495,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode = localError(c.graph.config, n.info, errFailedMove, ("cannot move '$1', passing '$1' to a sink parameter introduces an implicit copy") % $n) else: - if c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}: + if c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}: assert(not containsManagedMemory(nTyp)) if nTyp.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}: localError(c.graph.config, n.info, "cannot create an implicit openArray copy to be passed to a sink parameter") @@ -926,7 +926,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing if n[0].kind == nkSym and n[0].sym.magic in {mNew, mNewFinalize}: result[0] = copyTree(n[0]) - if c.graph.config.selectedGC in {gcHooks, gcArc, gcAtomicArc, gcOrc}: + if c.graph.config.selectedGC in {gcHooks, gcArc, gcAtomicArc, gcOrc, gcYrc}: let destroyOld = c.genDestroy(result[1]) result = newTree(nkStmtList, destroyOld, result) else: diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 24badb364a..3d4467991a 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -163,7 +163,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool, if c.filterDiscriminator != nil: return let f = n.sym let b = if c.kind == attachedTrace: y else: y.dotField(f) - if (sfCursor in f.flags and c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcHooks}) or + if (sfCursor in f.flags and c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc, gcHooks}) or enforceDefaultOp: defaultOp(c, f.typ, body, x.dotField(f), b) else: @@ -735,9 +735,9 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let elemType = t.elementType createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) - let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(c.g, elemType) + let isCyclic = c.g.config.selectedGC in {gcOrc, gcYrc} and types.canFormAcycle(c.g, elemType) - let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and + let isInheritableAcyclicRef = c.g.config.selectedGC in {gcOrc, gcYrc} and (not isPureObject(elemType)) and tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags # dynamic Acyclic refs need to use dyn decRef @@ -819,7 +819,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let xenv = genBuiltin(c, mAccessEnv, "accessEnv", x) xenv.typ = getSysType(c.g, c.info, tyPointer) - let isCyclic = c.g.config.selectedGC == gcOrc + let isCyclic = c.g.config.selectedGC in {gcOrc, gcYrc} let tmp = if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}: declareTempOf(c, body, xenv) @@ -937,7 +937,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = call[1] = y body.add newAsgnStmt(x, call) elif (optOwnedRefs in c.g.config.globalOptions and - optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}: + optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}: let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) xx.typ = getSysType(c.g, c.info, tyPointer) case c.kind @@ -1125,7 +1125,7 @@ proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttache result.typ.addParam src - if g.config.selectedGC == gcOrc and + if g.config.selectedGC in {gcOrc, gcYrc} and cyclicType(g, typ.skipTypes(abstractInst)): let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"), idgen, result, info) @@ -1152,7 +1152,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), idgen, result, info) - if kind == attachedDestructor and g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and + if kind == attachedDestructor and g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc} and ((g.config.isDefined("nimPreviewNonVarDestructor") and not isDiscriminant) or (typ.kind in {tyRef, tyString, tySequence})): dest.typ = typ else: @@ -1168,7 +1168,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp if kind notin {attachedDestructor, attachedWasMoved}: result.typ.addParam src - if kind == attachedAsgn and g.config.selectedGC == gcOrc and + if kind == attachedAsgn and g.config.selectedGC in {gcOrc, gcYrc} and cyclicType(g, typ.skipTypes(abstractInst)): let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"), idgen, result, info) @@ -1226,7 +1226,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; else: var tk: TTypeKind var skipped: PType = nil - if g.config.selectedGC in {gcArc, gcOrc, gcHooks, gcAtomicArc}: + if g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcHooks, gcAtomicArc}: skipped = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}) tk = skipped.kind else: @@ -1348,7 +1348,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf # we do not generate '=trace' procs if we # have the cycle detection disabled, saves code size. - let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace + let lastAttached = if g.config.selectedGC in {gcOrc, gcYrc}: attachedTrace else: attachedSink # bug #15122: We need to produce all prototypes before entering the diff --git a/compiler/options.nim b/compiler/options.nim index 993c90205d..fc15ee9792 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -195,6 +195,7 @@ type gcRegions = "regions" gcArc = "arc" gcOrc = "orc" + gcYrc = "yrc" # thread-safe ORC (concurrent cycle collector) gcAtomicArc = "atomicArc" gcMarkAndSweep = "markAndSweep" gcHooks = "hooks" diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 10d3f73bc0..6b3f96cf03 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -231,7 +231,7 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; if optOwnedRefs in oldGlobalOptions: conf.globalOptions.incl {optTinyRtti, optOwnedRefs, optSeqDestructors} defineSymbol(conf.symbols, "nimv2") - if conf.selectedGC in {gcArc, gcOrc, gcAtomicArc}: + if conf.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}: conf.globalOptions.incl {optTinyRtti, optSeqDestructors} defineSymbol(conf.symbols, "nimv2") defineSymbol(conf.symbols, "gcdestructors") @@ -241,6 +241,8 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; defineSymbol(conf.symbols, "gcarc") of gcOrc: defineSymbol(conf.symbols, "gcorc") + of gcYrc: + defineSymbol(conf.symbols, "gcyrc") of gcAtomicArc: defineSymbol(conf.symbols, "gcatomicarc") else: diff --git a/lib/system.nim b/lib/system.nim index b19f6d828b..6104c1b928 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -125,7 +125,7 @@ proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = const ThisIsSystem = true -const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) or defined(gcYrc) when defined(nimAllowNonVarDestructor) and arcLikeMem: proc new*[T](a: var ref T, finalizer: proc (x: T) {.nimcall.}) {. @@ -356,7 +356,7 @@ proc low*(x: string): int {.magic: "Low", noSideEffect.} ## See also: ## * `high(string) <#high,string>`_ -when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): +when not defined(gcArc) and not defined(gcOrc) and not defined(gcYrc) and not defined(gcAtomicArc): proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} ## Use this instead of `=` for a `shallow copy`:idx:. ## @@ -407,7 +407,7 @@ when defined(nimHasDup): proc `=sink`*[T](x: var T; y: T) {.inline, nodestroy, magic: "Asgn".} = ## Generic `sink`:idx: implementation that can be overridden. - when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + when defined(gcArc) or defined(gcOrc) or defined(gcYrc) or defined(gcAtomicArc): x = y else: shallowCopy(x, y) @@ -2561,7 +2561,7 @@ when compileOption("rangechecks"): else: template rangeCheck*(cond) = discard -when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): +when not defined(gcArc) and not defined(gcOrc) and not defined(gcYrc) and not defined(gcAtomicArc): proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not ## perform deep copies of `s`. @@ -2630,7 +2630,7 @@ when hasAlloc or defined(nimscript): setLen(x, xl+item.len) var j = xl-1 while j >= i: - when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + when defined(gcArc) or defined(gcOrc) or defined(gcYrc) or defined(gcAtomicArc): x[j+item.len] = move x[j] else: shallowCopy(x[j+item.len], x[j]) diff --git a/lib/system/arc.nim b/lib/system/arc.nim index 14da1531c2..31f0039850 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -16,7 +16,7 @@ runtime type and only contains a reference count. {.push raises: [], rangeChecks: off.} -when defined(gcOrc): +when defined(gcOrc) or defined(gcYrc): const rcIncrement = 0b10000 # so that lowest 4 bits are not touched rcMask = 0b1111 @@ -36,12 +36,12 @@ type rc: int # the object header is now a single RC field. # we could remove it in non-debug builds for the 'owned ref' # design but this seems unwise. - when defined(gcOrc): + when defined(gcOrc) or defined(gcYrc): rootIdx: int # thanks to this we can delete potential cycle roots # in O(1) without doubly linked lists when defined(nimArcDebug) or defined(nimArcIds): refId: int - when defined(gcOrc) and orcLeakDetector: + when (defined(gcOrc) or defined(gcYrc)) and orcLeakDetector: filename: cstring line: int @@ -119,7 +119,7 @@ proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} = else: result = cast[ptr RefHeader](alignedAlloc(s, alignment) +! hdrSize) head(result).rc = 0 - when defined(gcOrc): + when defined(gcOrc) or defined(gcYrc): head(result).rootIdx = 0 when defined(nimArcDebug): head(result).refId = gRefId @@ -157,7 +157,7 @@ proc nimIncRef(p: pointer) {.compilerRtl, inl.} = when traceCollector: cprintf("[INCREF] %p\n", head(p)) -when not defined(gcOrc) or defined(nimThinout): +when not (defined(gcOrc) or defined(gcYrc)) or defined(nimThinout): proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = # This is only used by the old RTTI mechanism and we know # that 'dest[]' is nil and needs no destruction. Which is really handy @@ -208,7 +208,9 @@ proc nimDestroyAndDispose(p: pointer) {.compilerRtl, quirky, raises: [].} = cstderr.rawWrite "has destructor!\n" nimRawDispose(p, rti.align) -when defined(gcOrc): +when defined(gcYrc): + include yrc +elif defined(gcOrc): when defined(nimThinout): include cyclebreaker else: @@ -251,7 +253,7 @@ proc GC_ref*[T](x: ref T) = ## New runtime only supports this operation for 'ref T'. if x != nil: nimIncRef(cast[pointer](x)) -when not defined(gcOrc): +when not (defined(gcOrc) or defined(gcYrc)): template GC_fullCollect* = ## Forces a full garbage collection pass. With `--mm:arc` a nop. discard diff --git a/lib/system/yrc.nim b/lib/system/yrc.nim new file mode 100644 index 0000000000..c7c589f681 --- /dev/null +++ b/lib/system/yrc.nim @@ -0,0 +1,426 @@ +# +# YRC: Thread-safe ORC (concurrent cycle collector). +# Same API as orc.nim but with striped queues and global lock for merge/collect. +# Destructors for refs run at collection time, not immediately on last decRef. +# + +{.push raises: [].} + +include cellseqs_v2 + +import std/locks + +const + NumStripes = 64 + QueueSize = 128 + RootsThreshold = 10 + + colBlack = 0b000 + colGray = 0b001 + colWhite = 0b010 + maybeCycle = 0b100 + jumpStackFlag = 0b1000 + colorMask = 0b011 + logOrc = defined(nimArcIds) + +type + TraceProc = proc (p, env: pointer) {.nimcall, benign, raises: [].} + DisposeProc = proc (p: pointer) {.nimcall, benign, raises: [].} + +template color(c): untyped = c.rc and colorMask +template setColor(c, col) = + when col == colBlack: + c.rc = c.rc and not colorMask + else: + c.rc = c.rc and not colorMask or col + +const + optimizedOrc = false + useJumpStack = false + +type + GcEnv = object + traceStack: CellSeq[ptr pointer] + when useJumpStack: + jumpStack: CellSeq[ptr pointer] + toFree: CellSeq[Cell] + freed, touched, edges, rcSum: int + keepThreshold: bool + +proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = + if desc.traceImpl != nil: + var p = s +! sizeof(RefHeader) + cast[TraceProc](desc.traceImpl)(p, addr(j)) + +include threadids + +type + Stripe = object + lock: Lock + toInc: array[QueueSize, Cell] + toIncLen: int + toDec: array[QueueSize, (Cell, PNimTypeV2)] + toDecLen: int + +var + gYrcGlobalLock: Lock + roots: CellSeq[Cell] # merged roots, used under global lock + toIncStripes: array[NumStripes, Stripe] + toDecStripes: array[NumStripes, Stripe] + rootsThreshold: int = 128 + defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128 + +proc getStripeIdx(): int {.inline.} = + getThreadId() and (NumStripes - 1) + +proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = + let h = head(p) + when optimizedOrc: + if cyclic: h.rc = h.rc or maybeCycle + let idx = getStripeIdx() + while true: + var overflow = false + withLock toIncStripes[idx].lock: + if toIncStripes[idx].toIncLen < QueueSize: + toIncStripes[idx].toInc[toIncStripes[idx].toIncLen] = h + toIncStripes[idx].toIncLen += 1 + else: + overflow = true + if overflow: + withLock gYrcGlobalLock: + for i in 0..", importc.} + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) + +proc unregisterCycle(s: Cell) = + let rootIdx = s.rootIdx + let idx = rootIdx -% 1 + let last = roots.len -% 1 + roots.d[idx] = roots.d[last] + roots.d[idx][0].rootIdx = rootIdx + roots.len = last + s.rootIdx = 0 + +proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + s.setColor colBlack + let until = j.traceStack.len + trace(s, desc, j) + when logOrc: writeCell("root still alive", s, desc) + while j.traceStack.len > until: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + t.rc = t.rc +% rcIncrement + if t.color != colBlack: + t.setColor colBlack + trace(t, desc, j) + when logOrc: writeCell("child still alive", t, desc) + +proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + if s.color != colGray: + s.setColor colGray + j.touched = j.touched +% 1 + j.rcSum = j.rcSum +% (s.rc shr rcShift) +% 1 + orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + t.rc = t.rc -% rcIncrement + j.edges = j.edges +% 1 + if t.color != colGray: + t.setColor colGray + j.touched = j.touched +% 1 + j.rcSum = j.rcSum +% (t.rc shr rcShift) +% 2 + trace(t, desc, j) + +proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + if s.color == colGray: + if (s.rc shr rcShift) >= 0: + scanBlack(s, desc, j) + else: + orcAssert(j.traceStack.len == 0, "scan: trace stack not empty") + s.setColor(colWhite) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + if t.color == colGray: + if (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + else: + t.setColor(colWhite) + trace(t, desc, j) + +proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = + if s.color == col and s.rootIdx == 0: + orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty") + s.setColor(colBlack) + j.toFree.add(s, desc) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil + if t.color == col and t.rootIdx == 0: + j.toFree.add(t, desc) + t.setColor(colBlack) + trace(t, desc, j) + +proc collectCyclesBacon(j: var GcEnv; lowMark: int) = + let last = roots.len -% 1 + when logOrc: + for i in countdown(last, lowMark): + writeCell("root", roots.d[i][0], roots.d[i][1]) + for i in countdown(last, lowMark): + markGray(roots.d[i][0], roots.d[i][1], j) + var colToCollect = colWhite + if j.rcSum == j.edges: + colToCollect = colGray + j.keepThreshold = true + else: + for i in countdown(last, lowMark): + scan(roots.d[i][0], roots.d[i][1], j) + init j.toFree + for i in 0 ..< roots.len: + let s = roots.d[i][0] + s.rootIdx = 0 + collectColor(s, roots.d[i][1], colToCollect, j) + when not defined(nimStressOrc): + let oldThreshold = rootsThreshold + rootsThreshold = high(int) + roots.len = 0 + for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) + free(j.toFree.d[i][0], j.toFree.d[i][1]) + when not defined(nimStressOrc): + rootsThreshold = oldThreshold + j.freed = j.freed +% j.toFree.len + deinit j.toFree + +when defined(nimOrcStats): + var freedCyclicObjects {.threadvar.}: int + +proc collectCycles() = + when logOrc: + cfprintf(cstderr, "[collectCycles] begin\n") + withLock gYrcGlobalLock: + mergePendingRoots() + if roots.len >= RootsThreshold: + var j: GcEnv + init j.traceStack + collectCyclesBacon(j, 0) + if roots.len == 0 and roots.d != nil: + deinit roots + when not defined(nimStressOrc): + if j.keepThreshold: + discard + elif j.freed *% 2 >= j.touched: + when not defined(nimFixedOrc): + rootsThreshold = max(rootsThreshold div 3 *% 2, 16) + else: + rootsThreshold = 0 + elif rootsThreshold < high(int) div 4: + rootsThreshold = (if rootsThreshold <= 0: defaultThreshold else: rootsThreshold) + rootsThreshold = rootsThreshold div 2 +% rootsThreshold + when logOrc: + cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld\n", j.freed, rootsThreshold) + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + deinit j.traceStack + +when defined(nimOrcStats): + type + OrcStats* = object + freedCyclicObjects*: int + proc GC_orcStats*(): OrcStats = + result = OrcStats(freedCyclicObjects: freedCyclicObjects) + +proc GC_runOrc* = + withLock gYrcGlobalLock: + mergePendingRoots() + var j: GcEnv + init j.traceStack + collectCyclesBacon(j, 0) + deinit j.traceStack + roots.len = 0 + when logOrc: orcAssert roots.len == 0, "roots not empty!" + +proc GC_enableOrc*() = + when not defined(nimStressOrc): + rootsThreshold = 0 + +proc GC_disableOrc*() = + when not defined(nimStressOrc): + rootsThreshold = high(int) + +proc GC_prepareOrc*(): int {.inline.} = + withLock gYrcGlobalLock: + mergePendingRoots() + result = roots.len + +proc GC_partialCollect*(limit: int) = + withLock gYrcGlobalLock: + mergePendingRoots() + if roots.len > limit: + var j: GcEnv + init j.traceStack + collectCyclesBacon(j, limit) + deinit j.traceStack + roots.len = limit + +proc GC_fullCollect* = + collectCycles() + +proc GC_enableMarkAndSweep*() = GC_enableOrc() +proc GC_disableMarkAndSweep*() = GC_disableOrc() + +const acyclicFlag = 1 + +when optimizedOrc: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 and (s.rc and maybeCycle) != 0 +else: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 + +proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline.} = + if isDestroyAction: + if s.rootIdx > 0: + unregisterCycle(s) + else: + if s.rootIdx == 0 and markedAsCyclic(s, desc): + s.setColor colBlack + let idx = getStripeIdx() + while true: + var overflow = false + withLock toDecStripes[idx].lock: + if toDecStripes[idx].toDecLen < QueueSize: + toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (s, desc) + toDecStripes[idx].toDecLen += 1 + else: + overflow = true + if overflow: + collectCycles() + else: + break + +proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = + result = false + if p != nil: + let cell = head(p) + let desc = cast[ptr PNimTypeV2](p)[] + let idx = getStripeIdx() + while true: + var overflow = false + withLock toDecStripes[idx].lock: + if toDecStripes[idx].toDecLen < QueueSize: + toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (cell, desc) + toDecStripes[idx].toDecLen += 1 + else: + overflow = true + if overflow: + collectCycles() + else: + break + +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + nimDecRefIsLastCyclicDyn(p) + +proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = + result = false + if p != nil: + let cell = head(p) + let idx = getStripeIdx() + while true: + var overflow = false + withLock toDecStripes[idx].lock: + if toDecStripes[idx].toDecLen < QueueSize: + toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (cell, desc) + toDecStripes[idx].toDecLen += 1 + else: + overflow = true + if overflow: + collectCycles() + else: + break + +proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = + dest[] = src + if src != nil: nimIncRefCyclic(src, true) + +proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = + when optimizedOrc: + if p != nil: + let h = head(p) + h.rc = h.rc or maybeCycle + +# Initialize locks at module load +initLock(gYrcGlobalLock) +for i in 0..