threadsafe cycle collection for Nim, the mm:yrc mode

compiler/ccgliterals.nim

@@ -16,7 +16,7 @@
 ## implementation.
 
 template detectVersion(field, corename) =
-  if m.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcHooks}:
+  if m.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc, gcHooks}:
     result = 2
   else:
     result = 1

compiler/ccgtypes.nim

@@ -277,7 +277,7 @@ proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
     of ctStruct:
       let t = skipTypes(rettype, typedescInst)
       if rettype.isImportedCppType or t.isImportedCppType or
-          (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc}):
+          (typ.callConv == ccCDecl and conf.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}):
         # prevents nrvo for cdecl procs; # bug #23401
         result = false
       else:
@@ -1692,7 +1692,7 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result:
         echo "ayclic but has this =trace ", t, " ", theProc.ast
   else:
     when false:
-      if op == attachedTrace and m.config.selectedGC == gcOrc and
+      if op == attachedTrace and m.config.selectedGC in {gcOrc, gcYrc} and
           containsGarbageCollectedRef(t):
         # unfortunately this check is wrong for an object type that only contains
         # .cursor fields like 'Node' inside 'cycleleak'.

compiler/commands.nim

@@ -245,7 +245,7 @@ proc processCompile(conf: ConfigRef; filename: string) =
   extccomp.addExternalFileToCompile(conf, found)
 
 const
-  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
+  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'yrc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console', 'lib' or 'staticlib' expected, but '$1' found"
   errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
@@ -266,6 +266,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "destructors", "arc": result = conf.selectedGC == gcArc
     of "orc": result = conf.selectedGC == gcOrc
+    of "yrc": result = conf.selectedGC == gcYrc
     of "hooks": result = conf.selectedGC == gcHooks
     of "go": result = conf.selectedGC == gcGo
     of "none": result = conf.selectedGC == gcNone
@@ -570,6 +571,7 @@ proc unregisterArcOrc*(conf: ConfigRef) =
   undefSymbol(conf.symbols, "gcdestructors")
   undefSymbol(conf.symbols, "gcarc")
   undefSymbol(conf.symbols, "gcorc")
+  undefSymbol(conf.symbols, "gcyrc")
   undefSymbol(conf.symbols, "gcatomicarc")
   undefSymbol(conf.symbols, "nimSeqsV2")
   undefSymbol(conf.symbols, "nimV2")
@@ -603,6 +605,10 @@ proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass,
       conf.selectedGC = gcOrc
       defineSymbol(conf.symbols, "gcorc")
       registerArcOrc(pass, conf)
+    of "yrc":
+      conf.selectedGC = gcYrc
+      defineSymbol(conf.symbols, "gcyrc")
+      registerArcOrc(pass, conf)
     of "atomicarc":
       conf.selectedGC = gcAtomicArc
       defineSymbol(conf.symbols, "gcatomicarc")

compiler/injectdestructors.nim

@@ -69,7 +69,7 @@ proc hasDestructor(c: Con; t: PType): bool {.inline.} =
   result = ast.hasDestructor(t)
   when toDebug.len > 0:
     # for more effective debugging
-    if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
+    if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}:
       assert(not containsGarbageCollectedRef(t))
 
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
@@ -165,7 +165,7 @@ proc isLastReadImpl(n: PNode; c: var Con; scope: var Scope): bool =
 
 template hasDestructorOrAsgn(c: var Con, typ: PType): bool =
   # bug #23354; an object type could have a non-trivial assignements when it is passed to a sink parameter
-  hasDestructor(c, typ) or (c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and
+  hasDestructor(c, typ) or (c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc} and
         typ.kind == tyObject and not isTrivial(getAttachedOp(c.graph, typ, attachedAsgn)))
 
 proc isLastRead(n: PNode; c: var Con; s: var Scope): bool =
@@ -329,14 +329,14 @@ proc isCriticalLink(dest: PNode): bool {.inline.} =
   result = dest.kind != nkSym
 
 proc finishCopy(c: var Con; result, dest: PNode; flags: set[MoveOrCopyFlag]; isFromSink: bool) =
-  if c.graph.config.selectedGC == gcOrc and IsExplicitSink notin flags:
+  if c.graph.config.selectedGC in {gcOrc, gcYrc} and IsExplicitSink notin flags:
     # add cyclic flag, but not to sink calls, which IsExplicitSink generates
     let t = dest.typ.skipTypes(tyUserTypeClasses + {tyGenericInst, tyAlias, tySink, tyDistinct})
     if cyclicType(c.graph, t):
       result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest))
 
 proc genMarkCyclic(c: var Con; result, dest: PNode) =
-  if c.graph.config.selectedGC == gcOrc:
+  if c.graph.config.selectedGC in {gcOrc, gcYrc}:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
     if cyclicType(c.graph, t):
       if t.kind == tyRef:
@@ -495,7 +495,7 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
       localError(c.graph.config, n.info, errFailedMove,
         ("cannot move '$1', passing '$1' to a sink parameter introduces an implicit copy") % $n)
   else:
-    if c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
+    if c.graph.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}:
       assert(not containsManagedMemory(nTyp))
     if nTyp.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
       localError(c.graph.config, n.info, "cannot create an implicit openArray copy to be passed to a sink parameter")
@@ -926,7 +926,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
 
       if n[0].kind == nkSym and n[0].sym.magic in {mNew, mNewFinalize}:
         result[0] = copyTree(n[0])
-        if c.graph.config.selectedGC in {gcHooks, gcArc, gcAtomicArc, gcOrc}:
+        if c.graph.config.selectedGC in {gcHooks, gcArc, gcAtomicArc, gcOrc, gcYrc}:
           let destroyOld = c.genDestroy(result[1])
           result = newTree(nkStmtList, destroyOld, result)
       else:

compiler/liftdestructors.nim

@@ -163,7 +163,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool,
     if c.filterDiscriminator != nil: return
     let f = n.sym
     let b = if c.kind == attachedTrace: y else: y.dotField(f)
-    if (sfCursor in f.flags and c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcHooks}) or
+    if (sfCursor in f.flags and c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc, gcHooks}) or
         enforceDefaultOp:
       defaultOp(c, f.typ, body, x.dotField(f), b)
     else:
@@ -735,9 +735,9 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let elemType = t.elementType
 
   createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen)
-  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(c.g, elemType)
+  let isCyclic = c.g.config.selectedGC in {gcOrc, gcYrc} and types.canFormAcycle(c.g, elemType)
 
-  let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and
+  let isInheritableAcyclicRef = c.g.config.selectedGC in {gcOrc, gcYrc} and
                       (not isPureObject(elemType)) and
                       tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
   # dynamic Acyclic refs need to use dyn decRef
@@ -819,7 +819,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xenv = genBuiltin(c, mAccessEnv, "accessEnv", x)
   xenv.typ = getSysType(c.g, c.info, tyPointer)
 
-  let isCyclic = c.g.config.selectedGC == gcOrc
+  let isCyclic = c.g.config.selectedGC in {gcOrc, gcYrc}
   let tmp =
     if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
       declareTempOf(c, body, xenv)
@@ -937,7 +937,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     call[1] = y
     body.add newAsgnStmt(x, call)
   elif (optOwnedRefs in c.g.config.globalOptions and
-      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
+      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}:
     let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
     xx.typ = getSysType(c.g, c.info, tyPointer)
     case c.kind
@@ -1125,7 +1125,7 @@ proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttache
 
   result.typ.addParam src
 
-  if g.config.selectedGC == gcOrc and
+  if g.config.selectedGC in {gcOrc, gcYrc} and
     cyclicType(g, typ.skipTypes(abstractInst)):
     let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
                             idgen, result, info)
@@ -1152,7 +1152,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"),
                    idgen, result, info)
 
-  if kind == attachedDestructor and g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and
+  if kind == attachedDestructor and g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc} and
      ((g.config.isDefined("nimPreviewNonVarDestructor") and not isDiscriminant) or (typ.kind in {tyRef, tyString, tySequence})):
     dest.typ = typ
   else:
@@ -1168,7 +1168,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   if kind notin {attachedDestructor, attachedWasMoved}:
     result.typ.addParam src
 
-  if kind == attachedAsgn and g.config.selectedGC == gcOrc and
+  if kind == attachedAsgn and g.config.selectedGC in {gcOrc, gcYrc} and
       cyclicType(g, typ.skipTypes(abstractInst)):
     let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
                             idgen, result, info)
@@ -1226,7 +1226,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
   else:
     var tk: TTypeKind
     var skipped: PType = nil
-    if g.config.selectedGC in {gcArc, gcOrc, gcHooks, gcAtomicArc}:
+    if g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcHooks, gcAtomicArc}:
       skipped = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink})
       tk = skipped.kind
     else:
@@ -1348,7 +1348,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
 
   # we do not generate '=trace' procs if we
   # have the cycle detection disabled, saves code size.
-  let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace
+  let lastAttached = if g.config.selectedGC in {gcOrc, gcYrc}: attachedTrace
                      else: attachedSink
 
   # bug #15122: We need to produce all prototypes before entering the

compiler/options.nim

@@ -195,6 +195,7 @@ type
     gcRegions = "regions"
     gcArc = "arc"
     gcOrc = "orc"
+    gcYrc = "yrc"       # thread-safe ORC (concurrent cycle collector)
     gcAtomicArc = "atomicArc"
     gcMarkAndSweep = "markAndSweep"
     gcHooks = "hooks"

compiler/scriptconfig.nim

@@ -231,7 +231,7 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   if optOwnedRefs in oldGlobalOptions:
     conf.globalOptions.incl {optTinyRtti, optOwnedRefs, optSeqDestructors}
     defineSymbol(conf.symbols, "nimv2")
-  if conf.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
+  if conf.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}:
     conf.globalOptions.incl {optTinyRtti, optSeqDestructors}
     defineSymbol(conf.symbols, "nimv2")
     defineSymbol(conf.symbols, "gcdestructors")
@@ -241,6 +241,8 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
       defineSymbol(conf.symbols, "gcarc")
     of gcOrc:
       defineSymbol(conf.symbols, "gcorc")
+    of gcYrc:
+      defineSymbol(conf.symbols, "gcyrc")
     of gcAtomicArc:
       defineSymbol(conf.symbols, "gcatomicarc")
     else:

lib/system.nim

@@ -125,7 +125,7 @@ proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} =
 
 const ThisIsSystem = true
 
-const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc)
+const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) or defined(gcYrc)
 
 when defined(nimAllowNonVarDestructor) and arcLikeMem:
   proc new*[T](a: var ref T, finalizer: proc (x: T) {.nimcall.}) {.
@@ -356,7 +356,7 @@ proc low*(x: string): int {.magic: "Low", noSideEffect.}
   ## See also:
   ## * `high(string) <#high,string>`_
 
-when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc):
+when not defined(gcArc) and not defined(gcOrc) and not defined(gcYrc) and not defined(gcAtomicArc):
   proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".}
     ## Use this instead of `=` for a `shallow copy`:idx:.
     ##
@@ -407,7 +407,7 @@ when defined(nimHasDup):
 
 proc `=sink`*[T](x: var T; y: T) {.inline, nodestroy, magic: "Asgn".} =
   ## Generic `sink`:idx: implementation that can be overridden.
-  when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
+  when defined(gcArc) or defined(gcOrc) or defined(gcYrc) or defined(gcAtomicArc):
     x = y
   else:
     shallowCopy(x, y)
@@ -2561,7 +2561,7 @@ when compileOption("rangechecks"):
 else:
   template rangeCheck*(cond) = discard
 
-when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc):
+when not defined(gcArc) and not defined(gcOrc) and not defined(gcYrc) and not defined(gcAtomicArc):
   proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} =
     ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not
     ## perform deep copies of `s`.
@@ -2630,7 +2630,7 @@ when hasAlloc or defined(nimscript):
     setLen(x, xl+item.len)
     var j = xl-1
     while j >= i:
-      when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
+      when defined(gcArc) or defined(gcOrc) or defined(gcYrc) or defined(gcAtomicArc):
         x[j+item.len] = move x[j]
       else:
         shallowCopy(x[j+item.len], x[j])

lib/system/arc.nim

@@ -16,7 +16,7 @@ runtime type and only contains a reference count.
 
 {.push raises: [], rangeChecks: off.}
 
-when defined(gcOrc):
+when defined(gcOrc) or defined(gcYrc):
   const
     rcIncrement = 0b10000 # so that lowest 4 bits are not touched
     rcMask = 0b1111
@@ -36,12 +36,12 @@ type
     rc: int # the object header is now a single RC field.
             # we could remove it in non-debug builds for the 'owned ref'
             # design but this seems unwise.
-    when defined(gcOrc):
+    when defined(gcOrc) or defined(gcYrc):
       rootIdx: int # thanks to this we can delete potential cycle roots
                    # in O(1) without doubly linked lists
     when defined(nimArcDebug) or defined(nimArcIds):
       refId: int
-    when defined(gcOrc) and orcLeakDetector:
+    when (defined(gcOrc) or defined(gcYrc)) and orcLeakDetector:
       filename: cstring
       line: int
 
@@ -119,7 +119,7 @@ proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} =
   else:
     result = cast[ptr RefHeader](alignedAlloc(s, alignment) +! hdrSize)
   head(result).rc = 0
-  when defined(gcOrc):
+  when defined(gcOrc) or defined(gcYrc):
     head(result).rootIdx = 0
   when defined(nimArcDebug):
     head(result).refId = gRefId
@@ -157,7 +157,7 @@ proc nimIncRef(p: pointer) {.compilerRtl, inl.} =
   when traceCollector:
     cprintf("[INCREF] %p\n", head(p))
 
-when not defined(gcOrc) or defined(nimThinout):
+when not (defined(gcOrc) or defined(gcYrc)) or defined(nimThinout):
   proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} =
     # This is only used by the old RTTI mechanism and we know
     # that 'dest[]' is nil and needs no destruction. Which is really handy
@@ -208,7 +208,9 @@ proc nimDestroyAndDispose(p: pointer) {.compilerRtl, quirky, raises: [].} =
       cstderr.rawWrite "has destructor!\n"
   nimRawDispose(p, rti.align)
 
-when defined(gcOrc):
+when defined(gcYrc):
+  include yrc
+elif defined(gcOrc):
   when defined(nimThinout):
     include cyclebreaker
   else:
@@ -251,7 +253,7 @@ proc GC_ref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
   if x != nil: nimIncRef(cast[pointer](x))
 
-when not defined(gcOrc):
+when not (defined(gcOrc) or defined(gcYrc)):
   template GC_fullCollect* =
     ## Forces a full garbage collection pass. With `--mm:arc` a nop.
     discard

lib/system/yrc.nim

@@ -0,0 +1,426 @@
+#
+# YRC: Thread-safe ORC (concurrent cycle collector).
+# Same API as orc.nim but with striped queues and global lock for merge/collect.
+# Destructors for refs run at collection time, not immediately on last decRef.
+#
+
+{.push raises: [].}
+
+include cellseqs_v2
+
+import std/locks
+
+const
+  NumStripes = 64
+  QueueSize = 128
+  RootsThreshold = 10
+
+  colBlack = 0b000
+  colGray = 0b001
+  colWhite = 0b010
+  maybeCycle = 0b100
+  jumpStackFlag = 0b1000
+  colorMask = 0b011
+  logOrc = defined(nimArcIds)
+
+type
+  TraceProc = proc (p, env: pointer) {.nimcall, benign, raises: [].}
+  DisposeProc = proc (p: pointer) {.nimcall, benign, raises: [].}
+
+template color(c): untyped = c.rc and colorMask
+template setColor(c, col) =
+  when col == colBlack:
+    c.rc = c.rc and not colorMask
+  else:
+    c.rc = c.rc and not colorMask or col
+
+const
+  optimizedOrc = false
+  useJumpStack = false
+
+type
+  GcEnv = object
+    traceStack: CellSeq[ptr pointer]
+    when useJumpStack:
+      jumpStack: CellSeq[ptr pointer]
+    toFree: CellSeq[Cell]
+    freed, touched, edges, rcSum: int
+    keepThreshold: bool
+
+proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} =
+  if desc.traceImpl != nil:
+    var p = s +! sizeof(RefHeader)
+    cast[TraceProc](desc.traceImpl)(p, addr(j))
+
+include threadids
+
+type
+  Stripe = object
+    lock: Lock
+    toInc: array[QueueSize, Cell]
+    toIncLen: int
+    toDec: array[QueueSize, (Cell, PNimTypeV2)]
+    toDecLen: int
+
+var
+  gYrcGlobalLock: Lock
+  roots: CellSeq[Cell]  # merged roots, used under global lock
+  toIncStripes: array[NumStripes, Stripe]
+  toDecStripes: array[NumStripes, Stripe]
+  rootsThreshold: int = 128
+  defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128
+
+proc getStripeIdx(): int {.inline.} =
+  getThreadId() and (NumStripes - 1)
+
+proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} =
+  let h = head(p)
+  when optimizedOrc:
+    if cyclic: h.rc = h.rc or maybeCycle
+  let idx = getStripeIdx()
+  while true:
+    var overflow = false
+    withLock toIncStripes[idx].lock:
+      if toIncStripes[idx].toIncLen < QueueSize:
+        toIncStripes[idx].toInc[toIncStripes[idx].toIncLen] = h
+        toIncStripes[idx].toIncLen += 1
+      else:
+        overflow = true
+    if overflow:
+      withLock gYrcGlobalLock:
+        for i in 0..<NumStripes:
+          withLock toIncStripes[i].lock:
+            for j in 0..<toIncStripes[i].toIncLen:
+              toIncStripes[i].toInc[j].rc = toIncStripes[i].toInc[j].rc +% rcIncrement
+            toIncStripes[i].toIncLen = 0
+    else:
+      break
+
+proc mergePendingRoots() =
+  for i in 0..<NumStripes:
+    withLock toIncStripes[i].lock:
+      for j in 0..<toIncStripes[i].toIncLen:
+        toIncStripes[i].toInc[j].rc = toIncStripes[i].toInc[j].rc +% rcIncrement
+      toIncStripes[i].toIncLen = 0
+    withLock toDecStripes[i].lock:
+      for j in 0..<toDecStripes[i].toDecLen:
+        let (c, desc) = toDecStripes[i].toDec[j]
+        c.rc = c.rc -% rcIncrement
+        if c.rootIdx == 0:
+          if roots.d == nil: init(roots)
+          add(roots, c, desc)
+          c.rootIdx = roots.len
+      toDecStripes[i].toDecLen = 0
+
+proc collectCycles()
+
+when logOrc or orcLeakDetector:
+  proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) =
+    when orcLeakDetector:
+      cfprintf(cstderr, "%s %s file: %s:%ld; color: %ld; thread: %ld\n",
+        msg, desc.name, s.filename, s.line, s.color, getThreadId())
+    else:
+      cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n",
+        msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId())
+
+proc free(s: Cell; desc: PNimTypeV2) {.inline.} =
+  when traceCollector:
+    cprintf("[From ] %p rc %ld color %ld\n", s, s.rc shr rcShift, s.color)
+  let p = s +! sizeof(RefHeader)
+  when logOrc: writeCell("free", s, desc)
+  if desc.destructor != nil:
+    cast[DestructorProc](desc.destructor)(p)
+  nimRawDispose(p, desc.align)
+
+template orcAssert(cond, msg) =
+  when logOrc:
+    if not cond:
+      cfprintf(cstderr, "[Bug!] %s\n", msg)
+      rawQuit 1
+
+when logOrc:
+  proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.}
+
+proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} =
+  let p = cast[ptr pointer](q)
+  if p[] != nil:
+    orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!"
+    var j = cast[ptr GcEnv](env)
+    j.traceStack.add(p, desc)
+
+proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} =
+  let p = cast[ptr pointer](q)
+  if p[] != nil:
+    var j = cast[ptr GcEnv](env)
+    j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[])
+
+proc unregisterCycle(s: Cell) =
+  let rootIdx = s.rootIdx
+  let idx = rootIdx -% 1
+  let last = roots.len -% 1
+  roots.d[idx] = roots.d[last]
+  roots.d[idx][0].rootIdx = rootIdx
+  roots.len = last
+  s.rootIdx = 0
+
+proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
+  s.setColor colBlack
+  let until = j.traceStack.len
+  trace(s, desc, j)
+  when logOrc: writeCell("root still alive", s, desc)
+  while j.traceStack.len > until:
+    let (entry, desc) = j.traceStack.pop()
+    let t = head entry[]
+    t.rc = t.rc +% rcIncrement
+    if t.color != colBlack:
+      t.setColor colBlack
+      trace(t, desc, j)
+      when logOrc: writeCell("child still alive", t, desc)
+
+proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
+  if s.color != colGray:
+    s.setColor colGray
+    j.touched = j.touched +% 1
+    j.rcSum = j.rcSum +% (s.rc shr rcShift) +% 1
+    orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty")
+    trace(s, desc, j)
+    while j.traceStack.len > 0:
+      let (entry, desc) = j.traceStack.pop()
+      let t = head entry[]
+      t.rc = t.rc -% rcIncrement
+      j.edges = j.edges +% 1
+      if t.color != colGray:
+        t.setColor colGray
+        j.touched = j.touched +% 1
+        j.rcSum = j.rcSum +% (t.rc shr rcShift) +% 2
+        trace(t, desc, j)
+
+proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
+  if s.color == colGray:
+    if (s.rc shr rcShift) >= 0:
+      scanBlack(s, desc, j)
+    else:
+      orcAssert(j.traceStack.len == 0, "scan: trace stack not empty")
+      s.setColor(colWhite)
+      trace(s, desc, j)
+      while j.traceStack.len > 0:
+        let (entry, desc) = j.traceStack.pop()
+        let t = head entry[]
+        if t.color == colGray:
+          if (t.rc shr rcShift) >= 0:
+            scanBlack(t, desc, j)
+          else:
+            t.setColor(colWhite)
+            trace(t, desc, j)
+
+proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) =
+  if s.color == col and s.rootIdx == 0:
+    orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty")
+    s.setColor(colBlack)
+    j.toFree.add(s, desc)
+    trace(s, desc, j)
+    while j.traceStack.len > 0:
+      let (entry, desc) = j.traceStack.pop()
+      let t = head entry[]
+      entry[] = nil
+      if t.color == col and t.rootIdx == 0:
+        j.toFree.add(t, desc)
+        t.setColor(colBlack)
+        trace(t, desc, j)
+
+proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
+  let last = roots.len -% 1
+  when logOrc:
+    for i in countdown(last, lowMark):
+      writeCell("root", roots.d[i][0], roots.d[i][1])
+  for i in countdown(last, lowMark):
+    markGray(roots.d[i][0], roots.d[i][1], j)
+  var colToCollect = colWhite
+  if j.rcSum == j.edges:
+    colToCollect = colGray
+    j.keepThreshold = true
+  else:
+    for i in countdown(last, lowMark):
+      scan(roots.d[i][0], roots.d[i][1], j)
+  init j.toFree
+  for i in 0 ..< roots.len:
+    let s = roots.d[i][0]
+    s.rootIdx = 0
+    collectColor(s, roots.d[i][1], colToCollect, j)
+  when not defined(nimStressOrc):
+    let oldThreshold = rootsThreshold
+    rootsThreshold = high(int)
+  roots.len = 0
+  for i in 0 ..< j.toFree.len:
+    when orcLeakDetector:
+      writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1])
+    free(j.toFree.d[i][0], j.toFree.d[i][1])
+  when not defined(nimStressOrc):
+    rootsThreshold = oldThreshold
+  j.freed = j.freed +% j.toFree.len
+  deinit j.toFree
+
+when defined(nimOrcStats):
+  var freedCyclicObjects {.threadvar.}: int
+
+proc collectCycles() =
+  when logOrc:
+    cfprintf(cstderr, "[collectCycles] begin\n")
+  withLock gYrcGlobalLock:
+    mergePendingRoots()
+    if roots.len >= RootsThreshold:
+      var j: GcEnv
+      init j.traceStack
+      collectCyclesBacon(j, 0)
+      if roots.len == 0 and roots.d != nil:
+        deinit roots
+      when not defined(nimStressOrc):
+        if j.keepThreshold:
+          discard
+        elif j.freed *% 2 >= j.touched:
+          when not defined(nimFixedOrc):
+            rootsThreshold = max(rootsThreshold div 3 *% 2, 16)
+          else:
+            rootsThreshold = 0
+        elif rootsThreshold < high(int) div 4:
+          rootsThreshold = (if rootsThreshold <= 0: defaultThreshold else: rootsThreshold)
+          rootsThreshold = rootsThreshold div 2 +% rootsThreshold
+      when logOrc:
+        cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld\n", j.freed, rootsThreshold)
+      when defined(nimOrcStats):
+        inc freedCyclicObjects, j.freed
+      deinit j.traceStack
+
+when defined(nimOrcStats):
+  type
+    OrcStats* = object
+      freedCyclicObjects*: int
+  proc GC_orcStats*(): OrcStats =
+    result = OrcStats(freedCyclicObjects: freedCyclicObjects)
+
+proc GC_runOrc* =
+  withLock gYrcGlobalLock:
+    mergePendingRoots()
+    var j: GcEnv
+    init j.traceStack
+    collectCyclesBacon(j, 0)
+    deinit j.traceStack
+    roots.len = 0
+  when logOrc: orcAssert roots.len == 0, "roots not empty!"
+
+proc GC_enableOrc*() =
+  when not defined(nimStressOrc):
+    rootsThreshold = 0
+
+proc GC_disableOrc*() =
+  when not defined(nimStressOrc):
+    rootsThreshold = high(int)
+
+proc GC_prepareOrc*(): int {.inline.} =
+  withLock gYrcGlobalLock:
+    mergePendingRoots()
+    result = roots.len
+
+proc GC_partialCollect*(limit: int) =
+  withLock gYrcGlobalLock:
+    mergePendingRoots()
+    if roots.len > limit:
+      var j: GcEnv
+      init j.traceStack
+      collectCyclesBacon(j, limit)
+      deinit j.traceStack
+      roots.len = limit
+
+proc GC_fullCollect* =
+  collectCycles()
+
+proc GC_enableMarkAndSweep*() = GC_enableOrc()
+proc GC_disableMarkAndSweep*() = GC_disableOrc()
+
+const acyclicFlag = 1
+
+when optimizedOrc:
+  template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool =
+    (desc.flags and acyclicFlag) == 0 and (s.rc and maybeCycle) != 0
+else:
+  template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool =
+    (desc.flags and acyclicFlag) == 0
+
+proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline.} =
+  if isDestroyAction:
+    if s.rootIdx > 0:
+      unregisterCycle(s)
+  else:
+    if s.rootIdx == 0 and markedAsCyclic(s, desc):
+      s.setColor colBlack
+      let idx = getStripeIdx()
+      while true:
+        var overflow = false
+        withLock toDecStripes[idx].lock:
+          if toDecStripes[idx].toDecLen < QueueSize:
+            toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (s, desc)
+            toDecStripes[idx].toDecLen += 1
+          else:
+            overflow = true
+        if overflow:
+          collectCycles()
+        else:
+          break
+
+proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} =
+  result = false
+  if p != nil:
+    let cell = head(p)
+    let desc = cast[ptr PNimTypeV2](p)[]
+    let idx = getStripeIdx()
+    while true:
+      var overflow = false
+      withLock toDecStripes[idx].lock:
+        if toDecStripes[idx].toDecLen < QueueSize:
+          toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (cell, desc)
+          toDecStripes[idx].toDecLen += 1
+        else:
+          overflow = true
+      if overflow:
+        collectCycles()
+      else:
+        break
+
+proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} =
+  nimDecRefIsLastCyclicDyn(p)
+
+proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} =
+  result = false
+  if p != nil:
+    let cell = head(p)
+    let idx = getStripeIdx()
+    while true:
+      var overflow = false
+      withLock toDecStripes[idx].lock:
+        if toDecStripes[idx].toDecLen < QueueSize:
+          toDecStripes[idx].toDec[toDecStripes[idx].toDecLen] = (cell, desc)
+          toDecStripes[idx].toDecLen += 1
+        else:
+          overflow = true
+      if overflow:
+        collectCycles()
+      else:
+        break
+
+proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} =
+  dest[] = src
+  if src != nil: nimIncRefCyclic(src, true)
+
+proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} =
+  when optimizedOrc:
+    if p != nil:
+      let h = head(p)
+      h.rc = h.rc or maybeCycle
+
+# Initialize locks at module load
+initLock(gYrcGlobalLock)
+for i in 0..<NumStripes:
+  initLock(toIncStripes[i].lock)
+  initLock(toDecStripes[i].lock)
+
+{.pop.}