diff --git a/lib/system/yrc.nim b/lib/system/yrc.nim index 29b6f70ef4..eb9551a568 100644 --- a/lib/system/yrc.nim +++ b/lib/system/yrc.nim @@ -12,10 +12,10 @@ # Multiple mutators may hold this read lock simultaneously. # # The cycle collector acquires the exclusive write lock for the entire -# mark/scan/collect phase. This means the heap topology is *completely -# frozen* during collection: no `nimAsgnYrc` or seq operation can mutate -# any pointer field while the three passes run. This gives the Bacon -# algorithm the stable subgraph it requires without full write barriers. +# collection. This means the heap topology is *completely frozen* during +# collection: no `nimAsgnYrc` or seq operation can mutate any pointer field +# while the collector runs. This gives the algorithm below the stable +# subgraph it requires without write barriers. # # Consequence for incRef in `nimAsgnYrc`: # Because the collector is blocked, the incRef can be a direct atomic @@ -25,28 +25,50 @@ # (`yrcDec`) still use the `toDec` stripe queue so that objects whose RC # might reach zero are handled by the collector's cycle-detection logic. # -# ## Why No Write Barrier Is Needed +# ## The Collection Algorithm # -# The classic concurrent-GC hazard is the "lost object" problem: during -# collection the mutator executes `A.field = B` where A is already scanned -# (black), B is reachable only through an unscanned (gray) object C, and then -# C's reference to B is removed. The collector never discovers B and frees it -# while A still points to it. Traditional concurrent collectors need write -# barriers to prevent this. +# Instead of the classic trial-deletion three-pass dance (markGray / scan / +# collectWhite) which mutates the rc words in place, collection is split +# into three phases that leave the heap untouched until the outcome is +# decided: # -# This problem structurally cannot arise in YRC for two reasons: +# 1. Capture: one Tarjan SCC traversal over everything reachable from the +# candidate roots. Node -> dense index lookup is O(1) without hashing: +# the spare `rootIdx` header word (unused by YRC otherwise) is stamped +# with an epoch-tagged discovery index, so stale stamps never need +# clearing. Everything else lives in side arrays (SoA layout). # -# 1. The mutator lock freezes the topology during all three passes, so no -# concurrent field write can race with markGray/scan/collectWhite. +# 2. Deadness: pure array work on the captured SCC condensation, no heap +# access. An SCC is garbage iff it has no references beyond its internal +# ones and no live SCC points to it: +# external(S) = sum(refcounts of members) - internal edges - edges +# from garbage SCCs +# Tarjan emits SCCs sinks-first, so one linear scan in reverse emission +# order settles every SCC (sources before their targets). # -# 2. Even without the lock, the cycle collector only frees *closed cycles* — -# subgraphs where every reference to every member comes from within the -# group, with zero external references. To execute `A.field = B` the -# mutator must hold a reference to A (external ref), which `scan` would -# rescue. The two conditions are mutually exclusive. +# 3. Commit: only members of dead SCCs are touched. Every slot of a dead +# member is nil'ed; slots pointing at survivors decrement the survivor's +# rc for real (edges inside the dead group die with the group). Then the +# members are destroyed and freed. Because the slots are nil by the time +# destructors run, destructors cannot re-enter the decRef machinery for +# the already-processed edges. # -# In practice reason (1) makes reason (2) a belt-and-suspenders safety -# argument rather than the primary mechanism. +# This structure visits each edge at most twice (capture + commit of the +# dead subset) instead of up to three times, and — because the outcome is +# decided on captured data and the heap is only written in commit — it is +# the stepping stone towards optimistic, lock-free collection: replace the +# frozen-heap invariant with a SATB write barrier plus commit-time +# validation (rc word unchanged since capture, no barrier hit on a member) +# and the same three phases run concurrently with mutators. +# +# ## Why No Lost Objects +# +# The collector only frees *closed cycles* — subgraphs where every +# reference to every member comes from within the group, with zero external +# references. To mutate the graph a mutator must hold a reference to some +# object (an external ref), which the deadness computation observes in the +# rc word. The two conditions are mutually exclusive; the frozen topology +# makes this argument airtight without any barrier. {.push raises: [].} @@ -73,14 +95,6 @@ type when defined(nimYrcAtomicIncs): template color(c): untyped = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) and colorMask - template setColor(c, col) = - block: - var expected = atomicLoadN(addr c.rc, ATOMIC_RELAXED) - while true: - let desired = (expected and not colorMask) or col - if atomicCompareExchangeN(addr c.rc, addr expected, desired, true, - ATOMIC_ACQ_REL, ATOMIC_RELAXED): - break template loadRc(c): int = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) template trialDec(c) = discard atomicFetchAdd(addr c.rc, -rcIncrement, ATOMIC_ACQ_REL) @@ -104,11 +118,6 @@ when defined(nimYrcAtomicIncs): break else: template color(c): untyped = c.rc and colorMask - template setColor(c, col) = - when col == colBlack: - c.rc = c.rc and not colorMask - else: - c.rc = c.rc and not colorMask or col template loadRc(c): int = c.rc template trialDec(c) = c.rc = c.rc -% rcIncrement template trialInc(c) = c.rc = c.rc +% rcIncrement @@ -117,22 +126,133 @@ else: const optimizedOrc = false - useJumpStack = false + +# ---------------- side structure for capture ---------------- type + RawSeq[T] = object + ## growable array of plain scalars, allocation idiom as in CellSeq + len, cap: int + d: ptr UncheckedArray[T] + +proc resize[T](s: var RawSeq[T]; minCap: int) = + s.cap = max(minCap, s.cap div 2 +% s.cap) + let newSize = s.cap *% sizeof(T) + when compileOption("threads"): + s.d = cast[ptr UncheckedArray[T]](reallocShared(s.d, cast[Natural](newSize))) + else: + s.d = cast[ptr UncheckedArray[T]](realloc(s.d, cast[Natural](newSize))) + +proc add[T](s: var RawSeq[T]; v: T) {.inline.} = + if s.len >= s.cap: resize(s, s.len +% 1) + s.d[s.len] = v + s.len = s.len +% 1 + +proc pop[T](s: var RawSeq[T]): T {.inline.} = + s.len = s.len -% 1 + result = s.d[s.len] + +proc init[T](s: var RawSeq[T]; cap: int = 256) = + s.len = 0 + s.cap = cap + when compileOption("threads"): + s.d = cast[ptr UncheckedArray[T]](allocShared(cast[Natural](s.cap *% sizeof(T)))) + else: + s.d = cast[ptr UncheckedArray[T]](alloc(cast[Natural](s.cap *% sizeof(T)))) + +proc deinit[T](s: var RawSeq[T]) = + if s.d != nil: + when compileOption("threads"): + deallocShared(s.d) + else: + dealloc(s.d) + s.d = nil + s.len = 0 + s.cap = 0 + +proc setLenZeroed[T](s: var RawSeq[T]; n: int) = + if s.cap < n: resize(s, n) + s.len = n + zeroMem(s.d, n *% sizeof(T)) + +proc setLenUninit[T](s: var RawSeq[T]; n: int) = + if s.cap < n: resize(s, n) + s.len = n + +type + TarjanFrame = object + u: int32 # dense index of the cell this frame belongs to + base: int # traceStack.len before this cell's trace ran + + CaptureRec = object + ## per captured cell, position == Tarjan discovery index; one record so + ## a node costs a single append during the DFS + cell: Cell + desc: PNimTypeV2 + rcWord: int # rc word as captured + lowlink: int32 + sccOf: int32 # -1 while the cell is on the Tarjan stack + + CaptureBufs = object + ## side structure of a collection; persistent across collections (only + ## the collector, under the global write lock, ever touches it) so that + ## frequent small collections don't pay per-collection allocations + recs: RawSeq[CaptureRec] + tstack: RawSeq[int32] + frames: RawSeq[TarjanFrame] + edges: RawSeq[int64] # (u shl 32) or v, dense indices + sccMemStart: RawSeq[int32] + sccMembers: RawSeq[int32] + sumRefs: RawSeq[int] # per SCC: sum of member reference counts + internal: RawSeq[int] # per SCC: number of intra-SCC edges + deadIn: RawSeq[int] # per SCC: number of edges from dead SCCs + sccFlags: RawSeq[uint8] + crossOff: RawSeq[int32] # condensation cross edges, bucketed by source + crossTgt: RawSeq[int32] + crossCursor: RawSeq[int32] + GcEnv = object traceStack: CellSeq[ptr pointer] - when useJumpStack: - jumpStack: CellSeq[ptr pointer] toFree: CellSeq[Cell] - freed, touched, edges, rcSum: int + nScc: int + nDeadScc: int + freed, touched: int keepThreshold: bool +var gCap: CaptureBufs + +const + flagDead = 1'u8 + flagForcedLive = 2'u8 + proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = if desc.traceImpl != nil: var p = s +! sizeof(RefHeader) cast[TraceProc](desc.traceImpl)(p, addr(j)) +# The spare rootIdx header word (unused by YRC's root registration, which +# relies on inRootsFlag) doubles as the capture stamp: it packs an epoch tag +# with the cell's dense discovery index, so node -> index lookup is one load +# and stale stamps from earlier collections never need clearing. +var gCaptureEpoch: int = 1 + +when sizeof(int) == 8: + # 31-bit epoch, wraps after 2^31 collections (decades of uptime); on wrap + # a stale stamp collision is astronomically unlikely but not impossible. + template bumpEpoch() = + gCaptureEpoch = (gCaptureEpoch +% 1) and 0x7FFFFFFF + if gCaptureEpoch == 0: gCaptureEpoch = 1 + template isStamped(c: Cell): bool = (c.rootIdx shr 32) == gCaptureEpoch + template stamp(c: Cell; idx: int) = + c.rootIdx = gCaptureEpoch shl 32 or idx + template denseIdx(c: Cell): int32 = int32(c.rootIdx and 0xFFFFFFFF) +else: + # no room for an epoch: stamps are cleared at the end of each collection + template bumpEpoch() = discard + template isStamped(c: Cell): bool = c.rootIdx != 0 + template stamp(c: Cell; idx: int) = c.rootIdx = idx +% 1 + template denseIdx(c: Cell): int32 = int32(c.rootIdx -% 1) + type Stripe = object when not defined(yrcAtomics): @@ -232,7 +352,7 @@ proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = proc mergePendingRoots() = # Merge buffered RC operations. Note: Unlike truly concurrent collectors, - # we don't need to set color to black on incRef because collection runs + # we don't need any color handling on incRef because collection runs # under the global lock, so no concurrent mutations happen during collection. for i in 0.. until: - let (entry, desc) = j.traceStack.pop() - let t = head entry[] - trialInc(t) - if t.color != colBlack: - t.setColor colBlack - trace(t, desc, j) - when logOrc: writeCell("child still alive", t, desc) - -proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = - if s.color != colGray: - s.setColor colGray - j.touched = j.touched +% 1 - j.rcSum = j.rcSum +% (loadRc(s) shr rcShift) +% 1 - orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") - trace(s, desc, j) - while j.traceStack.len > 0: - let (entry, desc) = j.traceStack.pop() - let t = head entry[] - trialDec(t) - j.edges = j.edges +% 1 - if t.color != colGray: - t.setColor colGray - j.touched = j.touched +% 1 - j.rcSum = j.rcSum +% (loadRc(t) shr rcShift) +% 2 - trace(t, desc, j) - -proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = - if s.color == colGray: - if (loadRc(s) shr rcShift) >= 0: - scanBlack(s, desc, j) + gCap.frames.add TarjanFrame(u: root, base: 0) + while gCap.frames.len > 0: + let u = gCap.frames.d[gCap.frames.len -% 1].u + let base = gCap.frames.d[gCap.frames.len -% 1].base + if j.traceStack.len > base: + let (slot, tdesc) = j.traceStack.pop() + let t = head(slot[]) + if isStamped(t): + let v = denseIdx(t) + gCap.edges.add (int64(u) shl 32) or int64(v) + if gCap.recs.d[v].sccOf < 0 and v < gCap.recs.d[u].lowlink: + gCap.recs.d[u].lowlink = v + else: + let childBase = j.traceStack.len + let v = pushCell(t, tdesc) + gCap.edges.add (int64(u) shl 32) or int64(v) + trace(t, tdesc, j) + gCap.frames.add TarjanFrame(u: v, base: childBase) else: - orcAssert(j.traceStack.len == 0, "scan: trace stack not empty") - s.setColor(colWhite) - trace(s, desc, j) - while j.traceStack.len > 0: - let (entry, desc) = j.traceStack.pop() - let t = head entry[] - if t.color == colGray: - if (loadRc(t) shr rcShift) >= 0: - scanBlack(t, desc, j) - else: - t.setColor(colWhite) - trace(t, desc, j) + gCap.frames.len = gCap.frames.len -% 1 + if gCap.frames.len > 0: + let pu = gCap.frames.d[gCap.frames.len -% 1].u + if gCap.recs.d[u].lowlink < gCap.recs.d[pu].lowlink: + gCap.recs.d[pu].lowlink = gCap.recs.d[u].lowlink + if gCap.recs.d[u].lowlink == u: + # u is the root of an SCC: pop the members off the Tarjan stack + gCap.sccMemStart.add int32(gCap.sccMembers.len) + var sum = 0 + while true: + let w = gCap.tstack.pop() + gCap.recs.d[w].sccOf = int32(j.nScc) + gCap.sccMembers.add w + sum = sum +% (gCap.recs.d[w].rcWord shr rcShift) +% 1 + if w == u: break + gCap.sumRefs.add sum + inc j.nScc -proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = - if s.color == col and (loadRc(s) and inRootsFlag) == 0: - orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty") - s.setColor(colBlack) - j.toFree.add(s, desc) - trace(s, desc, j) - while j.traceStack.len > 0: - let (entry, desc) = j.traceStack.pop() - let t = head entry[] - entry[] = nil - if t.color == col and (loadRc(t) and inRootsFlag) == 0: - j.toFree.add(t, desc) - t.setColor(colBlack) - trace(t, desc, j) +# ---------------- phase 2: deadness, side arrays only ---------------- -proc collectCyclesBacon(j: var GcEnv; lowMark: int) = - # YRC defers all destruction to collection time - process ALL roots through Bacon's algorithm - # This is different from ORC which handles immediate garbage (rc == 0) directly +proc computeDeadness(j: var GcEnv) = + let nScc = j.nScc + setLenZeroed gCap.internal, nScc + setLenZeroed gCap.deadIn, nScc + setLenZeroed gCap.sccFlags, nScc + setLenZeroed gCap.crossOff, nScc + 1 + setLenUninit gCap.crossCursor, nScc + # classify captured edges: internal to an SCC vs condensation cross edges + var nCross = 0 + for i in 0 ..< gCap.edges.len: + let e = gCap.edges.d[i] + let su = gCap.recs.d[int32(e shr 32)].sccOf + let sv = gCap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf + if su == sv: + inc gCap.internal.d[su] + else: + inc gCap.crossOff.d[su] + inc nCross + var total = 0'i32 + for s in 0 ..< nScc: + let c = gCap.crossOff.d[s] + gCap.crossOff.d[s] = total + gCap.crossCursor.d[s] = total + total = total +% c + gCap.crossOff.d[nScc] = total + setLenUninit gCap.crossTgt, nCross + for i in 0 ..< gCap.edges.len: + let e = gCap.edges.d[i] + let su = gCap.recs.d[int32(e shr 32)].sccOf + let sv = gCap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf + if su != sv: + gCap.crossTgt.d[gCap.crossCursor.d[su]] = sv + inc gCap.crossCursor.d[su] + # cells that stay registered as roots (partial collection) count as + # externally referenced: the roots buffer itself points at them + for mi in 0 ..< gCap.sccMembers.len: + let m = gCap.sccMembers.d[mi] + if (loadRc(gCap.recs.d[m].cell) and inRootsFlag) != 0: + let s = gCap.recs.d[m].sccOf + gCap.sccFlags.d[s] = gCap.sccFlags.d[s] or flagForcedLive + # deadness over the condensation. Tarjan emits sinks first, so higher SCC + # ids are sources and every cross edge goes from a higher id to a lower + # one: one reverse scan settles everything. + for s in countdown(nScc - 1, 0): + let ext = gCap.sumRefs.d[s] -% gCap.internal.d[s] -% gCap.deadIn.d[s] + when logOrc: + cfprintf(cstderr, "[scc %ld] members %ld sumRefs %ld internal %ld deadIn %ld ext %ld forced %ld\n", + s, gCap.sccMemStart.d[s+1] - gCap.sccMemStart.d[s], gCap.sumRefs.d[s], + gCap.internal.d[s], gCap.deadIn.d[s], ext, int(gCap.sccFlags.d[s])) + if (gCap.sccFlags.d[s] and flagForcedLive) == 0 and ext == 0: + gCap.sccFlags.d[s] = gCap.sccFlags.d[s] or flagDead + inc j.nDeadScc + for k in gCap.crossOff.d[s] ..< gCap.crossOff.d[s+1]: + inc gCap.deadIn.d[gCap.crossTgt.d[k]] + else: + # a live SCC keeps everything it points to alive + for k in gCap.crossOff.d[s] ..< gCap.crossOff.d[s+1]: + let t = gCap.crossTgt.d[k] + gCap.sccFlags.d[t] = gCap.sccFlags.d[t] or flagForcedLive + +# ---------------- phase 3: commit ---------------- + +proc commitDead(j: var GcEnv) = + init j.toFree + template deadCell(t: Cell): bool = + isStamped(t) and (gCap.sccFlags.d[gCap.recs.d[denseIdx(t)].sccOf] and flagDead) != 0 + let allDead = j.nDeadScc == j.nScc + for s in 0 ..< j.nScc: + if (gCap.sccFlags.d[s] and flagDead) != 0: + for mi in gCap.sccMemStart.d[s] ..< gCap.sccMemStart.d[s+1]: + let m = gCap.sccMembers.d[mi] + let cell = gCap.recs.d[m].cell + let desc = gCap.recs.d[m].desc + j.toFree.add(cell, desc) + # nil every slot so the destructor cannot dec these edges again; + # references to survivors are decremented for real, references into + # the dead group die with the group (already accounted by deadIn) + orcAssert(j.traceStack.len == 0, "commitDead: trace stack not empty") + trace(cell, desc, j) + if allDead: + # everything captured dies: no survivor can occur, just nil + while j.traceStack.len > 0: + let (slot, _) = j.traceStack.pop() + slot[] = nil + else: + while j.traceStack.len > 0: + let (slot, _) = j.traceStack.pop() + let t = head(slot[]) + slot[] = nil + if not deadCell(t): + trialDec(t) + when sizeof(int) != 8: + # no epoch in the stamp: clear them while all cells are still alive + for i in 0 ..< gCap.recs.len: + gCap.recs.d[i].cell.rootIdx = 0 + for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) + free(j.toFree.d[i][0], j.toFree.d[i][1]) + j.freed = j.toFree.len + deinit j.toFree + +proc collectCyclesImpl(j: var GcEnv; lowMark: int) = + # All destruction is deferred to collection time: plain rc==0 garbage in + # the roots buffer forms singleton SCCs with external count 0 and is freed + # by the same machinery as the cycles. if lockState == Collecting: return lockState = Collecting @@ -375,39 +614,30 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) = for i in countdown(last, lowMark): writeCell("root", roots.d[i][0], roots.d[i][1]) - # Process all roots through markGray (Bacon's algorithm) + bumpEpoch() + init j.traceStack + prepareCapture() + j.nScc = 0 + for i in countdown(last, lowMark): - markGray(roots.d[i][0], roots.d[i][1], j) + capture(roots.d[i][0], roots.d[i][1], j) + gCap.sccMemStart.add int32(gCap.sccMembers.len) # sentinel + j.touched = gCap.recs.len - var colToCollect = colWhite - if j.rcSum == j.edges: - # Short-cut: we know everything is garbage - colToCollect = colGray - j.keepThreshold = true - else: - # Normal scan phase - for i in countdown(last, lowMark): - scan(roots.d[i][0], roots.d[i][1], j) + # Unregister the processed roots before computing deadness: only cells that + # STAY registered (below lowMark, partial collection) count as externally + # referenced by the roots buffer. Doing this before freeing anything also + # ensures a nested collectCycles() (triggered from a destructor) cannot + # access freed cells. + for i in lowMark ..< roots.len: + rcClearFlag(roots.d[i][0], inRootsFlag) + roots.len = lowMark - # Collect phase: free all garbage objects - init j.toFree - for i in 0 ..< roots.len: - let s = roots.d[i][0] - rcClearFlag(s, inRootsFlag) - collectColor(s, roots.d[i][1], colToCollect, j) + computeDeadness(j) + commitDead(j) + j.keepThreshold = j.freed == j.touched and j.touched > 0 - # Clear roots before freeing to prevent nested collectCycles() from accessing freed cells - roots.len = 0 - - # Free all collected objects - # Destructors must not call nimDecRefIsLastCyclicStatic (add to toDec) during this phase - for i in 0 ..< j.toFree.len: - let s = j.toFree.d[i][0] - when orcLeakDetector: - writeCell("CYCLIC OBJECT FREED", s, j.toFree.d[i][1]) - free(s, j.toFree.d[i][1]) - j.freed = j.freed +% j.toFree.len - deinit j.toFree + deinit j.traceStack when defined(nimOrcStats): var freedCyclicObjects {.threadvar.}: int @@ -420,8 +650,7 @@ proc collectCycles() = if roots.len >= rootsThreshold and mayRunCycleCollect(): let nRoots = roots.len var j: GcEnv - init j.traceStack - collectCyclesBacon(j, 0) + collectCyclesImpl(j, 0) if roots.len == 0 and roots.d != nil: deinit roots when not defined(nimStressOrc): @@ -444,7 +673,6 @@ proc collectCycles() = cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld\n", j.freed, rootsThreshold) when defined(nimOrcStats): inc freedCyclicObjects, j.freed - deinit j.traceStack when defined(nimOrcStats): type @@ -458,10 +686,7 @@ proc GC_runOrc* = mergePendingRoots() if roots.len > 0 and mayRunCycleCollect(): var j: GcEnv - init j.traceStack - collectCyclesBacon(j, 0) - deinit j.traceStack - roots.len = 0 + collectCyclesImpl(j, 0) when logOrc: orcAssert roots.len == 0, "roots not empty!" proc GC_enableOrc*() = @@ -482,10 +707,7 @@ proc GC_partialCollect*(limit: int) = mergePendingRoots() if roots.len > limit and mayRunCycleCollect(): var j: GcEnv - init j.traceStack - collectCyclesBacon(j, limit) - deinit j.traceStack - roots.len = limit + collectCyclesImpl(j, limit) proc GC_fullCollect* = GC_runOrc()