mirror of
https://github.com/nim-lang/Nim.git
synced 2026-08-02 13:39:03 +00:00
YRC: use a side-table for topology and Tarjan's algorithm for cycle collection
This commit is contained in:
Araq
@@ -12,10 +12,10 @@
|
||||
# Multiple mutators may hold this read lock simultaneously.
|
||||
#
|
||||
# The cycle collector acquires the exclusive write lock for the entire
|
||||
# mark/scan/collect phase. This means the heap topology is *completely
|
||||
# frozen* during collection: no `nimAsgnYrc` or seq operation can mutate
|
||||
# any pointer field while the three passes run. This gives the Bacon
|
||||
# algorithm the stable subgraph it requires without full write barriers.
|
||||
# collection. This means the heap topology is *completely frozen* during
|
||||
# collection: no `nimAsgnYrc` or seq operation can mutate any pointer field
|
||||
# while the collector runs. This gives the algorithm below the stable
|
||||
# subgraph it requires without write barriers.
|
||||
#
|
||||
# Consequence for incRef in `nimAsgnYrc`:
|
||||
# Because the collector is blocked, the incRef can be a direct atomic
|
||||
@@ -25,28 +25,50 @@
|
||||
# (`yrcDec`) still use the `toDec` stripe queue so that objects whose RC
|
||||
# might reach zero are handled by the collector's cycle-detection logic.
|
||||
#
|
||||
# ## Why No Write Barrier Is Needed
|
||||
# ## The Collection Algorithm
|
||||
#
|
||||
# The classic concurrent-GC hazard is the "lost object" problem: during
|
||||
# collection the mutator executes `A.field = B` where A is already scanned
|
||||
# (black), B is reachable only through an unscanned (gray) object C, and then
|
||||
# C's reference to B is removed. The collector never discovers B and frees it
|
||||
# while A still points to it. Traditional concurrent collectors need write
|
||||
# barriers to prevent this.
|
||||
# Instead of the classic trial-deletion three-pass dance (markGray / scan /
|
||||
# collectWhite) which mutates the rc words in place, collection is split
|
||||
# into three phases that leave the heap untouched until the outcome is
|
||||
# decided:
|
||||
#
|
||||
# This problem structurally cannot arise in YRC for two reasons:
|
||||
# 1. Capture: one Tarjan SCC traversal over everything reachable from the
|
||||
# candidate roots. Node -> dense index lookup is O(1) without hashing:
|
||||
# the spare `rootIdx` header word (unused by YRC otherwise) is stamped
|
||||
# with an epoch-tagged discovery index, so stale stamps never need
|
||||
# clearing. Everything else lives in side arrays (SoA layout).
|
||||
#
|
||||
# 1. The mutator lock freezes the topology during all three passes, so no
|
||||
# concurrent field write can race with markGray/scan/collectWhite.
|
||||
# 2. Deadness: pure array work on the captured SCC condensation, no heap
|
||||
# access. An SCC is garbage iff it has no references beyond its internal
|
||||
# ones and no live SCC points to it:
|
||||
# external(S) = sum(refcounts of members) - internal edges - edges
|
||||
# from garbage SCCs
|
||||
# Tarjan emits SCCs sinks-first, so one linear scan in reverse emission
|
||||
# order settles every SCC (sources before their targets).
|
||||
#
|
||||
# 2. Even without the lock, the cycle collector only frees *closed cycles* —
|
||||
# subgraphs where every reference to every member comes from within the
|
||||
# group, with zero external references. To execute `A.field = B` the
|
||||
# mutator must hold a reference to A (external ref), which `scan` would
|
||||
# rescue. The two conditions are mutually exclusive.
|
||||
# 3. Commit: only members of dead SCCs are touched. Every slot of a dead
|
||||
# member is nil'ed; slots pointing at survivors decrement the survivor's
|
||||
# rc for real (edges inside the dead group die with the group). Then the
|
||||
# members are destroyed and freed. Because the slots are nil by the time
|
||||
# destructors run, destructors cannot re-enter the decRef machinery for
|
||||
# the already-processed edges.
|
||||
#
|
||||
# In practice reason (1) makes reason (2) a belt-and-suspenders safety
|
||||
# argument rather than the primary mechanism.
|
||||
# This structure visits each edge at most twice (capture + commit of the
|
||||
# dead subset) instead of up to three times, and — because the outcome is
|
||||
# decided on captured data and the heap is only written in commit — it is
|
||||
# the stepping stone towards optimistic, lock-free collection: replace the
|
||||
# frozen-heap invariant with a SATB write barrier plus commit-time
|
||||
# validation (rc word unchanged since capture, no barrier hit on a member)
|
||||
# and the same three phases run concurrently with mutators.
|
||||
#
|
||||
# ## Why No Lost Objects
|
||||
#
|
||||
# The collector only frees *closed cycles* — subgraphs where every
|
||||
# reference to every member comes from within the group, with zero external
|
||||
# references. To mutate the graph a mutator must hold a reference to some
|
||||
# object (an external ref), which the deadness computation observes in the
|
||||
# rc word. The two conditions are mutually exclusive; the frozen topology
|
||||
# makes this argument airtight without any barrier.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
@@ -73,14 +95,6 @@ type
|
||||
|
||||
when defined(nimYrcAtomicIncs):
|
||||
template color(c): untyped = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) and colorMask
|
||||
template setColor(c, col) =
|
||||
block:
|
||||
var expected = atomicLoadN(addr c.rc, ATOMIC_RELAXED)
|
||||
while true:
|
||||
let desired = (expected and not colorMask) or col
|
||||
if atomicCompareExchangeN(addr c.rc, addr expected, desired, true,
|
||||
ATOMIC_ACQ_REL, ATOMIC_RELAXED):
|
||||
break
|
||||
template loadRc(c): int = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE)
|
||||
template trialDec(c) =
|
||||
discard atomicFetchAdd(addr c.rc, -rcIncrement, ATOMIC_ACQ_REL)
|
||||
@@ -104,11 +118,6 @@ when defined(nimYrcAtomicIncs):
|
||||
break
|
||||
else:
|
||||
template color(c): untyped = c.rc and colorMask
|
||||
template setColor(c, col) =
|
||||
when col == colBlack:
|
||||
c.rc = c.rc and not colorMask
|
||||
else:
|
||||
c.rc = c.rc and not colorMask or col
|
||||
template loadRc(c): int = c.rc
|
||||
template trialDec(c) = c.rc = c.rc -% rcIncrement
|
||||
template trialInc(c) = c.rc = c.rc +% rcIncrement
|
||||
@@ -117,22 +126,133 @@ else:
|
||||
|
||||
const
|
||||
optimizedOrc = false
|
||||
useJumpStack = false
|
||||
|
||||
# ---------------- side structure for capture ----------------
|
||||
|
||||
type
|
||||
RawSeq[T] = object
|
||||
## growable array of plain scalars, allocation idiom as in CellSeq
|
||||
len, cap: int
|
||||
d: ptr UncheckedArray[T]
|
||||
|
||||
proc resize[T](s: var RawSeq[T]; minCap: int) =
|
||||
s.cap = max(minCap, s.cap div 2 +% s.cap)
|
||||
let newSize = s.cap *% sizeof(T)
|
||||
when compileOption("threads"):
|
||||
s.d = cast[ptr UncheckedArray[T]](reallocShared(s.d, cast[Natural](newSize)))
|
||||
else:
|
||||
s.d = cast[ptr UncheckedArray[T]](realloc(s.d, cast[Natural](newSize)))
|
||||
|
||||
proc add[T](s: var RawSeq[T]; v: T) {.inline.} =
|
||||
if s.len >= s.cap: resize(s, s.len +% 1)
|
||||
s.d[s.len] = v
|
||||
s.len = s.len +% 1
|
||||
|
||||
proc pop[T](s: var RawSeq[T]): T {.inline.} =
|
||||
s.len = s.len -% 1
|
||||
result = s.d[s.len]
|
||||
|
||||
proc init[T](s: var RawSeq[T]; cap: int = 256) =
|
||||
s.len = 0
|
||||
s.cap = cap
|
||||
when compileOption("threads"):
|
||||
s.d = cast[ptr UncheckedArray[T]](allocShared(cast[Natural](s.cap *% sizeof(T))))
|
||||
else:
|
||||
s.d = cast[ptr UncheckedArray[T]](alloc(cast[Natural](s.cap *% sizeof(T))))
|
||||
|
||||
proc deinit[T](s: var RawSeq[T]) =
|
||||
if s.d != nil:
|
||||
when compileOption("threads"):
|
||||
deallocShared(s.d)
|
||||
else:
|
||||
dealloc(s.d)
|
||||
s.d = nil
|
||||
s.len = 0
|
||||
s.cap = 0
|
||||
|
||||
proc setLenZeroed[T](s: var RawSeq[T]; n: int) =
|
||||
if s.cap < n: resize(s, n)
|
||||
s.len = n
|
||||
zeroMem(s.d, n *% sizeof(T))
|
||||
|
||||
proc setLenUninit[T](s: var RawSeq[T]; n: int) =
|
||||
if s.cap < n: resize(s, n)
|
||||
s.len = n
|
||||
|
||||
type
|
||||
TarjanFrame = object
|
||||
u: int32 # dense index of the cell this frame belongs to
|
||||
base: int # traceStack.len before this cell's trace ran
|
||||
|
||||
CaptureRec = object
|
||||
## per captured cell, position == Tarjan discovery index; one record so
|
||||
## a node costs a single append during the DFS
|
||||
cell: Cell
|
||||
desc: PNimTypeV2
|
||||
rcWord: int # rc word as captured
|
||||
lowlink: int32
|
||||
sccOf: int32 # -1 while the cell is on the Tarjan stack
|
||||
|
||||
CaptureBufs = object
|
||||
## side structure of a collection; persistent across collections (only
|
||||
## the collector, under the global write lock, ever touches it) so that
|
||||
## frequent small collections don't pay per-collection allocations
|
||||
recs: RawSeq[CaptureRec]
|
||||
tstack: RawSeq[int32]
|
||||
frames: RawSeq[TarjanFrame]
|
||||
edges: RawSeq[int64] # (u shl 32) or v, dense indices
|
||||
sccMemStart: RawSeq[int32]
|
||||
sccMembers: RawSeq[int32]
|
||||
sumRefs: RawSeq[int] # per SCC: sum of member reference counts
|
||||
internal: RawSeq[int] # per SCC: number of intra-SCC edges
|
||||
deadIn: RawSeq[int] # per SCC: number of edges from dead SCCs
|
||||
sccFlags: RawSeq[uint8]
|
||||
crossOff: RawSeq[int32] # condensation cross edges, bucketed by source
|
||||
crossTgt: RawSeq[int32]
|
||||
crossCursor: RawSeq[int32]
|
||||
|
||||
GcEnv = object
|
||||
traceStack: CellSeq[ptr pointer]
|
||||
when useJumpStack:
|
||||
jumpStack: CellSeq[ptr pointer]
|
||||
toFree: CellSeq[Cell]
|
||||
freed, touched, edges, rcSum: int
|
||||
nScc: int
|
||||
nDeadScc: int
|
||||
freed, touched: int
|
||||
keepThreshold: bool
|
||||
|
||||
var gCap: CaptureBufs
|
||||
|
||||
const
|
||||
flagDead = 1'u8
|
||||
flagForcedLive = 2'u8
|
||||
|
||||
proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} =
|
||||
if desc.traceImpl != nil:
|
||||
var p = s +! sizeof(RefHeader)
|
||||
cast[TraceProc](desc.traceImpl)(p, addr(j))
|
||||
|
||||
# The spare rootIdx header word (unused by YRC's root registration, which
|
||||
# relies on inRootsFlag) doubles as the capture stamp: it packs an epoch tag
|
||||
# with the cell's dense discovery index, so node -> index lookup is one load
|
||||
# and stale stamps from earlier collections never need clearing.
|
||||
var gCaptureEpoch: int = 1
|
||||
|
||||
when sizeof(int) == 8:
|
||||
# 31-bit epoch, wraps after 2^31 collections (decades of uptime); on wrap
|
||||
# a stale stamp collision is astronomically unlikely but not impossible.
|
||||
template bumpEpoch() =
|
||||
gCaptureEpoch = (gCaptureEpoch +% 1) and 0x7FFFFFFF
|
||||
if gCaptureEpoch == 0: gCaptureEpoch = 1
|
||||
template isStamped(c: Cell): bool = (c.rootIdx shr 32) == gCaptureEpoch
|
||||
template stamp(c: Cell; idx: int) =
|
||||
c.rootIdx = gCaptureEpoch shl 32 or idx
|
||||
template denseIdx(c: Cell): int32 = int32(c.rootIdx and 0xFFFFFFFF)
|
||||
else:
|
||||
# no room for an epoch: stamps are cleared at the end of each collection
|
||||
template bumpEpoch() = discard
|
||||
template isStamped(c: Cell): bool = c.rootIdx != 0
|
||||
template stamp(c: Cell; idx: int) = c.rootIdx = idx +% 1
|
||||
template denseIdx(c: Cell): int32 = int32(c.rootIdx -% 1)
|
||||
|
||||
type
|
||||
Stripe = object
|
||||
when not defined(yrcAtomics):
|
||||
@@ -232,7 +352,7 @@ proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} =
|
||||
|
||||
proc mergePendingRoots() =
|
||||
# Merge buffered RC operations. Note: Unlike truly concurrent collectors,
|
||||
# we don't need to set color to black on incRef because collection runs
|
||||
# we don't need any color handling on incRef because collection runs
|
||||
# under the global lock, so no concurrent mutations happen during collection.
|
||||
for i in 0..<NumStripes:
|
||||
when not defined(nimYrcAtomicIncs):
|
||||
@@ -299,74 +419,193 @@ proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} =
|
||||
var j = cast[ptr GcEnv](env)
|
||||
j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[])
|
||||
|
||||
proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
|
||||
s.setColor colBlack
|
||||
let until = j.traceStack.len
|
||||
# ---------------- phase 1: capture ----------------
|
||||
|
||||
proc prepareCapture() =
|
||||
if gCap.recs.d == nil:
|
||||
init gCap.recs
|
||||
init gCap.tstack
|
||||
init gCap.frames
|
||||
init gCap.edges
|
||||
init gCap.sccMemStart
|
||||
init gCap.sccMembers
|
||||
init gCap.sumRefs
|
||||
init gCap.internal
|
||||
init gCap.deadIn
|
||||
init gCap.sccFlags
|
||||
init gCap.crossOff
|
||||
init gCap.crossTgt
|
||||
init gCap.crossCursor
|
||||
else:
|
||||
gCap.recs.len = 0
|
||||
gCap.tstack.len = 0
|
||||
gCap.frames.len = 0
|
||||
gCap.edges.len = 0
|
||||
gCap.sccMemStart.len = 0
|
||||
gCap.sccMembers.len = 0
|
||||
gCap.sumRefs.len = 0
|
||||
|
||||
proc pushCell(c: Cell; desc: PNimTypeV2): int32 {.inline.} =
|
||||
result = int32(gCap.recs.len)
|
||||
stamp(c, gCap.recs.len)
|
||||
gCap.recs.add CaptureRec(cell: c, desc: desc, rcWord: loadRc(c),
|
||||
lowlink: result, sccOf: -1'i32)
|
||||
gCap.tstack.add result
|
||||
|
||||
proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
|
||||
## Iterative Tarjan SCC over everything reachable from `s`. A frame's
|
||||
## pending out-edges are the traceStack entries above frame.base; a child
|
||||
## pushes and drains its own segment above ours, so when the child's frame
|
||||
## pops, the stack is back at our segment and we resume popping our edges.
|
||||
if isStamped(s): return
|
||||
orcAssert(j.traceStack.len == 0, "capture: trace stack not empty")
|
||||
let root = pushCell(s, desc)
|
||||
trace(s, desc, j)
|
||||
when logOrc: writeCell("root still alive", s, desc)
|
||||
while j.traceStack.len > until:
|
||||
let (entry, desc) = j.traceStack.pop()
|
||||
let t = head entry[]
|
||||
trialInc(t)
|
||||
if t.color != colBlack:
|
||||
t.setColor colBlack
|
||||
trace(t, desc, j)
|
||||
when logOrc: writeCell("child still alive", t, desc)
|
||||
|
||||
proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
|
||||
if s.color != colGray:
|
||||
s.setColor colGray
|
||||
j.touched = j.touched +% 1
|
||||
j.rcSum = j.rcSum +% (loadRc(s) shr rcShift) +% 1
|
||||
orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty")
|
||||
trace(s, desc, j)
|
||||
while j.traceStack.len > 0:
|
||||
let (entry, desc) = j.traceStack.pop()
|
||||
let t = head entry[]
|
||||
trialDec(t)
|
||||
j.edges = j.edges +% 1
|
||||
if t.color != colGray:
|
||||
t.setColor colGray
|
||||
j.touched = j.touched +% 1
|
||||
j.rcSum = j.rcSum +% (loadRc(t) shr rcShift) +% 2
|
||||
trace(t, desc, j)
|
||||
|
||||
proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) =
|
||||
if s.color == colGray:
|
||||
if (loadRc(s) shr rcShift) >= 0:
|
||||
scanBlack(s, desc, j)
|
||||
gCap.frames.add TarjanFrame(u: root, base: 0)
|
||||
while gCap.frames.len > 0:
|
||||
let u = gCap.frames.d[gCap.frames.len -% 1].u
|
||||
let base = gCap.frames.d[gCap.frames.len -% 1].base
|
||||
if j.traceStack.len > base:
|
||||
let (slot, tdesc) = j.traceStack.pop()
|
||||
let t = head(slot[])
|
||||
if isStamped(t):
|
||||
let v = denseIdx(t)
|
||||
gCap.edges.add (int64(u) shl 32) or int64(v)
|
||||
if gCap.recs.d[v].sccOf < 0 and v < gCap.recs.d[u].lowlink:
|
||||
gCap.recs.d[u].lowlink = v
|
||||
else:
|
||||
let childBase = j.traceStack.len
|
||||
let v = pushCell(t, tdesc)
|
||||
gCap.edges.add (int64(u) shl 32) or int64(v)
|
||||
trace(t, tdesc, j)
|
||||
gCap.frames.add TarjanFrame(u: v, base: childBase)
|
||||
else:
|
||||
orcAssert(j.traceStack.len == 0, "scan: trace stack not empty")
|
||||
s.setColor(colWhite)
|
||||
trace(s, desc, j)
|
||||
while j.traceStack.len > 0:
|
||||
let (entry, desc) = j.traceStack.pop()
|
||||
let t = head entry[]
|
||||
if t.color == colGray:
|
||||
if (loadRc(t) shr rcShift) >= 0:
|
||||
scanBlack(t, desc, j)
|
||||
else:
|
||||
t.setColor(colWhite)
|
||||
trace(t, desc, j)
|
||||
gCap.frames.len = gCap.frames.len -% 1
|
||||
if gCap.frames.len > 0:
|
||||
let pu = gCap.frames.d[gCap.frames.len -% 1].u
|
||||
if gCap.recs.d[u].lowlink < gCap.recs.d[pu].lowlink:
|
||||
gCap.recs.d[pu].lowlink = gCap.recs.d[u].lowlink
|
||||
if gCap.recs.d[u].lowlink == u:
|
||||
# u is the root of an SCC: pop the members off the Tarjan stack
|
||||
gCap.sccMemStart.add int32(gCap.sccMembers.len)
|
||||
var sum = 0
|
||||
while true:
|
||||
let w = gCap.tstack.pop()
|
||||
gCap.recs.d[w].sccOf = int32(j.nScc)
|
||||
gCap.sccMembers.add w
|
||||
sum = sum +% (gCap.recs.d[w].rcWord shr rcShift) +% 1
|
||||
if w == u: break
|
||||
gCap.sumRefs.add sum
|
||||
inc j.nScc
|
||||
|
||||
proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) =
|
||||
if s.color == col and (loadRc(s) and inRootsFlag) == 0:
|
||||
orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty")
|
||||
s.setColor(colBlack)
|
||||
j.toFree.add(s, desc)
|
||||
trace(s, desc, j)
|
||||
while j.traceStack.len > 0:
|
||||
let (entry, desc) = j.traceStack.pop()
|
||||
let t = head entry[]
|
||||
entry[] = nil
|
||||
if t.color == col and (loadRc(t) and inRootsFlag) == 0:
|
||||
j.toFree.add(t, desc)
|
||||
t.setColor(colBlack)
|
||||
trace(t, desc, j)
|
||||
# ---------------- phase 2: deadness, side arrays only ----------------
|
||||
|
||||
proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
|
||||
# YRC defers all destruction to collection time - process ALL roots through Bacon's algorithm
|
||||
# This is different from ORC which handles immediate garbage (rc == 0) directly
|
||||
proc computeDeadness(j: var GcEnv) =
|
||||
let nScc = j.nScc
|
||||
setLenZeroed gCap.internal, nScc
|
||||
setLenZeroed gCap.deadIn, nScc
|
||||
setLenZeroed gCap.sccFlags, nScc
|
||||
setLenZeroed gCap.crossOff, nScc + 1
|
||||
setLenUninit gCap.crossCursor, nScc
|
||||
# classify captured edges: internal to an SCC vs condensation cross edges
|
||||
var nCross = 0
|
||||
for i in 0 ..< gCap.edges.len:
|
||||
let e = gCap.edges.d[i]
|
||||
let su = gCap.recs.d[int32(e shr 32)].sccOf
|
||||
let sv = gCap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf
|
||||
if su == sv:
|
||||
inc gCap.internal.d[su]
|
||||
else:
|
||||
inc gCap.crossOff.d[su]
|
||||
inc nCross
|
||||
var total = 0'i32
|
||||
for s in 0 ..< nScc:
|
||||
let c = gCap.crossOff.d[s]
|
||||
gCap.crossOff.d[s] = total
|
||||
gCap.crossCursor.d[s] = total
|
||||
total = total +% c
|
||||
gCap.crossOff.d[nScc] = total
|
||||
setLenUninit gCap.crossTgt, nCross
|
||||
for i in 0 ..< gCap.edges.len:
|
||||
let e = gCap.edges.d[i]
|
||||
let su = gCap.recs.d[int32(e shr 32)].sccOf
|
||||
let sv = gCap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf
|
||||
if su != sv:
|
||||
gCap.crossTgt.d[gCap.crossCursor.d[su]] = sv
|
||||
inc gCap.crossCursor.d[su]
|
||||
# cells that stay registered as roots (partial collection) count as
|
||||
# externally referenced: the roots buffer itself points at them
|
||||
for mi in 0 ..< gCap.sccMembers.len:
|
||||
let m = gCap.sccMembers.d[mi]
|
||||
if (loadRc(gCap.recs.d[m].cell) and inRootsFlag) != 0:
|
||||
let s = gCap.recs.d[m].sccOf
|
||||
gCap.sccFlags.d[s] = gCap.sccFlags.d[s] or flagForcedLive
|
||||
# deadness over the condensation. Tarjan emits sinks first, so higher SCC
|
||||
# ids are sources and every cross edge goes from a higher id to a lower
|
||||
# one: one reverse scan settles everything.
|
||||
for s in countdown(nScc - 1, 0):
|
||||
let ext = gCap.sumRefs.d[s] -% gCap.internal.d[s] -% gCap.deadIn.d[s]
|
||||
when logOrc:
|
||||
cfprintf(cstderr, "[scc %ld] members %ld sumRefs %ld internal %ld deadIn %ld ext %ld forced %ld\n",
|
||||
s, gCap.sccMemStart.d[s+1] - gCap.sccMemStart.d[s], gCap.sumRefs.d[s],
|
||||
gCap.internal.d[s], gCap.deadIn.d[s], ext, int(gCap.sccFlags.d[s]))
|
||||
if (gCap.sccFlags.d[s] and flagForcedLive) == 0 and ext == 0:
|
||||
gCap.sccFlags.d[s] = gCap.sccFlags.d[s] or flagDead
|
||||
inc j.nDeadScc
|
||||
for k in gCap.crossOff.d[s] ..< gCap.crossOff.d[s+1]:
|
||||
inc gCap.deadIn.d[gCap.crossTgt.d[k]]
|
||||
else:
|
||||
# a live SCC keeps everything it points to alive
|
||||
for k in gCap.crossOff.d[s] ..< gCap.crossOff.d[s+1]:
|
||||
let t = gCap.crossTgt.d[k]
|
||||
gCap.sccFlags.d[t] = gCap.sccFlags.d[t] or flagForcedLive
|
||||
|
||||
# ---------------- phase 3: commit ----------------
|
||||
|
||||
proc commitDead(j: var GcEnv) =
|
||||
init j.toFree
|
||||
template deadCell(t: Cell): bool =
|
||||
isStamped(t) and (gCap.sccFlags.d[gCap.recs.d[denseIdx(t)].sccOf] and flagDead) != 0
|
||||
let allDead = j.nDeadScc == j.nScc
|
||||
for s in 0 ..< j.nScc:
|
||||
if (gCap.sccFlags.d[s] and flagDead) != 0:
|
||||
for mi in gCap.sccMemStart.d[s] ..< gCap.sccMemStart.d[s+1]:
|
||||
let m = gCap.sccMembers.d[mi]
|
||||
let cell = gCap.recs.d[m].cell
|
||||
let desc = gCap.recs.d[m].desc
|
||||
j.toFree.add(cell, desc)
|
||||
# nil every slot so the destructor cannot dec these edges again;
|
||||
# references to survivors are decremented for real, references into
|
||||
# the dead group die with the group (already accounted by deadIn)
|
||||
orcAssert(j.traceStack.len == 0, "commitDead: trace stack not empty")
|
||||
trace(cell, desc, j)
|
||||
if allDead:
|
||||
# everything captured dies: no survivor can occur, just nil
|
||||
while j.traceStack.len > 0:
|
||||
let (slot, _) = j.traceStack.pop()
|
||||
slot[] = nil
|
||||
else:
|
||||
while j.traceStack.len > 0:
|
||||
let (slot, _) = j.traceStack.pop()
|
||||
let t = head(slot[])
|
||||
slot[] = nil
|
||||
if not deadCell(t):
|
||||
trialDec(t)
|
||||
when sizeof(int) != 8:
|
||||
# no epoch in the stamp: clear them while all cells are still alive
|
||||
for i in 0 ..< gCap.recs.len:
|
||||
gCap.recs.d[i].cell.rootIdx = 0
|
||||
for i in 0 ..< j.toFree.len:
|
||||
when orcLeakDetector:
|
||||
writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1])
|
||||
free(j.toFree.d[i][0], j.toFree.d[i][1])
|
||||
j.freed = j.toFree.len
|
||||
deinit j.toFree
|
||||
|
||||
proc collectCyclesImpl(j: var GcEnv; lowMark: int) =
|
||||
# All destruction is deferred to collection time: plain rc==0 garbage in
|
||||
# the roots buffer forms singleton SCCs with external count 0 and is freed
|
||||
# by the same machinery as the cycles.
|
||||
if lockState == Collecting:
|
||||
return
|
||||
lockState = Collecting
|
||||
@@ -375,39 +614,30 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) =
|
||||
for i in countdown(last, lowMark):
|
||||
writeCell("root", roots.d[i][0], roots.d[i][1])
|
||||
|
||||
# Process all roots through markGray (Bacon's algorithm)
|
||||
bumpEpoch()
|
||||
init j.traceStack
|
||||
prepareCapture()
|
||||
j.nScc = 0
|
||||
|
||||
for i in countdown(last, lowMark):
|
||||
markGray(roots.d[i][0], roots.d[i][1], j)
|
||||
capture(roots.d[i][0], roots.d[i][1], j)
|
||||
gCap.sccMemStart.add int32(gCap.sccMembers.len) # sentinel
|
||||
j.touched = gCap.recs.len
|
||||
|
||||
var colToCollect = colWhite
|
||||
if j.rcSum == j.edges:
|
||||
# Short-cut: we know everything is garbage
|
||||
colToCollect = colGray
|
||||
j.keepThreshold = true
|
||||
else:
|
||||
# Normal scan phase
|
||||
for i in countdown(last, lowMark):
|
||||
scan(roots.d[i][0], roots.d[i][1], j)
|
||||
# Unregister the processed roots before computing deadness: only cells that
|
||||
# STAY registered (below lowMark, partial collection) count as externally
|
||||
# referenced by the roots buffer. Doing this before freeing anything also
|
||||
# ensures a nested collectCycles() (triggered from a destructor) cannot
|
||||
# access freed cells.
|
||||
for i in lowMark ..< roots.len:
|
||||
rcClearFlag(roots.d[i][0], inRootsFlag)
|
||||
roots.len = lowMark
|
||||
|
||||
# Collect phase: free all garbage objects
|
||||
init j.toFree
|
||||
for i in 0 ..< roots.len:
|
||||
let s = roots.d[i][0]
|
||||
rcClearFlag(s, inRootsFlag)
|
||||
collectColor(s, roots.d[i][1], colToCollect, j)
|
||||
computeDeadness(j)
|
||||
commitDead(j)
|
||||
j.keepThreshold = j.freed == j.touched and j.touched > 0
|
||||
|
||||
# Clear roots before freeing to prevent nested collectCycles() from accessing freed cells
|
||||
roots.len = 0
|
||||
|
||||
# Free all collected objects
|
||||
# Destructors must not call nimDecRefIsLastCyclicStatic (add to toDec) during this phase
|
||||
for i in 0 ..< j.toFree.len:
|
||||
let s = j.toFree.d[i][0]
|
||||
when orcLeakDetector:
|
||||
writeCell("CYCLIC OBJECT FREED", s, j.toFree.d[i][1])
|
||||
free(s, j.toFree.d[i][1])
|
||||
j.freed = j.freed +% j.toFree.len
|
||||
deinit j.toFree
|
||||
deinit j.traceStack
|
||||
|
||||
when defined(nimOrcStats):
|
||||
var freedCyclicObjects {.threadvar.}: int
|
||||
@@ -420,8 +650,7 @@ proc collectCycles() =
|
||||
if roots.len >= rootsThreshold and mayRunCycleCollect():
|
||||
let nRoots = roots.len
|
||||
var j: GcEnv
|
||||
init j.traceStack
|
||||
collectCyclesBacon(j, 0)
|
||||
collectCyclesImpl(j, 0)
|
||||
if roots.len == 0 and roots.d != nil:
|
||||
deinit roots
|
||||
when not defined(nimStressOrc):
|
||||
@@ -444,7 +673,6 @@ proc collectCycles() =
|
||||
cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld\n", j.freed, rootsThreshold)
|
||||
when defined(nimOrcStats):
|
||||
inc freedCyclicObjects, j.freed
|
||||
deinit j.traceStack
|
||||
|
||||
when defined(nimOrcStats):
|
||||
type
|
||||
@@ -458,10 +686,7 @@ proc GC_runOrc* =
|
||||
mergePendingRoots()
|
||||
if roots.len > 0 and mayRunCycleCollect():
|
||||
var j: GcEnv
|
||||
init j.traceStack
|
||||
collectCyclesBacon(j, 0)
|
||||
deinit j.traceStack
|
||||
roots.len = 0
|
||||
collectCyclesImpl(j, 0)
|
||||
when logOrc: orcAssert roots.len == 0, "roots not empty!"
|
||||
|
||||
proc GC_enableOrc*() =
|
||||
@@ -482,10 +707,7 @@ proc GC_partialCollect*(limit: int) =
|
||||
mergePendingRoots()
|
||||
if roots.len > limit and mayRunCycleCollect():
|
||||
var j: GcEnv
|
||||
init j.traceStack
|
||||
collectCyclesBacon(j, limit)
|
||||
deinit j.traceStack
|
||||
roots.len = limit
|
||||
collectCyclesImpl(j, limit)
|
||||
|
||||
proc GC_fullCollect* =
|
||||
GC_runOrc()
|
||||
|
||||
Reference in New Issue
Block a user