Compare commits

..

1 Commits

Author SHA1 Message Date
Araq
0aa486753b IC: recompile batches in one process to speed things up 2026-07-20 21:06:58 +02:00
24 changed files with 366 additions and 831 deletions

View File

@@ -3143,12 +3143,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
localError(p.config, e.info,
"for --mm:arc|atomicArc|orc 'deepcopy' support has to be enabled with --deepcopy:on")
let typ = e[1].typ.skipTypes({tyVar, tyRef, tyGenericInst, tyTypeDesc,
tyAlias, tyInferred, tySink, tyLent, tyOwned})
if hasDisabledAsgn(p.module.g.graph, typ):
localError(p.config, e.info,
"'deepCopy' is not available for type <" & typeToString(typ) & ">")
let x = if e[1].kind in {nkAddr, nkHiddenAddr}: e[1][0] else: e[1]
var a = initLocExpr(p, x)
var b = initLocExpr(p, e[2])

View File

@@ -741,6 +741,130 @@ proc computeSCCs(c: DepContext): seq[seq[int]] =
if work.len > 0:
lowlink[work[^1].v] = min(lowlink[work[^1].v], lowlink[v])
proc nodeIsStale(c: DepContext; node: Node): bool =
## Driver-time estimate of "this module's `nim m` will (re)run this round",
## matching what nifmake decides from file mtimes. The driver runs BEFORE the
## nifmake pass, so the `.p.nif` parsed file still reflects the *previous* run
## (runNifler even deletes a source-newer one); the stable signal available
## here is the source `.nim` against the last semmed NIF (`.s.bif`).
##
## Only the estimate's *precision* matters, never correctness: nifmake still
## mtime-checks every emitted rule, so an over-estimate produces a batch it
## then skips, and an under-estimate leaves a singleton rule it rebuilds
## anyway (see computeBatches).
for f in node.files: # main file + its includes
let semmed = c.semmedFile(f)
if not fileExists(semmed): return true # never semmed (cold)
if not fileExists(c.parsedFile(f)): return true # parsed dropped by runNifler
let semmedTime = getLastModificationTime(semmed)
if fileExists(f.nimFile) and getLastModificationTime(f.nimFile) > semmedTime:
return true # edited since last sem
result = false
proc computeBatches(c: DepContext): seq[seq[int]] =
## Group SCCs into build batches for the frontend `nim m` rules. A batch is a
## *dirty module together with the transitive closure of its users* (the
## modules that import it, directly or transitively). Rationale: editing a
## module flips its cookies and forces every dependent to re-sem, and that
## re-sem set is a deep import chain that nifmake schedules one depth-level at
## a time — so `--parallel` never speeds it up while every process pays full
## startup + import-NIF-closure load. Compiling the whole closure in one
## `nim m --icGroup` process (source-compiling every member, resolving the
## imports in memory) amortizes that overhead across the affected set.
##
## Safety (why a wrong dirty estimate cannot corrupt output):
## * every rule still declares its real inputs/outputs, so nifmake skips a
## batch whose files are all fresh and rebuilds a missed module through its
## own singleton rule;
## * batches are a partition of the SCC condensation, so each NIF keeps a
## single writer (no two processes mint divergent instance ids into one NIF);
## * the closure is convex in the condensation DAG (any node on an import path
## between two batch members also reaches the seed, so it is in the batch),
## so dropping intra-batch edges cannot create a batch<->external nifmake
## cycle.
##
## `-d:icNoBatch` restores pure per-SCC grouping (the pre-batching behaviour).
let sccs = computeSCCs(c)
if isDefined(c.config, "icNoBatch") or sccs.len == 0:
return sccs
let numSccs = sccs.len
var sccOf = newSeq[int](c.nodes.len)
for sid, comp in sccs:
for nodeIdx in comp: sccOf[nodeIdx] = sid
# SCCs that must never be merged into a batch: system.nim's folded closure and
# the `--import:` implicit modules. They stay their own groups exactly as
# today, so the cmdM bootstrap that NIF-loads `system` is untouched; on a cold
# build the rest of the program forms one big batch that NIF-loads system
# rather than recompiling it in every process.
var pinned = newSeq[bool](numSccs)
if c.systemNodeId >= 0: pinned[sccOf[c.systemNodeId]] = true
for id in c.implicitNodeIds: pinned[sccOf[id]] = true
# Condensation edges. `rev[b]` = SCCs that import b (its users); `fwd` is used
# only to union connected dirty SCCs into one component.
var fwd = newSeq[HashSet[int]](numSccs)
var rev = newSeq[HashSet[int]](numSccs)
for v in 0 ..< c.nodes.len:
for w in c.nodes[v].deps:
let a = sccOf[v]
let b = sccOf[w]
if a != b:
fwd[a].incl b
rev[b].incl a
# Seed = a non-pinned SCC with any stale member (edited/missing NIF).
# mustRecompile = seeds plus every SCC that transitively imports a seed,
# walked over reverse condensation edges (a seed's users).
var must = newSeq[bool](numSccs)
var queue: seq[int] = @[]
for sid, comp in sccs:
if pinned[sid]: continue
for nodeIdx in comp:
if nodeIsStale(c, c.nodes[nodeIdx]):
must[sid] = true
queue.add sid
break
while queue.len > 0:
let s = queue.pop()
for u in rev[s]:
if not must[u] and not pinned[u]:
must[u] = true
queue.add u
# Union-Find the mustRecompile SCCs connected by any condensation edge; each
# connected component becomes one batch, every other SCC stays its own batch.
var parent = newSeq[int](numSccs)
for i in 0 ..< numSccs: parent[i] = i
proc find(x: int): int =
var x = x
while parent[x] != x:
parent[x] = parent[parent[x]] # path halving
x = parent[x]
x
for a in 0 ..< numSccs:
if not must[a]: continue
for b in fwd[a]:
if must[b]:
let ra = find(a)
let rb = find(b)
if ra != rb: parent[ra] = rb
# Assemble: one entry per batch, in first-seen (reverse-topological) SCC order.
# A merged batch is keyed by its union-find root (a `must` sid); a singleton is
# keyed by its own sid (a non-`must` sid) — the two key spaces are disjoint, so
# they never collide.
result = @[]
var batchIndex = initTable[int, int]()
for sid in 0 ..< numSccs:
let key = if must[sid]: find(sid) else: sid
let bi = batchIndex.getOrDefault(key, -1)
if bi == -1:
batchIndex[key] = result.len
result.add sccs[sid]
else:
for nodeIdx in sccs[sid]: result[bi].add nodeIdx
proc computeForwardedArgs(c: DepContext): seq[string] =
## Config/define forwarding shared by the frontend (`nim m`) and backend
## (`nim nifc`) child commands. Depends only on the driver's config, not on
@@ -868,21 +992,28 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
# Build rules for semantic checking (nim m).
#
# Modules are grouped into strongly-connected components: a module that is not
# in an import cycle is its own singleton group and compiles in its own
# `nim m <mod>` invocation as before. A cycle (A imports B, B imports A) cannot
# be ordered for separate per-module compilation, so the whole component is
# handed to a single `nim m` invocation: the first member is the project file,
# every member is passed via `--icGroup:<path>` so the compiler compiles them
# all from source in one process (resolving the recursion in-memory) and writes
# a NIF for each. Only dependencies *outside* the component become build-graph
# inputs — intra-component edges are produced by this very rule and listing
# them would reintroduce the cycle nifmake just rejected.
let sccs = computeSCCs(c)
var sccOf = newSeq[int](c.nodes.len)
for sccId, comp in sccs:
for nodeIdx in comp: sccOf[nodeIdx] = sccId
for comp in sccs:
# Modules are grouped into BATCHES (computeBatches). A batch is one or more
# strongly-connected components merged together, handed to a single `nim m`
# invocation: the first member is the project file, every member is passed via
# `--icGroup:<path>` so the compiler compiles them all from source in one
# process (resolving imports/recursion in-memory) and writes a NIF for each.
# Only dependencies *outside* the batch become build-graph inputs — intra-batch
# edges are produced by this very rule and listing them would reintroduce a
# cycle nifmake would reject.
#
# Two things drive a multi-module batch:
# * an import CYCLE (A imports B, B imports A) cannot be ordered for separate
# per-module compilation, so its whole SCC is one group (as before); and
# * a DIRTY module together with its transitive USERS — editing a module
# forces every dependent to re-sem, a serial import chain that per-process
# fan-out cannot speed up; batching compiles the whole affected closure in
# one process. A module that is neither in a cycle nor in a dirty closure
# stays its own singleton `nim m <mod>` rule.
let batches = computeBatches(c)
var batchOf = newSeq[int](c.nodes.len)
for batchId, comp in batches:
for nodeIdx in comp: batchOf[nodeIdx] = batchId
for comp in batches:
# Representative (project file for this invocation) = smallest node id, so a
# component containing the root (node 0) is driven by the root.
var members = comp
@@ -932,7 +1063,7 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
var stack: seq[int] = @[]
for m in members:
for depIdx in c.nodes[m].deps:
if sccOf[depIdx] != sccOf[members[0]]: stack.add depIdx
if batchOf[depIdx] != batchOf[members[0]]: stack.add depIdx
var visited = initHashSet[int]()
while stack.len > 0:
let n = stack.pop()
@@ -946,7 +1077,7 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
var directDeps = initHashSet[string]()
for m in members:
for depIdx in c.nodes[m].deps:
if sccOf[depIdx] == sccOf[m]: continue # intra-component edge
if batchOf[depIdx] == batchOf[m]: continue # intra-batch edge
let depName = c.nodes[depIdx].files[0].modname
directDeps.incl depName
let depFile =

View File

@@ -579,6 +579,19 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
# first NIF import is processed. See finalizeLoadedModules.
finalizeLoadedModules(graph)
discard graph.compilePipelineModule(projectFile, {sfMainModule})
# A batch (`--icGroup`) may hold several independent "top" modules that are
# not all reachable by import from the representative project file: a dirty
# module together with its users forms a DAG, not a cycle, so descending
# from one rep need not touch every member. Compile each remaining member
# explicitly so it writes its NIF. compilePipelineModule is idempotent
# (returns the cached module for one already reached through the rep's
# imports), and an unreached member is in `icGroup` so it is source-compiled
# here rather than NIF-loaded; resolving it pulls in its in-batch deps on
# demand, so no explicit ordering is needed.
for path in graph.config.icGroup:
let memberIdx = fileInfoIdx(graph.config, AbsoluteFile path)
if memberIdx != projectFile:
discard graph.compilePipelineModule(memberIdx, {})
else:
graph.compilePipelineSystemModule()
discard graph.compilePipelineModule(projectFile, {sfMainModule})

View File

@@ -108,16 +108,6 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
markUsed(c, a.info, a[0].sym)
template isAutoReturnType(t: PType): bool =
# `auto` return types are copied and marked so they are not generic params.
t.kind == tyAnything and tfRetType in t.flags
template isUnresolvedAutoReturnType(c: PContext; t: PType): bool =
# During return-type inference a recursive call has the routine's exact
# `auto` placeholder type. It contributes no type information of its own.
c.p != nil and c.p.owner != nil and c.p.owner.typ != nil and
c.p.owner.typ.returnType == t and isAutoReturnType(t)
proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
if arg.typ.isNil:
localError(c.config, arg.info, "expression has no type: " &
@@ -135,10 +125,6 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
if sameType(ch.typ.skipTypes({tyVar, tyLent}), formal):
return ch
typeMismatch(c.config, info, formal, arg.typ, arg)
elif isUnresolvedAutoReturnType(c, arg.typ):
# A concrete sibling branch supplies the missing type for this branch.
result = arg
changeType(c, result, formal, check=true)
else:
result = indexTypesMatch(c, formal, arg.typ, arg)
if result == nil:
@@ -172,10 +158,8 @@ proc commonType*(c: PContext; x, y: PType): PType =
var a = skipTypes(x, {tyGenericInst, tyAlias, tySink})
var b = skipTypes(y, {tyGenericInst, tyAlias, tySink})
result = x
# Recursive calls cannot contribute to their own `auto` return type, so let
# the other branch determine the common type when it has concrete evidence.
if a.kind in {tyUntyped, tyNil} or isUnresolvedAutoReturnType(c, a): result = y
elif b.kind in {tyUntyped, tyNil} or isUnresolvedAutoReturnType(c, b): result = x
if a.kind in {tyUntyped, tyNil}: result = y
elif b.kind in {tyUntyped, tyNil}: result = x
elif a.kind == tyTyped: result = a
elif b.kind == tyTyped: result = b
elif a.kind == tyTypeDesc:

View File

@@ -924,11 +924,6 @@ proc semResolvedCall(c: PContext, x: var TCandidate,
result[0] = newSymNode(finalCallee, getCallLineInfo(result[0]))
if finalCallee.magic notin {mArrGet, mArrPut}:
result.typ = finalCallee.typ.returnType
# Remember that this body contains a self-call still sharing its unresolved
# `auto` placeholder; a later concrete return must resolve that placeholder.
if c.p != nil and result.typ != nil and finalCallee == c.p.owner and
isAutoReturnType(result.typ):
c.p.hasUnresolvedAutoCall = true
updateDefaultParams(c, result)
proc canDeref(n: PNode): bool {.inline.} =

View File

@@ -43,7 +43,6 @@ type
mapping*: SymMapping
caseContext*: seq[tuple[n: PNode, idx: int]]
localBindStmts*: seq[PNode]
hasUnresolvedAutoCall*: bool # a self-call still uses the `auto` return placeholder
TMatchedConcept* = object
candidateType*: PType

View File

@@ -2125,15 +2125,6 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
internalAssert c.config, c.p.resultSym != nil
# Make sure the type is valid for the result variable
typeAllowedCheck(c, n.info, rhsTyp, skResult)
# Earlier self-calls retain the old placeholder pointer. Resolve it
# in place as an alias before the routine switches to the concrete
# type, so those already-typed calls see the inferred type too.
if c.p.hasUnresolvedAutoCall and not rhsTyp.isMetaType and
isAutoReturnType(lhs.sym.typ):
let resolved = newTypeS(tyAlias, c)
rawAddSon(resolved, rhsTyp)
assignType(lhs.sym.typ, resolved)
c.p.hasUnresolvedAutoCall = false
lhs.typ = rhsTyp
c.p.resultSym.typ = rhsTyp
c.p.owner.typ.setReturnType rhsTyp
@@ -2205,11 +2196,7 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
" flags=", c.p.resultSym.typ.flags,
" uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item,
" state=", c.p.resultSym.typ.state
# With no concrete return, the recursive placeholder is still circular.
if c.p.hasUnresolvedAutoCall:
localError(c.config, c.p.resultSym.info, errCannotInferReturnType %
c.p.owner.name.s)
elif isEmptyType(result.typ):
if isEmptyType(result.typ):
# we inferred a 'void' return type:
c.p.resultSym.typ = errorType(c)
c.p.owner.typ.setReturnType nil

View File

@@ -36,12 +36,7 @@ type
rc: int # the object header is now a single RC field.
# we could remove it in non-debug builds for the 'owned ref'
# design but this seems unwise.
when defined(gcYrc):
rootIdx: int64 # the collector's claim word: collection tag or epoch
# stamp packed with the dense capture index. Explicitly
# 64 bit so that 32-bit targets run the same concurrent
# claim and epoch-stamp algorithms
elif defined(gcOrc):
when defined(gcOrc) or defined(gcYrc):
rootIdx: int # thanks to this we can delete potential cycle roots
# in O(1) without doubly linked lists
when defined(nimArcDebug) or defined(nimArcIds):

View File

@@ -75,12 +75,10 @@
#
# 1. Capture: one Tarjan SCC traversal over everything reachable from the
# candidate roots. Node -> dense index lookup is O(1) without hashing:
# the spare `rootIdx` header word (a full int64 on every target so the
# concurrent claim and epoch-stamp packing work identically on 32-bit
# archs) is CAS-claimed with the collection's tag packed with the
# discovery index; stale tags of retired collections never need
# clearing. Per-cell data lives in one record array indexed by that
# discovery index; per-SCC data in one record array indexed by SCC id.
# the spare `rootIdx` header word is CAS-claimed with the collection's
# tag packed with the discovery index; stale tags of retired
# collections never need clearing. Everything else lives in side
# arrays (SoA layout).
#
# 2. Deadness: pure array work on the captured SCC condensation, no heap
# access. An SCC is garbage iff it has no references beyond its internal
@@ -152,17 +150,10 @@ const
type
TraceProc = proc (p, env: pointer) {.nimcall, gcsafe, raises: [].}
# The write barrier's incRef normally goes through a lock-free per-stripe
# queue (drained by the collector), matching the deferred decRef. Define
# `nimYrcDirectIncs` to bypass that queue and apply incRefs directly with an
# atomic RMW instead — simpler, but the collector then observes every incRef
# through commit-time rc validation rather than the queue peek.
const useIncQueue = not defined(nimYrcDirectIncs)
# With lock-free ref assignments, rc words are mutated concurrently with the
# collector (direct atomic incRefs), so all collector-side rc accesses must
# be atomic whenever threads exist.
const useAtomicRc = not useIncQueue or hasThreadSupport
const useAtomicRc = defined(nimYrcAtomicIncs) or hasThreadSupport
when useAtomicRc:
template color(c): untyped = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) and colorMask
@@ -274,28 +265,13 @@ type
CaptureRec = object
## per captured cell, position == Tarjan discovery index; one record so
## a node costs a single append during the DFS. Kept at 32 bytes: this
## is the hottest array in the capture DFS, so cold data that is read at
## most once per survivor (e.g. the survival age) stays in a side array.
## a node costs a single append during the DFS
cell: Cell
desc: PNimTypeV2
rcWord: int # rc word as captured
lowlink: int32
sccOf: int32 # -1 while the cell is on the Tarjan stack
SccRec = object
## per SCC of the condensation; the deadness pass reads all of these
## fields together, so one record per SCC beats parallel arrays. The
## array carries a sentinel record at [nScc]: memStart/crossOff are
## prefix offsets, an SCC's slice is [s]..<[s+1].
sumRefs: int # sum of member reference counts
internal: int # number of intra-SCC edges
deadIn: int # number of edges from dead SCCs
memStart: int32 # offset into sccMembers
crossOff: int32 # offset into crossTgt (cross edges by source)
crossCursor: int32
flags: uint8
CaptureBufs = object
## side structure of a collection; per collector thread (gCap is a
## threadvar) and persistent across collections, so that frequent
@@ -304,9 +280,15 @@ type
tstack: RawSeq[int32]
frames: RawSeq[TarjanFrame]
edges: RawSeq[int64] # (u shl 32) or v, dense indices
sccs: RawSeq[SccRec]
sccMemStart: RawSeq[int32]
sccMembers: RawSeq[int32]
sumRefs: RawSeq[int] # per SCC: sum of member reference counts
internal: RawSeq[int] # per SCC: number of intra-SCC edges
deadIn: RawSeq[int] # per SCC: number of edges from dead SCCs
sccFlags: RawSeq[uint8]
crossOff: RawSeq[int32] # condensation cross edges, bucketed by source
crossTgt: RawSeq[int32]
crossCursor: RawSeq[int32]
crossPend: CellSeq[Cell] # edge targets owned by other active collections
prunedSrc: RawSeq[int32] # dense indices of cells with pruned out-edges
ages: RawSeq[int32] # per captured cell: captures survived so far
@@ -349,22 +331,25 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} =
# long-lived live structures are traced once per epoch instead of once per
# collection. Roots always bypass the stamp: every death has a dec-witness
# that gets registered, and registered cells are always scanned as roots.
const
MaxPar {.intdefine.} = 8 # max concurrent collections
ParSlots = MaxPar
const MaxPar {.intdefine.} = 8 # max concurrent collections
when sizeof(int) == 8:
const ParSlots = MaxPar
else:
const ParSlots = 1 # no room to pack tags: single collector
var
gMergeLock: Lock # protects the tag slots + orphaned roots
gActiveTags: array[ParSlots, int64] # 0 = free slot
gActiveTags: array[ParSlots, int] # 0 = free slot
gSlotPhase: array[ParSlots, int] # 0 idle, 1 capturing, 2 committing
gSoloCapture: int # a solo collection is in its capture phase
gTagCounter: int64
gMyTag {.threadvar.}: int64
gTagCounter: int
gMyTag {.threadvar.}: int
gMySlot {.threadvar.}: int
gAmSolo {.threadvar.}: bool
gEpoch: int # advanced every YrcEpochLen collections
gCollectionCounter: int
gMyEpochStamp {.threadvar.}: int64 # this collection's epoch, as a stamp word
gMyEpochStamp {.threadvar.}: int # this collection's epoch, as a stamp word
gWaitLock: Lock # pairs gWaitCond's wait/broadcast; leaf
gWaitCond: Cond # signaled on capture-end and collection-finish
@@ -372,22 +357,13 @@ const
SpinBeforePark = 4000
YrcEpochLen {.intdefine.} = 64 # collections per epoch; bounds how long a
# stale "proven live" stamp defers rescans.
# Short epochs (≲ 4) resonate with the
# adaptive threshold: pruned collections are
# cheap, so collections speed up — and with
# them the epoch clock, forcing full re-traces
# MORE often than no stamps at all. 64 sits
# clear of that. A WORK-based clock (advance
# per N cells traced, decoupled from the
# collection count) was tried to kill the
# resonance at its root; it lost on webbench
# at every threshold — both slower (~65-90 vs
# ~50 ms) and ~30% more float — because
# pruning shrinks trace work, so the clock
# stalls exactly when a long-lived web should
# be re-examined. The resonance only bites
# below ~4, which 64 already avoids, so there
# was no live problem to trade float for.
# Short epochs resonate badly with the
# adaptive threshold: pruned collections
# are cheap, so collections speed up — and
# with them the epoch clock, forcing full
# re-traces MORE often than no stamps at
# all. (A work-based clock would fix this
# properly.)
YrcPromoteAge {.intdefine.} = 3 # captures a cell must survive before its
# stamp prunes. Die-young data must never
# be deferred: torcbench-style lists are
@@ -400,9 +376,9 @@ const
epochMask = 0x3FFFFFFF
# stamp layout: high word = epochBase|epoch, low word = survival age
template epochStamp(e: int): int64 = int64(epochBase or (e and epochMask)) shl 32
template stampAge(w: int64): int = int(w and 0xFFFFFFFF)
template isEpochStamp(w: int64): bool = (w shr 32) >= epochBase
template epochStamp(e: int): int = (epochBase or (e and epochMask)) shl 32
template stampAge(w: int): int = w and 0xFFFFFFFF
template isEpochStamp(w: int): bool = (w shr 32) >= epochBase
template parkUntil(cond: untyped) =
## Bounded spin (collections transition in microseconds when the system is
@@ -432,20 +408,24 @@ proc anySlotFree(): bool {.inline.} =
if atomicLoadN(addr gActiveTags[sl], ATOMIC_ACQUIRE) == 0:
return true
template isStamped(c: Cell): bool =
# "stamped" means: claimed by THIS collection. A relaxed load suffices:
# only this thread ever stores gMyTag, and any stale read of a foreign
# value routes into claimCell which re-validates with acquire + CAS.
(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == gMyTag
template denseIdx(c: Cell): int32 =
int32(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) and 0xFFFFFFFF)
when sizeof(int) == 8:
template isStamped(c: Cell): bool =
# "stamped" means: claimed by THIS collection. A relaxed load suffices:
# only this thread ever stores gMyTag, and any stale read of a foreign
# value routes into claimCell which re-validates with acquire + CAS.
(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == gMyTag
template denseIdx(c: Cell): int32 =
int32(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) and 0xFFFFFFFF)
proc isActiveTag(t: int64): bool {.inline.} =
result = false
if t != 0:
for s in 0 ..< ParSlots:
if atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) == t:
return true
proc isActiveTag(t: int): bool {.inline.} =
result = false
if t != 0:
for s in 0 ..< ParSlots:
if atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) == t:
return true
else:
template isStamped(c: Cell): bool = c.rootIdx != 0
template denseIdx(c: Cell): int32 = int32(c.rootIdx -% 1)
type
Stripe = object
@@ -528,7 +508,9 @@ proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} =
let h = head(p)
when optimizedOrc:
if cyclic: h.rc = h.rc or maybeCycle
when useIncQueue:
when defined(nimYrcAtomicIncs):
discard atomicFetchAdd(addr h.rc, rcIncrement, ATOMIC_ACQ_REL)
else:
# LOCK-FREE producer: reserve, then publish with an atomic exchange
# (see the Stripe declaration for why an RMW and not a release store)
let idx = getStripeIdx()
@@ -540,8 +522,6 @@ proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} =
# collection observes it through the commit-time rc validation.
# Buffering resumes once the next drain resets the queue.
trialInc(h)
else:
discard atomicFetchAdd(addr h.rc, rcIncrement, ATOMIC_ACQ_REL)
when defined(nimOrcStats):
var
@@ -563,7 +543,7 @@ proc registerLocal(c: Cell; desc: PNimTypeV2) {.inline.} =
## live by every collection (deadness checks the flag), so no buffer
## entry can ever dangle.
if rcTestSetFlag(c, inRootsFlag):
when defined(nimOrcStats):
when defined(nimOrcStats) and sizeof(int) == 8:
let st = atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED)
if st == 0: bumpStat gStatRegFresh
elif gMyTag != 0 and (st shr 32) == gMyTag: bumpStat gStatRegSelf
@@ -584,7 +564,7 @@ proc drainStripe(i: int) =
## a producer is publishing into. Reservations past QueueSize never
## wrote anything, so an overflowed counter is hard-reset.
withLock stripes[i].consumerLock:
when useIncQueue:
when not defined(nimYrcAtomicIncs):
# apply pending incs FIRST: an inc entry means the rc word
# under-counts, so its cell must be raised before decs can free
var consumedInc = 0
@@ -701,9 +681,15 @@ proc prepareCapture() =
init gCap.tstack
init gCap.frames
init gCap.edges
init gCap.sccs
init gCap.sccMemStart
init gCap.sccMembers
init gCap.sumRefs
init gCap.internal
init gCap.deadIn
init gCap.sccFlags
init gCap.crossOff
init gCap.crossTgt
init gCap.crossCursor
init gCap.crossPend
init gCap.prunedSrc
init gCap.ages
@@ -712,8 +698,9 @@ proc prepareCapture() =
gCap.tstack.len = 0
gCap.frames.len = 0
gCap.edges.len = 0
gCap.sccs.len = 0
gCap.sccMemStart.len = 0
gCap.sccMembers.len = 0
gCap.sumRefs.len = 0
gCap.crossPend.len = 0
gCap.prunedSrc.len = 0
gCap.ages.len = 0
@@ -721,46 +708,25 @@ proc prepareCapture() =
# rc is captured without the flag bits: the collector itself toggles
# inRootsFlag between capture and commit, which must not look like a
# mutation to the commit-time rc validation.
proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
pruneLive: bool): int32 =
## Dense index if this collection owns `c` (claiming and registering it
## if it was unclaimed), -1 if another ACTIVE collection owns it, or
## -2 if `pruneLive` and the cell was proven live in the current epoch
## (treat as an opaque live external, don't descend).
if gAmSolo:
# no other collection is (or can start) capturing: plain stores.
# This recovers the sequential capture speed of the single-collector
# design whenever collections do not actually overlap.
let old = c.rootIdx
if (old shr 32) == gMyTag:
return int32(old and 0xFFFFFFFF)
if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
stampAge(old) >= YrcPromoteAge:
return -2
let idx = cap.recs.len
c.rootIdx = (gMyTag shl 32) or int64(idx)
when defined(nimOrcStats):
bumpStat gStatCapTotal
if old != 0: bumpStat gStatCapRepeat
cap.recs.add CaptureRec(cell: c, desc: desc,
rcWord: loadRc(c) and not rcMask,
lowlink: int32(idx), sccOf: -1'i32)
cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
cap.tstack.add int32(idx)
return int32(idx)
while true:
var old = atomicLoadN(addr c.rootIdx, ATOMIC_ACQUIRE)
if (old shr 32) == gMyTag:
return int32(old and 0xFFFFFFFF)
if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
stampAge(old) >= YrcPromoteAge:
return -2
if isActiveTag(old shr 32):
return -1
let idx = cap.recs.len
if atomicCompareExchangeN(addr c.rootIdx, addr old,
(gMyTag shl 32) or int64(idx), false,
ATOMIC_ACQ_REL, ATOMIC_RELAXED):
when sizeof(int) == 8:
proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
pruneLive: bool): int32 =
## Dense index if this collection owns `c` (claiming and registering it
## if it was unclaimed), -1 if another ACTIVE collection owns it, or
## -2 if `pruneLive` and the cell was proven live in the current epoch
## (treat as an opaque live external, don't descend).
if gAmSolo:
# no other collection is (or can start) capturing: plain stores.
# This recovers the sequential capture speed of the single-collector
# design whenever collections do not actually overlap.
let old = c.rootIdx
if (old shr 32) == gMyTag:
return int32(old and 0xFFFFFFFF)
if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
stampAge(old) >= YrcPromoteAge:
return -2
let idx = cap.recs.len
c.rootIdx = (gMyTag shl 32) or idx
when defined(nimOrcStats):
bumpStat gStatCapTotal
if old != 0: bumpStat gStatCapRepeat
@@ -770,6 +736,41 @@ proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
cap.tstack.add int32(idx)
return int32(idx)
while true:
var old = atomicLoadN(addr c.rootIdx, ATOMIC_ACQUIRE)
if (old shr 32) == gMyTag:
return int32(old and 0xFFFFFFFF)
if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
stampAge(old) >= YrcPromoteAge:
return -2
if isActiveTag(old shr 32):
return -1
let idx = cap.recs.len
if atomicCompareExchangeN(addr c.rootIdx, addr old,
(gMyTag shl 32) or idx, false,
ATOMIC_ACQ_REL, ATOMIC_RELAXED):
when defined(nimOrcStats):
bumpStat gStatCapTotal
if old != 0: bumpStat gStatCapRepeat
cap.recs.add CaptureRec(cell: c, desc: desc,
rcWord: loadRc(c) and not rcMask,
lowlink: int32(idx), sccOf: -1'i32)
cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
cap.tstack.add int32(idx)
return int32(idx)
else:
proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
pruneLive: bool): int32 =
# no room to pack epoch stamps on 32 bit: pruneLive is ignored
if c.rootIdx != 0:
result = int32(c.rootIdx -% 1)
else:
result = int32(cap.recs.len)
c.rootIdx = cap.recs.len +% 1
cap.recs.add CaptureRec(cell: c, desc: desc,
rcWord: loadRc(c) and not rcMask,
lowlink: result, sccOf: -1'i32)
cap.tstack.add result
proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
## Iterative Tarjan SCC over everything reachable from `s`. A frame's
@@ -821,7 +822,7 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
cap.recs.d[pu].lowlink = cap.recs.d[u].lowlink
if cap.recs.d[u].lowlink == u:
# u is the root of an SCC: pop the members off the Tarjan stack
let memStart = int32(cap.sccMembers.len)
cap.sccMemStart.add int32(cap.sccMembers.len)
var sum = 0
while true:
let w = cap.tstack.pop()
@@ -829,17 +830,18 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
cap.sccMembers.add w
sum = sum +% (cap.recs.d[w].rcWord shr rcShift) +% 1
if w == u: break
cap.sccs.add SccRec(sumRefs: sum, memStart: memStart)
cap.sumRefs.add sum
inc j.nScc
# ---------------- phase 2: deadness, side arrays only ----------------
proc computeDeadness(j: var GcEnv; cap: ptr CaptureBufs) =
let nScc = j.nScc
# append the sentinel record ([nScc]); capture left internal/deadIn/flags
# zero-initialized and memStart valid. crossOff is filled below as a prefix
# sum, so the sentinel closes the last SCC's member and cross-edge slices.
cap.sccs.add SccRec(memStart: int32(cap.sccMembers.len))
setLenZeroed cap.internal, nScc
setLenZeroed cap.deadIn, nScc
setLenZeroed cap.sccFlags, nScc
setLenZeroed cap.crossOff, nScc + 1
setLenUninit cap.crossCursor, nScc
# classify captured edges: internal to an SCC vs condensation cross edges
var nCross = 0
for i in 0 ..< cap.edges.len:
@@ -847,58 +849,58 @@ proc computeDeadness(j: var GcEnv; cap: ptr CaptureBufs) =
let su = cap.recs.d[int32(e shr 32)].sccOf
let sv = cap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf
if su == sv:
inc cap.sccs.d[su].internal
inc cap.internal.d[su]
else:
inc cap.sccs.d[su].crossOff
inc cap.crossOff.d[su]
inc nCross
var total = 0'i32
for s in 0 ..< nScc:
let c = cap.sccs.d[s].crossOff
cap.sccs.d[s].crossOff = total
cap.sccs.d[s].crossCursor = total
let c = cap.crossOff.d[s]
cap.crossOff.d[s] = total
cap.crossCursor.d[s] = total
total = total +% c
cap.sccs.d[nScc].crossOff = total
cap.crossOff.d[nScc] = total
setLenUninit cap.crossTgt, nCross
for i in 0 ..< cap.edges.len:
let e = cap.edges.d[i]
let su = cap.recs.d[int32(e shr 32)].sccOf
let sv = cap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf
if su != sv:
cap.crossTgt.d[cap.sccs.d[su].crossCursor] = sv
inc cap.sccs.d[su].crossCursor
cap.crossTgt.d[cap.crossCursor.d[su]] = sv
inc cap.crossCursor.d[su]
# pruned out-edges taint the source SCC: pruning cannot cause a false
# "dead" (an untraced target only ever ADDS unexplained external refs),
# but a "live" verdict may lean on a stamp that went stale within the
# epoch, so validate re-registers surviving pruned SCCs
for i in 0 ..< cap.prunedSrc.len:
let s = cap.recs.d[cap.prunedSrc.d[i]].sccOf
cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagPruned
cap.sccFlags.d[s] = cap.sccFlags.d[s] or flagPruned
# cells that stay registered as roots (partial collection) count as
# externally referenced: the roots buffer itself points at them
for mi in 0 ..< cap.sccMembers.len:
let m = cap.sccMembers.d[mi]
if (loadRc(cap.recs.d[m].cell) and inRootsFlag) != 0:
let s = cap.recs.d[m].sccOf
cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagForcedLive
cap.sccFlags.d[s] = cap.sccFlags.d[s] or flagForcedLive
# deadness over the condensation. Tarjan emits sinks first, so higher SCC
# ids are sources and every cross edge goes from a higher id to a lower
# one: one reverse scan settles everything.
for s in countdown(nScc - 1, 0):
let ext = cap.sccs.d[s].sumRefs -% cap.sccs.d[s].internal -% cap.sccs.d[s].deadIn
let ext = cap.sumRefs.d[s] -% cap.internal.d[s] -% cap.deadIn.d[s]
when logOrc:
cfprintf(cstderr, "[scc %ld] members %ld sumRefs %ld internal %ld deadIn %ld ext %ld forced %ld\n",
s, cap.sccs.d[s+1].memStart - cap.sccs.d[s].memStart, cap.sccs.d[s].sumRefs,
cap.sccs.d[s].internal, cap.sccs.d[s].deadIn, ext, int(cap.sccs.d[s].flags))
if (cap.sccs.d[s].flags and flagForcedLive) == 0 and ext == 0:
cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagDead
s, cap.sccMemStart.d[s+1] - cap.sccMemStart.d[s], cap.sumRefs.d[s],
cap.internal.d[s], cap.deadIn.d[s], ext, int(cap.sccFlags.d[s]))
if (cap.sccFlags.d[s] and flagForcedLive) == 0 and ext == 0:
cap.sccFlags.d[s] = cap.sccFlags.d[s] or flagDead
inc j.nDeadScc
for k in cap.sccs.d[s].crossOff ..< cap.sccs.d[s+1].crossOff:
inc cap.sccs.d[cap.crossTgt.d[k]].deadIn
for k in cap.crossOff.d[s] ..< cap.crossOff.d[s+1]:
inc cap.deadIn.d[cap.crossTgt.d[k]]
else:
# a live SCC keeps everything it points to alive
for k in cap.sccs.d[s].crossOff ..< cap.sccs.d[s+1].crossOff:
for k in cap.crossOff.d[s] ..< cap.crossOff.d[s+1]:
let t = cap.crossTgt.d[k]
cap.sccs.d[t].flags = cap.sccs.d[t].flags or flagForcedLive
cap.sccFlags.d[t] = cap.sccFlags.d[t] or flagForcedLive
# ---------------- phase 3: validate & commit ----------------
@@ -911,9 +913,9 @@ proc markDirtyFromQueues(j: var GcEnv; cap: ptr CaptureBufs) =
let c = cp
if isStamped(c):
let s = cap.recs.d[denseIdx(c)].sccOf
cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagDirty
cap.sccFlags.d[s] = cap.sccFlags.d[s] or flagDirty
for i in 0..<NumStripes:
when useIncQueue:
when not defined(nimYrcAtomicIncs):
# lock-free peek. This one is LOAD-BEARING (an inc entry means the
# rc word under-counts a live reference), which is why the producer
# publishes with an RMW: every completed barrier's entry is globally
@@ -945,37 +947,28 @@ proc demoteTouchedDead(j: var GcEnv; cap: ptr CaptureBufs) =
## freed under a surviving reference. Targets have lower ids, so
## tainting them here demotes them (transitively) later in this loop.
for s in countdown(j.nScc - 1, 0):
if (cap.sccs.d[s].flags and flagDead) != 0:
var ok = (cap.sccs.d[s].flags and flagDirty) == 0
if (cap.sccFlags.d[s] and flagDead) != 0:
var ok = (cap.sccFlags.d[s] and flagDirty) == 0
if ok:
for mi in cap.sccs.d[s].memStart ..< cap.sccs.d[s+1].memStart:
for mi in cap.sccMemStart.d[s] ..< cap.sccMemStart.d[s+1]:
let m = cap.sccMembers.d[mi]
if (loadRc(cap.recs.d[m].cell) and not rcMask) != cap.recs.d[m].rcWord:
ok = false
break
if not ok:
cap.sccs.d[s].flags = cap.sccs.d[s].flags and not flagDead
cap.sccFlags.d[s] = cap.sccFlags.d[s] and not flagDead
inc j.nAborted
for k in cap.sccs.d[s].crossOff ..< cap.sccs.d[s+1].crossOff:
for k in cap.crossOff.d[s] ..< cap.crossOff.d[s+1]:
let t = cap.crossTgt.d[k]
if (cap.sccs.d[t].flags and flagDead) != 0:
cap.sccs.d[t].flags = cap.sccs.d[t].flags or flagDirty
let m = cap.sccMembers.d[cap.sccs.d[s].memStart]
if (cap.sccFlags.d[t] and flagDead) != 0:
cap.sccFlags.d[t] = cap.sccFlags.d[t] or flagDirty
let m = cap.sccMembers.d[cap.sccMemStart.d[s]]
registerLocal(cap.recs.d[m].cell, cap.recs.d[m].desc)
elif cap.prunedSrc.len > 0:
# A prune happened somewhere in THIS collection, so every "live" verdict
# it produced is suspect: a pruned cell is not traced, yet its out-edges
# still count toward its targets' rc. If that pruned cell is itself dead
# (promoted while live, died later this epoch), its phantom references
# inflate unrelated SCCs' external counts and misclassify genuinely dead
# SCCs as plain survivors. Such a survivor is not flagPruned, so without
# this it would be re-stamped, dropped from the retry set, and orphaned
# forever once its last flagPruned neighbor resolves. Keeping ONE member
# of every survivor registered guarantees it is re-examined until the
# epoch advances, captures the dead promoted cells, and decrements the
# phantom edges away. Cheap in practice: pruning keeps the captured set
# small, so "every survivor" is only the few cells actually traced.
let m = cap.sccMembers.d[cap.sccs.d[s].memStart]
elif (cap.sccFlags.d[s] and flagPruned) != 0:
# survives, but its liveness may rest on a stale epoch stamp: keep
# one member registered so the verdict is retried (fully re-examined
# once the epoch advances)
let m = cap.sccMembers.d[cap.sccMemStart.d[s]]
registerLocal(cap.recs.d[m].cell, cap.recs.d[m].desc)
proc validateDead(j: var GcEnv; cap: ptr CaptureBufs) =
@@ -1005,7 +998,7 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
for i in 0 ..< cap.crossPend.len:
registerLocal(cap.crossPend.d[i][0], cap.crossPend.d[i][1])
template deadCell(t: Cell): bool =
isStamped(t) and (cap.sccs.d[cap.recs.d[denseIdx(t)].sccOf].flags and flagDead) != 0
isStamped(t) and (cap.sccFlags.d[cap.recs.d[denseIdx(t)].sccOf] and flagDead) != 0
template graceWait() =
# Grace period: another collection still in its CAPTURE phase may hold
# stale (slot, value) snapshots referencing our dead cells; disposing
@@ -1013,12 +1006,13 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
# bounded and never wait on us. New captures cannot reach our dead
# cells: they are unreachable, and our tag stays active until after
# the frees.
for s in 0 ..< ParSlots:
if s != gMySlot:
let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE)
if tg != 0:
parkUntil(atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) != tg or
atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) != 1)
when sizeof(int) == 8:
for s in 0 ..< ParSlots:
if s != gMySlot:
let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE)
if tg != 0:
parkUntil(atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) != tg or
atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) != 1)
# A dead cell's reference to another active collection's cell must still
# be decremented (the target survives this round), so the all-dead fast
# path additionally requires that no cross-collection edge was seen.
@@ -1043,6 +1037,8 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
while j.traceStack.len > 0:
let (entry, _) = j.traceStack.pop()
entry.slot[] = nil
when sizeof(int) != 8:
cell.rootIdx = 0 # no epoch in the stamp: clear before the free
when orcLeakDetector:
writeCell("CYCLIC OBJECT FREED", cell, desc)
free(cell, desc)
@@ -1050,8 +1046,8 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
else:
init j.toFree
for s in 0 ..< j.nScc:
if (cap.sccs.d[s].flags and flagDead) != 0:
for mi in cap.sccs.d[s].memStart ..< cap.sccs.d[s+1].memStart:
if (cap.sccFlags.d[s] and flagDead) != 0:
for mi in cap.sccMemStart.d[s] ..< cap.sccMemStart.d[s+1]:
let m = cap.sccMembers.d[mi]
let cell = cap.recs.d[m].cell
let desc = cap.recs.d[m].desc
@@ -1069,23 +1065,29 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
entry.slot[] = nil
if not deadCell(t):
trialDec(t)
# a stamped target was not analyzed by THIS collection, so
# this dec may be the death blow: keep the cell examinable
if isEpochStamp(atomicLoadN(addr t.rootIdx, ATOMIC_RELAXED)):
registerLocal(t, tdesc)
# epoch-stamp what this collection PROVED live, carrying the cell's
# survival age: only cells that keep surviving get promoted to ages
# where captures prune them, so die-young data is never deferred.
# Demoted (dirty) and pruned SCCs stay unproven — leave their stale
# tags claimable. Our tag is still active, so no foreign claim can
# race these stores.
for s in 0 ..< j.nScc:
if (cap.sccs.d[s].flags and (flagDead or flagDirty or flagPruned)) == 0:
for mi in cap.sccs.d[s].memStart ..< cap.sccs.d[s+1].memStart:
let m = cap.sccMembers.d[mi]
atomicStoreN(addr cap.recs.d[m].cell.rootIdx,
gMyEpochStamp or int64(cap.ages.d[m] +% 1),
ATOMIC_RELAXED)
when sizeof(int) == 8:
# a stamped target was not analyzed by THIS collection, so
# this dec may be the death blow: keep the cell examinable
if isEpochStamp(atomicLoadN(addr t.rootIdx, ATOMIC_RELAXED)):
registerLocal(t, tdesc)
when sizeof(int) == 8:
# epoch-stamp what this collection PROVED live, carrying the cell's
# survival age: only cells that keep surviving get promoted to ages
# where captures prune them, so die-young data is never deferred.
# Demoted (dirty) and pruned SCCs stay unproven — leave their stale
# tags claimable. Our tag is still active, so no foreign claim can
# race these stores.
for s in 0 ..< j.nScc:
if (cap.sccFlags.d[s] and (flagDead or flagDirty or flagPruned)) == 0:
for mi in cap.sccMemStart.d[s] ..< cap.sccMemStart.d[s+1]:
let m = cap.sccMembers.d[mi]
atomicStoreN(addr cap.recs.d[m].cell.rootIdx,
gMyEpochStamp or (int(cap.ages.d[m]) +% 1),
ATOMIC_RELAXED)
else:
# no epoch in the stamp: clear them while all cells are still alive
for i in 0 ..< cap.recs.len:
cap.recs.d[i].cell.rootIdx = 0
graceWait()
for i in 0 ..< j.toFree.len:
when orcLeakDetector:
@@ -1125,7 +1127,7 @@ proc startCollection(minRoots, keepBelow: int; slice: var CellSeq[Cell];
drainStripe(getStripeIdx()) # the world moved while we waited
adoptOrphans()
else:
gTagCounter = (gTagCounter +% 1) and int64(epochBase - 1) # tags below the stamp namespace
gTagCounter = (gTagCounter +% 1) and (epochBase - 1) # tags below the stamp namespace
if gTagCounter == 0: gTagCounter = 1
gMyTag = gTagCounter
gMySlot = slot
@@ -1177,6 +1179,7 @@ proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) =
for i in countdown(last, 0):
capture(slice.d[i][0], slice.d[i][1], j, cap)
cap.sccMemStart.add int32(cap.sccMembers.len) # sentinel
j.touched = cap.recs.len
atomicStoreN(addr gSlotPhase[gMySlot], 2, ATOMIC_RELEASE) # capture done
if gAmSolo:

View File

@@ -724,38 +724,17 @@ theorem no_deadlock_from_total_order {n : Nat}
registered:
E1 roots never prune: a registered candidate is always fully
root-scanned, stamps notwithstanding;
E2 when a collection prunes ANY out-edge, it keeps one member of
EVERY surviving SCC registered — not just the SCCs that pruned an
edge themselves. This is broader than it first looks and the
breadth is load-bearing: a pruned cell is not traced, yet its
out-edges still count toward its targets' rc, so a pruned cell
that is ITSELF dead (promoted while live, died later this epoch)
contributes phantom refs that inflate an UNRELATED SCC's external
count and misclassify that genuinely-dead SCC as a plain (non-
prune-source) survivor. Registering only prune-source SCCs
(the original hook) let such a survivor be re-stamped, dropped
from the retry set, and orphaned permanently once its last
prune-source neighbour resolved — a real leak the dumpster `fuzz`
port surfaced (tests/yrc/tyrc_fuzz_graph.nim: ~1% of allocations
lost, ORC-clean, un-recoverable even by repeated GC_fullCollect).
Registering every survivor of a pruning collection closes it; the
cost is bounded because pruning keeps the captured set small, so
"every survivor" is only the handful actually traced;
E2 an SCC that pruned an out-edge and survives keeps one member
registered (flagPruned) — its "live" verdict may lean on a stamp
that went stale within the epoch;
E3 a commit-time dec into a stamped cell re-registers the target —
the dec may be the death blow to a cell no collection analyzed;
E4 explicit full collects advance the epoch first, so all stamps
are stale and nothing is pruned.
Not formalized (and E2's original narrow form was empirically wrong —
see above; the broadened form is validated by the fuzz port across seeds
and sizes, not machine-checked). The epoch clock counts collections
(YrcEpochLen=64);
Not formalized. Also noted: the epoch clock counts collections, and
short epochs (≲ 4) resonate with the adaptive threshold — pruned
collections are cheap, so collections and hence epoch turns speed up,
re-tracing MORE than with no stamps — but 64 sits clear of that. A
work-based clock (advance per N cells traced) was measured and lost on
long-lived structures: pruning shrinks trace work, so the clock stalls
exactly when a stale web should be re-examined, trading float for a
resonance the default length already avoids.
re-tracing MORE than with no stamps; a work-based clock would fix it.
Reference: D.F. Bacon and V.T. Rajan, "Concurrent Cycle Collection in
Reference Counted Systems", ECOOP 2001 — the deadness arithmetic is

View File

@@ -1,9 +0,0 @@
discard """
errormsg: "cannot infer the return type of 'foo'"
line: 6
"""
proc foo(n: int): auto =
return foo(n + 1)
discard foo(0)

View File

@@ -1,12 +0,0 @@
discard """
errormsg: "cannot infer the return type of 'foo'"
line: 6
"""
proc foo(n: int): auto =
if n > 0:
foo(n - 1)
else:
foo(n + 1)
discard foo(1)

View File

@@ -1,9 +0,0 @@
discard """
errormsg: "cannot infer the return type of 'foo'"
line: 6
"""
proc foo[T](x: T): auto =
foo(x)
discard foo(1)

View File

@@ -1,46 +0,0 @@
proc byReturn(n: int): auto =
if n < 5:
return byReturn(n + 1)
else:
return 9
proc byResult(n: int): auto =
if n < 5:
result = byResult(n + 1)
else:
result = 9
proc byExpression(n: int): auto =
if n < 5:
byExpression(n + 1)
else:
9
proc generic[T](x: T; n: int): auto =
if n < 5:
return generic(x, n + 1)
else:
return x
proc concreteFirst(n: int): auto =
if n >= 5:
return 9
else:
return concreteFirst(n + 1)
proc multipleRecursiveBranches(n: int): auto =
if n < 0:
return multipleRecursiveBranches(n + 1)
elif n < 5:
return multipleRecursiveBranches(n + 1)
else:
return 9
doAssert byReturn(3) == 9
doAssert byResult(3) == 9
doAssert byExpression(3) == 9
doAssert generic("ok", 3) == "ok"
doAssert concreteFirst(3) == 9
doAssert multipleRecursiveBranches(-1) == 9

View File

@@ -1,12 +0,0 @@
discard """
matrix: "--mm:refc; --mm:orc --deepcopy:on"
errormsg: "'deepCopy' is not available for type <NoCopy>"
file: "system.nim"
"""
type NoCopy = object
proc `=copy`(a: var NoCopy; b: NoCopy) {.error.}
var a = new NoCopy
var b = deepCopy(a)

View File

@@ -1,15 +0,0 @@
discard """
matrix: "--mm:refc; --mm:orc --deepcopy:on"
errormsg: "'deepCopy' is not available for type <Container>"
file: "system.nim"
"""
type
NoCopy = object
Container = object
value: NoCopy
proc `=copy`(a: var NoCopy; b: NoCopy) {.error.}
var a = new Container
var b = deepCopy(a)

View File

@@ -1,84 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Deterministic port of dumpster's `fuzz` test
# (https://claytonwramsey.com/blog/dumpster/): drive a mutable object graph
# through a long random sequence of node/edge inserts and removals, then drop
# every root and assert that *every allocation ever made is destroyed exactly
# once* -- no leak (count 0) and no double free (count > 1). The graph grows
# thick with overlapping and self cycles, so only the cycle collector can wind
# it down. A fixed LCG seed makes the shape reproducible across runs.
type
DropCount = object
id: int
live: bool # false in any moved-from temporary -> never miscounts
Node = ref object
refs: seq[Node]
dc: DropCount
var counts: seq[int] # counts[id] == times allocation `id` was destroyed
proc `=destroy`(x: DropCount) =
if x.live: inc counts[x.id]
var nextId = 0
proc newNode(): Node =
counts.add 0
result = Node(refs: @[], dc: DropCount(id: nextId, live: true))
inc nextId
# `child` is a by-value borrow (dumpster's `Gc::clone`): storing it copies the
# reference, leaving the caller's root slot still owning. Using `.refs.add`
# directly would move the root at its last read and change the graph shape.
proc link(parent, child: Node) = parent.refs.add child
# Small fixed-seed LCG (Numerical Recipes constants) for reproducible shape.
var rngState: uint32 = 12345
proc rnd(n: int): int =
rngState = rngState * 1664525'u32 + 1013904223'u32
int((rngState shr 16) mod uint32(n))
proc run =
const N = 20_000
var roots: seq[Node]
for i in 0 ..< 50: roots.add newNode()
for _ in 0 ..< N:
if roots.len == 0: roots.add newNode()
case rnd(4)
of 0: # allocate a fresh root
roots.add newNode()
of 1: # add edge from -> to (may self-loop)
let a = rnd(roots.len)
let b = rnd(roots.len)
link(roots[a], roots[b])
of 2: # drop a root handle (swap-remove)
let i = rnd(roots.len)
roots[i] = roots[roots.high]
roots.setLen roots.len - 1
else: # drop one outgoing edge of a root
let a = rnd(roots.len)
if roots[a].refs.len > 0:
let j = rnd(roots[a].refs.len)
roots[a].refs[j] = roots[a].refs[roots[a].refs.high]
roots[a].refs.setLen roots[a].refs.len - 1
roots.setLen 0 # release every remaining root
GC_fullCollect()
GC_fullCollect()
run()
var missing = 0
for id in 0 ..< nextId:
if counts[id] != 1:
inc missing
doAssert missing == 0, "graph not fully reclaimed: " & $missing & " of " &
$nextId & " allocations leaked or double-freed"
echo "ok"

View File

@@ -1,33 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Memory must stay bounded while creating cyclic garbage forever: the
# collector has to keep pace with allocation. A leak shows up as unbounded
# peak occupancy, which the assertion below catches.
type Node = ref object
next: Node
data: seq[int]
proc mk(n: int) =
var h = Node(data: newSeq[int](4))
var c = h
for i in 1 ..< n:
c.next = Node(data: newSeq[int](4))
c = c.next
c.next = h
var peak = 0
for round in 0 ..< 30:
for i in 0 ..< 10_000:
mk(8)
let occ = getOccupiedMem()
if occ > peak: peak = occ
doAssert peak < 64 * 1024 * 1024, "memory exploded: leak"
GC_fullCollect()
echo "ok"

View File

@@ -1,26 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "done"
valgrind: "leaks"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Smallest possible cycle: a three-node ring that is dead the instant `mk`
# returns. GC_fullCollect must reclaim it without touching freed memory.
type Node = ref object
next: Node
proc mk =
let a = Node()
let b = Node()
let c = Node()
a.next = b
b.next = c
c.next = a
mk()
GC_fullCollect()
echo "done"

View File

@@ -1,74 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
valgrind: "leaks"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# The "parallel_loop" complex graph from Clayton Ramsey's `dumpster` collector
# (https://claytonwramsey.com/blog/dumpster/). Four allocations form a single
# SCC built from two *overlapping* cycles that share nodes 1 and 4:
#
# 1 -> 4 4 -> 2, 4 -> 3 2 -> 1, 3 -> 1
#
# so 1->4->2->1 and 1->4->3->1 traverse the same 1 and 4. Every node keeps a
# nonzero refcount from *inside* the SCC, so plain reference counting can never
# free any of them; only cycle collection can, and only once the last external
# handle is gone. We drop the four root handles one at a time and assert that
# nothing is reclaimed until the final drop, then all four die together -- the
# exact assertion sequence dumpster's test makes.
type
# A field whose destructor bumps a per-node counter when the cell is freed;
# `slot` is nil in any moved-from temporary, so those don't miscount.
DropCount = object
slot: ptr int
Node = ref object
refs: seq[Node]
dc: DropCount
proc `=destroy`(x: DropCount) =
if x.slot != nil: inc x.slot[]
# Add an edge parent -> child. `child` is a by-value borrow, so the caller's
# handle keeps owning its reference -- this is Nim's equivalent of dumpster's
# `Gc::clone`. Building edges with `g1.refs.add g2` instead would *move* g2 at
# its last read and silently collapse the graph's root set.
proc link(parent, child: Node) = parent.refs.add child
# drops[0] is unused; nodes are 1..4 to mirror the blog's gc1..gc4. The four
# handles live in an array so each stays an independent, still-owning root.
var drops: array[5, int]
proc scenario =
var g: array[1..4, Node]
for i in 1..4: g[i] = Node(dc: DropCount(slot: addr drops[i]))
link(g[2], g[1]) # 2 -> 1
link(g[3], g[1]) # 3 -> 1
link(g[4], g[2]) # 4 -> 2
link(g[4], g[3]) # 4 -> 3
link(g[1], g[4]) # 1 -> 4 (closes both cycles)
GC_fullCollect()
doAssert drops == [0, 0, 0, 0, 0], "nothing dead yet"
g[1] = nil # node1 still held by node2 and node3
GC_fullCollect()
doAssert drops == [0, 0, 0, 0, 0], "dropping root 1 frees nothing"
g[2] = nil # node2 still held by node4
GC_fullCollect()
doAssert drops == [0, 0, 0, 0, 0], "dropping root 2 frees nothing"
g[3] = nil # node3 still held by node4
GC_fullCollect()
doAssert drops == [0, 0, 0, 0, 0], "dropping root 3 frees nothing"
g[4] = nil # last external handle gone: the whole SCC is garbage
GC_fullCollect()
doAssert drops == [0, 1, 1, 1, 1], "the full cycle is reclaimed at once"
scenario()
echo "ok"

View File

@@ -1,33 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
valgrind: "leaks"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Exercise the manual collection API: disable automatic collections, build a
# batch of dead cycles, then reclaim them in halves via GC_partialCollect and
# confirm the pending count shrinks accordingly.
type Node = ref object
next: Node
proc mk(n: int) =
var h = Node()
var c = h
for i in 1 ..< n: (c.next = Node(); c = c.next)
c.next = h
GC_disableOrc() # no automatic collections; exercise the partial API
for i in 0 ..< 300: mk(4)
let pending = GC_prepareOrc()
doAssert pending > 0
GC_partialCollect(pending div 2) # collect only the upper half
let remaining = GC_prepareOrc()
doAssert remaining <= pending div 2, $remaining & " vs " & $pending
GC_partialCollect(0) # collect the rest
doAssert GC_prepareOrc() == 0
GC_fullCollect()
echo "ok"

View File

@@ -1,60 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
valgrind: "leaks"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Functional test for the Tarjan-based collector: doubly-linked dead rings,
# self-referential cells, and one surviving ring whose integrity is checked
# after a full collect.
type
Node = ref object
next: Node
prev: Node
data: string
proc makeRing(n: int): Node =
result = Node(data: "head")
var cur = result
for i in 1 ..< n:
let x = Node(data: $i)
cur.next = x
x.prev = cur
cur = x
cur.next = result
result.prev = cur
proc dropRings =
for i in 0 ..< 2000:
discard makeRing(10) # dead immediately
proc keepOne: Node =
for i in 0 ..< 100:
discard makeRing(5)
result = makeRing(7) # survives
proc selfRef =
type S = ref object
self: S
buf: seq[int]
for i in 0 ..< 500:
let s = S(buf: newSeq[int](8))
s.self = s
dropRings()
selfRef()
let keep = keepOne()
GC_fullCollect()
doAssert keep.data == "head"
var cnt = 0
var it = keep
while true:
inc cnt
it = it.next
if it == keep: break
doAssert cnt == 7, "live ring corrupted: " & $cnt
echo "ok"

View File

@@ -1,72 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# Concurrent stress for the lock-free SATB collector: mutator threads rewire
# live cyclic structures (constant dirty traffic + capture aborts) and churn
# garbage cycles while a dedicated thread runs back-to-back collections. Live
# data corruption or a lost object trips a doAssert / a growing residual.
import std/typedthreads
type Node = ref object
next: Node # ring structure, stable
payload: Node # rewired constantly -> candidates + dirty SCCs
id: int
const NWorkers = 3
const Iters = 400_000
const RingLen = 64
var stopFlag: bool
var done: array[NWorkers, int]
proc mkRing(tag: int): seq[Node] =
result = newSeq[Node](RingLen)
for i in 0 ..< RingLen: result[i] = Node(id: tag + i)
for i in 0 ..< RingLen:
result[i].next = result[(i+1) mod RingLen]
result[i].payload = result[(i*13+7) mod RingLen]
proc verify(ring: seq[Node]; tag: int) =
for i in 0 ..< RingLen:
doAssert ring[i].id == tag + i, "node corrupted"
doAssert ring[i].next.id == tag + (i+1) mod RingLen, "ring broken"
doAssert ring[i].payload.id >= tag and ring[i].payload.id < tag + RingLen,
"payload points outside ring: live data corrupted"
proc worker(tid: int) {.thread.} =
var tag = tid * 1_000_000
var ring = mkRing(tag)
for i in 0 ..< Iters:
# lock-free barrier hot path: rewire a payload edge inside the live ring.
# decs the old target (rc > 0) -> candidate; collections capture the live
# ring concurrently and must rescue or abort, never free it.
ring[i mod RingLen].payload = ring[(i * 7 + 3) mod RingLen]
if (i and 8191) == 0:
verify(ring, tag)
if (i and 32767) == 0:
inc tag, RingLen
ring = mkRing(tag) # old ring becomes a garbage cycle tangle
verify(ring, tag)
done[tid] = 1
proc collector() {.thread.} =
while not stopFlag:
GC_runOrc()
var th: array[NWorkers, Thread[int]]
var col: Thread[void]
createThread(col, collector)
for i in 0 ..< NWorkers: createThread(th[i], worker, i)
joinThreads(th)
stopFlag = true
joinThread(col)
for i in 0 ..< NWorkers: doAssert done[i] == 1
GC_fullCollect()
GC_fullCollect()
echo "ok"

View File

@@ -1,60 +0,0 @@
discard """
cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file"
output: "ok"
disabled: "windows"
disabled: "freebsd"
disabled: "openbsd"
"""
# N threads each churn garbage cycles while maintaining one live ring that is
# verified continuously and replaced, plus explicit GC_runOrc collections from
# every thread. Corruption of live data trips a doAssert.
import std/typedthreads
type Node = ref object
next: Node
prev: Node
id: int
const NThreads = 4
const Iters = 30_000
proc mkRing(n, tag: int): Node =
result = Node(id: tag)
var c = result
for i in 1 ..< n:
let x = Node(id: tag + i)
c.next = x
x.prev = c
c = x
c.next = result
result.prev = c
proc checkRing(r: Node; n, tag: int) =
var c = r
for i in 0 ..< n:
doAssert c.id == tag + i, "ring corrupted!"
c = c.next
doAssert c == r, "ring not closed!"
var results: array[NThreads, int]
proc worker(tid: int) {.thread.} =
var keep = mkRing(5, tid * 1000)
for i in 0 ..< Iters:
discard mkRing(3 + (i and 7), 999999) # garbage
if (i and 255) == 0:
checkRing(keep, 5, tid * 1000)
keep = mkRing(5, tid * 1000) # old keep becomes garbage
if (i and 1023) == 0:
GC_runOrc() # explicit collections from all threads
checkRing(keep, 5, tid * 1000)
results[tid] = 1
var th: array[NThreads, Thread[int]]
for i in 0 ..< NThreads: createThread(th[i], worker, i)
joinThreads(th)
for i in 0 ..< NThreads: doAssert results[i] == 1
GC_fullCollect()
echo "ok"