diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b918b21050..a59bb5f91f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v10 + - uses: actions/stale@v11 with: days-before-pr-stale: 365 days-before-pr-close: 30 diff --git a/compiler/aliases.nim b/compiler/aliases.nim index fa1167753f..6877028c3a 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -21,6 +21,44 @@ type TAnalysisResult* = enum arNo, arMaybe, arYes + PartFlag* = enum + pfStructural ## use structural prefix-chain detection and tree-walk + pfBidirectional ## also check reverse direction per field in nkObjConstr + +func sameLocation(a, b: PNode): bool = + template sameConstIndex(a, b: PNode): bool = + a.kind in nkLiterals and b.kind in nkLiterals and a.intVal == b.intVal + var a = a + var b = b + while a.kind in {nkHiddenStdConv, nkHiddenSubConv, nkConv}: a = a[1] + while b.kind in {nkHiddenStdConv, nkHiddenSubConv, nkConv}: b = b[1] + if a.kind != b.kind: return false + case a.kind + of nkSym: result = a.sym.id == b.sym.id + of nkDotExpr, nkCheckedFieldExpr: + result = a[1].kind == nkSym and b[1].kind == nkSym and + sameLocation(a[0], b[0]) and a[1].sym.id == b[1].sym.id + of nkBracketExpr: + result = sameLocation(a[0], b[0]) and sameConstIndex(a[1], b[1]) + of nkObjUpConv, nkObjDownConv, nkDerefExpr, nkHiddenDeref: + result = sameLocation(a[0], b[0]) + else: result = false + +proc isAccessorPrefixOf(a, b: PNode): bool = + var cur = b + while cur.kind in {nkDotExpr, nkBracketExpr, nkCheckedFieldExpr, nkObjUpConv, + nkObjDownConv, nkHiddenDeref, nkDerefExpr, + nkHiddenStdConv, nkHiddenSubConv, nkConv}: + if sameLocation(cur, a): return true + case cur.kind + of nkDotExpr, nkBracketExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv, + nkHiddenDeref, nkDerefExpr: + cur = cur[0] + of nkHiddenStdConv, nkHiddenSubConv, nkConv: + cur = cur[1] + else: discard + result = sameLocation(cur, a) + proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult = @@ -70,14 +108,28 @@ proc isPartOf(a, b: PType): TAnalysisResult = # watch out: parameters reversed because I'm too lazy to change the code... result = isPartOfAux(b, a, marker) -proc isPartOf*(a, b: PNode): TAnalysisResult = - ## checks if location `a` can be part of location `b`. We treat seqs and - ## strings as pointers because the code gen often just passes them as such. +proc isPartOf*(a, b: PNode; flags: set[PartFlag] = {}): TAnalysisResult = + ## Checks if location `a` can be part of location `b`: i.e. whether writing to + ## `b` could affect what `a` reads. We treat seqs and strings as pointers + ## because the code gen often just passes them as such. ## ## Note: `a` can only be part of `b`, if `a`'s type can be part of `b`'s ## type. Since however type analysis is more expensive, we perform it only ## if necessary. ## + ## When `pfStructural` is set additional aliasing is detected: + ## * a structural prefix of an accessor chain is considered part of it + ## (e.g. `x.f <| x.f.g`). Normally `x.f !<| x.f.g` because the + ## same-kind `nkDotExpr` comparison treats the differing field names as + ## siblings, but `pfStructural` walks the chain to recognise the + ## relationship. + ## * Unrecognised node kinds are traversed recursively. + ## + ## When `pfBidirectional` is set: + ## * In `nkObjConstr` the reverse direction `isPartOf(value, a)` is also + ## checked per field value so that reads hidden behind calls/closures + ## are detected. + ## ## cases: ## ## YES-cases: @@ -86,13 +138,14 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = ## x[] <| x ## x[i] <| x ## x.f <| x + ## x.f <| x.f.g # when pfStructural (prefix chain) ## ``` ## ## NO-cases: ## ``` ## x !<| y # depending on type and symbol kind ## x[constA] !<| x[constB] - ## x.f !<| x.g + ## x.f !<| x.g # sibling fields at same level ## x.f !<| y.f iff x !<= y ## ``` ## @@ -121,7 +174,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = else: result = arNo of nkBracketExpr: - result = isPartOf(a[0], b[0]) + result = isPartOf(a[0], b[0], flags) if a.len >= 2 and b.len >= 2: # array accesses: if result == arYes and isDeepConstExpr(a[1]) and isDeepConstExpr(b[1]): @@ -131,7 +184,11 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = var y = if b[1].kind == nkHiddenStdConv: b[1][1] else: b[1] if sameValue(x, y): result = arYes + elif pfStructural in flags and isAccessorPrefixOf(a, b): + result = arYes else: result = arNo + elif pfStructural in flags and isAccessorPrefixOf(a, b): + result = arYes # else: maybe and no are accurate else: # pointer derefs: @@ -139,22 +196,25 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = if isPartOf(a.typ, b.typ) != arNo: result = arMaybe of nkDotExpr: - result = isPartOf(a[0], b[0]) + result = isPartOf(a[0], b[0], flags) if result != arNo: # if the fields are different, it's not the same location if a[1].sym.id != b[1].sym.id: - result = arNo + if pfStructural in flags and isAccessorPrefixOf(a, b): + result = arYes + else: + result = arNo of nkHiddenDeref, nkDerefExpr: - result = isPartOf(a[0], b[0]) + result = isPartOf(a[0], b[0], flags) # weaken because of indirection: if result != arYes: if isPartOf(a.typ, b.typ) != arNo: result = arMaybe of nkHiddenStdConv, nkHiddenSubConv, nkConv: - result = isPartOf(a[1], b[1]) + result = isPartOf(a[1], b[1], flags) of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr: - result = isPartOf(a[0], b[0]) + result = isPartOf(a[0], b[0], flags) else: result = arNo # Calls return a new location, so a default of ``arNo`` is fine. else: @@ -167,31 +227,31 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = case b.kind of Ix0Kinds: # a* !<| b.f iff a* !<| b - result = isPartOf(a, b[0]) + result = isPartOf(a, b[0], flags) of DerefKinds: # a* !<| b[] iff result = arNo if isPartOf(a.typ, b.typ) != arNo: - result = isPartOf(a, b[0]) + result = isPartOf(a, b[0], flags) if result == arNo: result = arMaybe of Ix1Kinds: # a* !<| T(b) iff a* !<| b - result = isPartOf(a, b[1]) + result = isPartOf(a, b[1], flags) of nkSym: # b is an atom, so we have to check a: case a.kind of Ix0Kinds: # a.f !<| b* iff a.f !<| b* - result = isPartOf(a[0], b) + result = isPartOf(a[0], b, flags) of Ix1Kinds: - result = isPartOf(a[1], b) + result = isPartOf(a[1], b, flags) of DerefKinds: if isPartOf(a.typ, b.typ) != arNo: - result = isPartOf(a[0], b) + result = isPartOf(a[0], b, flags) if result == arNo: result = arMaybe else: result = arNo @@ -199,20 +259,29 @@ proc isPartOf*(a, b: PNode): TAnalysisResult = of nkObjConstr: result = arNo for i in 1.. 0: - result = isPartOf(a, b[0]) + result = isPartOf(a, b[0], flags) else: result = arNo - else: result = arNo + else: + if pfStructural in flags: + for i in 0..= 0: var newlineKind = ltCrucialNewline if em.keepIndents > 0: - em.indentLevel = tok.indent + # Apply the requested --indent width to "don't touch" regions (if/block + # expressions) too: keep the relative offset from the enclosing block + # baseline, but rebase it onto indWidth. Otherwise a non-default + # --indent would leave these lines at the original column and inject + # invalid indentation (see #20078). + em.indentLevel = em.indentStack.high * em.indWidth + + (tok.indent - em.indentStack[^1]) elif (em.lastTok in (splitters + oprSet) and tok.tokType notin (closedPars - {tkBracketDotRi})): if tok.tokType in openPars and tok.indent > em.indentStack[^1]: diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 0529812c3a..032a4623f2 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -94,11 +94,21 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc genAddr(c: var TLiftCtx; x: PNode): PNode = - if x.kind == nkHiddenDeref: + # These synthesized addresses are always passed to codegen procs that expect a + # genuine pointer (nimAsgnYrc, nimSinkYrc, destructors, ...). `addr(deref x)` + # collapses to `x` only when `x` is a real pointer; on the C++ backend a `var` + # parameter is a C++ reference, so we must keep the `nkHiddenAddr` to actually + # take its address (`&dest`) instead of passing the reference's value. Likewise + # `tfVarIsPtr` keeps the C++ backend from lowering the synthesized address back + # to a reference and dropping the `&` (e.g. a closure's `tyPointer` env). See + # #26026 CI (yrc + cpp). + if x.kind == nkHiddenDeref and c.g.config.backend != backendCpp: checkSonsLen(x, 1, c.g.config) result = x[0] else: - result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ, c.idgen)) + let addrTyp = makeVarType(x.typ.owner, x.typ, c.idgen) + addrTyp.incl tfVarIsPtr + result = newNodeIT(nkHiddenAddr, x.info, addrTyp) result.add x proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode = @@ -796,8 +806,22 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) - # YRC uses dedicated runtime procs for the entire write barrier: - if c.g.config.selectedGC == gcYrc: + # YRC uses dedicated runtime procs for the entire write barrier -- but ONLY + # for refs that can actually form cycles. Routing an acyclic ref through + # `nimAsgnYrc` defeats the entire purpose of `.acyclic`: the barrier defers + # the dec into a stripe queue, `drainStripe` then hands the cell to + # `registerLocal`, and it enters the collector as a capture ROOT -- so a + # type annotated precisely to stay out of the cycle collector gets traced + # by it anyway. (The collector never reaches such a cell by TRAVERSAL: the + # attachedTrace hook below only emits `nimTraceRef` when `isCyclic`. The + # queued dec was the only way in.) + # + # Falling through instead gives acyclic refs the same prompt arc-style + # reclamation they get under --mm:arc/orc, which is also what lets a thread + # that avoids cycles at compile time avoid the collector entirely at run + # time. `canFormAcycle` is the same predicate ccgtypes.nim:1903 uses to set + # the descriptor's acyclic flag, so codegen and runtime cannot disagree. + if c.g.config.selectedGC == gcYrc and types.canFormAcycle(c.g, elemType): let desc = if isFinal(elemType): let ti = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index f9cd3b6f37..7b975268cd 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -606,10 +606,16 @@ proc setHookDisamb*(g: ModuleGraph; hook: PSym; opName: string; typ: PType) = break hook.disamb = h -proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool = - let op = getAttachedOp(g, t, attachedAsgn) +proc hasDisabledOp(g: ModuleGraph; t: PType; kind: TTypeAttachedOp): bool = + let op = getAttachedOp(g, t, kind) result = op != nil and sfError in op.flags +proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool = + result = hasDisabledOp(g, t, attachedAsgn) + +proc hasDisabledDup*(g: ModuleGraph; t: PType): bool = + result = hasDisabledOp(g, t, attachedDup) + proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = for k in low(TTypeAttachedOp)..high(TTypeAttachedOp): let op = getAttachedOp(g, src, k) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 07564aa171..fe424ffdcb 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2742,6 +2742,22 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags; expectedType: P else: result = semDirectOp(c, n, flags, expectedType) +proc semNimvmBranch(c: PContext, n: PNode, flags: TExprFlags): PNode = + let + oldOptionStack = c.optionStack[0..^1] + oldOptions = c.config.options + oldNotes = c.config.notes + oldWarningAsErrors = c.config.warningAsErrors + oldFeatures = c.features + try: + result = semExpr(c, n, flags) + finally: + c.optionStack = oldOptionStack + c.config.options = oldOptions + c.config.notes = oldNotes + c.config.warningAsErrors = oldWarningAsErrors + c.features = oldFeatures + proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = # If semCheck is set to false, ``when`` will return the verbatim AST of # the correct branch. Otherwise the AST will be passed through semStmt. @@ -2778,7 +2794,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = checkSonsLen(it, 2, c.config) if whenNimvm: if semCheck: - it[1] = semExpr(c, it[1], flags) + it[1] = semNimvmBranch(c, it[1], flags) typ = commonType(c, typ, it[1].typ) result = n # when nimvm is not elimited until codegen elif c.inGenericContext > 0: @@ -2809,7 +2825,8 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = discard elif result == nil or whenNimvm: if semCheck: - it[0] = semExpr(c, it[0], flags) + it[0] = if whenNimvm: semNimvmBranch(c, it[0], flags) + else: semExpr(c, it[0], flags) typ = commonType(c, typ, it[0].typ) if typ != nil and typ.kind != tyUntyped: it[0] = fitNode(c, typ, it[0], it[0].info) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index f5bf97c580..0e1eab91ea 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -693,5 +693,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, if n[1].kind in {nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt}: localError(c.config, n.info, "Nested expressions cannot be moved: '" & $n[1] & "'") + of mMove: + result = n + if isCursor(n[1]): + localError(c.config, n.info, errFailedMove, + "cannot move cursor '" & $n[1] & "'; a cursor does not own its value") else: result = n diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index a026fc994a..1c4c81c64c 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -219,9 +219,10 @@ proc semSet(c: PContext, n: PNode, prev: PType): PType = result = newOrPrevType(tySet, prev, c) if n.len == 2 and n[1].kind != nkEmpty: var base = semTypeNode(c, n[1], nil) + if base.kind == tyTypeDesc: base = base.base # unwrap from type traits like distinctBase addSonSkipIntLit(result, base, c.idgen) if base.kind in {tyGenericInst, tyAlias, tySink}: base = skipModifier(base) - if base.kind notin {tyGenericParam, tyGenericInvocation}: + if base.kind notin {tyGenericParam, tyGenericInvocation, tyFromExpr}: if base.kind == tyForward: c.forwardTypeUpdates.add (getCurrOwner(c), result, n) elif not isOrdinalType(base, allowEnumWithHoles = true): diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 211db1c863..8fc3282546 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2087,7 +2087,18 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, result = typeRel(c, f.base, a, flags) else: result = isGeneric - if result != isNone: put(c, f, aOrig) + if result != isNone: + if f.base.kind notin {tyNone, tyGenericParam} and + aOrig.kind == tyStatic and aOrig.n != nil and aOrig.n.typ != nil and + aOrig.n.typ.isEmptyContainer: + # we need to infer the inner type for empty containers + let literal = aOrig.n.copyTree + literal.typ = f.base + let staticArg = newTypeS(tyStatic, c.c, f.base) + staticArg.n = literal + put(c, f, staticArg) + else: + put(c, f, aOrig) elif aOrig.n != nil and aOrig.n.typ != nil: result = if f.base.kind != tyNone: typeRel(c, f.last, aOrig.n.typ, flags) diff --git a/compiler/trees.nim b/compiler/trees.nim index a42b616d97..24226e79d5 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -225,6 +225,17 @@ proc getRoot*(n: PNode): PSym = else: result = nil else: result = nil +proc isCursor*(n: PNode): bool = + case n.kind + of nkSym: + sfCursor in n.sym.flags + of nkDotExpr: + isCursor(n[1]) + of nkCheckedFieldExpr: + isCursor(n[0]) + else: + false + proc stupidStmtListExpr*(n: PNode): bool = for i in 0.. 0: incl(newEvents, Event.Write) p.selector.updateHandle(SocketHandle(fd), newEvents) - # Timer processing. - discard processTimers(p, result) - # Callback queue processing - processPendingCallbacks(p, result) + processCallbacksAndTimers(p, result) proc recv*(socket: AsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[string]) = diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 514a97f30c..213514564e 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -95,7 +95,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) template withValue*[A, B](t: var SharedTable[A, B], key: A, - value, body1, body2: untyped) = + value, body1, body2: untyped): untyped = ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. runnableExamples: diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 94d8721b96..8299ec9b7d 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -636,7 +636,7 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = body template withValue*[A, B](t: var Table[A, B], key: A, - value, body1, body2: untyped) = + value, body1, body2: untyped): untyped = ## Retrieves the value at `t[key]`. ## ## `value` can be modified in the scope of the `withValue` call. @@ -677,7 +677,7 @@ template withValue*[A, B](t: var Table[A, B], key: A, body2 template withValue*[A, B](t: Table[A, B], key: A, - value, body1, body2: untyped) = + value, body1, body2: untyped): untyped = ## Retrieves the value at `t[key]` if it exists, assigns ## it to the variable `value` and executes `body` runnableExamples: diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 818f1b37ac..9c652afdc1 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -256,12 +256,8 @@ else: cast[T](interlockedExchange(addr(location.value), cast[int64](desired))) proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = cast[T](interlockedCompareExchange(addr(location.value), cast[nonAtomicType(T)](desired), cast[nonAtomicType(T)](expected))) == expected - proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchange(location, expected, desired, order, order) proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = compareExchange(location, expected, desired, success, failure) - proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchangeWeak(location, expected, desired, order, order) proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = var currentValue = location.load() @@ -358,13 +354,9 @@ else: cast[T](atomic_exchange_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order)) proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = atomic_compare_exchange_strong_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) - proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchange(location, expected, desired, order, order) proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = atomic_compare_exchange_weak_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) - proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchangeWeak(location, expected, desired, order, order) # Numerical operations proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = @@ -378,6 +370,21 @@ else: proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + func compareExchangeFailureOrder(order: MemoryOrder): MemoryOrder {.inline.} = + case order + of moRelease: + moRelaxed + of moAcquireRelease: + moAcquire + else: + order + + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, compareExchangeFailureOrder(order)) + + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, compareExchangeFailureOrder(order)) + template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = while testAndSet(location.guard, moAcquire): discard try: @@ -411,10 +418,10 @@ else: compareExchange(location, expected, desired, success, failure) proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchange(location, expected, desired, order, order) + compareExchange(location, expected, desired, order, compareExchangeFailureOrder(order)) proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = - compareExchangeWeak(location, expected, desired, order, order) + compareExchangeWeak(location, expected, desired, order, compareExchangeFailureOrder(order)) proc atomicInc*[T: SomeInteger](location: var Atomic[T]; value: T = 1) {.inline.} = ## Atomically increments the atomic integer by some `value`. diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index fb6b4748e2..9d27fcba96 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -103,7 +103,6 @@ when defined(windows): import std/os const - DUPLICATE_SAME_ACCESS = 2 FOREGROUND_BLUE = 1 FOREGROUND_GREEN = 2 FOREGROUND_RED = 4 @@ -115,14 +114,7 @@ when defined(windows): FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 - type - SHORT = int16 - COORD = object - x: SHORT - y: SHORT - SMALL_RECT = object left: SHORT top: SHORT @@ -140,13 +132,6 @@ when defined(windows): dwSize: DWORD bVisible: WINBOOL - proc duplicateHandle(hSourceProcessHandle: Handle, hSourceHandle: Handle, - hTargetProcessHandle: Handle, lpTargetHandle: ptr Handle, - dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, - dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", - importc: "DuplicateHandle".} - proc getCurrentProcess(): Handle{.stdcall, dynlib: "kernel32", - importc: "GetCurrentProcess".} proc getConsoleScreenBufferInfo(hConsoleOutput: Handle, lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".} @@ -191,30 +176,6 @@ when defined(windows): if h > 0: return h return 0 - proc setConsoleCursorPosition(hConsoleOutput: Handle, - dwCursorPosition: COORD): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "SetConsoleCursorPosition".} - - proc fillConsoleOutputCharacter(hConsoleOutput: Handle, cCharacter: char, - nLength: DWORD, dwWriteCoord: COORD, - lpNumberOfCharsWritten: ptr DWORD): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "FillConsoleOutputCharacterA".} - - proc fillConsoleOutputAttribute(hConsoleOutput: Handle, wAttribute: int16, - nLength: DWORD, dwWriteCoord: COORD, - lpNumberOfAttrsWritten: ptr DWORD): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "FillConsoleOutputAttribute".} - - proc setConsoleTextAttribute(hConsoleOutput: Handle, - wAttributes: int16): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "SetConsoleTextAttribute".} - - proc getConsoleMode(hConsoleHandle: Handle, dwMode: ptr DWORD): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "GetConsoleMode".} - - proc setConsoleMode(hConsoleHandle: Handle, dwMode: DWORD): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "SetConsoleMode".} - proc getCursorPos(h: Handle): tuple [x, y: int] = var c: CONSOLE_SCREEN_BUFFER_INFO if getConsoleScreenBufferInfo(h, addr(c)) == 0: @@ -915,9 +876,6 @@ when defined(windows): var mode = DWORD 0 discard getConsoleMode(hi, addr mode) let origMode = mode - const - ENABLE_PROCESSED_INPUT = 1 - ENABLE_ECHO_INPUT = 4 mode = (mode or ENABLE_PROCESSED_INPUT) and not ENABLE_ECHO_INPUT discard setConsoleMode(hi, mode) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index bbb03ad447..f7deb9879f 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -913,17 +913,14 @@ proc findAll*(n: XmlNode, tag: string, caseInsensitive = false): seq[XmlNode] = proc xmlConstructor(a: NimNode): NimNode = if a.kind == nnkCall: - result = newCall("newXmlTree", toStrLit(a[0])) + result = newCall("newXmlTree", newStrLitNode($a[0])) var attrs = newNimNode(nnkBracket, a) var newStringTabCall = newCall(bindSym"newStringTable", attrs, bindSym"modeCaseSensitive") var elements = newNimNode(nnkBracket, a) for i in 1..a.len-1: if a[i].kind == nnkExprEqExpr: - # In order to support attributes like `data-lang` we have to - # replace whitespace because `toStrLit` gives `data - lang`. - let attrName = toStrLit(a[i][0]).strVal.replace(" ", "") - attrs.add(newStrLitNode(attrName)) + attrs.add(newStrLitNode($a[i][0])) attrs.add(a[i][1]) #echo repr(attrs) else: diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index beaf9d89a3..63a66cc182 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -17,19 +17,30 @@ when defined(nimPreviewSlimSystem): const whitespaces = {' ', '\t', '\v', '\r', '\l', '\f'} +const notJSnotNims = not defined(js) and not defined(nimscript) +template whenNotVmJsNims(normalBody, restrictedBody: untyped) = + ## hack, see: #12517 #12518; Edit together with identical in `system` + when nimvm: + restrictedBody + else: + when notJSnotNims: + normalBody + else: + restrictedBody + proc add*(x: var string, y: openArray[char]) = - ## Concatenates `x` and `y` in place. `y` must not overlap with `x` to - ## allow future `memcpy` optimizations. + ## Concatenates `x` and `y` in place. `y` must not overlap with `x` # Use `{.noalias.}` ? - let n = x.len - x.setLen n + y.len - # pending #19727 - # setLen unnecessarily zeros memory - var i = 0 - while i < y.len: - x[n + i] = y[i] - i.inc - # xxx use `nimCopyMem(x[n].addr, y[0].addr, y.len)` after some refactoring + if y.len == 0: return + let oldLen = x.len + x.setLenUninit(oldLen + y.len) + whenNotVmJsNims(): + {.cast(noSideEffect).}: + copyMem(beginStore(x, oldLen + y.len, oldLen), addr(y[0]), y.len) + endStore(x) + do: + for i, ch in y: + x[oldLen + i] = ch func stripSlice(s: openArray[char], leading = true, trailing = true, chars: set[char] = whitespaces): Slice[int] = ## Returns the slice range of `s` which is stripped `chars`. @@ -74,19 +85,14 @@ func setSlice*(s: var string, slice: Slice[int]) = if first > last: s.setLen(0) return - template impl = - for index in first .. last: - s[index - first] = s[index] if first > 0: - when nimvm: impl() - else: - # not JS and not Nimscript - when not declared(moveMem): - impl() - else: - let p = beginStore(s, s.len) - moveMem(p, addr p[first], last - first + 1) - endStore(s) + whenNotVmJsNims(): + let p = beginStore(s, s.len) + moveMem(p, addr p[first], last - first + 1) + endStore(s) + do: + for index in first .. last: + s[index - first] = s[index] s.setLen(last - first + 1) func strip*(a: var string, leading = true, trailing = true, chars: set[char] = whitespaces) {.inline.} = diff --git a/lib/system/arc.nim b/lib/system/arc.nim index d380aa621d..cd74e7f4fa 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -252,7 +252,36 @@ proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} = writeStackTrace() cfprintf(cstderr, "[DecRef] %p %ld\n", p, cell.count) - when (defined(gcAtomicArc) or defined(gcYrc)) and hasThreadSupport: + when defined(gcAtomicArc) and hasThreadSupport and + not defined(nimNoAtomicArcFastPath): + # Uniquely-referenced fast path: skip the RMW entirely. + # + # A counted reference can only be derived from the location being + # destroyed (which happens-before this destructor, or the program races + # on that location) or from another counted reference (whose + # contribution is already in `rc`, forcing the RMW below). So observing + # a zero count proves no other thread holds a reference to this cell and + # therefore none can be inside this destructor: there is nothing to + # adjudicate and no RMW is needed. This is only sound because + # `--mm:atomicArc` has no collector -- ORC/YRC mutate `rc` from a + # participant that holds no counted reference at all. + # + # The load must be ACQUIRE: the count may have reached zero because + # another thread's release-decrement got there first, and we have to see + # its writes before destroying the object. + # + # The slow path stays self-testing (it frees on the value the RMW + # returned, never on a separate load), which is what keeps this out of + # the nim-lang/threading#45 bug class. + if (atomicLoadN(addr cell.rc, ATOMIC_ACQUIRE) and not rcMask) == 0: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + elif atomicDec(cell.rc, rcIncrement) == -rcIncrement: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + elif (defined(gcAtomicArc) or defined(gcYrc)) and hasThreadSupport: # `atomicDec` returns the new value if atomicDec(cell.rc, rcIncrement) == -rcIncrement: result = true diff --git a/lib/system/yrc.nim b/lib/system/yrc.nim index 681fb1db94..67d589aad0 100644 --- a/lib/system/yrc.nim +++ b/lib/system/yrc.nim @@ -5,8 +5,10 @@ # See yrc_proof.lean for a machine-checked (Lean 4) proof of the core # invariants — garbage stability, validation soundness, capture partition # disjointness, grace periods, fence mutual exclusion, deadlock freedom — -# and yrc_tarjan_proof.lean for soundness AND completeness of the SCC -# deadness algorithm. +# yrc_tarjan_proof.lean for soundness AND completeness of the SCC +# deadness algorithm, and yrc_opt_proof.lean for the three optimizations +# that changed those invariants: SCC-uniform epoch ages, the demand-grown +# `gParSlots` pool, and deferred reclamation (gPendingCells). # # ## Synchronization at a Glance # @@ -25,7 +27,7 @@ # collection anywhere. A cell sits in at most one buffer, guarded by an # atomic test-and-set of inRootsFlag; buffered cells are forced live. # -# Up to MaxPar collections run CONCURRENTLY — with the mutators and with +# Up to one collection PER THREAD runs CONCURRENTLY — with the mutators and with # each other — each capturing a disjoint partition of the heap by CAS-ing # a claim tag into the rootIdx header word. gMergeLock covers only the # tag-slot claim and orphan adoption. All waiting (for a free slot, for a @@ -118,8 +120,21 @@ # while die-young data — never promoted — is never deferred. Roots always # bypass stamps (every death has a dec-witness that gets registered, and # registered cells are always scanned as roots), and explicit full -# collects advance the epoch first, staying exhaustive. The price is -# floating garbage bounded by ~2 epochs. +# collects advance the epoch first, staying exhaustive. +# +# Young→old edges are generational: when commit frees a dead cell and +# `trialDec`s a stamped survivor, it does NOT re-root that survivor for the +# current epoch unless the dec drove its RC to zero (a true death blow). +# Live old graphs just lose the young reference and stay pruned. An old +# *cycle* whose last external refs were young is remembered in the +# thread's `gCtx.genSuspects` buffer and promoted to the root set when the +# epoch advances — the major-collection half of the generational scheme. +# That buffer is a candidate buffer like `gLocalRoots` and takes the same +# `inRootsFlag` token: it both dedupes the entry and keeps the cell alive +# (forced live if captured, refused by `free`) until the flush. Marking the +# stamp word instead does not work — `claimCell` CASes the whole word to +# its tag on capture, losing the mark while the list entry survives. +# Floating garbage is thus bounded by ~1 epoch. # # ## Why No Lost Objects # @@ -140,7 +155,7 @@ import std/locks const NumStripes = 64 - QueueSize {.intdefine.} = 128 # override with -d:QueueSize=N + QueueSize {.intdefine.} = 256 # override with -d:QueueSize=N # rc-word flag bits, layout shared with ORC. YRC does no tricolor # marking; colorMask survives only for the debug printout. @@ -152,12 +167,12 @@ const type TraceProc = proc (p, env: pointer) {.nimcall, gcsafe, raises: [].} -# The write barrier's incRef normally goes through a lock-free per-stripe -# queue (drained by the collector), matching the deferred decRef. Define -# `nimYrcDirectIncs` to bypass that queue and apply incRefs directly with an -# atomic RMW instead — simpler, but the collector then observes every incRef -# through commit-time rc validation rather than the queue peek. -const useIncQueue = not defined(nimYrcDirectIncs) +# Direct atomic incRefs are the default: `nimAsgnYrc` already increments that +# way, and commit-time rc validation observes them. Define `nimYrcIncQueue` +# to also buffer incRefs in the stripe queues (matching deferred decs); the +# collector then peeks `toInc` at commit. `nimYrcDirectIncs` is kept as an +# explicit alias for the default. +const useIncQueue = defined(nimYrcIncQueue) and not defined(nimYrcDirectIncs) # With lock-free ref assignments, rc words are mutated concurrently with the # collector (direct atomic incRefs), so all collector-side rc accesses must @@ -228,6 +243,12 @@ proc add[T](s: var RawSeq[T]; v: T) {.inline.} = s.d[s.len] = v s.len = s.len +% 1 +proc addUnchecked[T](s: var RawSeq[T]; v: T) {.inline.} = + ## `add` without the capacity branch; the caller guarantees the room. + ## Used by `claimCell`, which reserves for all four of its arrays at once. + s.d[s.len] = v + s.len = s.len +% 1 + proc pop[T](s: var RawSeq[T]): T {.inline.} = s.len = s.len -% 1 result = s.d[s.len] @@ -255,10 +276,6 @@ proc setLenZeroed[T](s: var RawSeq[T]; n: int) = s.len = n zeroMem(s.d, n *% sizeof(T)) -proc setLenUninit[T](s: var RawSeq[T]; n: int) = - if s.cap < n: resize(s, n) - s.len = n - type TraceEntry = object ## (slot, value) snapshot taken at trace time. The value is read exactly @@ -270,6 +287,7 @@ type TarjanFrame = object u: int32 # dense index of the cell this frame belongs to + ebase: int32 # edges.len when this cell's frame was pushed base: int # traceStack.len before this cell's trace ran CaptureRec = object @@ -281,7 +299,7 @@ type desc: PNimTypeV2 rcWord: int # rc word as captured lowlink: int32 - sccOf: int32 # -1 while the cell is on the Tarjan stack + selfRefs: int32 # self edges, folded out of the edge array SccRec = object ## per SCC of the condensation; the deadness pass reads all of these @@ -293,7 +311,6 @@ type deadIn: int # number of edges from dead SCCs memStart: int32 # offset into sccMembers crossOff: int32 # offset into crossTgt (cross edges by source) - crossCursor: int32 flags: uint8 CaptureBufs = object @@ -301,15 +318,32 @@ type ## threadvar) and persistent across collections, so that frequent ## small collections don't pay per-collection allocations recs: RawSeq[CaptureRec] + sccIdx: RawSeq[int32] # per captured cell: its SCC, -1 while the cell + # is on the Tarjan stack. Deliberately NOT a + # CaptureRec field: it is read once per captured + # EDGE, and a 4-byte stride keeps that array + # L1-resident where a 32-byte record stride would + # miss on almost every edge. tstack: RawSeq[int32] frames: RawSeq[TarjanFrame] - edges: RawSeq[int64] # (u shl 32) or v, dense indices + edges: RawSeq[int32] # PENDING out-edge targets (dense indices) of the + # SCCs still being built. An SCC's edges are + # classified and dropped the moment it is emitted + # (see `capture`), so this stays proportional to + # the DFS frontier, not to the captured graph. sccs: RawSeq[SccRec] sccMembers: RawSeq[int32] crossTgt: RawSeq[int32] crossPend: CellSeq[Cell] # edge targets owned by other active collections prunedSrc: RawSeq[int32] # dense indices of cells with pruned out-edges + prunedTgt: CellSeq[Cell] # epoch-stamped targets we did not descend into ages: RawSeq[int32] # per captured cell: captures survived so far + slots: RawSeq[ptr pointer] + ## Field addresses seen in capture. The allDead commit path nils these + ## without re-tracing (dead cells are immutable, so the snapshot is + ## exact). The mixed path still re-traces — it needs live target + ## classification, and keeping full (slot,tgt,desc) logs in the DFS + ## hot path cost more than the re-trace saved. GcEnv = object traceStack: CellSeq[TraceEntry] @@ -336,10 +370,11 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = # The spare rootIdx header word (unused by YRC's root registration, which # relies on inRootsFlag) doubles as the capture claim: it packs the owning -# collection's tag with the cell's dense discovery index. Up to MaxPar -# collections run CONCURRENTLY, each capturing a disjoint partition of the -# heap: the first collection to CAS its tag into a cell owns it; everyone -# else treats the cell as an opaque survivor. Stale tags (from retired +# collection's tag with the cell's dense discovery index. Collections run +# CONCURRENTLY — one slot per collecting thread, see gParSlots — each +# capturing a disjoint partition of the heap: the first collection to CAS +# its tag into a cell owns it; everyone else treats the cell as an opaque +# survivor. Stale tags (from retired # collections) never need clearing, they are simply reclaimable. # # The word does double duty a second time: commit re-stamps proven-live @@ -349,22 +384,66 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = # long-lived live structures are traced once per epoch instead of once per # collection. Roots always bypass the stamp: every death has a dec-witness # that gets registered, and registered cells are always scanned as roots. +type + CollCtx = object + ## This thread's collector context, in ONE threadvar rather than one per + ## field. macOS resolves every `{.threadvar.}` access through a + ## `tlv_get_addr` thunk — an indirect call, not a register-relative load + ## — so N separate threadvars cost N calls in a function while N fields + ## of one cost ONE. The hot collector paths hoist `addr gCtx` once and + ## pass the pointer down; `claimCell` (entered once per captured cell) + ## and `rememberGenSuspect` (once per young→old dec) resolve no TLS at + ## all. Add new collector-thread state HERE rather than as a fresh + ## threadvar, or that win is given straight back. + ## + ## `tag`/`slot`/`epochStamp`/`amSolo` are per-collection and set by + ## `startCollection`; the rest persists across collections. + tag: int64 + slot: int + epochStamp: int64 ## this collection's epoch, as a stamp word + amSolo: bool + genSuspects: CellSeq[Cell] + ## Stamped cells that received a young→old commit dec this epoch + ## without an RC death blow. Not roots (so minor collects keep + ## pruning); flushed into `gLocalRoots` when `gEpoch` advances — see + ## `flushGenSuspects`. Holds `inRootsFlag` on every entry, which is + ## what keeps the cells alive while listed. + seenEpoch: int + ## Last epoch for which this thread flushed `genSuspects`. + genFlushReady: bool + ## False until the first `flushGenSuspects` call (a zeroed context has + ## `seenEpoch == 0 == gEpoch`, so the flag distinguishes "never + ## flushed" from "already flushed epoch 0"). + const - MaxPar {.intdefine.} = 8 # max concurrent collections + MaxPar {.intdefine.} = 256 + ## CAPACITY of the slot table, not a tuning knob: a hard ceiling on + ## concurrent collections, sized to stay out of the way (16 bytes of BSS + ## per entry, and scans only ever run over `gParSlots`, below). Raise it + ## only for programs with more than this many threads collecting AT ONCE. ParSlots = MaxPar var gMergeLock: Lock # protects the tag slots + orphaned roots gActiveTags: array[ParSlots, int64] # 0 = free slot gSlotPhase: array[ParSlots, int] # 0 idle, 1 capturing, 2 committing + gParSlots: int = 1 + ## How many slots are IN PLAY. Every scan below runs over this prefix + ## rather than over the whole table, and it is raised (under gMergeLock, + ## never lowered) exactly when a thread wants to collect and every slot + ## already in play is busy. It therefore converges on the number of + ## threads that actually collect CONCURRENTLY — one slot each — and then + ## stops: the parallelism is auto-tuned to the workload instead of being + ## a compile-time guess. A fixed bound throttled hard whenever the thread + ## count exceeded it, because the surplus threads did not just collect + ## later, they PARKED in `parkUntil(anySlotFree())` waiting for a slot + ## (32 threads against 8 slots: 6.3s vs 2.6s with a slot each). + ## Starting at 1 also means single-threaded programs scan one entry. gSoloCapture: int # a solo collection is in its capture phase gTagCounter: int64 - gMyTag {.threadvar.}: int64 - gMySlot {.threadvar.}: int - gAmSolo {.threadvar.}: bool + gCtx {.threadvar.}: CollCtx gEpoch: int # advanced every YrcEpochLen collections gCollectionCounter: int - gMyEpochStamp {.threadvar.}: int64 # this collection's epoch, as a stamp word gWaitLock: Lock # pairs gWaitCond's wait/broadcast; leaf gWaitCond: Cond # signaled on capture-end and collection-finish @@ -399,9 +478,19 @@ const epochBase = 0x40000000 # stamp namespace: tags stay below this epochMask = 0x3FFFFFFF -# stamp layout: high word = epochBase|epoch, low word = survival age +# Every packed word in this file squeezes two 32-bit fields into one int64 as +# `(hi shl 32) or lo`. The claim word (a cell's `rootIdx`, see above) is either +# hi = owning collection's tag, lo = the cell's dense capture index +# hi = epochBase|epoch (a stamp), lo = the cell's survival age +# and a `gPendingWatch` entry is hi = slot, lo = tag. The high half is read +# with a plain `shr 32` — every value stored there is below 2^31, so the shift +# keeps it positive and needs no mask. The low half is the one that needs +# masking: `shl 32` left the high half sitting above it, and `and 0xFFFFFFFF` +# is what clears those bits back out. +template loWord(w: int64): int64 = w and 0xFFFFFFFF + template epochStamp(e: int): int64 = int64(epochBase or (e and epochMask)) shl 32 -template stampAge(w: int64): int = int(w and 0xFFFFFFFF) +template stampAge(w: int64): int = int(loWord(w)) template isEpochStamp(w: int64): bool = (w shr 32) >= epochBase template parkUntil(cond: untyped) = @@ -426,24 +515,32 @@ proc collectorEvent() {.inline.} = broadcast gWaitCond release gWaitLock +proc slotsInPlay(): int {.inline.} = + ## Must be loaded AFTER whatever claim word the caller is validating: a + ## slot is put in play before the collection owning it can tag any cell, + ## so a load ordered after reading a tagged word is guaranteed to cover + ## the slot that wrote the tag. `gParSlots` only ever grows, so a scan + ## over this prefix can never shrink under a reader. + atomicLoadN(addr gParSlots, ATOMIC_ACQUIRE) + proc anySlotFree(): bool {.inline.} = result = false - for sl in 0 ..< ParSlots: + for sl in 0 ..< slotsInPlay(): if atomicLoadN(addr gActiveTags[sl], ATOMIC_ACQUIRE) == 0: return true -template isStamped(c: Cell): bool = +template isStamped(c: Cell; ctx: ptr CollCtx): bool = # "stamped" means: claimed by THIS collection. A relaxed load suffices: - # only this thread ever stores gMyTag, and any stale read of a foreign + # only this thread ever stores ctx.tag, and any stale read of a foreign # value routes into claimCell which re-validates with acquire + CAS. - (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == gMyTag + (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == ctx.tag template denseIdx(c: Cell): int32 = - int32(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) and 0xFFFFFFFF) + int32(loWord(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED))) proc isActiveTag(t: int64): bool {.inline.} = result = false if t != 0: - for s in 0 ..< ParSlots: + for s in 0 ..< slotsInPlay(): if atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) == t: return true @@ -485,6 +582,25 @@ var ## draining this thread's stripe queue registers candidates here, and ## this thread's collections steal it as their slice — no lock, and ## collections keep the cache locality of thread-local data. + gPendingCells {.threadvar.}: CellSeq[Cell] + ## Cells this thread committed dead — slots nil'ed, references already + ## decremented — but has not handed back to the allocator yet, because a + ## capture that was in flight at commit time may still hold a stale + ## (slot, value) snapshot pointing at them. Released at the start of this + ## thread's next collection; see `releasePending`. + gPendingWatch {.threadvar.}: RawSeq[int64] + ## The captures that batch must outlive, packed as (slot shl 32) or tag. + gPendingSlot {.threadvar.}: int + gPendingActive {.threadvar.}: bool + gSpareRoots {.threadvar.}: CellSeq[Cell] + ## The buffer a finished collection hands back, so that stealing + ## `gLocalRoots` costs no allocation in steady state. + gTraceBuf {.threadvar.}: CellSeq[TraceEntry] + gFreeBuf {.threadvar.}: CellSeq[Cell] + ## Storage for `GcEnv.traceStack` / `GcEnv.toFree`, kept across + ## collections like the `gCap` arrays: both grow to the size of the + ## captured graph, so re-allocating and re-growing them per collection + ## was a memcpy of the whole trace frontier every time. stripes: array[NumStripes, Stripe] rootsThreshold: int = 128 # shared adaptive heuristic; races are benign defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128 @@ -566,12 +682,50 @@ proc registerLocal(c: Cell; desc: PNimTypeV2) {.inline.} = when defined(nimOrcStats): let st = atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) if st == 0: bumpStat gStatRegFresh - elif gMyTag != 0 and (st shr 32) == gMyTag: bumpStat gStatRegSelf + elif gCtx.tag != 0 and (st shr 32) == gCtx.tag: bumpStat gStatRegSelf elif isActiveTag(st shr 32): bumpStat gStatRegCross else: bumpStat gStatRegRepeat if gLocalRoots.d == nil: init(gLocalRoots) add(gLocalRoots, c, desc) +proc rememberGenSuspect(c: Cell; desc: PNimTypeV2; + ctx: ptr CollCtx) {.inline.} = + ## Note a stamped cell that lost a young→old edge without an RC death + ## blow. This IS a candidate buffer — just one that is not scanned until + ## the epoch advances — so it takes the same ownership token as + ## `gLocalRoots`: winning `inRootsFlag` is what keeps the cell in exactly + ## one buffer AND what keeps it alive while listed (`computeDeadness` + ## forces flagged cells live, and `free` refuses to dispose them). + ## + ## Marking the stamp word instead does NOT work: `claimCell` CASes the + ## whole word to its tag when a later collection captures the cell, so + ## the mark is lost while the list entry survives — and the cell is then + ## freed under a list that still points at it. + if rcTestSetFlag(c, inRootsFlag): + if ctx.genSuspects.d == nil: init(ctx.genSuspects) + add(ctx.genSuspects, c, desc) + +proc spillGenSuspects(ctx: ptr CollCtx) {.inline.} = + ## Move suspects into the root set (epoch advance, thread exit, or a + ## forced major collect). The cells stay flagged; they merely change + ## buffers, so the one-buffer invariant holds — same handover as + ## `adoptOrphans`. Going through `registerLocal` would be wrong here: it + ## would lose the test-and-set it already owns and drop every entry. + if ctx.genSuspects.len == 0: return + if gLocalRoots.d == nil: init(gLocalRoots) + for i in 0 ..< ctx.genSuspects.len: + add(gLocalRoots, ctx.genSuspects.d[i][0], ctx.genSuspects.d[i][1]) + ctx.genSuspects.len = 0 + +proc flushGenSuspects(ctx: ptr CollCtx) {.inline.} = + ## Promote deferred young→old dec targets into the root set after an + ## epoch advance (major collection). No-op while the epoch is stable. + let e = atomicLoadN(addr gEpoch, ATOMIC_RELAXED) + if ctx.genFlushReady and e == ctx.seenEpoch: return + ctx.genFlushReady = true + ctx.seenEpoch = e + spillGenSuspects(ctx) + proc drainStripe(i: int) = ## Apply the pending RC operations of one stripe queue; freshly ## dead-looking cells become THIS thread's candidates. rc mutations are @@ -677,6 +831,62 @@ template orcAssert(cond, msg) = cfprintf(cstderr, "[Bug!] %s\n", msg) rawQuit 1 +proc graceSatisfied(): bool = + ## Has every capture recorded in `gPendingWatch` finished? A capture that + ## started later cannot hold a stale snapshot of the batch: it reads every + ## slot fresh, and the batch's cells are unreachable by then. + result = true + for i in 0 ..< gPendingWatch.len: + let w = gPendingWatch.d[i] + let s = int(w shr 32) + let tg = loWord(w) + if atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) == tg and + atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) == 1: + return false + +proc buildPendingWatch(): bool = + ## Record the collections that are in their CAPTURE phase right now: they + ## are the only ones that can hold a stale (slot, value) snapshot of the + ## cells we are about to release. `false` (nothing capturing) is the common + ## case below a handful of threads, and means the batch can be freed on the + ## spot with no deferral and no destructor-timing change at all. + if gPendingWatch.d == nil: init gPendingWatch + gPendingWatch.len = 0 + for s in 0 ..< slotsInPlay(): + if s != gCtx.slot: + let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) + if tg != 0 and atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) == 1: + gPendingWatch.add((int64(s) shl 32) or tg) + result = gPendingWatch.len > 0 + +proc releasePending() = + ## Hand this thread's parked batch back to the allocator and give its slot + ## up. Called at the start of every collection, so by the time it runs the + ## watched captures have had a whole collection's worth of time to finish + ## and the wait below is virtually always already satisfied — that is the + ## whole point: the wait moved off the commit path, where it blocked BOTH + ## the committing collector and (because commit runs inside the GC fence) + ## every mutator doing a seq operation. + if not gPendingActive: return + parkUntil(graceSatisfied()) + # Give the slot up FIRST: the batch is unreachable and no capture can hold + # a snapshot of it any more, so the tag has nothing left to protect. + gPendingActive = false + gPendingWatch.len = 0 + atomicStoreN(addr gActiveTags[gPendingSlot], 0, ATOMIC_SEQ_CST) + collectorEvent() + # Destructors run here. `Collecting` is the existing re-entrancy guard: a + # destructor-driven dec that overflows a stripe must drain it, not start a + # nested collection that would write into the batch we are walking. + let prev = lockState + lockState = Collecting + for i in 0 ..< gPendingCells.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", gPendingCells.d[i][0], gPendingCells.d[i][1]) + free(gPendingCells.d[i][0], gPendingCells.d[i][1]) + gPendingCells.len = 0 + lockState = prev + proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = let p = cast[ptr pointer](q) # read the slot exactly once: mutators may exchange it concurrently. @@ -698,6 +908,7 @@ proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = proc prepareCapture() = if gCap.recs.d == nil: init gCap.recs + init gCap.sccIdx init gCap.tstack init gCap.frames init gCap.edges @@ -706,98 +917,166 @@ proc prepareCapture() = init gCap.crossTgt init gCap.crossPend init gCap.prunedSrc + init gCap.prunedTgt init gCap.ages + init gCap.slots else: gCap.recs.len = 0 + gCap.sccIdx.len = 0 gCap.tstack.len = 0 gCap.frames.len = 0 gCap.edges.len = 0 gCap.sccs.len = 0 gCap.sccMembers.len = 0 + gCap.crossTgt.len = 0 gCap.crossPend.len = 0 gCap.prunedSrc.len = 0 + gCap.prunedTgt.len = 0 gCap.ages.len = 0 + gCap.slots.len = 0 + +proc growCaptureArrays(cap: ptr CaptureBufs) {.noinline.} = + ## `recs`, `sccIdx`, `ages` and `tstack` are appended to together, and only + ## by `claimCell` — one entry each per claimed cell. So they are grown + ## together and ONE capacity check on `recs` covers all four: `recs` drives + ## the growth, the other three are topped up to at least its capacity. + ## `tstack` is popped as SCCs are emitted, so its length only ever trails + ## `recs.len`; matching capacities keeps its appends unchecked too. + ## Marked `noinline` to keep the cold resize path out of `claimCell`. + resize(cap.recs, cap.recs.len +% 1) + let n = cap.recs.cap + if cap.sccIdx.cap < n: resize(cap.sccIdx, n) + if cap.ages.cap < n: resize(cap.ages, n) + if cap.tstack.cap < n: resize(cap.tstack, n) # rc is captured without the flag bits: the collector itself toggles # inRootsFlag between capture and commit, which must not look like a # mutation to the commit-time rc validation. proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs; - pruneLive: bool): int32 = + ctx: ptr CollCtx; pruneLive: bool; old0: int64): int32 = ## Dense index if this collection owns `c` (claiming and registering it ## if it was unclaimed), -1 if another ACTIVE collection owns it, or ## -2 if `pruneLive` and the cell was proven live in the current epoch ## (treat as an opaque live external, don't descend). - if gAmSolo: + ## + ## `old0` is `c`'s claim word as the caller already read it (acquire): the + ## DFS reads it to test ownership, and re-reading it here would be a second + ## dependent load of the same cold header word on every traversed edge. + if ctx.amSolo: # no other collection is (or can start) capturing: plain stores. # This recovers the sequential capture speed of the single-collector # design whenever collections do not actually overlap. - let old = c.rootIdx - if (old shr 32) == gMyTag: - return int32(old and 0xFFFFFFFF) - if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and + let old = old0 + if (old shr 32) == ctx.tag: + return int32(loWord(old)) + if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and stampAge(old) >= YrcPromoteAge: return -2 let idx = cap.recs.len - c.rootIdx = (gMyTag shl 32) or int64(idx) + c.rootIdx = (ctx.tag shl 32) or int64(idx) when defined(nimOrcStats): bumpStat gStatCapTotal if old != 0: bumpStat gStatCapRepeat - cap.recs.add CaptureRec(cell: c, desc: desc, + if idx >= cap.recs.cap: growCaptureArrays(cap) + cap.recs.addUnchecked CaptureRec(cell: c, desc: desc, rcWord: loadRc(c) and not rcMask, - lowlink: int32(idx), sccOf: -1'i32) - cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) - cap.tstack.add int32(idx) + lowlink: int32(idx), selfRefs: 0'i32) + cap.sccIdx.addUnchecked -1'i32 + cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) + cap.tstack.addUnchecked int32(idx) return int32(idx) + var old = old0 while true: - var old = atomicLoadN(addr c.rootIdx, ATOMIC_ACQUIRE) - if (old shr 32) == gMyTag: - return int32(old and 0xFFFFFFFF) - if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and + if (old shr 32) == ctx.tag: + return int32(loWord(old)) + if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and stampAge(old) >= YrcPromoteAge: return -2 - if isActiveTag(old shr 32): + # an epoch stamp is never a tag (tags are allocated below `epochBase`), + # so the scan over the active-tag slots is skipped for the common + # "cell survived an earlier collection" word + if not isEpochStamp(old) and isActiveTag(old shr 32): return -1 let idx = cap.recs.len if atomicCompareExchangeN(addr c.rootIdx, addr old, - (gMyTag shl 32) or int64(idx), false, + (ctx.tag shl 32) or int64(idx), false, ATOMIC_ACQ_REL, ATOMIC_RELAXED): when defined(nimOrcStats): bumpStat gStatCapTotal if old != 0: bumpStat gStatCapRepeat - cap.recs.add CaptureRec(cell: c, desc: desc, + if idx >= cap.recs.cap: growCaptureArrays(cap) + cap.recs.addUnchecked CaptureRec(cell: c, desc: desc, rcWord: loadRc(c) and not rcMask, - lowlink: int32(idx), sccOf: -1'i32) - cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) - cap.tstack.add int32(idx) + lowlink: int32(idx), selfRefs: 0'i32) + cap.sccIdx.addUnchecked -1'i32 + cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) + cap.tstack.addUnchecked int32(idx) return int32(idx) + # a failed CAS leaves the fresh claim word in `old`; loop with it proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = ## Iterative Tarjan SCC over everything reachable from `s`. A frame's ## pending out-edges are the traceStack entries above frame.base; a child ## pushes and drains its own segment above ours, so when the child's frame ## pops, the stack is back at our segment and we resume popping our edges. - if isStamped(s): return + ## + ## An SCC's out-edges are classified into internal/cross the moment the SCC + ## is emitted, not in a later pass over a whole-graph edge list. That works + ## because `cap.edges` is truncated back to a frame's `ebase` whenever that + ## frame's cell turns out to be an SCC root: edges of already-emitted + ## sub-SCCs are gone, so `edges[ebase(u) ..< len]` at u's emission holds + ## exactly the out-edges of u's SCC. Every one of them targets either a + ## member (u would not be an SCC root if a member pointed at a cell still + ## on the stack below u) or an SCC emitted earlier, so `sccIdx` is final + ## for all of them and the cross targets can be appended to `crossTgt` + ## contiguously — which makes `crossOff` a prefix offset for free. + # one TLS resolution for the whole traversal; `claimCell` and the per-edge + # ownership tests below read the context through this pointer + let ctx = addr gCtx + let rootWord = atomicLoadN(addr s.rootIdx, ATOMIC_ACQUIRE) + if (rootWord shr 32) == ctx.tag: return orcAssert(j.traceStack.len == 0, "capture: trace stack not empty") # roots never prune: a dec-witnessed suspicion overrides any epoch stamp - let root = claimCell(s, desc, cap, pruneLive = false) + let root = claimCell(s, desc, cap, ctx, pruneLive = false, old0 = rootWord) if root < 0: return # another active collection owns this candidate; it handles it trace(s, desc, j) - cap.frames.add TarjanFrame(u: root, base: 0) - while cap.frames.len > 0: - let u = cap.frames.d[cap.frames.len -% 1].u - let base = cap.frames.d[cap.frames.len -% 1].base + # The innermost frame is kept in `u`/`base` instead of being re-read from + # `cap.frames` on every iteration: the loop body runs once per captured + # EDGE, so reloading the frame there costs more than the frame stack itself. + # `cap.frames` therefore only holds the ANCESTORS of `u`. + var u = root + var base = 0 + var ebase = 0'i32 + while true: if j.traceStack.len > base: - let (entry, tdesc) = j.traceStack.pop() + # inlined pop: log every slot for commit (dead cells are immutable, so + # the capture-time value is what commit must nil/dec), then classify. + let last = j.traceStack.len -% 1 + j.traceStack.len = last + let entry = j.traceStack.d[last][0] + let tdesc = j.traceStack.d[last][1] let t = head(entry.val) - if isStamped(t): - let v = denseIdx(t) - cap.edges.add (int64(u) shl 32) or int64(v) - if cap.recs.d[v].sccOf < 0 and v < cap.recs.d[u].lowlink: - cap.recs.d[u].lowlink = v + # field address for the allDead nil pass (8 bytes; see CaptureBufs.slots) + cap.slots.add entry.slot + # one load of the target's claim word serves both the ownership test + # and the dense-index extraction + let cw = atomicLoadN(addr t.rootIdx, ATOMIC_ACQUIRE) + if (cw shr 32) == ctx.tag: + let v = int32(loWord(cw)) + if v == u: + # A self edge is internal by construction and its reference is + # already in `rcWord`, so it cancels out of `sumRefs - internal` + # exactly. Counting it here keeps it out of the edge array and out + # of the classification pass below. + inc cap.recs.d[u].selfRefs + else: + cap.edges.add v + if cap.sccIdx.d[v] < 0 and v < cap.recs.d[u].lowlink: + cap.recs.d[u].lowlink = v else: let childBase = j.traceStack.len - let v = claimCell(t, tdesc, cap, pruneLive = true) + let v = claimCell(t, tdesc, cap, ctx, pruneLive = true, old0 = cw) if v == -1: # cross-collection edge: the owner sees our reference in the rc # word and classifies the target live; we re-register it as a @@ -806,79 +1085,103 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = elif v == -2: # target proven live this epoch: opaque live external, no descent. # Taint u's SCC — its own "live" verdict may lean on the stamp. - cap.prunedSrc.add u + # The list is only ever read as a set (and for its emptiness), and + # one cell's out-edges are consumed consecutively, so suppressing a + # repeat of the previous entry removes nearly every duplicate a + # multi-pruned cell would otherwise contribute. + if cap.prunedSrc.len == 0 or + cap.prunedSrc.d[cap.prunedSrc.len -% 1] != u: + cap.prunedSrc.add u + # If the stamped target itself looks RC-dead, keep it examinable + # (roots bypass stamps). Do NOT register live stamped targets: + # that would force a full re-trace of the long-lived web every + # collection and defeat epoch pruning. Survivors that may be + # pinned by phantom edges from a dead stamped cell are handled + # in demoteTouchedDead. + # + # `t` is NOT owned by this collection (it is stamped, i.e. claimable + # by anyone), so parking a bare pointer until commit is unsafe: the + # grace period only watches collections in their CAPTURE phase + # (`buildPendingWatch`), so once we reach phase 2 another thread may + # capture `t`, find it dead and free it — under a list still holding + # it. Winning `inRootsFlag` here makes `prunedTgt` a proper candidate + # buffer: the cell is then forced live by any collection that + # captures it and refused by `free`, so it survives to our commit. + if (loadRc(t) and not rcMask) == 0 and rcTestSetFlag(t, inRootsFlag): + cap.prunedTgt.add(t, tdesc) when defined(nimOrcStats): bumpStat gStatCapPruned else: - cap.edges.add (int64(u) shl 32) or int64(v) + cap.edges.add v trace(t, tdesc, j) - cap.frames.add TarjanFrame(u: v, base: childBase) + cap.frames.add TarjanFrame(u: u, ebase: ebase, base: base) + u = v + ebase = int32(cap.edges.len) + base = childBase else: - cap.frames.len = cap.frames.len -% 1 - if cap.frames.len > 0: - let pu = cap.frames.d[cap.frames.len -% 1].u - if cap.recs.d[u].lowlink < cap.recs.d[pu].lowlink: - cap.recs.d[pu].lowlink = cap.recs.d[u].lowlink - if cap.recs.d[u].lowlink == u: + let lowU = cap.recs.d[u].lowlink + if lowU == u: # u is the root of an SCC: pop the members off the Tarjan stack + let sid = int32(j.nScc) let memStart = int32(cap.sccMembers.len) var sum = 0 while true: - let w = cap.tstack.pop() - cap.recs.d[w].sccOf = int32(j.nScc) - cap.sccMembers.add w - sum = sum +% (cap.recs.d[w].rcWord shr rcShift) +% 1 - if w == u: break - cap.sccs.add SccRec(sumRefs: sum, memStart: memStart) + let m = cap.tstack.pop() + cap.sccIdx.d[m] = sid + cap.sccMembers.add m + # `- selfRefs`: a self edge counts in both `sumRefs` and `internal` + # and was folded out of the edge array in the loop above + sum = sum +% (cap.recs.d[m].rcWord shr rcShift) +% 1 -% + cap.recs.d[m].selfRefs + if m == u: break + # classify this SCC's out-edges now that every target's SCC is final + let crossOff = int32(cap.crossTgt.len) + var internal = 0 + for i in ebase ..< int32(cap.edges.len): + let sv = cap.sccIdx.d[cap.edges.d[i]] + if sv == sid: inc internal + else: cap.crossTgt.add sv + cap.edges.len = ebase + cap.sccs.add SccRec(sumRefs: sum, internal: internal, + memStart: memStart, crossOff: crossOff) inc j.nScc + if cap.frames.len == 0: break + let pi = cap.frames.len -% 1 + cap.frames.len = pi + let pu = cap.frames.d[pi].u + if lowU < cap.recs.d[pu].lowlink: + cap.recs.d[pu].lowlink = lowU + u = pu + ebase = cap.frames.d[pi].ebase + base = cap.frames.d[pi].base # ---------------- phase 2: deadness, side arrays only ---------------- proc computeDeadness(j: var GcEnv; cap: ptr CaptureBufs) = let nScc = j.nScc - # append the sentinel record ([nScc]); capture left internal/deadIn/flags - # zero-initialized and memStart valid. crossOff is filled below as a prefix - # sum, so the sentinel closes the last SCC's member and cross-edge slices. - cap.sccs.add SccRec(memStart: int32(cap.sccMembers.len)) - # classify captured edges: internal to an SCC vs condensation cross edges - var nCross = 0 - for i in 0 ..< cap.edges.len: - let e = cap.edges.d[i] - let su = cap.recs.d[int32(e shr 32)].sccOf - let sv = cap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf - if su == sv: - inc cap.sccs.d[su].internal - else: - inc cap.sccs.d[su].crossOff - inc nCross - var total = 0'i32 - for s in 0 ..< nScc: - let c = cap.sccs.d[s].crossOff - cap.sccs.d[s].crossOff = total - cap.sccs.d[s].crossCursor = total - total = total +% c - cap.sccs.d[nScc].crossOff = total - setLenUninit cap.crossTgt, nCross - for i in 0 ..< cap.edges.len: - let e = cap.edges.d[i] - let su = cap.recs.d[int32(e shr 32)].sccOf - let sv = cap.recs.d[int32(e and 0xFFFFFFFF'i64)].sccOf - if su != sv: - cap.crossTgt.d[cap.sccs.d[su].crossCursor] = sv - inc cap.sccs.d[su].crossCursor + # append the sentinel record ([nScc]) that closes the last SCC's member and + # cross-edge slices; capture left deadIn/flags zero-initialized and filled + # sumRefs/internal/memStart/crossOff in already, so the condensation is + # complete the moment the DFS ends. + cap.sccs.add SccRec(memStart: int32(cap.sccMembers.len), + crossOff: int32(cap.crossTgt.len)) # pruned out-edges taint the source SCC: pruning cannot cause a false # "dead" (an untraced target only ever ADDS unexplained external refs), # but a "live" verdict may lean on a stamp that went stale within the # epoch, so validate re-registers surviving pruned SCCs for i in 0 ..< cap.prunedSrc.len: - let s = cap.recs.d[cap.prunedSrc.d[i]].sccOf + let s = cap.sccIdx.d[cap.prunedSrc.d[i]] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagPruned # cells that stay registered as roots (partial collection) count as - # externally referenced: the roots buffer itself points at them - for mi in 0 ..< cap.sccMembers.len: - let m = cap.sccMembers.d[mi] + # externally referenced: the roots buffer itself points at them. + # Scanned over `recs` rather than over `sccMembers`: every claimed cell is + # pushed to the Tarjan stack once and popped into `sccMembers` once, so the + # two cover exactly the same set, and the SCC comes from `sccIdx` either + # way. Going through `sccMembers` would only add a random 32-byte-stride + # gather in front of a load that already misses on the cell header. + for m in 0 ..< cap.recs.len: if (loadRc(cap.recs.d[m].cell) and inRootsFlag) != 0: - let s = cap.recs.d[m].sccOf + let s = cap.sccIdx.d[m] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagForcedLive # deadness over the condensation. Tarjan emits sinks first, so higher SCC # ids are sources and every cross edge goes from a higher id to a lower @@ -907,10 +1210,11 @@ proc markDirtyFromQueues(j: var GcEnv; cap: ptr CaptureBufs) = ## the last drain had its reference set changed during capture. Peek ## (don't drain!) the stripe queues and taint the affected SCCs; the ## entries stay queued and the next merge re-registers them as candidates. + let ctx = addr gCtx template taint(cp: Cell) = let c = cp - if isStamped(c): - let s = cap.recs.d[denseIdx(c)].sccOf + if isStamped(c, ctx): + let s = cap.sccIdx.d[denseIdx(c)] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagDirty for i in 0.. 0: - # A prune happened somewhere in THIS collection, so every "live" verdict - # it produced is suspect: a pruned cell is not traced, yet its out-edges - # still count toward its targets' rc. If that pruned cell is itself dead - # (promoted while live, died later this epoch), its phantom references - # inflate unrelated SCCs' external counts and misclassify genuinely dead - # SCCs as plain survivors. Such a survivor is not flagPruned, so without - # this it would be re-stamped, dropped from the retry set, and orphaned - # forever once its last flagPruned neighbor resolves. Keeping ONE member - # of every survivor registered guarantees it is re-examined until the - # epoch advances, captures the dead promoted cells, and decrements the - # phantom edges away. Cheap in practice: pruning keeps the captured set - # small, so "every survivor" is only the few cells actually traced. + # A prune happened somewhere in THIS collection, so every "live" + # verdict it produced is suspect: a pruned cell is not traced, yet + # its out-edges still count toward its targets' rc. If that pruned + # cell is itself dead (promoted while live, died later this epoch), + # its phantom references inflate unrelated SCCs' external counts. + # Keeping ONE member of every survivor examinable is what catches + # that. RC-dead stamped targets are additionally queued in prunedTgt + # (roots bypass stamps). + # + # A SUSPECT, not a root: the epoch advance is the only thing that can + # ever settle these. Re-examining a survivor without tracing the + # pruned cell reproduces the same verdict, so as roots they are + # captured, survive, and re-register every single collection — a loop + # that cannot converge and that grows the captured set without bound + # in exactly the workloads pruning is meant to speed up (on the + # generational bench, 56% of all captures and 84% of the repeats). + # The suspect buffer keeps the cell just as findable: same + # inRootsFlag ownership, forced live by computeDeadness and refused + # by free while listed, spilled into the root set by the epoch + # advance, by thread exit and by GC_fullCollect (which loops until + # quiet). Dropping the registration ENTIRELY instead is unsound and + # leaks: a survivor that is neither root nor suspect is invisible + # forever, and no later full collect can find it again. let m = cap.sccMembers.d[cap.sccs.d[s].memStart] - registerLocal(cap.recs.d[m].cell, cap.recs.d[m].desc) + rememberGenSuspect(cap.recs.d[m].cell, cap.recs.d[m].desc, addr gCtx) proc validateDead(j: var GcEnv; cap: ptr CaptureBufs) = ## Demote every dead SCC that a mutator touched during capture: dirty via @@ -999,56 +1314,76 @@ proc validateDead(j: var GcEnv; cap: ptr CaptureBufs) = proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) = validateDead(j, cap) + let ctx = addr gCtx # publish cross-collection edge targets as candidate roots BEFORE any of # our commit decs could make them collectible: they are owner-live this # round, and the registration keeps them examinable in a later round for i in 0 ..< cap.crossPend.len: registerLocal(cap.crossPend.d[i][0], cap.crossPend.d[i][1]) - template deadCell(t: Cell): bool = - isStamped(t) and (cap.sccs.d[cap.recs.d[denseIdx(t)].sccOf].flags and flagDead) != 0 - template graceWait() = - # Grace period: another collection still in its CAPTURE phase may hold - # stale (slot, value) snapshots referencing our dead cells; disposing - # them now could hand reused memory to its traversal. Captures are - # bounded and never wait on us. New captures cannot reach our dead - # cells: they are unreachable, and our tag stays active until after - # the frees. - for s in 0 ..< ParSlots: - if s != gMySlot: - let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) - if tg != 0: - parkUntil(atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) != tg or - atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) != 1) + # pruned (epoch-stamped) targets: roots bypass stamps, so a later + # collection will capture a target that has since died and clear the + # phantom edges that would otherwise pin unrelated survivors. Capture + # already won `inRootsFlag` on these (see the `prunedTgt.add` site), so + # they only change buffers here — `registerLocal` would lose the + # test-and-set it already holds and drop every one of them. + if cap.prunedTgt.len > 0: + if gLocalRoots.d == nil: init(gLocalRoots) + for i in 0 ..< cap.prunedTgt.len: + add(gLocalRoots, cap.prunedTgt.d[i][0], cap.prunedTgt.d[i][1]) + template deadCell(w: int64): bool = + ## `w` is the target's claim word, read once by the caller + (w shr 32) == ctx.tag and + (cap.sccs.d[cap.sccIdx.d[int32(loWord(w))]].flags and flagDead) != 0 + # Grace period: a collection still in its CAPTURE phase may hold stale + # (slot, value) snapshots referencing our dead cells; disposing them now + # could hand reused memory to its traversal. This used to BLOCK here until + # every such capture ended, which serialized each committing collector + # against every capturing one — and did so while holding the GC fence, so + # mutators doing seq operations spun for the duration too. Instead the + # batch is parked (`gPendingCells`) together with the set of captures it + # must outlive, and `releasePending` frees it at the start of this + # thread's next collection. Our tag stays in `gActiveTags` until then, so + # a capture that reaches a parked cell through a stale snapshot still sees + # it as owned by an active collection and cannot claim — and free — it. + # The batch's destructors therefore run one collection later than they + # used to; nothing else observes the delay, since the cells are + # unreachable, their slots are nil and their references already dropped. + template parkBatch(): bool = (if cap.recs.len == 0: false else: buildPendingWatch()) + template holdOrFree(c: Cell; d: PNimTypeV2; deferred: bool) = + if deferred: + gPendingCells.add(c, d) + else: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", c, d) + free(c, d) # A dead cell's reference to another active collection's cell must still # be decremented (the target survives this round), so the all-dead fast # path additionally requires that no cross-collection edge was seen. # pruned edges also disable the fused path: its nil-without-dec would - # leak rc on the stamped targets (and skip their re-registration) + # leak rc on the stamped targets (those still need trialDec) let allDead = j.nDeadScc == j.nScc and j.nAborted == 0 and cap.crossPend.len == 0 and cap.prunedSrc.len == 0 if allDead: # Everything captured dies and no slot can point outside the dead set: - # nil the slots and free in ONE pass over the cells — they went cold - # since capture, a second sweep would miss cache all over again. - # Freeing cell A before nil-ing a later cell B's slot that points at A - # is fine: nobody reads B's slots in between (mutators cannot reach the - # closed dead set, foreign captures never traverse our tagged cells, - # and the grace wait has retired stale snapshots before the first free). - graceWait() + # nil from the capture-time slot log (no re-trace) and free. Freeing + # cell A before nil-ing a later cell B's slot that points at A is fine: + # nobody reads B's slots in between (mutators cannot reach the closed + # dead set, foreign captures never traverse our tagged cells, and a + # parked batch is not touched until its watch list is clear). + let deferred = parkBatch() + if deferred and gPendingCells.d == nil: init gPendingCells + for i in 0 ..< cap.slots.len: + cap.slots.d[i][] = nil for m in 0 ..< cap.recs.len: - let cell = cap.recs.d[m].cell - let desc = cap.recs.d[m].desc - orcAssert(j.traceStack.len == 0, "commitDead: trace stack not empty") - trace(cell, desc, j) - while j.traceStack.len > 0: - let (entry, _) = j.traceStack.pop() - entry.slot[] = nil - when orcLeakDetector: - writeCell("CYCLIC OBJECT FREED", cell, desc) - free(cell, desc) + holdOrFree(cap.recs.d[m].cell, cap.recs.d[m].desc, deferred) j.freed = cap.recs.len + if deferred: + gPendingSlot = ctx.slot + gPendingActive = true else: - init j.toFree + if gFreeBuf.d == nil: init gFreeBuf + gFreeBuf.len = 0 + j.toFree = gFreeBuf for s in 0 ..< j.nScc: if (cap.sccs.d[s].flags and flagDead) != 0: for mi in cap.sccs.d[s].memStart ..< cap.sccs.d[s+1].memStart: @@ -1067,32 +1402,66 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) = let (entry, tdesc) = j.traceStack.pop() let t = head(entry.val) entry.slot[] = nil - if not deadCell(t): + let tw = atomicLoadN(addr t.rootIdx, ATOMIC_RELAXED) + if not deadCell(tw): trialDec(t) - # a stamped target was not analyzed by THIS collection, so - # this dec may be the death blow: keep the cell examinable - if isEpochStamp(atomicLoadN(addr t.rootIdx, ATOMIC_RELAXED)): - registerLocal(t, tdesc) - # epoch-stamp what this collection PROVED live, carrying the cell's - # survival age: only cells that keep surviving get promoted to ages - # where captures prune them, so die-young data is never deferred. - # Demoted (dirty) and pruned SCCs stay unproven — leave their stale - # tags claimable. Our tag is still active, so no foreign claim can - # race these stores. + # Stamped target: not analyzed by THIS collection. Generational + # young→old — re-root immediately only on a true RC death blow + # (refcount encoding: (rc shr rcShift) + 1 == #refs). Otherwise + # remember the cell as a suspect; epoch advance flushes + # suspects into the root set (major collection). + if isEpochStamp(tw): + if (loadRc(t) shr rcShift) < 0: + registerLocal(t, tdesc) + else: + rememberGenSuspect(t, tdesc, ctx) + # epoch-stamp what this collection PROVED live, carrying the survival + # age: only cells that keep surviving get promoted to ages where + # captures prune them, so die-young data is never deferred. Demoted + # (dirty) and pruned SCCs stay unproven — leave their stale tags + # claimable. Our tag is still active, so no foreign claim can race + # these stores. + # + # The age is the SCC's, not the cell's: the whole SCC gets the age of + # its YOUNGEST member, so promotion is all-or-nothing. A per-cell age + # lets one member of an SCC promote ahead of its own SCC-mates; the + # next capture then prunes that INTERNAL edge, which taints the SCC as + # flagPruned, which stops it from ever being stamped again — freezing + # every member's age at its current value and re-tracing the whole + # structure on every collection from then on. Members age at different + # rates whenever a structure is built incrementally (a list appended to + # across several collections), so this is the common case, not a corner + # one. Taking the minimum can only delay a promotion, never hasten one, + # so it cannot widen the floating-garbage bound. for s in 0 ..< j.nScc: if (cap.sccs.d[s].flags and (flagDead or flagDirty or flagPruned)) == 0: - for mi in cap.sccs.d[s].memStart ..< cap.sccs.d[s+1].memStart: - let m = cap.sccMembers.d[mi] - atomicStoreN(addr cap.recs.d[m].cell.rootIdx, - gMyEpochStamp or int64(cap.ages.d[m] +% 1), - ATOMIC_RELAXED) - graceWait() - for i in 0 ..< j.toFree.len: - when orcLeakDetector: - writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) - free(j.toFree.d[i][0], j.toFree.d[i][1]) + let memStart = cap.sccs.d[s].memStart + let memEnd = cap.sccs.d[s+1].memStart + var age = high(int32) + for mi in memStart ..< memEnd: + let a = cap.ages.d[cap.sccMembers.d[mi]] + if a < age: age = a + let stamp = ctx.epochStamp or int64(age +% 1) + for mi in memStart ..< memEnd: + atomicStoreN(addr cap.recs.d[cap.sccMembers.d[mi]].cell.rootIdx, + stamp, ATOMIC_RELAXED) j.freed = j.toFree.len - deinit j.toFree + if j.toFree.len > 0 and buildPendingWatch(): + # park the whole batch by swapping buffers: the collection keeps the + # (now empty) buffer the previous batch used, so neither side allocates + let spare = gPendingCells + gPendingCells = j.toFree + j.toFree = spare + j.toFree.len = 0 + gPendingSlot = ctx.slot + gPendingActive = true + else: + for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) + free(j.toFree.d[i][0], j.toFree.d[i][1]) + j.toFree.len = 0 + gFreeBuf = j.toFree proc startCollection(minRoots, keepBelow: int; slice: var CellSeq[Cell]; wait: bool; drainAll = false): bool = @@ -1100,50 +1469,65 @@ proc startCollection(minRoots, keepBelow: int; slice: var CellSeq[Cell]; ## collect) and try to become a collector over THIS THREAD's candidates: ## claim a tag slot — the only step still under gMergeLock — and steal ## the thread-local buffer as this collection's slice, lock-free. When - ## there is enough work but all ParSlots collections are running, `wait` - ## decides between parking until a slot frees (backpressure for - ## overflowing mutators) and giving up. Either way the drain happened, + ## there is enough work but the slot table is full (more than MaxPar + ## threads collecting at once), `wait` decides between parking until a + ## slot frees and giving up. Either way the drain happened, ## so the caller's overflowing queue has room again. result = false + releasePending() # last collection's batch: its watch list is long clear if drainAll: drainAllStripes() else: drainStripe(getStripeIdx()) adoptOrphans() + flushGenSuspects(addr gCtx) # epoch advanced ⇒ suspects become roots while gLocalRoots.len >= minRoots and gLocalRoots.len > keepBelow and mayRunCycleCollect(): acquire gMergeLock var slot = -1 - for sl in 0 ..< ParSlots: + let inPlay = gParSlots + for sl in 0 ..< inPlay: if atomicLoadN(addr gActiveTags[sl], ATOMIC_RELAXED) == 0: slot = sl break + if slot < 0 and inPlay < ParSlots: + # Every slot in play is busy and the table has room: widen the pool + # instead of throttling this thread. Publishing the wider bound before + # the tag lands in the new slot is what makes `slotsInPlay` safe. + slot = inPlay + atomicStoreN(addr gParSlots, inPlay +% 1, ATOMIC_SEQ_CST) if slot < 0: release gMergeLock if not wait: break - # backpressure: all ParSlots collections are running; park until one - # finishes (finishCollection broadcasts) instead of burning a core + # the table itself is full (more than MaxPar threads collecting at + # once): park until one finishes (finishCollection broadcasts) + # instead of burning a core parkUntil(anySlotFree()) drainStripe(getStripeIdx()) # the world moved while we waited adoptOrphans() else: gTagCounter = (gTagCounter +% 1) and int64(epochBase - 1) # tags below the stamp namespace if gTagCounter == 0: gTagCounter = 1 - gMyTag = gTagCounter - gMySlot = slot - gMyEpochStamp = epochStamp(atomicLoadN(addr gEpoch, ATOMIC_RELAXED)) + gCtx.tag = gTagCounter + gCtx.slot = slot + gCtx.epochStamp = epochStamp(atomicLoadN(addr gEpoch, ATOMIC_RELAXED)) var othersActive = false - for sl in 0 ..< ParSlots: + for sl in 0 ..< gParSlots: if sl != slot and atomicLoadN(addr gActiveTags[sl], ATOMIC_RELAXED) != 0: othersActive = true - gAmSolo = not othersActive - if gAmSolo: + gCtx.amSolo = not othersActive + if gCtx.amSolo: atomicStoreN(addr gSoloCapture, 1, ATOMIC_RELEASE) atomicStoreN(addr gSlotPhase[slot], 1, ATOMIC_RELEASE) - atomicStoreN(addr gActiveTags[slot], gMyTag, ATOMIC_SEQ_CST) + atomicStoreN(addr gActiveTags[slot], gCtx.tag, ATOMIC_SEQ_CST) release gMergeLock # our buffer, our slice: no lock needed if keepBelow == 0: slice = gLocalRoots # steal the whole buffer - init(gLocalRoots) + if gSpareRoots.d != nil: + gLocalRoots = gSpareRoots + gLocalRoots.len = 0 + gSpareRoots = default(CellSeq[Cell]) + else: + init(gLocalRoots) else: init(slice, max(gLocalRoots.len - keepBelow, 8)) for i in keepBelow ..< gLocalRoots.len: @@ -1155,10 +1539,18 @@ proc startCollection(minRoots, keepBelow: int; slice: var CellSeq[Cell]; proc finishCollection() = if atomicAddFetch(addr gCollectionCounter, 1, ATOMIC_RELAXED) mod YrcEpochLen == 0: discard atomicAddFetch(addr gEpoch, 1, ATOMIC_RELAXED) - atomicStoreN(addr gActiveTags[gMySlot], 0, ATOMIC_SEQ_CST) - atomicStoreN(addr gSlotPhase[gMySlot], 0, ATOMIC_RELEASE) - gMyTag = 0 - gAmSolo = false + if gPendingActive: + # A batch is parked under this collection's tag. Clear only the PHASE — + # so nobody's grace check waits on us — and leave the tag in + # `gActiveTags`: it is what stops a foreign capture from claiming, and + # then freeing, a cell that is sitting in the batch. `releasePending` + # gives the slot back. + atomicStoreN(addr gSlotPhase[gCtx.slot], 0, ATOMIC_RELEASE) + else: + atomicStoreN(addr gActiveTags[gCtx.slot], 0, ATOMIC_SEQ_CST) + atomicStoreN(addr gSlotPhase[gCtx.slot], 0, ATOMIC_RELEASE) + gCtx.tag = 0 + gCtx.amSolo = false collectorEvent() # wake backpressure and grace waiters proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) = @@ -1170,7 +1562,9 @@ proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) = for i in countdown(last, 0): writeCell("root", slice.d[i][0], slice.d[i][1]) - init j.traceStack + if gTraceBuf.d == nil: init gTraceBuf + gTraceBuf.len = 0 + j.traceStack = gTraceBuf prepareCapture() let cap = addr gCap # hoist the TLS lookup out of the hot loops j.nScc = 0 @@ -1178,8 +1572,8 @@ proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) = for i in countdown(last, 0): capture(slice.d[i][0], slice.d[i][1], j, cap) j.touched = cap.recs.len - atomicStoreN(addr gSlotPhase[gMySlot], 2, ATOMIC_RELEASE) # capture done - if gAmSolo: + atomicStoreN(addr gSlotPhase[gCtx.slot], 2, ATOMIC_RELEASE) # capture done + if gCtx.amSolo: atomicStoreN(addr gSoloCapture, 0, ATOMIC_RELEASE) collectorEvent() # wake solo-gate and grace waiters @@ -1194,13 +1588,13 @@ proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) = commitDead(j, cap) j.keepThreshold = j.freed == j.touched and j.touched > 0 - deinit j.traceStack + gTraceBuf = j.traceStack # hand the (possibly grown) buffer back proc runCollection(j: var GcEnv; slice: var CellSeq[Cell]) = ## Runs one collection over the stolen slice, concurrently with mutators - ## AND with up to ParSlots-1 other collections over disjoint partitions. + ## AND with the other collecting threads over disjoint partitions. yrcGcFenceEnter() # freeze seq structure mutations, not ref writes - if not gAmSolo: + if not gCtx.amSolo: # a solo collection claims with plain stores; nobody else may claim # cells until its capture phase is over parkUntil(atomicLoadN(addr gSoloCapture, ATOMIC_ACQUIRE) == 0) @@ -1210,7 +1604,10 @@ proc runCollection(j: var GcEnv; slice: var CellSeq[Cell]) = lockState = prev yrcGcFenceExit() finishCollection() - deinit slice + if gSpareRoots.d == nil and slice.d != nil: + gSpareRoots = slice # recycle it as the next steal's replacement + else: + deinit slice when defined(nimOrcStats): var freedCyclicObjects {.threadvar.}: int @@ -1274,15 +1671,57 @@ when defined(nimOrcStats): capRepeat: atomicLoadN(addr gStatCapRepeat, ATOMIC_RELAXED), capPruned: atomicLoadN(addr gStatCapPruned, ATOMIC_RELAXED)) +proc releaseCollectorScratch() = + ## Drop TLS collector scratch (capture side structure, trace/free buffers, + ## suspect list). Persistent across ordinary collections so small captures + ## don't reallocate; released on thread exit and after `GC_runOrc` so a + ## single large capture cannot pin tens of MB for the process lifetime. + ## `deinit` nils `d`, which is the sentinel `prepareCapture` tests. + deinit(gCtx.genSuspects) + deinit(gSpareRoots) + deinit(gTraceBuf) + deinit(gFreeBuf) + deinit(gPendingCells) + deinit(gPendingWatch) + deinit(gCap.recs) + deinit(gCap.sccIdx) + deinit(gCap.tstack) + deinit(gCap.frames) + deinit(gCap.edges) + deinit(gCap.sccs) + deinit(gCap.sccMembers) + deinit(gCap.crossTgt) + deinit(gCap.crossPend) + deinit(gCap.prunedSrc) + deinit(gCap.prunedTgt) + deinit(gCap.ages) + deinit(gCap.slots) + proc GC_runOrc* = if lockState == Collecting: return # an explicit collect must be exhaustive: age out every liveness stamp - # so nothing is pruned + # so nothing is pruned, and young→old suspects become roots. Commit of + # one round may `rememberGenSuspect` further cells (e.g. a dying bridge + # dropping its last edge into a stamped web), so loop until quiet. discard atomicAddFetch(addr gEpoch, 1, ATOMIC_RELAXED) var slice: CellSeq[Cell] - if startCollection(1, 0, slice, wait = true, drainAll = true): + while true: + spillGenSuspects(addr gCtx) + if not startCollection(1, 0, slice, wait = true, drainAll = true): + break var j: GcEnv runCollection(j, slice) + when defined(nimOrcStats): + # collectCycles updates this; GC_runOrc must too (tests/benches read it) + inc freedCyclicObjects, j.freed + releasePending() # GC_fullCollect must not leave a batch parked + # A single large capture (e.g. reclaiming an 80k-node stamped web after + # epoch advance) otherwise leaves tens of MB of TLS RawSeq capacity + # resident for the rest of the process. Partial collects keep the + # buffers; only an exhaustive collect drops them. Spill first so a + # last-round suspect is not discarded with the list. + spillGenSuspects(addr gCtx) + releaseCollectorScratch() # note: aborted SCCs and cross-collection targets legitimately leave # re-registered roots behind; other RUNNING threads' local candidates # are theirs to collect (exiting threads spill to the orphan buffer) @@ -1315,7 +1754,11 @@ proc nimYrcThreadTeardown() = ## of ours is stranded in a queue no other thread hashes to, then spill ## our candidate buffer to the global orphan buffer, where the next ## collection on any thread adopts it. + releasePending() # nobody else can release this thread's parked batch drainStripe(getStripeIdx()) + # Suspects → roots before the orphan spill, otherwise young→old dec + # targets on this thread would die with the TLS list. + spillGenSuspects(addr gCtx) if gLocalRoots.len > 0: acquire gMergeLock if roots.d == nil: init(roots) @@ -1326,6 +1769,7 @@ proc nimYrcThreadTeardown() = deinit(gLocalRoots) gLocalRoots.d = nil gLocalRoots.len = 0 + releaseCollectorScratch() proc GC_enableMarkAndSweep*() = GC_enableOrc() proc GC_disableMarkAndSweep*() = GC_disableOrc() @@ -1343,10 +1787,10 @@ proc enqueueDec(cell: Cell; desc: PNimTypeV2) {.inline.} = ## LOCK-FREE producer for the deferred-dec queue: reserve a slot with a ## fetch-add, store the cell, then publish by storing the desc (release; ## desc != nil is the ready marker consumers wait for/skip). A reservation - ## past QueueSize never writes anything — the reserver makes room by - ## draining (collectCycles always at least drains our own stripe, even - ## when nested in a collection or a seq critical section) and retries - ## with a fresh reservation. + ## past QueueSize never writes anything — the reserver drains our stripe + ## to make room and only starts a full collection when the candidate set + ## is already at threshold (bursty allocators otherwise paid a collect + ## on every overflow even when a plain drain would suffice). let idx = getStripeIdx() while true: let slot = atomicFetchAdd(addr stripes[idx].toDecLen, 1, ATOMIC_ACQ_REL) @@ -1354,7 +1798,9 @@ proc enqueueDec(cell: Cell; desc: PNimTypeV2) {.inline.} = stripes[idx].toDec[slot][0] = cell atomicStoreN(addr stripes[idx].toDec[slot][1], desc, ATOMIC_RELEASE) break - collectCycles() + drainStripe(idx) + if gLocalRoots.len >= rootsThreshold: + collectCycles() proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = result = false @@ -1362,7 +1808,27 @@ proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = enqueueDec(head(p), cast[ptr PNimTypeV2](p)[]) proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = - nimDecRefIsLastCyclicDyn(p) + ## ACYCLIC ref: prompt reclamation, exactly as under --mm:arc. This used to + ## forward to `nimDecRefIsLastCyclicDyn`, which enqueued the dec and so + ## dragged every `.acyclic` type through capture/deadness/commit -- the + ## precise opposite of what the annotation asks for. + ## + ## No grace period is needed here, and that is not an accident: the + ## collector has no way to be holding this cell. It cannot reach it by + ## traversal, because liftdestructors only emits `nimTraceRef` for fields + ## whose type is cyclic; and it cannot hold it as a capture root, because + ## roots come only from `registerLocal` on a drained dec, and an acyclic + ## dec is never queued now that `nimAsgnYrc` is gated on `canFormAcycle`. + ## Both halves must stay true together -- prompt reclamation here is only + ## sound while nothing else puts an acyclic cell into the collector. + result = false + if p != nil: + when hasThreadSupport: + result = atomicDec(head(p).rc, rcIncrement) == -rcIncrement + else: + let cell = head(p) + if (cell.rc and not rcMask) == 0: result = true + else: cell.rc = cell.rc -% rcIncrement proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = result = false diff --git a/lib/system/yrc_opt_proof.lean b/lib/system/yrc_opt_proof.lean new file mode 100644 index 0000000000..79bec0d009 --- /dev/null +++ b/lib/system/yrc_opt_proof.lean @@ -0,0 +1,668 @@ +/- + YRC Optimization Proofs — SCC-uniform ages, demand-grown slots, deferred + reclamation + ======================================================================= + Self-contained, no Mathlib. Checked with Lean 4 (v4.32.0). + + Companion to yrc_proof.lean (core safety: garbage stability, validation + soundness, partitions, grace, fence, deadlock freedom) and to + yrc_tarjan_proof.lean (soundness AND completeness of the SCC deadness + scan). This file models the three changes of the "YRC: optimizations" + round, each of which touches an invariant the other two files rely on: + + §A SCC-UNIFORM EPOCH AGES (commitDead's re-stamp loop). + Before: every proven-live cell was stamped with its OWN survival + age + 1. One member of an SCC could then reach `YrcPromoteAge` + ahead of its SCC-mates; the next capture pruned that INTERNAL edge, + tainting the SCC `flagPruned`, which stops it from ever being + re-stamped — freezing every member's age and re-tracing the whole + structure on every collection from then on. Members age at + different rates whenever a structure is built incrementally, so + this was the common case. + After: the whole SCC is stamped with the age of its YOUNGEST + member + 1, so promotion is all-or-nothing. + + §B DEMAND-GROWN COLLECTOR SLOTS (`gParSlots`, startCollection). + Before: `MaxPar` was a fixed 8 and the 9th collecting thread PARKED + in `parkUntil(anySlotFree())` instead of collecting. After: + `gParSlots` counts the slots in play, starts at 1, and is raised + under gMergeLock exactly when a thread wants to collect and every + slot in play is busy; `MaxPar` (256) is only the table capacity. + Every scan (`isActiveTag`, `anySlotFree`, `buildPendingWatch`) now + runs over a prefix that GROWS under the reader, which is sound only + because of the ordering rule at `slotsInPlay`. + + §C DEFERRED RECLAMATION (`gPendingCells` / `releasePending`). + Before: commitDead BLOCKED until every concurrent capture ended + (the grace period), inside the GC fence — stalling both the + committing collector and every mutator doing a seq operation. + After: the dead batch is parked together with a watch list of the + captures it must outlive, and released at the start of this + thread's next collection. The parked collection's TAG STAYS in + `gActiveTags` (only the phase is cleared): that is what stops a + foreign capture from claiming — and then freeing — a parked cell. + + Nothing here weakens yrc_proof.lean: §A only changes which cells a + capture skips (pruning shrinks the captured set, the conservative + direction), §B only changes how many slots a scan covers, and §C only + moves the free of an already-validated, already-closed dead set later + in time, where garbage stability (yrc_proof.lean §3) keeps it dead. +-/ + +abbrev Obj := Nat +abbrev Tag := Nat +-- Slot indices and times are plain `Nat`: `omega` ignores atoms whose type +-- is an abbreviation, and both appear in arithmetic below. + +/-! ## §A SCC-uniform epoch ages + + The spare header word (`rootIdx`) is a three-way namespace: 0 for a + cell no collection ever touched, a claim tag while a collection owns + the cell, or an epoch stamp (`epochBase | epoch` in the high word, the + survival age in the low word). Tags are allocated strictly below + `epochBase`, so a stamp is never mistaken for a tag — modelled here by + keeping the three cases as separate constructors. -/ + +inductive Word where + | fresh -- 0: never claimed, never stamped + | tag (t : Tag) -- a collection's claim word + | stamp (e : Nat) (age : Nat) -- an epoch stamp written by commitDead + +/-- The age a capture reads out of a claim word (`cap.ages`): a stamp + contributes its recorded age, anything else contributes 0. -/ +def ageOf : Word → Nat + | .stamp _ a => a + | _ => 0 + +def capturedAge (w : Obj → Word) (x : Obj) : Nat := ageOf (w x) + +/-- `claimCell(pruneLive = true)` prunes at a target whose stamp is of the + CURRENT epoch and whose age reached `YrcPromoteAge`: the target is + treated as an opaque live external and is not descended into. Roots + are always claimed with `pruneLive = false`, so this never applies to + them. -/ +def prunable (curEpoch promoteAge : Nat) : Word → Prop + | .stamp e a => e = curEpoch ∧ promoteAge ≤ a + | _ => False + +/-- `high(int32)`, the seed of commitDead's per-SCC minimum. -/ +def bigAge : Nat := 2147483647 + +/-- Minimum of a list with a default (the `var age = high(int32)` fold). -/ +def listMin : List Nat → Nat → Nat + | [], d => d + | a :: l, d => Nat.min a (listMin l d) + +theorem listMin_le : ∀ (l : List Nat) (d a : Nat), a ∈ l → listMin l d ≤ a := by + intro l + induction l with + | nil => intro d a ha; cases ha + | cons b t ih => + intro d a ha + cases List.mem_cons.mp ha with + | inl h => + simp only [listMin, Nat.min_def] + split <;> omega + | inr h => + have := ih d a h + simp only [listMin, Nat.min_def] + split <;> omega + +theorem listMin_ge : ∀ (l : List Nat) (d b : Nat), b ≤ d → + (∀ x, x ∈ l → b ≤ x) → b ≤ listMin l d := by + intro l + induction l with + | nil => intro d b hd _; simpa [listMin] using hd + | cons a t ih => + intro d b hd hall + have h1 : b ≤ a := hall a (by simp) + have h2 := ih d b hd (fun x hx => hall x (List.mem_cons_of_mem a hx)) + simp only [listMin, Nat.min_def] + split <;> omega + +theorem listMin_const (l : List Nat) (d a : Nat) (hne : l ≠ []) + (hall : ∀ x, x ∈ l → x = a) (had : a ≤ d) : listMin l d = a := by + have hlo : a ≤ listMin l d := + listMin_ge l d a had (fun x hx => by have := hall x hx; omega) + cases l with + | nil => exact absurd rfl hne + | cons b t => + have hb : b ∈ b :: t := by simp + have hhi := listMin_le (b :: t) d b hb + have := hall b hb + omega + +/-- **The change.** commitDead stamps a proven-live SCC with ONE word: the + current epoch and 1 + the age of its YOUNGEST member. -/ +def sccAge (w : Obj → Word) (ms : List Obj) : Nat := + listMin (ms.map (fun m => capturedAge w m)) bigAge + 1 + +/-- The post-state of commitDead's re-stamp loop, as committed: every + member of the SCC gets the same word, nothing else is touched. -/ +structure StampedUniform (w w' : Obj → Word) (e : Nat) (ms : List Obj) : Prop where + members : ∀ m, m ∈ ms → w' m = .stamp e (sccAge w ms) + others : ∀ x, x ∉ ms → w' x = w x + +/-- The previous, per-cell scheme, for contrast. -/ +structure StampedPerCell (w w' : Obj → Word) (e : Nat) (ms : List Obj) : Prop where + members : ∀ m, m ∈ ms → w' m = .stamp e (capturedAge w m + 1) + others : ∀ x, x ∉ ms → w' x = w x + +/-- **A1 Uniformity**: after commit, all members of a stamped SCC carry + the identical claim word — same epoch AND same age. -/ +theorem stamped_uniform (w w' : Obj → Word) (e : Nat) (ms : List Obj) + (h : StampedUniform w w' e ms) : + ∀ x y, x ∈ ms → y ∈ ms → w' x = w' y := by + intro x y hx hy + rw [h.members x hx, h.members y hy] + +/-- **A2 No internal prune**: a capture descends into a member `u` of an + SCC only through an edge that was NOT pruned (or because `u` is a + root, which bypasses stamps entirely). With uniform words, every + other member `v` is then unprunable too — so no INTERNAL edge of the + SCC can be pruned, and the SCC is never tainted `flagPruned` by its + own topology. This is precisely the failure the per-cell age caused. -/ +theorem uniform_no_internal_prune + (w : Obj → Word) (curEpoch promoteAge : Nat) (ms : List Obj) + (huni : ∀ x y, x ∈ ms → y ∈ ms → w x = w y) + (u v : Obj) (hu : u ∈ ms) (hv : v ∈ ms) + (hdescended : ¬ prunable curEpoch promoteAge (w u)) : + ¬ prunable curEpoch promoteAge (w v) := by + rw [← huni u v hu hv] + exact hdescended + +/-- A2 applied to the state commitDead actually leaves behind. -/ +theorem committed_scc_no_internal_prune + (w w' : Obj → Word) (e curEpoch promoteAge : Nat) (ms : List Obj) + (hst : StampedUniform w w' e ms) + (u v : Obj) (hu : u ∈ ms) (hv : v ∈ ms) + (hdescended : ¬ prunable curEpoch promoteAge (w' u)) : + ¬ prunable curEpoch promoteAge (w' v) := + uniform_no_internal_prune w' curEpoch promoteAge ms + (stamped_uniform w w' e ms hst) u v hu hv hdescended + +/-- **A3 The per-cell scheme diverges**: two members of ONE SCC whose + captured ages differ (the normal case for a structure built + incrementally across collections) end up with different ages. -/ +theorem perCell_ages_diverge (w w'' : Obj → Word) (e : Nat) (ms : List Obj) + (hp : StampedPerCell w w'' e ms) (u v : Obj) (hu : u ∈ ms) (hv : v ∈ ms) + (hdiff : capturedAge w u ≠ capturedAge w v) : + ageOf (w'' u) ≠ ageOf (w'' v) := by + rw [hp.members u hu, hp.members v hv] + simp only [ageOf] + omega + +/-- ...and diverged ages inside one SCC mean exactly one prunable end of + an internal edge: the promoted member is pruned while its unpromoted + SCC-mate is still being traced. -/ +theorem diverged_ages_prune_internally (curEpoch promoteAge au av : Nat) + (hu : au < promoteAge) (hv : promoteAge ≤ av) : + ¬ prunable curEpoch promoteAge (.stamp curEpoch au) ∧ + prunable curEpoch promoteAge (.stamp curEpoch av) := by + constructor + · intro h + obtain ⟨-, h2⟩ := h + omega + · exact ⟨rfl, hv⟩ + +/-- **A4 The minimum can only delay a promotion, never hasten one**, so + taking it cannot widen the floating-garbage bound (~2 epochs). -/ +theorem uniform_age_le_perCell (w w' w'' : Obj → Word) (e : Nat) (ms : List Obj) + (hu : StampedUniform w w' e ms) (hp : StampedPerCell w w'' e ms) + (m : Obj) (hm : m ∈ ms) : + ageOf (w' m) ≤ ageOf (w'' m) := by + have hmem : capturedAge w m ∈ ms.map (fun x => capturedAge w x) := + List.mem_map_of_mem hm + have hle := listMin_le (ms.map (fun x => capturedAge w x)) bigAge + (capturedAge w m) hmem + rw [hu.members m hm, hp.members m hm] + simp only [ageOf, sccAge] + omega + +/-- Corollary: wherever the SCC-uniform stamp prunes, the per-cell stamp + would have pruned too. Pruning is the only thing a stamp does, so the + change cannot make any capture skip MORE than before. -/ +theorem uniform_never_hastens_promotion (w w' w'' : Obj → Word) (e : Nat) + (ms : List Obj) (promoteAge : Nat) + (hu : StampedUniform w w' e ms) (hp : StampedPerCell w w'' e ms) + (m : Obj) (hm : m ∈ ms) (h : promoteAge ≤ ageOf (w' m)) : + promoteAge ≤ ageOf (w'' m) := by + have := uniform_age_le_perCell w w' w'' e ms hu hp m hm + omega + +/-- **A5 The uniform age is exactly the common age + 1**, so a surviving + SCC's age advances by one per collection and stays uniform: the + invariant of A1/A2 is inductive. -/ +theorem uniform_age_succ (w w' : Obj → Word) (e a : Nat) (ms : List Obj) + (hne : ms ≠ []) (hcap : a ≤ bigAge) + (hall : ∀ m, m ∈ ms → capturedAge w m = a) + (h : StampedUniform w w' e ms) : + ∀ m, m ∈ ms → ageOf (w' m) = a + 1 := by + intro m hm + have hmin : listMin (ms.map (fun x => capturedAge w x)) bigAge = a := by + apply listMin_const + · cases ms with + | nil => exact absurd rfl hne + | cons b t => simp + · intro x hx + obtain ⟨y, hy, rfl⟩ := List.mem_map.mp hx + exact hall y hy + · exact hcap + rw [h.members m hm] + simp only [ageOf, sccAge, hmin] + +/-- **A6 The freeze**: an SCC that is never re-stamped (the `flagPruned` + taint excludes it from commitDead's stamp loop) keeps its age + forever. Below `YrcPromoteAge` that means it is fully re-traced by + every collection until the epoch turns — the regression A2 removes. -/ +theorem age_frozen_if_never_restamped (age : Nat → Nat) (a : Nat) + (h0 : age 0 = a) (hfreeze : ∀ n, age (n + 1) = age n) : + ∀ n, age n = a := by + intro n + induction n with + | zero => exact h0 + | succ n ih => rw [hfreeze n, ih] + +/-- **A7 Progress**: an SCC that IS re-stamped every round promotes after + `YrcPromoteAge` collections, and by A2 stays promoted uniformly — so + a long-lived structure is traced once per epoch, not once per + collection. -/ +theorem age_promotes_if_restamped (age : Nat → Nat) (a promoteAge : Nat) + (h0 : age 0 = a) (hstep : ∀ n, age (n + 1) = age n + 1) : + promoteAge ≤ age promoteAge := by + have h : ∀ n, age n = a + n := by + intro n + induction n with + | zero => simpa using h0 + | succ n ih => rw [hstep n, ih]; omega + rw [h promoteAge] + omega + +/-! ## §B Demand-grown collector slots + + `gParSlots` counts the slots IN PLAY. It starts at 1, is raised under + gMergeLock when a thread wants to collect and every slot in play is + busy, and is NEVER lowered. Two things must hold: + + (i) a scan over the prefix `0 ..< slotsInPlay()` must never MISS an + active tag — a missed tag would let a second collection claim a + cell another collection already owns, breaking partition + disjointness (yrc_proof.lean §5) and admitting a double free; + (ii) a thread must not park while the table still has room — that was + the throttle the fixed `MaxPar = 8` imposed. + + (i) rests on the publication order in startCollection: the wider bound + is stored BEFORE the tag lands in the new slot, and `gParSlots` only + grows. So a load of `slotsInPlay()` ordered AFTER the read of a tagged + claim word is guaranteed to cover the slot that wrote that tag. -/ + +/-- `gParSlots` never shrinks, so a prefix scan cannot shrink under a + reader either. -/ +theorem in_play_persists (parSlots : Nat → Nat) + (hmono : ∀ i j, i ≤ j → parSlots i ≤ parSlots j) + (k : Nat) (t t' : Nat) (h : t ≤ t') (hk : k < parSlots t) : + k < parSlots t' := by + have := hmono t t' h + omega + +/-- **B1 The scan covers the slot that wrote the tag.** `tGrow` is when + the slot was put in play (under gMergeLock), `tTag` the tag store, + `tRead` the reader's load of the claim word, `tScan` its subsequent + load of `slotsInPlay()`. -/ +theorem scan_covers_tagged_slot (parSlots : Nat → Nat) + (hmono : ∀ i j, i ≤ j → parSlots i ≤ parSlots j) + (k : Nat) (tGrow tTag tRead tScan : Nat) + (hgrow : k < parSlots tGrow) + (hpub : tGrow ≤ tTag) -- wider bound published before the tag store + (hread : tTag ≤ tRead) -- the reader observed the tag + (hafter : tRead ≤ tScan) :-- slotsInPlay() loaded AFTER the claim word + k < parSlots tScan := by + have := hmono tGrow tScan (by omega) + omega + +/-- **B2 `isActiveTag` is complete**: an active tag is always found by the + prefix scan, so `claimCell` never claims a cell another collection + owns. -/ +theorem active_tag_never_missed (parSlots : Nat → Nat) + (tagAt : Nat → Nat → Tag) + (hmono : ∀ i j, i ≤ j → parSlots i ≤ parSlots j) + (k : Nat) (t : Tag) (tGrow tTag tRead tScan : Nat) + (hgrow : k < parSlots tGrow) (hpub : tGrow ≤ tTag) + (hread : tTag ≤ tRead) (hafter : tRead ≤ tScan) + (hheld : tagAt k tScan = t) : + ∃ j, j < parSlots tScan ∧ tagAt j tScan = t := + ⟨k, scan_covers_tagged_slot parSlots hmono k tGrow tTag tRead tScan + hgrow hpub hread hafter, hheld⟩ + +/-- **B3 The ordering rule is load-bearing**: a `slotsInPlay()` load + ordered BEFORE the read of the claim word can legitimately miss the + slot, because the pool may widen in between. -/ +theorem scan_before_read_may_miss : + ∃ (parSlots : Nat → Nat) (k : Nat) (tScan tGrow : Nat), + (∀ i j, i ≤ j → parSlots i ≤ parSlots j) ∧ tScan < tGrow ∧ + k < parSlots tGrow ∧ ¬ (k < parSlots tScan) := by + refine ⟨fun t => t + 1, 1, 0, 1, ?_, ?_, ?_, ?_⟩ + · intro i j h + show i + 1 ≤ j + 1 + omega + · decide + · decide + · decide + +/-- startCollection's slot decision. -/ +inductive SlotOutcome where + | reuse (k : Nat) -- a slot already in play was free + | grow (k : Nat) -- pool widened; the new slot is `k = inPlay` + | park -- table full: MaxPar collections already running + +inductive SlotClaim (busy : Nat → Prop) (inPlay maxPar : Nat) : SlotOutcome → Prop where + | reuse (k : Nat) (hk : k < inPlay) (hfree : ¬ busy k) : + SlotClaim busy inPlay maxPar (.reuse k) + | grow (hfull : ∀ k, k < inPlay → busy k) (hroom : inPlay < maxPar) : + SlotClaim busy inPlay maxPar (.grow inPlay) + | park (hfull : ∀ k, k < inPlay → busy k) (hno : maxPar ≤ inPlay) : + SlotClaim busy inPlay maxPar .park + +/-- **B4 Parking means saturation**, not throttling: a thread blocks in + `parkUntil(anySlotFree())` only when `MaxPar` collections are running + concurrently. Under the old fixed bound this triggered at the 9th + collecting thread (32 threads vs 8 slots: 6.3s against 2.6s). -/ +theorem park_only_when_saturated (busy : Nat → Prop) (inPlay maxPar : Nat) + (h : SlotClaim busy inPlay maxPar .park) : + maxPar ≤ inPlay ∧ ∀ k, k < inPlay → busy k := by + cases h with + | park hfull hno => exact ⟨hno, hfull⟩ + +/-- **B5 No parking below capacity.** -/ +theorem no_park_below_capacity (busy : Nat → Prop) (inPlay maxPar : Nat) + (hroom : inPlay < maxPar) : ¬ SlotClaim busy inPlay maxPar .park := by + intro h + cases h with + | park hfull hno => omega + +/-- **B6 A grown slot is fresh**: `k = inPlay` is distinct from every slot + already in play, so the widening thread cannot collide with a + collection that is already running. -/ +theorem grown_slot_not_in_play (inPlay k : Nat) (hk : k < inPlay) : + inPlay ≠ k := by omega + +/-! ## §C Deferred reclamation + + commitDead used to spin until every concurrent capture had ended before + freeing, INSIDE the GC fence. Now the batch is parked in + `gPendingCells` with a watch list `gPendingWatch` of the (slot, tag) + pairs that were in capture phase at commit time, and `releasePending` + — the FIRST thing startCollection does, outside the fence and holding + no lock — waits out that list and then frees. + + Three obligations: + (C1) the watch list is COMPLETE: every capture that could hold a stale + `(slot, value)` snapshot of the batch is on it; + (C2) the grace check is SOUND: `graceSatisfied` reports "finished" only + when the watched capture really has finished; + (C3) the batch is PROTECTED while parked: no foreign capture may claim + (and hence free) one of its cells. This is why finishCollection + clears only `gSlotPhase` and leaves the tag in `gActiveTags`. -/ + +/-- One slot's published state. -/ +structure SlotState where + tag : Tag + phase : Nat -- 0 idle, 1 capturing, 2 committing + +/-- finishCollection with a batch parked: phase → 0, tag RETAINED. -/ +def finishParked (st : SlotState) : SlotState := { st with phase := 0 } + +/-- releasePending, after the grace wait: the tag is given up first, then + the destructors and `nimRawDispose` run. -/ +def releaseSlot (st : SlotState) : SlotState := { st with tag := 0 } + +theorem finishParked_keeps_tag (st : SlotState) : + (finishParked st).tag = st.tag := rfl + +/-- A parked slot blocks nobody's grace period: its phase is 0, so every + other collector's `graceSatisfied` passes over it. -/ +theorem parked_slot_blocks_nobody (st : SlotState) (tg : Tag) : + ¬ ((finishParked st).tag = tg ∧ (finishParked st).phase = 1) := by + intro h + simp [finishParked] at h + +/-- ...yet the tag survives, which is what protects the parked cells. -/ +theorem parked_slot_still_tagged (st : SlotState) (t : Tag) (h : st.tag = t) : + (finishParked st).tag = t := h + +/-- `gPendingWatch`, as `(slot shl 32) or tag` pairs. -/ +abbrev WatchList := List (Nat × Tag) + +/-- `graceSatisfied()` evaluated at time `t`. -/ +def graceSatisfied (tagAt : Nat → Nat → Tag) (phaseAt : Nat → Nat → Nat) + (W : WatchList) (t : Nat) : Prop := + ∀ p, p ∈ W → ¬ (tagAt p.1 t = p.2 ∧ phaseAt p.1 t = 1) + +/-- **C1 The watch list is complete.** A capture that was in flight at + commit time is running on a slot that was in play when its tag was + stored; by B1 the `buildPendingWatch` scan — which loads + `slotsInPlay()` after reading the claim state — covers that slot, and + the capture's phase is 1, so it is recorded. -/ +theorem watch_covers_inflight_capture (parSlots : Nat → Nat) + (tagAt : Nat → Nat → Tag) (phaseAt : Nat → Nat → Nat) + (hmono : ∀ i j, i ≤ j → parSlots i ≤ parSlots j) + (s : Nat) (tg : Tag) (tGrow tTag commitT : Nat) + (hgrow : s < parSlots tGrow) (hpub : tGrow ≤ tTag) (hread : tTag ≤ commitT) + (hcapturing : tagAt s commitT = tg ∧ phaseAt s commitT = 1) : + s < parSlots commitT ∧ tagAt s commitT = tg ∧ phaseAt s commitT = 1 := + ⟨scan_covers_tagged_slot parSlots hmono s tGrow tTag commitT commitT + hgrow hpub hread (Nat.le_refl commitT), hcapturing.1, hcapturing.2⟩ + +/-- **C2 The grace check is sound**: if `graceSatisfied` holds at `t` and + a watched capture was still running at `t`, we have a contradiction — + so every watched capture finished strictly before `t`. There is no + ABA on the (slot, tag) pair: tags come from a monotonic counter + (yrc_proof.lean §5 `tags_distinct`), so a later collection on the + same slot carries a different tag. -/ +theorem grace_check_sound (tagAt : Nat → Nat → Tag) (phaseAt : Nat → Nat → Nat) + (W : WatchList) (s : Nat) (tg : Tag) (start finish t : Nat) + (hw : (s, tg) ∈ W) + (hcap : ∀ u, start ≤ u → u ≤ finish → tagAt s u = tg ∧ phaseAt s u = 1) + (hstart : start ≤ t) + (hsat : graceSatisfied tagAt phaseAt W t) : + finish < t := by + cases Nat.lt_or_ge finish t with + | inl h => exact h + | inr h => exact absurd (hcap t hstart h) (hsat (s, tg) hw) + +/-- A capture's window and the values it ever snapshots (TraceEntry), as + in yrc_proof.lean §6. -/ +structure CaptureWindow where + start : Nat + finish : Nat + snap : Obj → Prop + derefs : Obj → Nat → Prop + +/-- **C3 Grace safety survives the deferral.** `commitT` is when the dead + set validated, `releaseT` when releasePending's wait succeeded (C2), + `freeT` when the batch is actually disposed. The only change from + yrc_proof.lean's `grace_no_use_after_free` is that `freeT` moved + LATER — the wait is unchanged in strength, it just happens off the + commit path. -/ +theorem deferred_grace_no_use_after_free + (C : CaptureWindow) (D : Obj → Prop) (commitT releaseT freeT : Nat) + (h_deref : ∀ x t, C.derefs x t → C.start ≤ t ∧ t ≤ C.finish ∧ C.snap x) + (h_watched : C.start < commitT → C.finish < releaseT) + (h_release : releaseT ≤ freeT) + (h_miss : commitT ≤ C.start → ∀ x, D x → ¬ C.snap x) : + ∀ x t, D x → C.derefs x t → t < freeT := by + intro x t hD hd + obtain ⟨h1, h2, h3⟩ := h_deref x t hd + cases Nat.lt_or_ge C.start commitT with + | inl h => have := h_watched h; omega + | inr h => exact absurd h3 (h_miss h x hD) + +/-- claimCell's decision on a cell, as a relation over its claim word. -/ +inductive ClaimResult where + | mine -- our own tag: already captured this round + | refuse -- owned by another ACTIVE collection (-1, crossPend) + | prune -- proven live this epoch (-2, opaque live external) + | claim -- CAS it into our partition + +inductive ClaimDecision (myTag curEpoch promoteAge : Nat) (active : Tag → Prop) : + Word → ClaimResult → Prop where + | mine (t : Tag) (h : t = myTag) : + ClaimDecision myTag curEpoch promoteAge active (.tag t) .mine + | refuse (t : Tag) (hne : t ≠ myTag) (ha : active t) : + ClaimDecision myTag curEpoch promoteAge active (.tag t) .refuse + | claimTag (t : Tag) (hne : t ≠ myTag) (ha : ¬ active t) : + ClaimDecision myTag curEpoch promoteAge active (.tag t) .claim + | prune (e a : Nat) (hp : prunable curEpoch promoteAge (.stamp e a)) : + ClaimDecision myTag curEpoch promoteAge active (.stamp e a) .prune + | claimStamp (e a : Nat) (hp : ¬ prunable curEpoch promoteAge (.stamp e a)) : + ClaimDecision myTag curEpoch promoteAge active (.stamp e a) .claim + | claimFresh : + ClaimDecision myTag curEpoch promoteAge active .fresh .claim + +/-- **C4 A cell carrying an active foreign tag is always refused.** -/ +theorem tagged_cell_refused (myTag curEpoch promoteAge : Nat) (active : Tag → Prop) + (ownerTag : Tag) (hne : ownerTag ≠ myTag) (ha : active ownerTag) + (r : ClaimResult) + (h : ClaimDecision myTag curEpoch promoteAge active (.tag ownerTag) r) : + r = .refuse := by + cases h with + | mine t ht => exact absurd ht hne + | refuse t hne' ha' => rfl + | claimTag t hne' ha' => exact absurd ha ha' + +/-- **C5 No double free while parked.** The parked cells still carry the + parking collection's tag, and finishCollection left that tag in + `gActiveTags` (only the phase was cleared, C-`finishParked`). So a + foreign capture that reaches a parked cell through a stale snapshot + refuses it: it can neither traverse it nor claim it, hence never + classifies it dead and never frees it. Without the tag retention this + is exactly a double free — the batch's owner will free it too. -/ +theorem no_foreign_claim_while_parked + (activeAt : Nat → Tag → Prop) (ownerTag myTag : Tag) + (curEpoch promoteAge : Nat) (commitT releaseT u : Nat) + (hretain : ∀ v, commitT ≤ v → v ≤ releaseT → activeAt v ownerTag) + (hne : ownerTag ≠ myTag) (h1 : commitT ≤ u) (h2 : u ≤ releaseT) + (r : ClaimResult) + (h : ClaimDecision myTag curEpoch promoteAge (activeAt u) (.tag ownerTag) r) : + r = .refuse := + tagged_cell_refused myTag curEpoch promoteAge (activeAt u) ownerTag hne + (hretain u h1 h2) r h + +/-- **C6 Deferring the free is safe.** The batch was closed (unreachable) + at commit time — that is what validation established (yrc_proof.lean + §4 `validated_closed`) — and `hstable` is exactly + yrc_proof.lean §3 `garbage_stability`: a closed set stays closed + under every mutator step, allocation and foreign free. Freeing one + collection later therefore satisfies the same §1 free condition as + freeing immediately. -/ +theorem deferred_free_safe (unreachable : Nat → Obj → Prop) (D : Obj → Prop) + (commitT freeT : Nat) + (hcommit : ∀ x, D x → unreachable commitT x) + (hstable : ∀ x t t', D x → t ≤ t' → unreachable t x → unreachable t' x) + (hlater : commitT ≤ freeT) : + ∀ x, D x → unreachable freeT x := + fun x hx => hstable x commitT freeT hx hlater (hcommit x hx) + +/-- **C7 The grace wait left the GC fence.** It now runs at the start of + startCollection, before `yrcGcFenceEnter`, so no mutator seq + operation can be stalled by it — the property the deferral was made + for. -/ +theorem grace_wait_outside_fence (waitStart waitEnd fenceEnter fenceExit t : Nat) + (horder : waitEnd < fenceEnter) (hw : waitStart ≤ t ∧ t ≤ waitEnd) : + ¬ (fenceEnter ≤ t ∧ t ≤ fenceExit) := by + intro hf + omega + +/-- `waits a b`: thread `a` is blocked in releasePending on thread `b`'s + capture. -/ +def waits (capturing releasing : Nat → Prop) (a b : Nat) : Prop := + releasing a ∧ capturing b + +/-- **C8 The new wait cannot deadlock.** releasePending runs BEFORE this + thread claims a slot, so a releasing thread is never itself + capturing; the wait-for graph therefore has depth one and cannot + contain a cycle of any length. (Captures never wait on anything: + claimCell returns -1 immediately on contention.) -/ +theorem release_wait_depth_one (capturing releasing : Nat → Prop) + (hexcl : ∀ t, releasing t → ¬ capturing t) + (a b c : Nat) (h1 : waits capturing releasing a b) : + ¬ waits capturing releasing b c := by + intro h2 + exact hexcl b h2.1 h1.2 + +/-- Corollary: no 2-cycle, hence no mutual wait between two parked + collectors. -/ +theorem release_wait_acyclic (capturing releasing : Nat → Prop) + (hexcl : ∀ t, releasing t → ¬ capturing t) (a b : Nat) + (h1 : waits capturing releasing a b) : + ¬ waits capturing releasing b a := + release_wait_depth_one capturing releasing hexcl a b a h1 + +/-! ## Summary of verified properties (all QED, no sorry) + + §A `stamped_uniform` — a committed SCC's members carry one identical + claim word. + `uniform_no_internal_prune`, `committed_scc_no_internal_prune` — a + capture that descends into a member cannot prune an internal edge, + so an SCC is never tainted `flagPruned` by its own topology. + `perCell_ages_diverge` + `diverged_ages_prune_internally` — the + pre-fix scheme admits exactly that taint. + `uniform_age_le_perCell`, `uniform_never_hastens_promotion` — the + minimum only delays promotions, so the float bound is unchanged. + `uniform_age_succ` — uniformity is inductive: age advances by one + and stays uniform. + `age_frozen_if_never_restamped` / `age_promotes_if_restamped` — the + frozen-age regression versus the intended once-per-epoch trace. + + §B `in_play_persists`, `scan_covers_tagged_slot`, + `active_tag_never_missed` — a growing `gParSlots` prefix scan never + misses an active tag, PROVIDED `slotsInPlay()` is loaded after the + claim word; `scan_before_read_may_miss` shows the order is + load-bearing. + `park_only_when_saturated`, `no_park_below_capacity`, + `grown_slot_not_in_play` — a collector parks only when `MaxPar` + collections run at once, and a widened slot collides with nobody. + + §C `watch_covers_inflight_capture` — the watch list records every + capture that could hold a stale snapshot of the batch (via §B). + `grace_check_sound` — `graceSatisfied` reports "finished" only when + the watched capture has finished. + `deferred_grace_no_use_after_free` — no capture dereferences a + parked cell at or after its free time. + `tagged_cell_refused`, `no_foreign_claim_while_parked` — retaining + the tag (finishCollection clears only the phase) is what prevents a + foreign collection from claiming and freeing a parked cell. + `parked_slot_blocks_nobody`, `parked_slot_still_tagged` — the two + halves of that split: protection without blocking. + `deferred_free_safe` — deferring the free preserves the §1 free + condition, by garbage stability. + `grace_wait_outside_fence` — the wait no longer overlaps the GC + fence, so it cannot stall a mutator's seq operation. + `release_wait_depth_one`, `release_wait_acyclic` — the new wait + adds no cycle to the wait-for structure of yrc_proof.lean §8. + + ## What is NOT proved + + • Slot-retention liveness. A parked batch holds its tag slot until the + owning thread's NEXT collection (or GC_runOrc / nimYrcThreadTeardown, + both of which call releasePending). A thread that parks a batch and + then never collects again keeps a slot occupied; with enough such + threads the table could saturate and other collectors would park + (§B4). Bounded in practice by `MaxPar = 256` and by teardown, not + formalized. + • That `buildPendingWatch` returning false (nothing capturing) really + is the common case — a performance claim, measured, not proved. + • Destructor timing. Deferral runs a dead batch's destructors one + collection later. Safety is C6; the observable-behaviour claim + ("nothing else observes the delay, the cells are unreachable and + their references already dropped") is an argument about the Nim + language semantics, not modelled here. + • Conservatism of tag retention. While a batch is parked, the tag also + protects SURVIVORS that kept a stale tag (dirty and pruned SCCs are + deliberately not re-stamped), so a foreign capture refuses them for + one extra collection. That delays their re-examination; it cannot + lose them, because §A's E1–E4 hooks keep a dec-witness registered. + • Everything already listed as unproved in yrc_proof.lean (rc-exactness + mechanics, the C11 memory model, liveness/completeness of the retry + loop, tag wrap-around). +-/ diff --git a/lib/system/yrc_proof.lean b/lib/system/yrc_proof.lean index 64a501ed03..81d84197b6 100644 --- a/lib/system/yrc_proof.lean +++ b/lib/system/yrc_proof.lean @@ -459,9 +459,18 @@ theorem cross_target_live (D : Obj → Prop) (claimedB : Obj → Prop) A concurrent capture holds raw `(slot, value)` snapshots (TraceEntry); the value pointer is dereferenced later (header read in claimCell). A capture that overlapped our validation may have snapshotted a slot - that USED to point into our dead set. commitDead therefore waits, for - every other slot that is in capture phase (gSlotPhase == 1), until - that capture ends — captures never wait on anyone, so this is bounded. + that USED to point into our dead set. The dead batch must therefore + outlive every other slot that was in capture phase (gSlotPhase == 1) + at commit time — captures never wait on anyone, so this is bounded. + + commitDead no longer BLOCKS on that: it parks the batch + (`gPendingCells`) with a watch list of those captures and + `releasePending` frees it at the start of this thread's next + collection, off the commit path and outside the GC fence. The parking + collection's tag stays in `gActiveTags` so a foreign capture cannot + claim a parked cell. See yrc_opt_proof.lean §C for the model of the + deferral; the theorems below are the invariant it preserves, with the + free time merely moved later. Two obligations: (a) captures that started BEFORE our commit are waited out — temporal @@ -498,9 +507,10 @@ structure CaptureWindow where /-- **Grace safety**: no capture dereferences a dead cell at or after its free time. `commitT` is when the dead set validated; `freeT` is - when commitDead's free loop runs. The premises are exactly the - protocol: (grace) commitDead's spin means any capture that started - before commit has finished before we free; (miss) §6(b) above. -/ + when the free loop runs (in releasePending, one collection later). + The premises are exactly the protocol: (grace) the watch list means + any capture that started before commit has finished before we free; + (miss) §6(b) above. -/ theorem grace_no_use_after_free (C : CaptureWindow) (D : Obj → Prop) (commitT freeT : Nat) (h_deref : ∀ x t, C.derefs x t → C.start ≤ t ∧ t ≤ C.finish ∧ C.snap x) @@ -676,8 +686,8 @@ theorem no_deadlock_from_total_order {n : Nat} referenced across a partition boundary is never freed by its owner this round (soundness of claimCell's -1 + crossPend). §6 `post_commit_snap_misses_dead`, `grace_no_use_after_free` — with - commitDead's grace spin, no capture ever dereferences freed - memory. + the grace period (now enforced by the deferred batch's watch list, + yrc_opt_proof.lean §C), no capture ever dereferences freed memory. §7 `fence_mutual_exclusion` — the SEQ_CST Dekker pairing in seqs_v2.nim excludes seq structure mutation during collection. §8 `lockLevel_injective`, `mergeLock_level_min`, @@ -714,7 +724,12 @@ theorem no_deadlock_from_total_order {n : Nat} Commit re-stamps proven-live cells with (epochBase|epoch, survivalAge) in the claim word; a capture treats a current-epoch stamp of age ≥ YrcPromoteAge on a DESCENDANT as an opaque live external and does not - descend. Soundness needs no new lemmas: a pruned cell is simply an + descend. The age is the SCC's, not the cell's — every member is + stamped with the age of the SCC's YOUNGEST member, so promotion is + all-or-nothing and no INTERNAL edge is ever pruned; see + yrc_opt_proof.lean §A, which also shows the minimum can only delay a + promotion, so the float bound below is unaffected. Soundness needs no + new lemmas: a pruned cell is simply an uncaptured cell, so the captured set shrinks and every §3–§6 statement quantifies over a smaller S. Pruning can only ADD unexplained external refs to captured SCCs (a pruned predecessor's refs are never explained diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 2704a6111e..b5f09b9a16 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -26,6 +26,7 @@ type WinChar* = Utf16Char # See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type Handle* = int + SHORT* = int16 LONG* = int32 ULONG* = int32 PULONG* = ptr int @@ -1013,10 +1014,57 @@ type uChar*: int16 dwControlKeyState*: DWORD + # https://learn.microsoft.com/en-us/windows/console/coord-str + COORD* = object + x*: SHORT + y*: SHORT + +const + # used by std/terminal + # https://learn.microsoft.com/en-us/windows/console/setconsolemode + ENABLE_ECHO_INPUT* = 0x0004 + ENABLE_INSERT_MODE* = 0x0020 + ENABLE_LINE_INPUT* = 0x0002 + ENABLE_MOUSE_INPUT* = 0x0010 + ENABLE_PROCESSED_INPUT* = 0x0001 + ENABLE_QUICK_EDIT_MODE* = 0x0040 + ENABLE_WINDOW_INPUT* = 0x0008 + ENABLE_VIRTUAL_TERMINAL_INPUT* = 0x0200 + + ENABLE_PROCESSED_OUTPUT* = 0x0001 + ENABLE_WRAP_AT_EOL_OUTPUT* = 0x0002 + ENABLE_VIRTUAL_TERMINAL_PROCESSING* = 0x0004 + DISABLE_NEWLINE_AUTO_RETURN* = 0x0008 + ENABLE_LVB_GRID_WORLDWIDE* = 0x0010 + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, lpNumberOfEventsRead: ptr cint): cint {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} +proc getConsoleMode*(hConsoleHandle: Handle, dwMode: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetConsoleMode".} + +proc setConsoleMode*(hConsoleHandle: Handle, dwMode: DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleMode".} + +proc setConsoleCursorPosition*(hConsoleOutput: Handle, + dwCursorPosition: COORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleCursorPosition".} + +proc fillConsoleOutputCharacter*(hConsoleOutput: Handle, cCharacter: char, + nLength: DWORD, dwWriteCoord: COORD, + lpNumberOfCharsWritten: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "FillConsoleOutputCharacterA".} + +proc fillConsoleOutputAttribute*(hConsoleOutput: Handle, wAttribute: int16, + nLength: DWORD, dwWriteCoord: COORD, + lpNumberOfAttrsWritten: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "FillConsoleOutputAttribute".} + +proc setConsoleTextAttribute*(hConsoleOutput: Handle, + wAttributes: int16): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleTextAttribute".} + type LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} diff --git a/testament/categories.nim b/testament/categories.nim index 17e9a48a86..7ed98b8aa9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -190,8 +190,10 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # ------------------------- async tests --------------------------------------- proc asyncTests(r: var TResults, cat: Category, options: string) = + # Run async with yrc instead of the default orc; the CI already runs long + # enough that we cannot afford to test both. template test(filename: untyped) = - testSpec r, makeTest(filename, options, cat) + testSpec r, makeTest(filename, options & " --mm:yrc", cat) for t in os.walkFiles("tests/async/t*.nim"): test(t) @@ -528,6 +530,7 @@ proc mmRaise(kind: TResultEnum, expected, given: string) = raise e proc isMetamorphicIcTest(content: string): bool = + result = false for line in content.splitLines: if line.strip == "#? metamorphic": return true @@ -559,7 +562,7 @@ proc stableBinary(path: string): string = ## so two builds seconds apart differ there even with identical codegen. Skipping ## a generous fixed window keeps the clean-vs-incremental check about codegen. const headerSkip = 4096 - var f: File + var f: File = nil if not open(f, path, fmRead): raise newException(IOError, "cannot open: " & path) defer: close(f) diff --git a/tests/arc/t26010.nim b/tests/arc/t26010.nim new file mode 100644 index 0000000000..619991e52a --- /dev/null +++ b/tests/arc/t26010.nim @@ -0,0 +1,23 @@ +discard """ + action: reject + matrix: "--mm:orc; --mm:refc" + errormsg: "cannot move cursor 'a'; a cursor does not own its value" +""" + +# bug #26010: a cursor is a non-owning alias and cannot transfer ownership. + +type Xxx = object + +proc `=destroy`(v: var Xxx) = + debugEcho "dest" + +proc test(v: ref Xxx) = + var a {.cursor.} = v + var b = move(a) + discard + +proc main() = + var x = new Xxx + test(x) + +main() diff --git a/tests/arc/tconcurrentdecref.nim b/tests/arc/tconcurrentdecref.nim new file mode 100644 index 0000000000..2222fe51b7 --- /dev/null +++ b/tests/arc/tconcurrentdecref.nim @@ -0,0 +1,71 @@ +discard """ + matrix: "--mm:atomicArc --threads:on" + output: "ok" +""" + +# Every thread here holds its OWN counted reference to the same cell and drops +# it concurrently with the others. Exactly one free per object must happen: a +# leak (nobody frees) and a double free (two threads free) are both caught. +# +# This is the shape that went wrong in nim-lang/threading#45, where the +# destructor decided who frees from a separate load and discarded the result +# of the read-modify-write, so the role could be dropped by every participant +# at once. `nimDecRefIsLast` must always decide on the value its own RMW +# returned. The uniquely-referenced fast path added on top of it may only +# skip the RMW when the load proves no other thread holds a reference. + +import std/atomics + +type + Payload = object + id: int + Obj = ref Payload + +var freeCount: Atomic[int] + +proc `=destroy`(p: Payload) = + discard freeCount.fetchAdd(1, moRelease) + +const + NumObjects = 2000 + NumThreads = 6 + Rounds = 3 + +type + Arg = object + refs: seq[Obj] + +var + go: Atomic[bool] + threads: array[NumThreads, Thread[ptr Arg]] + args: array[NumThreads, Arg] + +proc worker(a: ptr Arg) {.thread.} = + while not go.load(moAcquire): cpuRelax() + a.refs.setLen(0) # drop them all, as fast as possible + +proc main = + var expected = 0 + for round in 1..Rounds: + var mine = newSeq[Obj](NumObjects) + for i in 0 ..< NumObjects: + mine[i] = Obj(id: i) + for t in 0 ..< NumThreads: + args[t].refs = newSeq[Obj](NumObjects) + for i in 0 ..< NumObjects: + args[t].refs[i] = mine[i] # counted copy + go.store(false, moRelease) + for t in 0 ..< NumThreads: + createThread(threads[t], worker, addr args[t]) + go.store(true, moRelease) # everybody drops at once... + mine.setLen(0) # ...including this thread + joinThreads(threads) + expected += NumObjects + let got = freeCount.load(moAcquire) + if got != expected: + echo "round ", round, ": got ", got, " frees, expected ", expected, + (if got < expected: " (leak)" else: " (double free)") + quit 1 + echo "ok" + +main() diff --git a/tests/arc/torcmisc.nim b/tests/arc/torcmisc.nim index fc0eaf2ff7..079d99c133 100644 --- a/tests/arc/torcmisc.nim +++ b/tests/arc/torcmisc.nim @@ -65,3 +65,17 @@ method handleConn*(myParam: PubSub, proto: string) {.base, async.} = myParam.peers.withValue(conn.peerInfo.peerId, peer): let peerB = peer[] + + + +block: + type M = object + + proc `=dup`(_: M): M {.error.} + proc take(_: sink M) = discard + + proc test() = + var value: M + take(value) + + test() diff --git a/tests/async/t23615.nim b/tests/async/t23615.nim new file mode 100644 index 0000000000..f8d7a8d797 --- /dev/null +++ b/tests/async/t23615.nim @@ -0,0 +1,52 @@ +discard """ + valgrind: true + cmd: '''nim c --mm:orc -d:nimAllocStats -d:useMalloc $file''' + output: '''ok''' +""" + +# bug #23615: exceptions caught by a typed except branch in a closure +# iterator (and thus in any async proc) leaked under ARC/ORC. + +import std/[asyncdispatch, importutils] + +privateAccess(AllocStats) + +block: # pure closure iterator, the minimal form of the bug + proc runIter() = + iterator it(): int {.closure.} = + try: + yield 1 + raise newException(ValueError, "x") + except ValueError: + discard + yield 2 + var f = it + doAssert f() == 1 + doAssert f() == 2 + let base = getAllocStats() + runIter() + GC_fullCollect() + let after = getAllocStats() + doAssert after.allocCount - after.deallocCount == + base.allocCount - base.deallocCount, $base & " " & $after + +block: # the async incarnation from the issue + proc err {.async.} = + raise newException(ValueError, "err1") + + proc amain {.async.} = + await sleepAsync(1) + for _ in 0..<50: + try: + await err() + except ValueError: + discard + + waitFor amain() + doAssert not hasPendingOperations() + setGlobalDispatcher(nil) + GC_fullCollect() + +let stats = getAllocStats() +doAssert stats.allocCount - stats.deallocCount < 10, $stats +echo "ok" diff --git a/tests/async/tasyncclosestall.nim b/tests/async/tasyncclosestall.nim index 05348587c1..ea4865fc79 100644 --- a/tests/async/tasyncclosestall.nim +++ b/tests/async/tasyncclosestall.nim @@ -4,6 +4,7 @@ discard """ exitcode: 0 """ import asyncdispatch, asyncnet +import std/strutils when defined(windows): from winlean import ERROR_NETNAME_DELETED @@ -14,6 +15,7 @@ else: # even when the socket is closed. const timeout = 2000 + messagePaddingSize = 64 * 1024 var port = Port(0) var sent = 0 @@ -31,10 +33,12 @@ proc isExpectedDisconnectionError(errCode: int32): bool = errCode == EBADF or errCode == ECONNRESET or errCode == EPIPE proc keepSendingTo(c: AsyncSocket) {.async.} = + let messagePadding = repeat('x', messagePaddingSize) while true: - # This write will eventually get stuck because the client is not reading - # its messages. - let sendFut = c.send("Foobar" & $sent & "\n", flags = {}) + # Larger writes reach socket backpressure quickly even on slow CI machines. + # This write will eventually get stuck because the client is not reading. + # Keep the padding after the newline so recvLine does not drain it. + let sendFut = c.send("Foobar" & $sent & "\n" & messagePadding, flags = {}) var sendTimedOut = false try: # On some platforms (notably macOS ARM64), the kernel may return diff --git a/tests/async/tasyncdispatchordering.nim b/tests/async/tasyncdispatchordering.nim new file mode 100644 index 0000000000..5b226bad8a --- /dev/null +++ b/tests/async/tasyncdispatchordering.nim @@ -0,0 +1,32 @@ +discard """ + action: run +""" + +import asyncdispatch, os + +proc wrap(fut: Future[void]): Future[void] = + result = newFuture[void]("wrap") + let retFuture = result + fut.addCallback proc () = + if fut.failed: + retFuture.fail(fut.error) + else: + retFuture.complete() + +block: + let root = newFuture[void]("root") + let wrapped = wrap(wrap(wrap(root))) + let completedBeforeDeadline = withTimeout(wrapped, 20) + + # Completion has happened at the bottom of the future chain, but its + # callbacks cannot propagate until control reaches the dispatcher. + root.complete() + sleep(40) + + doAssert waitFor(completedBeforeDeadline) + +block: + var callbackRan = false + sleepAsync(0).addCallback proc () = callbackRan = true + poll(0) + doAssert callbackRan diff --git a/tests/benchmarks/yrcbech.nim b/tests/benchmarks/yrcbech.nim new file mode 100644 index 0000000000..587a0dc5be --- /dev/null +++ b/tests/benchmarks/yrcbech.nim @@ -0,0 +1,113 @@ +discard """ + output: '''true peak memory: true''' + cmd: "nim c --mm:orc -d:release --threads:on $file" +""" + +## torcbench (tests/arc/torcbench.nim), threaded — plus the smallest changes +## that let the generational scheme show up in the number. +## +## Each thread runs its own private copy of the torcbench workload: one long +## doubly-linked list of strings, and a stream of short-lived cyclic trees +## whose every node embeds a copy of the list header — that is a reference +## into the list per tree node. Nothing is shared between threads, so the same +## program is a fair measurement under --mm:orc and --mm:yrc. +## +## Two changes vs torcbench, each needed to make the young -> old pattern +## measurable rather than incidental: +## +## 1. The list is built once per THREAD, not once per outer iteration, so it +## survives long enough to be promoted. It is the old generation; the trees +## are the young one. +## 2. Collection runs at a fixed cadence (GC_partialCollect per outer +## iteration) instead of being left to each collector's threshold +## heuristic. Without this the benchmark measures how often each collector +## decides to collect rather than what a collection over this heap costs — +## and ORC's threshold scales with heap size, so a bigger list makes it +## collect LESS and the re-trace it is supposed to be paying never appears. +## +## Nothing seeds the promotion: the tree stream itself is what ages the list, +## which is why the workload can stay an ordinary one. `DoublyLinkedNode.prev` +## and `DoublyLinkedList.tail` are `{.cursor.}`, so copying a `parent` header +## incRefs `head` alone, and the list is a chain of one-node SCCs rather than +## a single big one. Dirtiness is per-SCC, so the churn only ever dirties +## `head`; every node behind it is traced clean by the collections the trees +## trigger anyway and promotes after YrcPromoteAge of them. The capture then +## prunes one edge in — at `head.next` — instead of walking 60000 nodes. +## +## --mm:orc every collection follows `parent` into the list and re-traces +## all ListLen nodes of it. +## --mm:yrc once the list is promoted, capture prunes at the epoch-stamp +## boundary and walks only the young frontier. +## +## Sizing matters: the win is the ratio of old-generation size to young work +## per collection, so ListLen is large and TreeIters small. Total young work +## is about the same as torcbench's (200x51 trees vs 25x401). NumThreads=4 is +## the default because 8 threads on a 4-performance-core machine dilutes the +## result (1.15x vs 1.69x measured on an M1). +## +## nim c -r --mm:orc -d:release --threads:on yrcbech.nim +## nim c -r --mm:yrc -d:release --threads:on yrcbech.nim +## +## Add -d:yrcBenchTime for a wall-clock line, -d:nimOrcStats for capture and +## prune counts (YRC only). + +import std/[lists, monotimes, times] + +const + NumThreads {.intdefine.} = 4 + OuterIters {.intdefine.} = 200 ## per thread; also the collection count + ListLen {.intdefine.} = 60000 ## the old generation + TreeIters {.intdefine.} = 50 ## young trees per collection + TreeDepth {.intdefine.} = 8 + +type + Node = ref object + parent: DoublyLinkedList[string] ## copy of the header: a ref into the list + le, ri: Node + self: Node ## self-cycle, forces cycle detection + +proc buildTree(parent: DoublyLinkedList[string]; depth: int): Node = + if depth == 0: + result = nil + elif depth == 1: + result = Node(parent: parent) + result.self = result + else: + result = Node(parent: parent, + le: buildTree(parent, depth - 1), + ri: buildTree(parent, depth - 2)) + result.self = result + +proc threadWork() {.thread.} = + # (1) built once per thread: the old generation + var leakList = initDoublyLinkedList[string]() + for j in 1 .. ListLen: + leakList.append(newString(200)) + + for i in 1 .. OuterIters: + for k in 0 .. TreeIters: + discard buildTree(leakList, TreeDepth) # young: dead the moment it returns + GC_partialCollect(0) # (2) fixed cadence + +var threads: array[NumThreads, Thread[void]] + +let t0 = getMonoTime() +for i in 0 ..< NumThreads: + createThread(threads[i], threadWork) +joinThreads(threads) +GC_fullCollect() +let dtMs = inMilliseconds(getMonoTime() - t0) + +when defined(yrcBenchTime): + echo "wall_ms ", dtMs + +when not defined(useMalloc): + echo getOccupiedMem() < 10 * 1024 * 1024, " peak memory: ", + getMaxMem() < 256 * 1024 * 1024 +else: + echo "true peak memory: true" + +when defined(nimOrcStats) and defined(gcYrc): + let s = GC_orcStats() + echo "capTotal ", s.capTotal, " capPruned ", s.capPruned, + " capRepeat ", s.capRepeat diff --git a/tests/ccgbugs/tobjconstr_self_alias.nim b/tests/ccgbugs/tobjconstr_self_alias.nim new file mode 100644 index 0000000000..fef409b356 --- /dev/null +++ b/tests/ccgbugs/tobjconstr_self_alias.nim @@ -0,0 +1,85 @@ +discard """ + matrix: "--mm:refc; --mm:arc; --mm:orc" + output: '''42 +55 +42 +42 +42 +42''' +""" + +# bug #25993 : an object constructor assigned to a location zeroed the +# destination before evaluating a field value that reads from inside that same +# destination, so `tp.h = H(a: tp.h.a)` produced `a == 0`. + +type + Inner = object + a: int + b: int + Mid = object + inner: Inner + x: int + RefT = ref object + h: Inner + other: Inner + m: Mid + +# -------------------------------------------------------------------------- +# bug demonstrations: each printed 0 before the fix +# -------------------------------------------------------------------------- + +proc refDotField(v: int) = + # dest `t.h` is a field of a ref; value reads `t.h.a` (nested in dest) + let t = RefT() + t.h.a = v + t.h = Inner(a: t.h.a) + echo t.h.a + +proc nestedConstr(v: int) = + # dest `t.m`; nested constructor value reads `t.m.inner.a` (nested in dest) + let t = RefT() + t.m.inner.a = v + t.m = Mid(inner: Inner(a: t.m.inner.a), x: 0) + echo t.m.inner.a + +proc refDeepField(v: int) = + # dest `t.m.inner`; value reads `t.m.inner.a` (nested in dest) + let t = RefT() + t.m.inner.a = v + t.m.inner = Inner(a: t.m.inner.a) + echo t.m.inner.a + +var gT: RefT + +proc readsField(t: RefT): int = t.h.a + +proc viaCall(v: int) = + # read of dest hidden behind a call whose argument is the root ref + let t = RefT() + t.h.a = v + t.h = Inner(a: readsField(t)) + echo t.h.a + +proc viaClosureGlobal(v: int) = + # read of dest hidden behind a closure reaching it through a global + let t = RefT() + t.h.a = v + gT = t + let cl = proc(): int = gT.h.a + t.h = Inner(a: cl()) + echo t.h.a + +proc viaClosureCapture(v: int) = + # read of dest hidden behind a closure that captures the root ref + let t = RefT() + t.h.a = v + let cl = proc(): int = t.h.a + t.h = Inner(a: cl()) + echo t.h.a + +refDotField(42) +nestedConstr(55) +refDeepField(42) +viaCall(42) +viaClosureGlobal(42) +viaClosureCapture(42) diff --git a/tests/set/tset_range_trait.nim b/tests/set/tset_range_trait.nim new file mode 100644 index 0000000000..0eafdca1f8 --- /dev/null +++ b/tests/set/tset_range_trait.nim @@ -0,0 +1,39 @@ +# Test that set[] accepts range types via typedesc[R], and set[typedesc[R]] +# must unwrap the typedesc wrapper before checking ordinality. + +import std/typetraits + +type + TestDistinctRange = distinct range[0 .. 63] + +block: # explicit range type as set base + type S = set[range[0 .. 63]] + var s: S = {0, 1} + doAssert 0 in s + +block: # distinctBase result as set base (non-generic) + type S = set[TestDistinctRange.distinctBase] + var s: S = {0, 1} + doAssert 0 in s + +block: # range alias as set base + type RangeAlias = range[0 .. 63] + type S = set[RangeAlias] + var s: S = {0, 1} + doAssert 0 in s + +block: # set[T.distinctBase] in generic body type position + proc test[T: TestDistinctRange]() = + var s: set[T.distinctBase] + s = {0, 1} + doAssert 0 in s + + test[TestDistinctRange]() + +block: # passing set[T.distinctBase] to a proc expecting set[0..63] + proc accept(x: typedesc[set[0 .. 63]]) = discard + + proc pass[T: TestDistinctRange](p: typedesc[set[T]]) = + accept(set[T.distinctBase]) + + pass(set[TestDistinctRange]) diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index fbab9e7b80..dc828f312b 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -457,3 +457,20 @@ block: # bug #22600 var x: c[2] x.init() + +block: + # bug #25938 + + proc p(h: static set[bool] = {}) = + discard false in h + + p() + p({}) + +block: + # bug #25942 + + proc p(h: static set[bool]) = discard len(h) + p({}) + + diff --git a/tests/stdlib/concurrency/tatomics.nim b/tests/stdlib/concurrency/tatomics.nim index 08f2e7d3ee..44760395eb 100644 --- a/tests/stdlib/concurrency/tatomics.nim +++ b/tests/stdlib/concurrency/tatomics.nim @@ -49,7 +49,7 @@ block trivialExchange: doAssert location.load == 6 -block trivialCompareExchangeDoesExchange: +block trivialCompareExchangeDoesExchange: # bug #26027 var location: Atomic[int] var expected = 1 location.store(1) @@ -115,11 +115,11 @@ block trivialCompareExchangeSuccessFailureDoesExchange: doAssert expected == 3 doAssert location.load == 4 expected = 4 - doAssert location.compareExchange(expected, 5, moRelease, moRelease) + doAssert location.compareExchange(expected, 5, moRelease, moRelaxed) doAssert expected == 4 doAssert location.load == 5 expected = 5 - doAssert location.compareExchange(expected, 6, moAcquireRelease, moAcquireRelease) + doAssert location.compareExchange(expected, 6, moAcquireRelease, moAcquire) doAssert expected == 5 doAssert location.load == 6 @@ -140,11 +140,11 @@ block trivialCompareExchangeSuccessFailureDoesNotExchange: doAssert expected == 1 doAssert location.load == 1 expected = 10 - doAssert not location.compareExchange(expected, 5, moRelease, moRelease) + doAssert not location.compareExchange(expected, 5, moRelease, moRelaxed) doAssert expected == 1 doAssert location.load == 1 expected = 10 - doAssert not location.compareExchange(expected, 6, moAcquireRelease, moAcquireRelease) + doAssert not location.compareExchange(expected, 6, moAcquireRelease, moAcquire) doAssert expected == 1 doAssert location.load == 1 @@ -215,11 +215,11 @@ block trivialCompareExchangeWeakSuccessFailureDoesExchange: doAssert expected == 3 doAssert location.load == 4 expected = 4 - doAssert location.compareExchangeWeak(expected, 5, moRelease, moRelease) + doAssert location.compareExchangeWeak(expected, 5, moRelease, moRelaxed) doAssert expected == 4 doAssert location.load == 5 expected = 5 - doAssert location.compareExchangeWeak(expected, 6, moAcquireRelease, moAcquireRelease) + doAssert location.compareExchangeWeak(expected, 6, moAcquireRelease, moAcquire) doAssert expected == 5 doAssert location.load == 6 @@ -240,11 +240,11 @@ block trivialCompareExchangeWeakSuccessFailureDoesNotExchange: doAssert expected == 1 doAssert location.load == 1 expected = 10 - doAssert not location.compareExchangeWeak(expected, 5, moRelease, moRelease) + doAssert not location.compareExchangeWeak(expected, 5, moRelease, moRelaxed) doAssert expected == 1 doAssert location.load == 1 expected = 10 - doAssert not location.compareExchangeWeak(expected, 6, moAcquireRelease, moAcquireRelease) + doAssert not location.compareExchangeWeak(expected, 6, moAcquireRelease, moAcquire) doAssert expected == 1 doAssert location.load == 1 @@ -349,11 +349,11 @@ block objectCompareExchangeSuccessFailureDoesExchange: doAssert expected == Object(val: 3) doAssert location.load == Object(val: 4) expected = Object(val: 4) - doAssert location.compareExchange(expected, Object(val: 5), moRelease, moRelease) + doAssert location.compareExchange(expected, Object(val: 5), moRelease, moRelaxed) doAssert expected == Object(val: 4) doAssert location.load == Object(val: 5) expected = Object(val: 5) - doAssert location.compareExchange(expected, Object(val: 6), moAcquireRelease, moAcquireRelease) + doAssert location.compareExchange(expected, Object(val: 6), moAcquireRelease, moAcquire) doAssert expected == Object(val: 5) doAssert location.load == Object(val: 6) @@ -374,11 +374,11 @@ block objectCompareExchangeSuccessFailureDoesNotExchange: doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) expected = Object(val: 10) - doAssert not location.compareExchange(expected, Object(val: 5), moRelease, moRelease) + doAssert not location.compareExchange(expected, Object(val: 5), moRelease, moRelaxed) doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) expected = Object(val: 10) - doAssert not location.compareExchange(expected, Object(val: 6), moAcquireRelease, moAcquireRelease) + doAssert not location.compareExchange(expected, Object(val: 6), moAcquireRelease, moAcquire) doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) @@ -449,11 +449,11 @@ block objectCompareExchangeWeakSuccessFailureDoesExchange: doAssert expected == Object(val: 3) doAssert location.load == Object(val: 4) expected = Object(val: 4) - doAssert location.compareExchangeWeak(expected, Object(val: 5), moRelease, moRelease) + doAssert location.compareExchangeWeak(expected, Object(val: 5), moRelease, moRelaxed) doAssert expected == Object(val: 4) doAssert location.load == Object(val: 5) expected = Object(val: 5) - doAssert location.compareExchangeWeak(expected, Object(val: 6), moAcquireRelease, moAcquireRelease) + doAssert location.compareExchangeWeak(expected, Object(val: 6), moAcquireRelease, moAcquire) doAssert expected == Object(val: 5) doAssert location.load == Object(val: 6) @@ -474,11 +474,11 @@ block objectCompareExchangeWeakSuccessFailureDoesNotExchange: doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) expected = Object(val: 10) - doAssert not location.compareExchangeWeak(expected, Object(val: 5), moRelease, moRelease) + doAssert not location.compareExchangeWeak(expected, Object(val: 5), moRelease, moRelaxed) doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) expected = Object(val: 10) - doAssert not location.compareExchangeWeak(expected, Object(val: 6), moAcquireRelease, moAcquireRelease) + doAssert not location.compareExchangeWeak(expected, Object(val: 6), moAcquireRelease, moAcquire) doAssert expected == Object(val: 1) doAssert location.load == Object(val: 1) diff --git a/tests/stdlib/tstrbasics.nim b/tests/stdlib/tstrbasics.nim index bfae55fde9..e812e20ee4 100644 --- a/tests/stdlib/tstrbasics.nim +++ b/tests/stdlib/tstrbasics.nim @@ -90,7 +90,8 @@ proc main() = var a0 = "hi" var b0 = "foobar" when nimvm: - discard # pending bug #15952 + a0.add b0.toOpenArray(1,3) + doAssert a0 == "hioob" else: a0.add b0.toOpenArray(1,3) doAssert a0 == "hioob" diff --git a/tests/stdlib/ttables.nim b/tests/stdlib/ttables.nim index ca5c96a593..58ec246504 100644 --- a/tests/stdlib/ttables.nim +++ b/tests/stdlib/ttables.nim @@ -33,6 +33,22 @@ s2[p2] = 45_000 s3[p1] = 30_000 s3[p2] = 45_000 +block: # two-argument form of withValue forms expression + block: # Present + let sal = salaries.withValue(p1, sal): + sal[] + do: + 0 + doAssert sal == 30_000 + block: # Missing + let sal = salaries.withValue(Person(), sal): + sal[] + do: + 0 + doAssert sal == 0 + block: # Short form + doAssert salaries.withValue(p1, sal, sal[], 0) == 30_000 + block: # Ordered table should preserve order after deletion var s4 = initOrderedTable[int, int]() diff --git a/tests/stdlib/txmltree.nim b/tests/stdlib/txmltree.nim index add12a3fc0..138cfe3166 100644 --- a/tests/stdlib/txmltree.nim +++ b/tests/stdlib/txmltree.nim @@ -118,3 +118,14 @@ block: #21541 doAssert temp.text == "Hello!" temp.text = "Hola!" doAssert temp.text == "Hola!" + +block: #26039 + let tree = <>rss( + "xmlns:atom" = "http://www.w3.org/2005/Atom", + <>"atom:link"( + `data-dummy` = "test", + ), + ) + doAssert $tree == """ + +""" diff --git a/tests/whenstmt/twhen_nimvm_push.nim b/tests/whenstmt/twhen_nimvm_push.nim new file mode 100644 index 0000000000..7991239fcc --- /dev/null +++ b/tests/whenstmt/twhen_nimvm_push.nim @@ -0,0 +1,21 @@ +discard """ + output: "ok" +""" + +var overflowDetected = false +when nimvm: + {.push overflowChecks: off.} +else: + var branchX = high(int) + try: + inc branchX + except OverflowDefect: + overflowDetected = true + +doAssert overflowDetected + +var x = high(int) +try: + inc x +except OverflowDefect: + echo "ok" diff --git a/tests/whenstmt/twhen_nimvm_push_pop.nim b/tests/whenstmt/twhen_nimvm_push_pop.nim new file mode 100644 index 0000000000..31f44c4058 --- /dev/null +++ b/tests/whenstmt/twhen_nimvm_push_pop.nim @@ -0,0 +1,8 @@ +discard """ + errormsg: "{.pop.} without a corresponding {.push.}" + line: 8 +""" + +when nimvm: + {.push checks: off.} +else: {.pop.} diff --git a/tests/yrc/tyrc_generational.nim b/tests/yrc/tyrc_generational.nim new file mode 100644 index 0000000000..d608f1707f --- /dev/null +++ b/tests/yrc/tyrc_generational.nim @@ -0,0 +1,178 @@ +discard """ + cmd: "nim c --mm:yrc -d:useMalloc --threads:on $file" + output: "ok" + disabled: "windows" + disabled: "freebsd" + disabled: "openbsd" +""" + +# Generational epoch stamps, young -> old. +# +# Each thread keeps a long-lived cyclic web and promotes it past +# YrcPromoteAge with a few seeded partial collects. After that, every +# iteration allocates die-young cyclic rings that reference the web -- the +# classic new-refers-to-old pattern. Capture then prunes at the stamp +# boundary and commit `trialDec`s the young -> web edges WITHOUT re-rooting +# the web, so from that point on the web's liveness no longer rests on being +# traced: it rests on the deferred machinery (the per-thread suspect buffer +# and the pruned-target list) keeping those cells examinable and alive until +# the epoch advances. +# +# Two properties are asserted. The surviving web is walked in full at the +# end, so a web that was collected or partially collected out from under the +# deferred machinery shows up as a nil edge, a corrupted id or a short node +# count. And after every web is dropped, a full collect must reclaim all of +# it -- deferring reclamation to the epoch boundary must not turn into never +# reclaiming. +# +# Sized so the shared epoch clock (YrcEpochLen collections) turns over +# repeatedly mid-run: the steady-state suspect flush is on the path under +# test, not just the one forced by the final GC_fullCollect. Measured on the +# current collector this run remembers ~200 suspects across ~80 flushes. +# +# NOTE: this is a functional test of the generational path, not a regression +# test for the dangling-suspect use-after-free that path once had. It was +# tried in that role and does not reproduce it: the suspects it creates are +# nearly always flushed before they die, so the bad ordering never comes up. +# tests/async/tasyncawait.nim reproduces that one reliably. + +const + NumThreads = 4 + WebSize = 8_000 ## the web that survives to the integrity check + DoomedSize = 800 ## promoted, given young -> old edges, then dropped + WebDegree = 4 + SeedProbes = 6 ## must exceed YrcPromoteAge (3) to promote a web + OuterIters = 40 ## with NumThreads, enough collections to cross epochs + YoungBatches = 8 + YoungRing = 100 + +type + WebNode = ref object + id: int32 + seen: int32 ## walk marker, plain data: no GC interaction + edges: array[WebDegree, WebNode] + + Bridge = ref object + toWeb: WebNode + self: Bridge + + YoungNode = ref object + next: YoungNode + hub: WebNode ## the young -> old edge under test + self: YoungNode + +var probeSlot {.threadvar.}: Bridge + +proc buildWebNodes(n: int): seq[WebNode] = + ## Strongly connected mesh: one incoming edge pulls the whole web into any + ## collector that does not prune at the stamp boundary. + result = newSeq[WebNode](n) + for i in 0 ..< n: + result[i] = WebNode(id: int32(i)) + for i in 0 ..< n: + for d in 0 ..< WebDegree: + result[i].edges[d] = result[(i + 1 + d * 97) mod n] + +proc buildWeb(n: int): WebNode = buildWebNodes(n)[0] + +proc checkWeb(root: WebNode; n: int) = + ## Every node reachable exactly once, every edge intact. A web that was + ## collected out from under us fails here instead of faulting later. + var stack = @[root] + root.seen = 1 + var count = 0 + while stack.len > 0: + let x = stack.pop() + inc count + doAssert x.id >= 0'i32 and x.id < int32(n), "web node corrupted: id " & $x.id + for d in 0 ..< WebDegree: + let e = x.edges[d] + doAssert e != nil, "web edge nil'ed at node " & $x.id + if e.seen != 1: + e.seen = 1 + stack.add e + doAssert count == n, "web lost nodes: " & $count & " of " & $n + +proc paintYoung(hub: WebNode; n: int) = + ## Ring of `n` self-referential nodes, each pointing at the web. When the + ## seq drops, the ring is garbage whose only external edges go into the + ## live (and by now stamp-pruned) web. + var nodes = newSeq[YoungNode](n) + for i in 0 ..< n: + nodes[i] = YoungNode(hub: hub) + for i in 0 ..< n: + nodes[i].next = nodes[(i + 1) mod n] + nodes[i].self = nodes[i] + +proc paintYoungSpread(web: seq[WebNode]; n: int) = + ## Same, but every young node targets a DIFFERENT old cell, so the commit + ## deposits many distinct cells in the suspect buffer instead of just the + ## web root. Breadth here is what makes the "suspect dies before the epoch + ## flush" ordering likely rather than incidental. + var nodes = newSeq[YoungNode](n) + for i in 0 ..< n: + nodes[i] = YoungNode(hub: web[(i * 7) mod web.len]) + for i in 0 ..< n: + nodes[i].next = nodes[(i + 1) mod n] + nodes[i].self = nodes[i] + +proc probeBridge(b: Bridge) {.noinline.} = + ## Seeded false alarm so a partial collect traces -- and stamps -- the web. + ## The threadvar slot is deliberate: a stack temporary is not a reliable + ## way to get the bridge registered as a candidate root. + probeSlot = b + probeSlot = nil + +proc promote(b: Bridge) = + ## Trace-and-stamp the bridge's web often enough that its cells pass + ## YrcPromoteAge and captures start pruning at them. + for _ in 1 .. SeedProbes: + probeBridge(b) + # Deliberately not GC_fullCollect: that advances the epoch and wipes the + # stamps this test needs. + GC_partialCollect(0) + +proc cycleDoomedWeb() = + ## Promote a web, hand it young -> old edges so its cells land in the + ## deferred suspect buffer, then drop it. Those cells are now garbage + ## while still listed, and an ordinary collection reclaims them well + ## before the epoch advance that flushes the buffer. THIS is the case a + ## live-forever web never produces: the list has to not be holding + ## pointers to cells anyone else was free to reclaim. + let doomedNodes = buildWebNodes(DoomedSize) + let b = Bridge(toWeb: doomedNodes[0]) + b.self = b + promote(b) + for _ in 1 .. YoungBatches: + paintYoungSpread(doomedNodes, YoungRing) + GC_partialCollect(0) + # `doomedNodes` and `b` die with this scope: every cell that just landed + # in the suspect buffer is now garbage while still listed there. + +proc threadWork() {.thread.} = + let web = buildWeb(WebSize) + let bridge = Bridge(toWeb: web) + bridge.self = bridge + promote(bridge) + + for i in 1 .. OuterIters: + cycleDoomedWeb() + for _ in 1 .. YoungBatches: + paintYoung(web, YoungRing) + GC_partialCollect(0) + + checkWeb(web, WebSize) + doAssert bridge.toWeb == web, "bridge lost its web" + +var threads: array[NumThreads, Thread[void]] +for i in 0 ..< NumThreads: + createThread(threads[i], threadWork) +joinThreads(threads) + +# Every web is unreachable now. The full collect advances the epoch, which +# flushes the suspect buffers into the root set -- the major-collection half +# of the scheme -- so all of it must come back. +GC_fullCollect() +doAssert getOccupiedMem() < 8 * 1024 * 1024, + "webs not reclaimed: " & $(getOccupiedMem() div 1024) & " KiB still occupied" +echo "ok"