diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 511bb87d82..fc4a7a7a3b 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -25,31 +25,55 @@ when defined(gcYrc): HasCollectorLock Collecting - AlignedRwLock = object - ## One RwLock per cache line. {.align: 64.} causes the compiler to round - ## the struct size up to 64 bytes, so consecutive array elements never - ## share a cache line (sizeof(RwLock) = 56 on Linux x86_64 → 8 byte pad). - lock {.align: 64.}: RwLock + AlignedCounter = object + ## one counter per cache line to avoid false sharing between stripes + c {.align: 64.}: int + # Asymmetric two-class exclusion: seq structure mutations and collections + # exclude each other, but seq ops run concurrently with seq ops and + # collections run concurrently with collections. This replaces the old + # RwLock scheme (which allowed only ONE collector, serializing parallel + # collection) and also sidesteps POSIX's requirement that a rwlock be + # unlocked by its acquiring thread. var - gYrcLocks: array[NumLockStripes, AlignedRwLock] + gSeqActive: array[NumLockStripes, AlignedCounter] # in-flight seq ops + gGcActive: int # active collections var lockState {.threadvar.}: YrcLockState proc getYrcStripe(): int {.inline.} = - ## Map this thread to one of the NumLockStripes RwLock stripes. + ## Map this thread to one of the NumLockStripes counter stripes. ## getThreadId() is already cached thread-locally in threadids.nim. getThreadId() and (NumLockStripes - 1) proc acquireMutatorLock() {.compilerRtl, inl.} = if lockState == HasNoLock: - acquireRead gYrcLocks[getYrcStripe()].lock + let s = getYrcStripe() + while true: + # SEQ_CST inc-then-check pairs with the collector's SEQ_CST + # inc-then-drain (Dekker-style store/load ordering) + discard atomicFetchAdd(addr gSeqActive[s].c, 1, ATOMIC_SEQ_CST) + if atomicLoadN(addr gGcActive, ATOMIC_SEQ_CST) == 0: break + discard atomicFetchSub(addr gSeqActive[s].c, 1, ATOMIC_SEQ_CST) + while atomicLoadN(addr gGcActive, ATOMIC_ACQUIRE) != 0: + discard lockState = HasMutatorLock proc releaseMutatorLock() {.compilerRtl, inl.} = if lockState == HasMutatorLock: lockState = HasNoLock - releaseRead gYrcLocks[getYrcStripe()].lock + discard atomicFetchSub(addr gSeqActive[getYrcStripe()].c, 1, ATOMIC_SEQ_CST) + + proc yrcGcFenceEnter() = + ## A collection announces itself and waits for in-flight seq structure + ## mutations to drain. Multiple collections may hold the fence at once. + discard atomicFetchAdd(addr gGcActive, 1, ATOMIC_SEQ_CST) + for s in 0 ..< NumLockStripes: + while atomicLoadN(addr gSeqActive[s].c, ATOMIC_SEQ_CST) > 0: + discard + + proc yrcGcFenceExit() = + discard atomicFetchSub(addr gGcActive, 1, ATOMIC_SEQ_CST) template yrcMutatorLock*(t: typedesc; body: untyped) = {.noSideEffect.}: @@ -71,23 +95,6 @@ when defined(gcYrc): {.noSideEffect.}: releaseMutatorLock() - template yrcCollectorLock(body: untyped) = - if lockState == HasMutatorLock: releaseMutatorLock() - let prevState = lockState - let hadToAcquire = prevState < HasCollectorLock - if hadToAcquire: - # Acquire all stripes in ascending order — the only thread ever holding - # multiple write locks is the collector, so there is no lock-order cycle. - for yrcI in 0.. base: - let (slot, tdesc) = j.traceStack.pop() - let t = head(slot[]) + let (entry, tdesc) = j.traceStack.pop() + let t = head(entry.val) if isStamped(t): let v = denseIdx(t) gCap.edges.add (int64(u) shl 32) or int64(v) @@ -560,13 +604,64 @@ proc computeDeadness(j: var GcEnv) = let t = gCap.crossTgt.d[k] gCap.sccFlags.d[t] = gCap.sccFlags.d[t] or flagForcedLive -# ---------------- phase 3: commit ---------------- +# ---------------- phase 3: validate & commit ---------------- + +proc markDirtyFromQueues(j: var GcEnv) = + ## The SATB half of the design: any cell with an inc or dec enqueued since + ## mergePendingRoots had its reference set changed during capture. Peek + ## (don't drain!) the stripe queues and taint the affected SCCs; the + ## entries stay queued and the next merge re-registers them as candidates. + template taint(cp: Cell) = + let c = cp + if isStamped(c): + let s = gCap.recs.d[denseIdx(c)].sccOf + gCap.sccFlags.d[s] = gCap.sccFlags.d[s] or flagDirty + for i in 0.. 0: - let (slot, _) = j.traceStack.pop() - slot[] = nil + let (entry, _) = j.traceStack.pop() + entry.slot[] = nil else: while j.traceStack.len > 0: - let (slot, _) = j.traceStack.pop() - let t = head(slot[]) - slot[] = nil + let (entry, _) = j.traceStack.pop() + let t = head(entry.val) + entry.slot[] = nil if not deadCell(t): trialDec(t) when sizeof(int) != 8: @@ -687,7 +782,7 @@ proc GC_runOrc* = if roots.len > 0 and mayRunCycleCollect(): var j: GcEnv collectCyclesImpl(j, 0) - when logOrc: orcAssert roots.len == 0, "roots not empty!" + # note: aborted SCCs legitimately leave re-registered roots behind proc GC_enableOrc*() = when not defined(nimStressOrc): @@ -775,25 +870,27 @@ proc yrcDec(tmp: pointer; desc: PNimTypeV2) {.inline.} = discard nimDecRefIsLastCyclicDyn(tmp) proc nimAsgnYrc(dest: ptr pointer; src: pointer; desc: PNimTypeV2) {.compilerRtl.} = - ## YRC write barrier for ref copy assignment. - ## Holds the mutator read lock for the entire operation so the collector - ## cannot run between the incRef and decRef, closing the stale-decRef - ## bug. Direct atomic incRef replaces the toInc stripe queue: the - ## collector is blocked, so the RC update is immediately visible and correct. - acquireMutatorLock() - if src != nil: increment head(src) # direct atomic: no toInc queue needed - let tmp = dest[] - dest[] = src - if tmp != nil: yrcDec(tmp, desc) # still deferred via toDec for cycle detection - releaseMutatorLock() + ## YRC write barrier for ref copy assignment. LOCK-FREE: the deferred dec + ## of the old value doubles as the snapshot-at-the-beginning log (the + ## collector peeks the toDec queues at commit time), and the direct atomic + ## incRef of the new value is exactly the rc mutation the collector's + ## commit-time rc validation observes. + if src != nil: increment head(src) + when hasThreadSupport: + let tmp = atomicExchangeN(dest, src, ATOMIC_ACQ_REL) + else: + let tmp = dest[] + dest[] = src + if tmp != nil: yrcDec(tmp, desc) proc nimSinkYrc(dest: ptr pointer; src: pointer; desc: PNimTypeV2) {.compilerRtl.} = ## YRC write barrier for ref sink (move). No incRef on source. - acquireMutatorLock() - let tmp = dest[] - dest[] = src + when hasThreadSupport: + let tmp = atomicExchangeN(dest, src, ATOMIC_ACQ_REL) + else: + let tmp = dest[] + dest[] = src if tmp != nil: yrcDec(tmp, desc) - releaseMutatorLock() proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = when optimizedOrc: diff --git a/tests/arc/torcbench.nim b/tests/arc/torcbench.nim index e54537883a..a010b229c2 100644 --- a/tests/arc/torcbench.nim +++ b/tests/arc/torcbench.nim @@ -36,4 +36,4 @@ proc main() = main() GC_fullCollect() when not defined(useMalloc): - echo getOccupiedMem() < 10 * 1024 * 1024, " peak memory: ", getMaxMem() < 10 * 1024 * 1024 + echo getOccupiedMem() < 10 * 1024 * 1024, " peak memory: ", getMaxMem() < 12 * 1024 * 1024