From 3d0bac8e5b21b8e0372f083bb4d4c5763cd14edd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 31 Jul 2026 05:34:47 +0200 Subject: [PATCH] YRC: micro optimizations (#26056) --- lib/system/yrc.nim | 163 ++++++++++++++++++++++++++++++--------------- 1 file changed, 111 insertions(+), 52 deletions(-) diff --git a/lib/system/yrc.nim b/lib/system/yrc.nim index 0a73fd72ea..deb4550443 100644 --- a/lib/system/yrc.nim +++ b/lib/system/yrc.nim @@ -230,6 +230,12 @@ proc add[T](s: var RawSeq[T]; v: T) {.inline.} = s.d[s.len] = v s.len = s.len +% 1 +proc addUnchecked[T](s: var RawSeq[T]; v: T) {.inline.} = + ## `add` without the capacity branch; the caller guarantees the room. + ## Used by `claimCell`, which reserves for all four of its arrays at once. + s.d[s.len] = v + s.len = s.len +% 1 + proc pop[T](s: var RawSeq[T]): T {.inline.} = s.len = s.len -% 1 result = s.d[s.len] @@ -358,6 +364,13 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = # long-lived live structures are traced once per epoch instead of once per # collection. Roots always bypass the stamp: every death has a dec-witness # that gets registered, and registered cells are always scanned as roots. +type + CollCtx = object + tag: int64 + slot: int + epochStamp: int64 ## this collection's epoch, as a stamp word + amSolo: bool + const MaxPar {.intdefine.} = 256 ## CAPACITY of the slot table, not a tuning knob: a hard ceiling on @@ -384,12 +397,9 @@ var ## Starting at 1 also means single-threaded programs scan one entry. gSoloCapture: int # a solo collection is in its capture phase gTagCounter: int64 - gMyTag {.threadvar.}: int64 - gMySlot {.threadvar.}: int - gAmSolo {.threadvar.}: bool + gCtx {.threadvar.}: CollCtx gEpoch: int # advanced every YrcEpochLen collections gCollectionCounter: int - gMyEpochStamp {.threadvar.}: int64 # this collection's epoch, as a stamp word gWaitLock: Lock # pairs gWaitCond's wait/broadcast; leaf gWaitCond: Cond # signaled on capture-end and collection-finish @@ -475,11 +485,11 @@ proc anySlotFree(): bool {.inline.} = if atomicLoadN(addr gActiveTags[sl], ATOMIC_ACQUIRE) == 0: return true -template isStamped(c: Cell): bool = +template isStamped(c: Cell; ctx: ptr CollCtx): bool = # "stamped" means: claimed by THIS collection. A relaxed load suffices: - # only this thread ever stores gMyTag, and any stale read of a foreign + # only this thread ever stores ctx.tag, and any stale read of a foreign # value routes into claimCell which re-validates with acquire + CAS. - (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == gMyTag + (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == ctx.tag template denseIdx(c: Cell): int32 = int32(loWord(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED))) @@ -628,7 +638,7 @@ proc registerLocal(c: Cell; desc: PNimTypeV2) {.inline.} = when defined(nimOrcStats): let st = atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) if st == 0: bumpStat gStatRegFresh - elif gMyTag != 0 and (st shr 32) == gMyTag: bumpStat gStatRegSelf + elif gCtx.tag != 0 and (st shr 32) == gCtx.tag: bumpStat gStatRegSelf elif isActiveTag(st shr 32): bumpStat gStatRegCross else: bumpStat gStatRegRepeat if gLocalRoots.d == nil: init(gLocalRoots) @@ -761,7 +771,7 @@ proc buildPendingWatch(): bool = if gPendingWatch.d == nil: init gPendingWatch gPendingWatch.len = 0 for s in 0 ..< slotsInPlay(): - if s != gMySlot: + if s != gCtx.slot: let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE) if tg != 0 and atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) == 1: gPendingWatch.add((int64(s) shl 32) or tg) @@ -839,11 +849,25 @@ proc prepareCapture() = gCap.prunedSrc.len = 0 gCap.ages.len = 0 +proc growCaptureArrays(cap: ptr CaptureBufs) {.noinline.} = + ## `recs`, `sccIdx`, `ages` and `tstack` are appended to together, and only + ## by `claimCell` — one entry each per claimed cell. So they are grown + ## together and ONE capacity check on `recs` covers all four: `recs` drives + ## the growth, the other three are topped up to at least its capacity. + ## `tstack` is popped as SCCs are emitted, so its length only ever trails + ## `recs.len`; matching capacities keeps its appends unchecked too. + ## Marked `noinline` to keep the cold resize path out of `claimCell`. + resize(cap.recs, cap.recs.len +% 1) + let n = cap.recs.cap + if cap.sccIdx.cap < n: resize(cap.sccIdx, n) + if cap.ages.cap < n: resize(cap.ages, n) + if cap.tstack.cap < n: resize(cap.tstack, n) + # rc is captured without the flag bits: the collector itself toggles # inRootsFlag between capture and commit, which must not look like a # mutation to the commit-time rc validation. proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs; - pruneLive: bool; old0: int64): int32 = + ctx: ptr CollCtx; pruneLive: bool; old0: int64): int32 = ## Dense index if this collection owns `c` (claiming and registering it ## if it was unclaimed), -1 if another ACTIVE collection owns it, or ## -2 if `pruneLive` and the cell was proven live in the current epoch @@ -852,33 +876,34 @@ proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs; ## `old0` is `c`'s claim word as the caller already read it (acquire): the ## DFS reads it to test ownership, and re-reading it here would be a second ## dependent load of the same cold header word on every traversed edge. - if gAmSolo: + if ctx.amSolo: # no other collection is (or can start) capturing: plain stores. # This recovers the sequential capture speed of the single-collector # design whenever collections do not actually overlap. let old = old0 - if (old shr 32) == gMyTag: + if (old shr 32) == ctx.tag: return int32(loWord(old)) - if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and + if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and stampAge(old) >= YrcPromoteAge: return -2 let idx = cap.recs.len - c.rootIdx = (gMyTag shl 32) or int64(idx) + c.rootIdx = (ctx.tag shl 32) or int64(idx) when defined(nimOrcStats): bumpStat gStatCapTotal if old != 0: bumpStat gStatCapRepeat - cap.recs.add CaptureRec(cell: c, desc: desc, + if idx >= cap.recs.cap: growCaptureArrays(cap) + cap.recs.addUnchecked CaptureRec(cell: c, desc: desc, rcWord: loadRc(c) and not rcMask, lowlink: int32(idx), selfRefs: 0'i32) - cap.sccIdx.add -1'i32 - cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) - cap.tstack.add int32(idx) + cap.sccIdx.addUnchecked -1'i32 + cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) + cap.tstack.addUnchecked int32(idx) return int32(idx) var old = old0 while true: - if (old shr 32) == gMyTag: + if (old shr 32) == ctx.tag: return int32(loWord(old)) - if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and + if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and stampAge(old) >= YrcPromoteAge: return -2 # an epoch stamp is never a tag (tags are allocated below `epochBase`), @@ -888,17 +913,18 @@ proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs; return -1 let idx = cap.recs.len if atomicCompareExchangeN(addr c.rootIdx, addr old, - (gMyTag shl 32) or int64(idx), false, + (ctx.tag shl 32) or int64(idx), false, ATOMIC_ACQ_REL, ATOMIC_RELAXED): when defined(nimOrcStats): bumpStat gStatCapTotal if old != 0: bumpStat gStatCapRepeat - cap.recs.add CaptureRec(cell: c, desc: desc, + if idx >= cap.recs.cap: growCaptureArrays(cap) + cap.recs.addUnchecked CaptureRec(cell: c, desc: desc, rcWord: loadRc(c) and not rcMask, lowlink: int32(idx), selfRefs: 0'i32) - cap.sccIdx.add -1'i32 - cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) - cap.tstack.add int32(idx) + cap.sccIdx.addUnchecked -1'i32 + cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0) + cap.tstack.addUnchecked int32(idx) return int32(idx) # a failed CAS leaves the fresh claim word in `old`; loop with it @@ -918,11 +944,14 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = ## on the stack below u) or an SCC emitted earlier, so `sccIdx` is final ## for all of them and the cross targets can be appended to `crossTgt` ## contiguously — which makes `crossOff` a prefix offset for free. + # one TLS resolution for the whole traversal; `claimCell` and the per-edge + # ownership tests below read the context through this pointer + let ctx = addr gCtx let rootWord = atomicLoadN(addr s.rootIdx, ATOMIC_ACQUIRE) - if (rootWord shr 32) == gMyTag: return + if (rootWord shr 32) == ctx.tag: return orcAssert(j.traceStack.len == 0, "capture: trace stack not empty") # roots never prune: a dec-witnessed suspicion overrides any epoch stamp - let root = claimCell(s, desc, cap, pruneLive = false, old0 = rootWord) + let root = claimCell(s, desc, cap, ctx, pruneLive = false, old0 = rootWord) if root < 0: return # another active collection owns this candidate; it handles it trace(s, desc, j) @@ -942,7 +971,7 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = # one load of the target's claim word serves both the ownership test # and the dense-index extraction let cw = atomicLoadN(addr t.rootIdx, ATOMIC_ACQUIRE) - if (cw shr 32) == gMyTag: + if (cw shr 32) == ctx.tag: let v = int32(loWord(cw)) if v == u: # A self edge is internal by construction and its reference is @@ -957,7 +986,7 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = else: let tdesc = j.traceStack.d[last][1] let childBase = j.traceStack.len - let v = claimCell(t, tdesc, cap, pruneLive = true, old0 = cw) + let v = claimCell(t, tdesc, cap, ctx, pruneLive = true, old0 = cw) if v == -1: # cross-collection edge: the owner sees our reference in the rc # word and classifies the target live; we re-register it as a @@ -966,7 +995,13 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) = elif v == -2: # target proven live this epoch: opaque live external, no descent. # Taint u's SCC — its own "live" verdict may lean on the stamp. - cap.prunedSrc.add u + # The list is only ever read as a set (and for its emptiness), and + # one cell's out-edges are consumed consecutively, so suppressing a + # repeat of the previous entry removes nearly every duplicate a + # multi-pruned cell would otherwise contribute. + if cap.prunedSrc.len == 0 or + cap.prunedSrc.d[cap.prunedSrc.len -% 1] != u: + cap.prunedSrc.add u when defined(nimOrcStats): bumpStat gStatCapPruned else: @@ -1031,9 +1066,13 @@ proc computeDeadness(j: var GcEnv; cap: ptr CaptureBufs) = let s = cap.sccIdx.d[cap.prunedSrc.d[i]] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagPruned # cells that stay registered as roots (partial collection) count as - # externally referenced: the roots buffer itself points at them - for mi in 0 ..< cap.sccMembers.len: - let m = cap.sccMembers.d[mi] + # externally referenced: the roots buffer itself points at them. + # Scanned over `recs` rather than over `sccMembers`: every claimed cell is + # pushed to the Tarjan stack once and popped into `sccMembers` once, so the + # two cover exactly the same set, and the SCC comes from `sccIdx` either + # way. Going through `sccMembers` would only add a random 32-byte-stride + # gather in front of a load that already misses on the cell header. + for m in 0 ..< cap.recs.len: if (loadRc(cap.recs.d[m].cell) and inRootsFlag) != 0: let s = cap.sccIdx.d[m] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagForcedLive @@ -1064,9 +1103,10 @@ proc markDirtyFromQueues(j: var GcEnv; cap: ptr CaptureBufs) = ## the last drain had its reference set changed during capture. Peek ## (don't drain!) the stripe queues and taint the affected SCCs; the ## entries stay queued and the next merge re-registers them as candidates. + let ctx = addr gCtx template taint(cp: Cell) = let c = cp - if isStamped(c): + if isStamped(c, ctx): let s = cap.sccIdx.d[denseIdx(c)] cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagDirty for i in 0..