YRC: micro optimizations (#26056)

lib/system/yrc.nim

@@ -230,6 +230,12 @@ proc add[T](s: var RawSeq[T]; v: T) {.inline.} =
   s.d[s.len] = v
   s.len = s.len +% 1
 
+proc addUnchecked[T](s: var RawSeq[T]; v: T) {.inline.} =
+  ## `add` without the capacity branch; the caller guarantees the room.
+  ## Used by `claimCell`, which reserves for all four of its arrays at once.
+  s.d[s.len] = v
+  s.len = s.len +% 1
+
 proc pop[T](s: var RawSeq[T]): T {.inline.} =
   s.len = s.len -% 1
   result = s.d[s.len]
@@ -358,6 +364,13 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} =
 # long-lived live structures are traced once per epoch instead of once per
 # collection. Roots always bypass the stamp: every death has a dec-witness
 # that gets registered, and registered cells are always scanned as roots.
+type
+  CollCtx = object
+    tag: int64
+    slot: int
+    epochStamp: int64   ## this collection's epoch, as a stamp word
+    amSolo: bool
+
 const
   MaxPar {.intdefine.} = 256
     ## CAPACITY of the slot table, not a tuning knob: a hard ceiling on
@@ -384,12 +397,9 @@ var
     ## Starting at 1 also means single-threaded programs scan one entry.
   gSoloCapture: int                      # a solo collection is in its capture phase
   gTagCounter: int64
-  gMyTag {.threadvar.}: int64
-  gMySlot {.threadvar.}: int
-  gAmSolo {.threadvar.}: bool
+  gCtx {.threadvar.}: CollCtx
   gEpoch: int                            # advanced every YrcEpochLen collections
   gCollectionCounter: int
-  gMyEpochStamp {.threadvar.}: int64     # this collection's epoch, as a stamp word
   gWaitLock: Lock                        # pairs gWaitCond's wait/broadcast; leaf
   gWaitCond: Cond                        # signaled on capture-end and collection-finish
 
@@ -475,11 +485,11 @@ proc anySlotFree(): bool {.inline.} =
     if atomicLoadN(addr gActiveTags[sl], ATOMIC_ACQUIRE) == 0:
       return true
 
-template isStamped(c: Cell): bool =
+template isStamped(c: Cell; ctx: ptr CollCtx): bool =
   # "stamped" means: claimed by THIS collection. A relaxed load suffices:
-  # only this thread ever stores gMyTag, and any stale read of a foreign
+  # only this thread ever stores ctx.tag, and any stale read of a foreign
   # value routes into claimCell which re-validates with acquire + CAS.
-  (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == gMyTag
+  (atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED) shr 32) == ctx.tag
 template denseIdx(c: Cell): int32 =
   int32(loWord(atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED)))
 
@@ -628,7 +638,7 @@ proc registerLocal(c: Cell; desc: PNimTypeV2) {.inline.} =
     when defined(nimOrcStats):
       let st = atomicLoadN(addr c.rootIdx, ATOMIC_RELAXED)
       if st == 0: bumpStat gStatRegFresh
-      elif gMyTag != 0 and (st shr 32) == gMyTag: bumpStat gStatRegSelf
+      elif gCtx.tag != 0 and (st shr 32) == gCtx.tag: bumpStat gStatRegSelf
       elif isActiveTag(st shr 32): bumpStat gStatRegCross
       else: bumpStat gStatRegRepeat
     if gLocalRoots.d == nil: init(gLocalRoots)
@@ -761,7 +771,7 @@ proc buildPendingWatch(): bool =
   if gPendingWatch.d == nil: init gPendingWatch
   gPendingWatch.len = 0
   for s in 0 ..< slotsInPlay():
-    if s != gMySlot:
+    if s != gCtx.slot:
       let tg = atomicLoadN(addr gActiveTags[s], ATOMIC_ACQUIRE)
       if tg != 0 and atomicLoadN(addr gSlotPhase[s], ATOMIC_ACQUIRE) == 1:
         gPendingWatch.add((int64(s) shl 32) or tg)
@@ -839,11 +849,25 @@ proc prepareCapture() =
     gCap.prunedSrc.len = 0
     gCap.ages.len = 0
 
+proc growCaptureArrays(cap: ptr CaptureBufs) {.noinline.} =
+  ## `recs`, `sccIdx`, `ages` and `tstack` are appended to together, and only
+  ## by `claimCell` — one entry each per claimed cell. So they are grown
+  ## together and ONE capacity check on `recs` covers all four: `recs` drives
+  ## the growth, the other three are topped up to at least its capacity.
+  ## `tstack` is popped as SCCs are emitted, so its length only ever trails
+  ## `recs.len`; matching capacities keeps its appends unchecked too.
+  ## Marked `noinline` to keep the cold resize path out of `claimCell`.
+  resize(cap.recs, cap.recs.len +% 1)
+  let n = cap.recs.cap
+  if cap.sccIdx.cap < n: resize(cap.sccIdx, n)
+  if cap.ages.cap < n: resize(cap.ages, n)
+  if cap.tstack.cap < n: resize(cap.tstack, n)
+
 # rc is captured without the flag bits: the collector itself toggles
 # inRootsFlag between capture and commit, which must not look like a
 # mutation to the commit-time rc validation.
 proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
-               pruneLive: bool; old0: int64): int32 =
+               ctx: ptr CollCtx; pruneLive: bool; old0: int64): int32 =
   ## Dense index if this collection owns `c` (claiming and registering it
   ## if it was unclaimed), -1 if another ACTIVE collection owns it, or
   ## -2 if `pruneLive` and the cell was proven live in the current epoch
@@ -852,33 +876,34 @@ proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
   ## `old0` is `c`'s claim word as the caller already read it (acquire): the
   ## DFS reads it to test ownership, and re-reading it here would be a second
   ## dependent load of the same cold header word on every traversed edge.
-  if gAmSolo:
+  if ctx.amSolo:
     # no other collection is (or can start) capturing: plain stores.
     # This recovers the sequential capture speed of the single-collector
     # design whenever collections do not actually overlap.
     let old = old0
-    if (old shr 32) == gMyTag:
+    if (old shr 32) == ctx.tag:
       return int32(loWord(old))
-    if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
+    if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and
         stampAge(old) >= YrcPromoteAge:
       return -2
     let idx = cap.recs.len
-    c.rootIdx = (gMyTag shl 32) or int64(idx)
+    c.rootIdx = (ctx.tag shl 32) or int64(idx)
     when defined(nimOrcStats):
       bumpStat gStatCapTotal
       if old != 0: bumpStat gStatCapRepeat
-    cap.recs.add CaptureRec(cell: c, desc: desc,
+    if idx >= cap.recs.cap: growCaptureArrays(cap)
+    cap.recs.addUnchecked CaptureRec(cell: c, desc: desc,
                              rcWord: loadRc(c) and not rcMask,
                              lowlink: int32(idx), selfRefs: 0'i32)
-    cap.sccIdx.add -1'i32
-    cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
-    cap.tstack.add int32(idx)
+    cap.sccIdx.addUnchecked -1'i32
+    cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
+    cap.tstack.addUnchecked int32(idx)
     return int32(idx)
   var old = old0
   while true:
-    if (old shr 32) == gMyTag:
+    if (old shr 32) == ctx.tag:
       return int32(loWord(old))
-    if pruneLive and (old shr 32) == (gMyEpochStamp shr 32) and
+    if pruneLive and (old shr 32) == (ctx.epochStamp shr 32) and
         stampAge(old) >= YrcPromoteAge:
       return -2
     # an epoch stamp is never a tag (tags are allocated below `epochBase`),
@@ -888,17 +913,18 @@ proc claimCell(c: Cell; desc: PNimTypeV2; cap: ptr CaptureBufs;
       return -1
     let idx = cap.recs.len
     if atomicCompareExchangeN(addr c.rootIdx, addr old,
-                               (gMyTag shl 32) or int64(idx), false,
+                               (ctx.tag shl 32) or int64(idx), false,
                                ATOMIC_ACQ_REL, ATOMIC_RELAXED):
       when defined(nimOrcStats):
         bumpStat gStatCapTotal
         if old != 0: bumpStat gStatCapRepeat
-      cap.recs.add CaptureRec(cell: c, desc: desc,
+      if idx >= cap.recs.cap: growCaptureArrays(cap)
+      cap.recs.addUnchecked CaptureRec(cell: c, desc: desc,
                                rcWord: loadRc(c) and not rcMask,
                                lowlink: int32(idx), selfRefs: 0'i32)
-      cap.sccIdx.add -1'i32
-      cap.ages.add int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
-      cap.tstack.add int32(idx)
+      cap.sccIdx.addUnchecked -1'i32
+      cap.ages.addUnchecked int32(if isEpochStamp(old): min(stampAge(old), 1000) else: 0)
+      cap.tstack.addUnchecked int32(idx)
       return int32(idx)
     # a failed CAS leaves the fresh claim word in `old`; loop with it
 
@@ -918,11 +944,14 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
   ## on the stack below u) or an SCC emitted earlier, so `sccIdx` is final
   ## for all of them and the cross targets can be appended to `crossTgt`
   ## contiguously — which makes `crossOff` a prefix offset for free.
+  # one TLS resolution for the whole traversal; `claimCell` and the per-edge
+  # ownership tests below read the context through this pointer
+  let ctx = addr gCtx
   let rootWord = atomicLoadN(addr s.rootIdx, ATOMIC_ACQUIRE)
-  if (rootWord shr 32) == gMyTag: return
+  if (rootWord shr 32) == ctx.tag: return
   orcAssert(j.traceStack.len == 0, "capture: trace stack not empty")
   # roots never prune: a dec-witnessed suspicion overrides any epoch stamp
-  let root = claimCell(s, desc, cap, pruneLive = false, old0 = rootWord)
+  let root = claimCell(s, desc, cap, ctx, pruneLive = false, old0 = rootWord)
   if root < 0:
     return   # another active collection owns this candidate; it handles it
   trace(s, desc, j)
@@ -942,7 +971,7 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
       # one load of the target's claim word serves both the ownership test
       # and the dense-index extraction
       let cw = atomicLoadN(addr t.rootIdx, ATOMIC_ACQUIRE)
-      if (cw shr 32) == gMyTag:
+      if (cw shr 32) == ctx.tag:
         let v = int32(loWord(cw))
         if v == u:
           # A self edge is internal by construction and its reference is
@@ -957,7 +986,7 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
       else:
         let tdesc = j.traceStack.d[last][1]
         let childBase = j.traceStack.len
-        let v = claimCell(t, tdesc, cap, pruneLive = true, old0 = cw)
+        let v = claimCell(t, tdesc, cap, ctx, pruneLive = true, old0 = cw)
         if v == -1:
           # cross-collection edge: the owner sees our reference in the rc
           # word and classifies the target live; we re-register it as a
@@ -966,7 +995,13 @@ proc capture(s: Cell; desc: PNimTypeV2; j: var GcEnv; cap: ptr CaptureBufs) =
         elif v == -2:
           # target proven live this epoch: opaque live external, no descent.
           # Taint u's SCC — its own "live" verdict may lean on the stamp.
-          cap.prunedSrc.add u
+          # The list is only ever read as a set (and for its emptiness), and
+          # one cell's out-edges are consumed consecutively, so suppressing a
+          # repeat of the previous entry removes nearly every duplicate a
+          # multi-pruned cell would otherwise contribute.
+          if cap.prunedSrc.len == 0 or
+              cap.prunedSrc.d[cap.prunedSrc.len -% 1] != u:
+            cap.prunedSrc.add u
           when defined(nimOrcStats):
             bumpStat gStatCapPruned
         else:
@@ -1031,9 +1066,13 @@ proc computeDeadness(j: var GcEnv; cap: ptr CaptureBufs) =
     let s = cap.sccIdx.d[cap.prunedSrc.d[i]]
     cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagPruned
   # cells that stay registered as roots (partial collection) count as
-  # externally referenced: the roots buffer itself points at them
-  for mi in 0 ..< cap.sccMembers.len:
-    let m = cap.sccMembers.d[mi]
+  # externally referenced: the roots buffer itself points at them.
+  # Scanned over `recs` rather than over `sccMembers`: every claimed cell is
+  # pushed to the Tarjan stack once and popped into `sccMembers` once, so the
+  # two cover exactly the same set, and the SCC comes from `sccIdx` either
+  # way. Going through `sccMembers` would only add a random 32-byte-stride
+  # gather in front of a load that already misses on the cell header.
+  for m in 0 ..< cap.recs.len:
     if (loadRc(cap.recs.d[m].cell) and inRootsFlag) != 0:
       let s = cap.sccIdx.d[m]
       cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagForcedLive
@@ -1064,9 +1103,10 @@ proc markDirtyFromQueues(j: var GcEnv; cap: ptr CaptureBufs) =
   ## the last drain had its reference set changed during capture. Peek
   ## (don't drain!) the stripe queues and taint the affected SCCs; the
   ## entries stay queued and the next merge re-registers them as candidates.
+  let ctx = addr gCtx
   template taint(cp: Cell) =
     let c = cp
-    if isStamped(c):
+    if isStamped(c, ctx):
       let s = cap.sccIdx.d[denseIdx(c)]
       cap.sccs.d[s].flags = cap.sccs.d[s].flags or flagDirty
   for i in 0..<NumStripes:
@@ -1156,6 +1196,7 @@ proc validateDead(j: var GcEnv; cap: ptr CaptureBufs) =
 
 proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
   validateDead(j, cap)
+  let ctx = addr gCtx
   # publish cross-collection edge targets as candidate roots BEFORE any of
   # our commit decs could make them collectible: they are owner-live this
   # round, and the registration keeps them examinable in a later round
@@ -1163,7 +1204,7 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
     registerLocal(cap.crossPend.d[i][0], cap.crossPend.d[i][1])
   template deadCell(w: int64): bool =
     ## `w` is the target's claim word, read once by the caller
-    (w shr 32) == gMyTag and
+    (w shr 32) == ctx.tag and
       (cap.sccs.d[cap.sccIdx.d[int32(loWord(w))]].flags and flagDead) != 0
   # Grace period: a collection still in its CAPTURE phase may hold stale
   # (slot, value) snapshots referencing our dead cells; disposing them now
@@ -1215,7 +1256,7 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
       holdOrFree(cell, desc, deferred)
     j.freed = cap.recs.len
     if deferred:
-      gPendingSlot = gMySlot
+      gPendingSlot = ctx.slot
       gPendingActive = true
   else:
     if gFreeBuf.d == nil: init gFreeBuf
@@ -1272,7 +1313,7 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
         for mi in memStart ..< memEnd:
           let a = cap.ages.d[cap.sccMembers.d[mi]]
           if a < age: age = a
-        let stamp = gMyEpochStamp or int64(age +% 1)
+        let stamp = ctx.epochStamp or int64(age +% 1)
         for mi in memStart ..< memEnd:
           atomicStoreN(addr cap.recs.d[cap.sccMembers.d[mi]].cell.rootIdx,
                        stamp, ATOMIC_RELAXED)
@@ -1284,7 +1325,7 @@ proc commitDead(j: var GcEnv; cap: ptr CaptureBufs) =
       gPendingCells = j.toFree
       j.toFree = spare
       j.toFree.len = 0
-      gPendingSlot = gMySlot
+      gPendingSlot = ctx.slot
       gPendingActive = true
     else:
       for i in 0 ..< j.toFree.len:
@@ -1336,18 +1377,18 @@ proc startCollection(minRoots, keepBelow: int; slice: var CellSeq[Cell];
     else:
       gTagCounter = (gTagCounter +% 1) and int64(epochBase - 1)  # tags below the stamp namespace
       if gTagCounter == 0: gTagCounter = 1
-      gMyTag = gTagCounter
-      gMySlot = slot
-      gMyEpochStamp = epochStamp(atomicLoadN(addr gEpoch, ATOMIC_RELAXED))
+      gCtx.tag = gTagCounter
+      gCtx.slot = slot
+      gCtx.epochStamp = epochStamp(atomicLoadN(addr gEpoch, ATOMIC_RELAXED))
       var othersActive = false
       for sl in 0 ..< gParSlots:
         if sl != slot and atomicLoadN(addr gActiveTags[sl], ATOMIC_RELAXED) != 0:
           othersActive = true
-      gAmSolo = not othersActive
-      if gAmSolo:
+      gCtx.amSolo = not othersActive
+      if gCtx.amSolo:
         atomicStoreN(addr gSoloCapture, 1, ATOMIC_RELEASE)
       atomicStoreN(addr gSlotPhase[slot], 1, ATOMIC_RELEASE)
-      atomicStoreN(addr gActiveTags[slot], gMyTag, ATOMIC_SEQ_CST)
+      atomicStoreN(addr gActiveTags[slot], gCtx.tag, ATOMIC_SEQ_CST)
       release gMergeLock
       # our buffer, our slice: no lock needed
       if keepBelow == 0:
@@ -1375,12 +1416,12 @@ proc finishCollection() =
     # `gActiveTags`: it is what stops a foreign capture from claiming, and
     # then freeing, a cell that is sitting in the batch. `releasePending`
     # gives the slot back.
-    atomicStoreN(addr gSlotPhase[gMySlot], 0, ATOMIC_RELEASE)
+    atomicStoreN(addr gSlotPhase[gCtx.slot], 0, ATOMIC_RELEASE)
   else:
-    atomicStoreN(addr gActiveTags[gMySlot], 0, ATOMIC_SEQ_CST)
-    atomicStoreN(addr gSlotPhase[gMySlot], 0, ATOMIC_RELEASE)
-  gMyTag = 0
-  gAmSolo = false
+    atomicStoreN(addr gActiveTags[gCtx.slot], 0, ATOMIC_SEQ_CST)
+    atomicStoreN(addr gSlotPhase[gCtx.slot], 0, ATOMIC_RELEASE)
+  gCtx.tag = 0
+  gCtx.amSolo = false
   collectorEvent()   # wake backpressure and grace waiters
 
 proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) =
@@ -1402,8 +1443,8 @@ proc collectCyclesImpl(j: var GcEnv; slice: var CellSeq[Cell]) =
   for i in countdown(last, 0):
     capture(slice.d[i][0], slice.d[i][1], j, cap)
   j.touched = cap.recs.len
-  atomicStoreN(addr gSlotPhase[gMySlot], 2, ATOMIC_RELEASE)  # capture done
-  if gAmSolo:
+  atomicStoreN(addr gSlotPhase[gCtx.slot], 2, ATOMIC_RELEASE)  # capture done
+  if gCtx.amSolo:
     atomicStoreN(addr gSoloCapture, 0, ATOMIC_RELEASE)
   collectorEvent()   # wake solo-gate and grace waiters
 
@@ -1424,7 +1465,7 @@ proc runCollection(j: var GcEnv; slice: var CellSeq[Cell]) =
   ## Runs one collection over the stolen slice, concurrently with mutators
   ## AND with the other collecting threads over disjoint partitions.
   yrcGcFenceEnter()      # freeze seq structure mutations, not ref writes
-  if not gAmSolo:
+  if not gCtx.amSolo:
     # a solo collection claims with plain stores; nobody else may claim
     # cells until its capture phase is over
     parkUntil(atomicLoadN(addr gSoloCapture, ATOMIC_ACQUIRE) == 0)
@@ -1560,6 +1601,24 @@ proc nimYrcThreadTeardown() =
   deinit(gFreeBuf)
   deinit(gPendingCells)
   deinit(gPendingWatch)
+  # The capture side structure is persistent across this thread's collections
+  # (see `prepareCapture`, which only resets the lengths) and grows to the
+  # peak captured graph, so it must be released here too — otherwise every
+  # thread that ever collected leaks its peak-sized arrays for the rest of
+  # the process. `deinit` nils `d`, which is exactly the sentinel
+  # `prepareCapture` tests, so a later collection on a reused thread slot
+  # re-initializes cleanly.
+  deinit(gCap.recs)
+  deinit(gCap.sccIdx)
+  deinit(gCap.tstack)
+  deinit(gCap.frames)
+  deinit(gCap.edges)
+  deinit(gCap.sccs)
+  deinit(gCap.sccMembers)
+  deinit(gCap.crossTgt)
+  deinit(gCap.crossPend)
+  deinit(gCap.prunedSrc)
+  deinit(gCap.ages)
 
 proc GC_enableMarkAndSweep*() = GC_enableOrc()
 proc GC_disableMarkAndSweep*() = GC_disableOrc()