diff --git a/lib/system/yrc.nim b/lib/system/yrc.nim index c18d01a45a..d3dee32be1 100644 --- a/lib/system/yrc.nim +++ b/lib/system/yrc.nim @@ -93,12 +93,49 @@ type TraceProc = proc (p, env: pointer) {.nimcall, benign, raises: [].} DisposeProc = proc (p: pointer) {.nimcall, benign, raises: [].} -template color(c): untyped = c.rc and colorMask -template setColor(c, col) = - when col == colBlack: - c.rc = c.rc and not colorMask - else: - c.rc = c.rc and not colorMask or col +when defined(nimYrcAtomicIncs): + template color(c): untyped = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) and colorMask + template setColor(c, col) = + block: + var expected = atomicLoadN(addr c.rc, ATOMIC_RELAXED) + while true: + let desired = (expected and not colorMask) or col + if atomicCompareExchangeN(addr c.rc, addr expected, desired, true, + ATOMIC_ACQ_REL, ATOMIC_RELAXED): + break + template loadRc(c): int = atomicLoadN(addr c.rc, ATOMIC_ACQUIRE) + template trialDec(c) = + discard atomicFetchAdd(addr c.rc, -rcIncrement, ATOMIC_ACQ_REL) + template trialInc(c) = + discard atomicFetchAdd(addr c.rc, rcIncrement, ATOMIC_ACQ_REL) + template rcClearFlag(c, flag) = + block: + var expected = atomicLoadN(addr c.rc, ATOMIC_RELAXED) + while true: + let desired = expected and not flag + if atomicCompareExchangeN(addr c.rc, addr expected, desired, true, + ATOMIC_ACQ_REL, ATOMIC_RELAXED): + break + template rcSetFlag(c, flag) = + block: + var expected = atomicLoadN(addr c.rc, ATOMIC_RELAXED) + while true: + let desired = expected or flag + if atomicCompareExchangeN(addr c.rc, addr expected, desired, true, + ATOMIC_ACQ_REL, ATOMIC_RELAXED): + break +else: + template color(c): untyped = c.rc and colorMask + template setColor(c, col) = + when col == colBlack: + c.rc = c.rc and not colorMask + else: + c.rc = c.rc and not colorMask or col + template loadRc(c): int = c.rc + template trialDec(c) = c.rc = c.rc -% rcIncrement + template trialInc(c) = c.rc = c.rc +% rcIncrement + template rcClearFlag(c, flag) = c.rc = c.rc and not flag + template rcSetFlag(c, flag) = c.rc = c.rc or flag const optimizedOrc = false @@ -181,7 +218,9 @@ proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = let h = head(p) when optimizedOrc: if cyclic: h.rc = h.rc or maybeCycle - when defined(yrcAtomics): + when defined(nimYrcAtomicIncs): + discard atomicFetchAdd(addr h.rc, rcIncrement, ATOMIC_ACQ_REL) + elif defined(yrcAtomics): let s = getStripeIdx() let slot = atomicFetchAdd(addr stripes[s].toIncLen, 1, ATOMIC_ACQ_REL) if slot < QueueSize: @@ -220,23 +259,25 @@ proc mergePendingRoots() = # we don't need to set color to black on incRef because collection runs # under the global lock, so no concurrent mutations happen during collection. for i in 0.. until: let (entry, desc) = j.traceStack.pop() let t = head entry[] - t.rc = t.rc +% rcIncrement + trialInc(t) if t.color != colBlack: t.setColor colBlack trace(t, desc, j) @@ -300,23 +341,23 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = if s.color != colGray: s.setColor colGray j.touched = j.touched +% 1 - j.rcSum = j.rcSum +% (s.rc shr rcShift) +% 1 + j.rcSum = j.rcSum +% (loadRc(s) shr rcShift) +% 1 orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") trace(s, desc, j) while j.traceStack.len > 0: let (entry, desc) = j.traceStack.pop() let t = head entry[] - t.rc = t.rc -% rcIncrement + trialDec(t) j.edges = j.edges +% 1 if t.color != colGray: t.setColor colGray j.touched = j.touched +% 1 - j.rcSum = j.rcSum +% (t.rc shr rcShift) +% 2 + j.rcSum = j.rcSum +% (loadRc(t) shr rcShift) +% 2 trace(t, desc, j) proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = if s.color == colGray: - if (s.rc shr rcShift) >= 0: + if (loadRc(s) shr rcShift) >= 0: scanBlack(s, desc, j) else: orcAssert(j.traceStack.len == 0, "scan: trace stack not empty") @@ -326,14 +367,14 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = let (entry, desc) = j.traceStack.pop() let t = head entry[] if t.color == colGray: - if (t.rc shr rcShift) >= 0: + if (loadRc(t) shr rcShift) >= 0: scanBlack(t, desc, j) else: t.setColor(colWhite) trace(t, desc, j) proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = - if s.color == col and (s.rc and inRootsFlag) == 0: + if s.color == col and (loadRc(s) and inRootsFlag) == 0: orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty") s.setColor(colBlack) j.toFree.add(s, desc) @@ -342,7 +383,7 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = let (entry, desc) = j.traceStack.pop() let t = head entry[] entry[] = nil - if t.color == col and (t.rc and inRootsFlag) == 0: + if t.color == col and (loadRc(t) and inRootsFlag) == 0: j.toFree.add(t, desc) t.setColor(colBlack) trace(t, desc, j) @@ -376,7 +417,7 @@ proc collectCyclesBacon(j: var GcEnv; lowMark: int) = init j.toFree for i in 0 ..< roots.len: let s = roots.d[i][0] - s.rc = s.rc and not inRootsFlag + rcClearFlag(s, inRootsFlag) collectColor(s, roots.d[i][1], colToCollect, j) # Clear roots before freeing to prevent nested collectCycles() from accessing freed cells @@ -559,7 +600,7 @@ proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = # Initialize locks at module load initRwLock(gYrcGlobalLock) for i in 0..