Compare commits

..

1 Commits

Author SHA1 Message Date
narimiran
c5bf6d55d6 stupid commit to please github 2026-06-23 14:48:17 +02:00
153 changed files with 2031 additions and 9125 deletions

View File

@@ -15,7 +15,7 @@ jobs:
name: ${{ matrix.platform }}-bisects
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install OpenSSL (Windows)
if: |

View File

@@ -53,7 +53,7 @@ jobs:
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2

View File

@@ -18,12 +18,12 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
batch: ["0_3", "1_3", "2_3"] # list of `index_num`
os: [ubuntu-latest, macos-14]
batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
include:
- os: ubuntu-latest
cpu: amd64
- os: macos-latest
- os: macos-14
cpu: arm64
name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
runs-on: ${{ matrix.os }}
@@ -33,12 +33,12 @@ jobs:
NIM_TESTAMENT_BATCH: ${{ matrix.batch }}
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: 'Install node.js'
uses: actions/setup-node@v7
uses: actions/setup-node@v6
with:
node-version: 24

View File

@@ -17,12 +17,12 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: 'Install node.js'
uses: actions/setup-node@v7
uses: actions/setup-node@v6
with:
node-version: 24

1
.gitignore vendored
View File

@@ -87,7 +87,6 @@ tweeter_test.db
/tests/megatest.nim
/tests/ic/*_temp.nim
/tests/ic/*_mm/
/tests/navigator/*_temp.nim

View File

@@ -79,8 +79,6 @@ parameter and result types, not just their source-level shape. Use
Modes include `Nim` (default, fully compatible) and two new experimental modes:
`Lax` and `Gnu` for different option parsing behaviors.
- `std/symlinks.expandSymlink` now supports Windows symlinks and junctions with
POSIX-like single-hop `readlink` semantics.
- `std/nre2` is added to replace deprecated NRE.
- `system.typeof` adds a new parameter `modifierMode` to specify how type modifiers are handled.

View File

@@ -332,10 +332,7 @@ when defined(nimsuggest):
result = s.allUsagesImpl
proc `allUsages=`*(s: PSym, val: sink seq[TLineInfo]) {.inline.} =
# No `assert s.state != Sealed`: `allUsagesImpl` is nimsuggest-only usage
# tracking, NOT part of the NIF-serialized symbol. nimsuggest loads symbols
# as `Sealed` (ast2nif.loadedState under cmdM) yet `suggestSym` legitimately
# records usages on them; the getter likewise doesn't assert.
assert s.state != Sealed
if s.state == Partial: loadSym(s)
s.allUsagesImpl = val
@@ -478,8 +475,6 @@ proc comment*(n: PNode): string =
else:
result = ""
nodeCommentReader = proc(n: PNode): string {.nimcall.} = comment(n)
proc `comment=`*(n: PNode, a: string) =
let id = n.nodeId
if a.len > 0:
@@ -495,8 +490,6 @@ proc `comment=`*(n: PNode, a: string) =
n.flags.excl nfHasComment
gconfig.comments.del(id)
nodeCommentWriter = proc(n: PNode; s: string) {.nimcall.} = n.comment = s
# BUGFIX: a module is overloadable so that a proc can have the
# same name as an imported module. This is necessary because of
# the poor naming choices in the standard library.
@@ -546,30 +539,12 @@ proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator =
proc nextSymId(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
when not defined(nimKochBootstrap):
if x.backendMinted:
# Share the loader's per-module backend counter so a freshly-minted
# backend sym never collides with an `@bk` sym loaded from the module's
# `.t.bif` (see ast2nif.nextBackendSymItem).
let it = nextBackendSymItem(program, x.module)
if it >= 0'i32:
return backendItemId(x.module, it)
inc x.symId
result = if x.backendMinted: backendItemId(x.module, x.symId)
else: itemId(x.module, x.symId)
proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
when not defined(nimKochBootstrap):
if x.backendMinted:
# Share the loader's per-module backend TYPE counter (seeded from the
# module's `(unusedid)`) so a freshly-minted backend type sits ABOVE every
# loaded type — never colliding with a frontend type's `toId` (the bug that
# crashed cgen's `getTypeDescAux` cycle check on `AsyncBufferRef`). Mirrors
# `nextSymId` (see ast2nif.nextBackendTypeItem).
let it = nextBackendTypeItem(program, x.module)
if it >= 0'i32:
return backendItemId(x.module, it)
inc x.typeId
result = if x.backendMinted: backendItemId(x.module, x.typeId)
else: itemId(x.module, x.typeId)
@@ -849,10 +824,6 @@ proc newSymNode*(sym: PSym): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typ
if result.typField == nil and nifcBackendActive:
# See the two-arg overload in astdef: in the NIF backend cg stage a sym node
# built from a not-yet-typed stub must track the symbol's type lazily.
result.flags.incl nfLazyType
result.info = sym.info
proc newOpenSym*(n: PNode): PNode {.inline.} =

File diff suppressed because it is too large Load Diff

View File

@@ -17,15 +17,6 @@ when defined(nimPreviewSlimSystem):
export int128
var nifcBackendActive* = false
## Set only while the per-module NIF backend codegen stage runs
## (`nifbackend.generateCgStage`, `cmd == cmdNifC`). It gates `newSymNode`'s
## lazy-type marking so it applies ONLY in the backend — where syms are loaded
## from NIF and a cg-stage transform can build a sym node from a not-yet-typed
## stub — and never during frontend sem, where the same marking would perturb
## effect/exception inference (it diverges from a non-IC build, e.g.
## `times.toDateTimeByWeek` gaining a spurious unlisted `Exception`).
import nodekinds
export nodekinds
@@ -339,14 +330,6 @@ type
# because openSym experimental switch is disabled
# gives warning instead
nfLazyType # node has a lazy type
nfLazyBody # IC: this node is a placeholder for a routine body (bodyPos son)
# not yet materialized. Reading its children (via `len`/`safeLen`)
# triggers `forceLazyBodyHook`. Process-local, stripped on serialize.
nfBroadcast # this `nkBracket` is a *broadcast* default array: a single son
# standing for `lengthOrd` identical zero copies (see
# `broadcastArrayThreshold`). The flag disambiguates it from an
# ordinary 1-element collection (e.g. a seq value that happens to
# carry an array type), so it must survive copies + serialization.
TNodeFlags* = set[TNodeFlag]
TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 47)
@@ -874,8 +857,7 @@ const
nfFromTemplate, nfDefaultRefsParam,
nfExecuteOnReload, nfLastRead,
nfFirstWrite, nfSkipFieldChecking,
nfDisabledOpenSym, nfLazyType,
nfBroadcast}
nfDisabledOpenSym, nfLazyType}
namePos* = 0
patternPos* = 1 # empty except for term rewriting macros
genericParamsPos* = 2
@@ -912,24 +894,7 @@ const
defaultOffset* = -1
var forceLazyBodyHook*: proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.}
## Set by the IC loader (ast2nif). When a node carries `nfLazyBody`, any access
## to its children through `len` materializes the deferred routine body in place.
## `safeLen` delegates to `len`, so it is covered transitively; a lazy body is
## never a leaf kind, so the `{nkNone..nkNilLit}` short-circuit never hides it.
##
## The type MUST be effect-free (`raises: []`/`tags: []`): `len` is a fundamental
## `PNode` accessor that the whole compiler — and every compiler-as-library
## consumer (nimble, nimsuggest, ...) — assumes cannot raise. An unannotated
## `proc` var defaults to `raises: [Exception]`, so the indirect call tainted
## `len`/`safeLen`/`items` with `Exception`, breaking any iterator/`{.raises.}`
## over a `PNode` (e.g. nimble's `extract {.raises: [CatchableError].}`).
## Materialization is a pure in-memory buffer transform; a corrupt buffer is a
## `Defect` (`raiseAssert`), which is outside exception tracking.
proc len*(n: PNode): int {.inline.} =
if nfLazyBody in n.flags and forceLazyBodyHook != nil:
forceLazyBodyHook(n)
result = n.sons.len
proc safeLen*(n: PNode): int {.inline.} =
@@ -1005,14 +970,6 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typImpl
if result.typField == nil and nifcBackendActive:
# In the per-module NIF backend cg stage a transform (chronos async
# closure-iterator lowering) builds `result = …` sym nodes from a not-yet-typed
# NIF stub; snapshotting the nil here would leave the node permanently typeless
# and the backend later reads `t.flags` off it and SIGSEGVs (injectdestructors
# hasDestructor). Mark it lazy so `typ` re-reads `sym.typ` once resolved. Gated
# on `nifcBackendActive` so frontend sem is untouched (see the flag's doc).
result.flags.incl nfLazyType
result.info = info
proc newStrNode*(kind: TNodeKind, strVal: string): PNode =
@@ -1193,11 +1150,3 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym =
if result == nil: break
if result.name.id == name.id: break
h = nextTry(h, high(t.data))
# --- doc-comment bridge for the NIF serializer -------------------------------
# `ast2nif` (the NIF reader/writer) cannot import `ast` (where the comment
# accessor and its `gconfig.comments` side table live) because `ast` imports
# `ast2nif`. These hooks are assigned by `ast` and let the serializer carry a
# decl's `##` doc comment across a NIF round-trip.
var nodeCommentReader*: proc(n: PNode): string {.nimcall.}
var nodeCommentWriter*: proc(n: PNode; s: string) {.nimcall.}

View File

@@ -1071,7 +1071,7 @@ proc genRecordField(p: BProc, e: PNode, d: var TLoc) =
proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc)
proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
var test, u, v: TLoc
for i in 1..<e.len:
var it = e[i]
@@ -1081,17 +1081,10 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
if op.magic == mNot: it = it[1]
let disc = it[2].skipConv
assert(disc.kind == nkSym)
# Re-navigate the discriminant in the object type: under `nim ic` `disc.sym` is
# a field-use stub whose `loc.snippet` is empty (the backend fills it on the
# canonical reclist field, not on per-use leaves). Look up the canonical field
# for the C member name; `disc`'s own node still supplies its type (TLoc.t).
# Byte-neutral for non-IC, where re-navigation returns the same field.
var rr = obj
let dfield = lookupFieldAgain(p, ty, disc.sym, rr)
test = initLoc(locNone, it, OnStack)
u = initLocExpr(p, it[1])
v = initLoc(locExpr, disc, OnUnknown)
v.snippet = dotField(obj, dfield.loc.snippet)
v.snippet = dotField(obj, disc.sym.loc.snippet)
genInExprAux(p, it, u, v, test)
var msg = ""
if optDeclaredLocs in p.config.globalOptions:
@@ -1101,7 +1094,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
# by encoding the file names separately from `file(line:col)`, essentially
# passing around `TLineInfo` + the set of files in the project.
msg.add toFileLineCol(p.config, e.info) & " "
msg.add genFieldDefect(p.config, field.name.s, dfield)
msg.add genFieldDefect(p.config, field.name.s, disc.sym)
var strLitBuilder = newBuilder("")
genStringLiteral(p.module, newStrNode(nkStrLit, msg), strLitBuilder)
let strLit = extract(strLitBuilder)
@@ -1165,7 +1158,7 @@ proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
if field.loc.snippet == "": fillObjectFields(p.module, ty)
if field.loc.snippet == "":
internalError(p.config, e.info, "genCheckedRecordField") # generate the checks:
genFieldCheck(p, e, r, field, ty)
genFieldCheck(p, e, r, field)
r = dotField(r, field.loc.snippet)
putIntoDest(p, d, e[0], r, a.storage)
r.freeze
@@ -1865,7 +1858,7 @@ proc genFieldObjConstr(p: BProc; ty: PType; useTemp, isRef: bool; nField, val, c
if field.loc.snippet == "": fillObjectFields(p.module, ty)
if field.loc.snippet == "": internalError(p.config, info, "genFieldObjConstr")
if check != nil and optFieldCheck in p.options:
genFieldCheck(p, check, r, field, ty)
genFieldCheck(p, check, r, field)
tmp2.snippet = dotField(tmp2.snippet, field.loc.snippet)
if useTemp:
tmp2.k = locTemp
@@ -2940,13 +2933,6 @@ proc genEnumToStr(p: BProc, e: PNode, d: var TLoc) =
proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
case op
of mAsgn:
let kind = if e[0].sym.name.s == "=sink": nkSinkAsgn else: nkAsgn
let lhs = e[1].skipHiddenAddr
let n = newTreeI(kind, e.info, lhs, e[2])
n.typ = e.typ
cow(p, e[2])
genAsgn(p, n, fastAsgn = kind != nkAsgn)
of mOr, mAnd: genAndOr(p, e, d, op)
of mNot..mUnaryMinusF64: unaryArith(p, e, d, op)
of mUnaryMinusI..mAbsI: unaryArithOverflow(p, e, d, op)
@@ -3964,13 +3950,6 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Builder)
let elemTyp = skipTypes(t.elementType, abstractRange+{tyOwned}-{tyTypeDesc})
if isOpaqueImportcType(elemTyp):
result.add "{0}"
elif toInt(lengthOrd(p.config, t.indexType)) > broadcastArrayThreshold and
elemTyp.kind in {tyInt..tyUInt64, tyBool, tyChar, tyFloat..tyFloat128,
tyPtr, tyPointer, tyCstring}:
# Large array of a scalar whose default is the zero representation: a single
# C `{0}` zero-fills all `lengthOrd` slots instead of emitting that many
# initializers (keeps huge SSZ-style zero buffers compact in the C output).
result.add "{0}"
else:
var arrInit: StructInitializer
result.addStructInitializer(arrInit, kind = siArray):
@@ -4257,13 +4236,7 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; resul
var d: TLoc = initLocExpr(p, n)
result.add rdLoc(d)
of tyArray, tyVarargs:
if isDefaultBroadcastArray(n, p.config):
# Compact zero/null-default array (see `isDefaultBroadcastArray`): the
# whole thing is the null value of every slot, so a single C `{0}`
# zero-fills all `lengthOrd` elements — no need to materialise them.
result.add "{0}"
else:
genConstSimpleList(p, n, isConst, result)
genConstSimpleList(p, n, isConst, result)
of tyTuple:
genConstTuple(p, n, isConst, typ, result)
of tyOpenArray:

View File

@@ -39,30 +39,11 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
if isExtern: Extern
elif lfExportLib in s.loc.flags: ExportLibVar
else: Private
if m.config.cmd == cmdNifC and vis == Private and not isExtern:
# A `{.threadvar.}`/`{.global.}` thread-local declared inside a routine is
# emitted by every module that emit-everywhere's its enclosing routine
# (e.g. libp2p's `var keys {.global.}: HashSet`), so its content-addressed
# name collides at link. Same fix as a plain global (genGlobalVarDecl):
# `extern` declaration + a droppable `'d'` definition unit the merge stage
# assigns one owner. The thread-local storage class rides on both.
let cname = stripCnifMarks(s.loc.snippet)
let td = getTypeDesc(m, s.loc.t)
# `extern` declaration via the full `addVar` overload — it knows the
# thread-local storage class (`NIM_THREADVAR`); the simple `addVar`'s
# `addVarHeader` does not implement `Threadvar`.
m.s[cfsVars].addVar(m, s, name = s.loc.snippet, typ = td,
kind = Threadvar, visibility = Extern)
m.s[cfsVars].add(cnifDefDirective(cname, "d", icNifName(m, s)))
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet, typ = td, kind = Threadvar, visibility = vis)
m.s[cfsVars].add(cnifEndDefs())
else:
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
proc generateThreadLocalStorage(m: BModule) =
if m.g.nimtv.buf.len != 0 and (usesThreadVars in m.flags or sfMainModule in m.module.flags):

View File

@@ -108,14 +108,7 @@ proc fillBackendName(m: BModule; s: PSym) =
var result: Rope
if s.kind in routineKinds and {optCDebug, optItaniumMangle} * m.g.config.globalOptions == {optCDebug, optItaniumMangle} and
m.g.config.symbolFiles == disabledSf:
# Under the per-module IC backend the bare-name uniqueness probe
# (`m.g.mangledPrcs`) only sees the routines of the CURRENT module, so the
# clean-vs-`makeUnique` decision is made independently per process: a
# method base mangles clean at its owner but loses the in-module race to
# its same-signature dispatcher elsewhere (clean `speak` defined twice ->
# "multiple definition"; demanders call `speak_u<n>` that nobody defines).
# Force the stable, disamb-based unique name so every process agrees.
result = mangleProc(m, s, makeUnique = m.config.cmd == cmdNifC).rope
result = mangleProc(m, s, false).rope
else:
let shared = sharedInstanceCName(m, s)
if shared.len > 0:
@@ -819,11 +812,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
# don't use fieldType here because we need the
# tyGenericInst for C++ template support
let noInit = sfNoInit in field.flags or (field.typ.sym != nil and sfNoInit in field.typ.sym.flags)
# Under `nim ic`, object fields are local NIF syms restored without an
# `owner`; `rectype` is the owning record type, so fall back to it rather
# than deref a nil `field.owner`.
let ownerTyp = if field.owner != nil: field.owner.typ else: rectype
if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, ownerTyp)):
if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, field.owner.typ)):
var didGenTemp = false
initializer = genCppInitializer(m, nil, fieldType, didGenTemp)
result.addField(field, sname, typ, isFlexArray, initializer)
@@ -1419,24 +1408,10 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
m.hcrCreateTypeInfosProc.addCast(typ = ptrType(CPointer)):
m.hcrCreateTypeInfosProc.add(cAddr(name))
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
if m.config.cmd == cmdNifC:
# Emit-everywhere (see genTypeInfoV1's perModuleCg gate): every demanding
# `cg` process emits this type info's tentative definition. Declare it
# `extern` first (the data analogue of a proc prototype) so a TU whose copy
# the merge stage drops still has a valid declaration; wrap the definition
# as a droppable `'d'` unit the merge stage assigns to a single owner so
# exactly one external-linkage tentative definition survives (preserving
# the RTTI pointer identity refc relies on).
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifDefDirective(name, "d", icNifName(m, origType)))
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifEndDefs())
m.icDataDefs.add (name, icNifName(m, origType))
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope;
info: TLineInfo) =
@@ -1529,25 +1504,8 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope;
m.s[cfsTypeInit3].addFieldAssignment(expr, "name", makeCString(field.name.s))
m.s[cfsTypeInit3].addFieldAssignment(expr, "sons", cAddr(subscript(tmp, cIntValue(0))))
m.s[cfsTypeInit3].addFieldAssignment(expr, "len", L)
if m.config.cmd == cmdNifC:
# The discriminator table has a content-addressed name
# (`NimDT_<hashType>_<field>`) and is emitted by every module that demands
# this variant type's RTTI (emit-everywhere; RTTI has no single owner —
# emission is lazy and often skipped). Declare it `extern` + wrap the
# tentative definition as a droppable `'d'` unit so the merge stage keeps
# exactly one external-linkage definition (mirrors the `TNimType` var and
# consts); otherwise the identical name collides across modules at link.
m.s[cfsData].addDeclWithVisibility(Extern):
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifDefDirective(tmp, "d", ""))
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifEndDefs())
m.icDataDefs.add (tmp, "")
else:
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
for i in 1..<n.len:
var b = n[i] # branch
var tmp2 = getNimNode(m)
@@ -1811,16 +1769,6 @@ proc generateRttiDestructor(g: ModuleGraph; typ: PType; owner: PSym; kind: TType
incl result.flagsImpl, sfFromGeneric
incl result.flagsImpl, sfGeneratedOp
# Under IC the `rttiDestroy` wrapper is generated independently in every cg
# process that emits `typ`'s RTTI (the type-info is emit-everywhere). A plain
# counter `disamb` renumbers per process, so the RTTI table baked in module A
# references `rttiDestroy_c<n>` while module B (the =destroy owner) defines a
# different number → undefined at link. Give it a content-derived `disamb`
# (stable across processes) + `HookDisambBit`, exactly like `symPrototype` does
# for the hook itself: same `typ` ⇒ same C name everywhere, and the bit makes
# `emitsBodyInThisModule` emit the body in every demander (merge dedups). The
# `"rttiDestroy"` op-name keeps its key disjoint from the real `=destroy` hook's.
setHookDisamb(g, result, "rttiDestroy", typ)
proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: var Builder) =
let theProc = getAttachedOp(m.g.graph, t, op)
@@ -2176,15 +2124,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
return prefixTI(result)
var owner = t.skipTypes(typedescPtrs).itemId.module
# In the per-module backend (`cg`) V1 RTTI is emit-everywhere like procs,
# consts and V2 type info: every demanding module emits the `'d'` definition
# (deduped to one owner by the merge stage). The owner-routing below would
# instead push the definition into the owner module's *unwritten* backend
# module (discarded in this process) and emit only an extern here, leaving the
# symbol undefined at link — the refc `NTI*` undefined-reference bug. (V2 got
# this gate in 8e0dd4bfb; V1, only reached under `--mm:refc`, was missed.)
let perModuleCg = m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"
if not perModuleCg and owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
dbgNti "extern:ownerRouted"
# make sure the type info is created in the owner module
discard genTypeInfoV1(m.g.mods[owner], origType, info)
@@ -2283,21 +2223,3 @@ proc genTypeSection(m: BModule, n: PNode) =
discard getTypeDescAux(m, s.typ, intSet, descKindFromSymKind(s.kind))
if m.g.generatedHeader != nil:
discard getTypeDescAux(m.g.generatedHeader, s.typ, intSet, descKindFromSymKind(s.kind))
# Unlike genCppInitializer which returns just the braced value list (e.g. "{a, b}"),
# genCppConstructorExpr returns a full type-prefixed expression (e.g. "Foo(a, b)").
# This is used when a standalone construction expression is needed — e.g. on the
# right-hand side of an assignment — whereas genCppInitializer is used in variable
# declarations where the type is already written separately before the initializer.
proc genCppConstructorExpr(m: BModule, prc: BProc; typ: PType; didGenTemp: var bool): Snippet =
var params = ""
if typ.itemId in m.g.graph.initializersPerType:
let call = m.g.graph.initializersPerType[typ.itemId]
if call != nil:
var p = prc
if p == nil:
p = BProc(module: m)
params = genCppParamsForCtor(p, call, didGenTemp)
if prc == nil:
assert p.blocks.len == 0, "BProc belongs to a struct doesnt have blocks"
result = getTypeDesc(m, typ, dkVar) & "(" & params & ")"

View File

@@ -11,7 +11,7 @@
import
ast, types, msgs, wordrecg,
platform, trees, options, cgendata, mangleutils, renderer, modulegraphs
platform, trees, options, cgendata, mangleutils, renderer
import std/[hashes, strutils, formatfloat]
@@ -114,29 +114,8 @@ proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
result = if name == "": s.name.s else: name
# keep backend-minted ids out of the `_u` namespace; their item counter
# restarts at 0 and would collide with loaded symbols' ids
if s.itemId.isBackendMinted:
result.add "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# A backend-minted sym whose `disamb` is content-derived (setHookDisamb gave
# it HookDisambBit) — e.g. the `rttiDestroy` wrapper. Its `itemId.item` is a
# PER-PROCESS backend counter, so using it makes the C name diverge across
# the emit-everywhere processes: the type's RTTI table (emit-everywhere,
# merge-deduped) ends up referencing one process's `_c<item>` while the
# wrapper is defined with another's -> undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes, so use it.
result.add $s.disamb
else:
result.add $s.itemId.item
else:
result.add "_u"
# Mirror `mangleProcNameExt`: use the per-(module,name) `disamb`, NOT
# `itemId.item`. Under the per-module IC backend the same symbol is loaded
# from a NIF in many processes and `itemId.item` is a fresh, load-order
# dependent counter — so a method base would mangle to `_u1` in one module,
# `_u3` in another and clean at its owner, none of which link. `disamb` is
# assigned deterministically per (module, name) and is serialized, so every
# process that touches the symbol derives the identical C name.
result.add $s.disamb
result.add(if s.itemId.isBackendMinted: "_c" else: "_u")
result.add $s.itemId.item
# module suffix LAST (a strippable trailing token; see `mangleProcNameExt`)
result.add "__"
result.add m.g.graph.ifaces[s.itemId.module].uniqueName

View File

@@ -125,45 +125,10 @@ proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
## owning-module top-level — they are minted on demand — so each demander emits
## them and the merge stage deduplicates by their content-addressed C name.
##
## A NESTED routine is not emitted on its own: it is lambda-lifted and emitted
## as part of its ENCLOSING routine's body, into the same TU. So the decision
## must follow the OUTERMOST enclosing routine (the one directly under the
## module — `skipGenericOwner` stops at a generic *instance*, not its
## originating generic), never the nested symbol's own identity. Otherwise a
## nested proc whose enclosing is a generic instance (content-addressed,
## emitted by every demander) — e.g. nim-serialization's per-field `readField`
## inside the `makeFieldReadersTable[R,W]` instance, whose address fills the
## returned table — is gated out (its own `itemId.module` is the minting module
## and its disamb is a plain counter), so the enclosing's lift degrades it to a
## prototype and its body lands in no TU → undefined at link.
if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
return true
# The symbol may ITSELF be content-addressed (a synthesized hook or a generic
# instance carries `Hook/InstanceDisambBit` on its OWN `disamb`): then it has no
# single owning module and every demander emits it (merge dedups by C name),
# regardless of what it is nested under. This must be checked on `prc` directly,
# not on `top`: a `=destroy`/`=sink` lifted while compiling some enclosing proc
# (e.g. system's `isZeroMemory` destroying a `ptr array`) has that PROC as its
# `skipGenericOwner`, so `top` walks up to a plain routine whose own disamb has
# no bit — gating the hook to that routine's owner module, which mints it
# on demand and emits it nowhere → undefined at link.
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32:
return true
var top = prc
while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
top = top.skipGenericOwner
result = top.itemId.module == m.module.position or
(top.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 or
# An INLINE iterator has no standalone body — it is expanded at each
# call site — so it is materialized in every module that iterates over
# it, never in its owner. A proc nested in one (e.g. std/uri's
# `parseData` inside `iterator decodeQuery`) is lambda-lifted into each
# of those consumer TUs and must be emitted there (its stable
# owner-suffixed name + `'u'` flag let the merge stage keep one); gating
# it to the iterator's owner module leaves it in no TU → undefined.
(top.kind == skIterator and top.typ != nil and
top.typ.callConv != ccClosure)
result = prc.itemId.module == m.module.position or
(prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32
proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
result = TLoc(k: k, storage: s, lode: lode,
@@ -185,13 +150,9 @@ proc fillLoc(a: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) {.inline.} =
a.storage = s
proc t(a: TLoc): PType {.inline.} =
if a.lode.kind == nkSym and a.lode.sym.typ != nil:
if a.lode.kind == nkSym:
result = a.lode.sym.typ
else:
# Under `nim ic` an object-field reference is a typeless leaf stub (its def
# lives in another seek; see ast2nif `FieldMarker`) that carries its type on
# the NODE instead. Fall back to the node type. Byte-neutral for non-IC, where
# a real sym always has a type.
result = a.lode.typ
proc lodeTyp(t: PType): PNode =
@@ -613,7 +574,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
if isImportedCppType(typ):
var didGenTemp = false
let rl = rdLoc(loc)
let init = genCppConstructorExpr(p.module, p, typ, didGenTemp)
let init = genCppInitializer(p.module, p, typ, didGenTemp)
p.s(cpsStmts).addAssignment(rl, init)
return
if optSeqDestructors in p.config.globalOptions and typ.kind in {tyString, tySequence}:
@@ -815,31 +776,12 @@ proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
typ = constType(typ)
if p.hcrOn:
typ = ptrType(typ)
if p.config.cmd == cmdNifC and vis == Private and sfImportc notin s.flags:
# A `{.global.}` var (e.g. chronos's per-call-site `var loc {.global.} =
# SrcLoc(...)`, or a gensym'd `var dummy`/`var topic` with no initializer)
# declared inside a routine is emitted by every module that emit-everywhere's
# its enclosing routine; its content-addressed name then collides at link.
# Declare it `extern` + wrap the definition as a droppable `'d'` unit so the
# merge stage keeps exactly one (like consts / TNimType / the NimDT
# discriminator tables / the threadvar path). This covers no-initializer
# globals too — they collide just the same. A module-level global has a
# single claimant → its sole emitter is the owner merge keeps.
let cname = stripCnifMarks(s.loc.snippet)
res.addDeclWithVisibility(Extern):
res.addVar(kind = Local, name = s.loc.snippet, typ = typ)
res.add(cnifDefDirective(cname, "d", icNifName(p.module, s)))
res.addVar(p.module, s,
name = s.loc.snippet, typ = typ, visibility = vis,
initializer = initializer, initializerKind = initializerKind)
res.add(cnifEndDefs())
else:
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
let s = n.sym
@@ -896,12 +838,8 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
else:
initializer = value
genGlobalVarDecl(p.module.s[cfsVars], p, n, td, initializer = initializer)
if p.withinLoop > 0 and value == "" and
s.loc.t.skipTypes(abstractInst).kind notin {tyVar, tyLent}:
if p.withinLoop > 0 and value == "":
# fixes tests/run/tzeroarray:
# Don't reset borrowed references (var/lent): the pointer itself is still
# uninitialized here, so resetLoc would dereference garbage. Such variables
# (e.g. the loop var of `mitems`) are always assigned before use anyway.
backendEnsureMutable s
resetLoc(p, s.locImpl)
@@ -1222,17 +1160,8 @@ proc closeNamespaceNim(result: var Builder) =
proc closureSetup(p: BProc, prc: PSym) =
if tfCapturesEnv notin prc.typ.flags: return
# prc.ast[paramsPos].last contains the type we're after — BUT a closure loaded
# from a `.t.bif` (a lambda-lifted nested proc / generic instance the `lower`
# stage transformed) can arrive with an EMPTY AST param node: the lifted hidden
# `:env` param lives in `typ.n`, the authoritative signature (`genProc` already
# reads `typ.n`, not the AST). The two param nodes diverge across the NIF
# boundary; fall back to `typ.n` so the env param resolves instead of indexing
# an empty container.
var params = prc.ast[paramsPos]
if params.safeLen == 0 and prc.typ.n != nil and prc.typ.n.kind == nkFormalParams:
params = prc.typ.n
var ls = lastSon(params)
# prc.ast[paramsPos].last contains the type we're after:
var ls = lastSon(prc.ast[paramsPos])
if ls.kind != nkSym:
internalError(p.config, prc.info, "closure generation failed")
var env = ls.sym
@@ -1477,21 +1406,8 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
var returnStmt: Snippet = ""
assert(prc.ast != nil)
# A body LOADED from `.t.bif` was already FULLY lowered by the `lower` stage —
# transformed AND destructor-injected (see nifbackend.generateLowerStage). The
# `.t.bif` is the authoritative backend artifact; re-injecting here would lower
# it twice (double `=destroy` calls) and, worse, re-lift the env hooks per cg
# process (owned by nobody → undefined at link). So inject ONLY when the body
# was re-derived in this process (`wasLoaded == false`). Capture before
# `transformBody`, which returns the cached body (non-nil) when it was loaded.
# ONLY under IC: in a normal `nim c` build `transformedBody` is the ordinary
# transform cache (set whenever `transformBody` already ran for `prc`, e.g. a
# CT-evaluated or earlier-referenced routine), NOT a `.t.bif` load — gating on
# it there would WRONGLY skip destructor injection and miscompile (orc
# decref-on-freed). The `.t.bif`-loaded-body concept exists only under cmdNifC.
let wasLoaded = m.config.cmd == cmdNifC and prc.transformedBody != nil
var procBody = transformBody(m.g.graph, m.idgen, prc, {})
if sfInjectDestructors in prc.flags and not wasLoaded:
if sfInjectDestructors in prc.flags:
procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
let tmpInfo = prc.info
@@ -1551,17 +1467,6 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
for i in 1..<prc.typ.n.len:
let param = prc.typ.n[i].sym
if param.typ.isCompileTimeOnly: continue
if prc.typ.callConv == ccClosure and param.name.s == ":envP":
# The hidden closure-env param is materialised by `closureSetup`, never a
# normal C parameter (`genProcParams` omits it from the signature). In a
# from-source build it lives only in the routine's AST params and never in
# `typ.n`, so this loop never reaches it. Under IC `closureParams` leaks it
# into `typ.n`; for a LOADED closure it is already present at header time
# (`genProcParams` fills its loc), but for a RE-DERIVED closure
# (`wasLoaded == false`) `transformBody` appends it only AFTER
# `genProcHeader` ran, so its `loc.snippet` is still empty here. Skip it to
# match the from-source invariant — `closureSetup` assigns its local below.
continue
assignParam(p, param, prc.typ.returnType)
closureSetup(p, prc)
genProcBody(p, procBody)
@@ -1663,15 +1568,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
useHeader(m, sym)
if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return
if lfDynamicLib in sym.loc.flags:
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Under IC per-module cg every demander emits the dynlib proc's DEFINITION
# locally (findPendingModule returns `m`, so symInDynamicLib follows this
# call and the merge stage keeps one def per C name). Emitting the
# cross-module `extern` proto here would register `sym.id` in
# `m.declaredThings` and thereby make that `symInDynamicLib` skip, leaving
# the `Dl_*` symbol declared-but-never-defined -> undefined at link.
discard "definition emitted by symInDynamicLib"
elif sym.itemId.module != m.module.position and
if sym.itemId.module != m.module.position and
not containsOrIncl(m.declaredThings, sym.id):
let vis = if isReloadable(m, sym): StaticProc else: Extern
let name = mangleDynLibProc(sym)

View File

@@ -160,17 +160,7 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
var witness: PSym = nil
if s.typ.firstParamType.owner.getModule != s.getModule and vtables in g.config.features and not
g.config.isDefined("nimInternalNonVtablesTesting") and sfFromGeneric notin s.flags:
# `sfFromGeneric` excepted: this is the same-module restriction for vtable
# slot placement, and it must be judged on the GENERIC method, not on an
# instance. The generic `method skip[T](x: Input[T])` never reaches here
# (`semMethodPrototype` registers generic methods via `addMethodToGeneric`,
# bypassing `methodDef`); only its instance `skip[string]` does, and that
# instance's first-param type `Input[string]` is owned by whichever module
# first instantiated it (`tparsecombnum`, which `import parsecomb`s and uses
# it), NOT by `Input[T]`'s defining module — so the comparison spuriously
# fails for a method that is perfectly legal at the generic level. (Concrete
# methods, `sfFromGeneric notin flags`, are still checked.)
g.config.isDefined("nimInternalNonVtablesTesting"):
localError(g.config, s.info, errGenerated, "method `" & s.name.s &
"` can be defined only in the same module with its type (" & s.typ.firstParamType.typeToString() & ")")
if sfImportc in s.flags:

View File

@@ -53,8 +53,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: ConfigRef;
graph: ModuleGraph): bool =
if self.suggestMode:
conf.setCmd cmdCheck
conf.ideActive = true
conf.setCmd cmdIdeTools
if conf.cmd == cmdNimscript:
incl(conf.globalOptions, optWasNimscript)
loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files

View File

@@ -509,7 +509,6 @@ proc parseCommand*(command: string): Command =
of "nifc": cmdNifC # generate C from NIF files
of "ic": cmdIc # generate .build.nif for nifmake
of "icconfig": cmdIcConfig # produce the precompiled config artifact
of "track": cmdTrack # IDE goto-def / find-usages over `nim ic`'s NIF output
else: cmdUnknown
proc setCmd*(conf: ConfigRef, cmd: Command) =
@@ -719,14 +718,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
of "usenimcache":
processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
of "ideimports":
# nimsuggest: where the import closure comes from. IC is opt-in.
# nif|on load unchanged imports from precompiled NIF (cmdM)
# source|off (default) recompile the whole closure from source (cmdCheck)
case arg.normalize
of "nif", "on", "": conf.ideImportsFromNif = true
of "source", "off": conf.ideImportsFromNif = false
else: localError(conf, info, "'--ideImports' expects 'nif' or 'source', got: '$1'" % arg)
of "docseesrcurl":
expectArg(conf, switch, arg, pass, info)
conf.docSeeSrcUrl = arg
@@ -826,8 +817,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
localError(conf, info, "expected nim|cpp but found " & arg)
of "compress":
conf.globalOptions.incl optCompress
of "genbif":
processOnOffSwitchG(conf, {optGenBif}, arg, pass, info)
of "g": # alias for --debugger:native
conf.globalOptions.incl optCDebug
conf.options.incl optLineDir
@@ -1327,16 +1316,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
# support UNIX style filenames everywhere for portable build scripts:
if config.projectName.len == 0:
config.projectName = unixToNativePath(p.key)
if config.cmd == cmdTrack:
# `nim track PROJ --def:...`: unlike a normal command (where everything
# after the project file is passed to the compiled program), `track`
# accepts its IDE-query switches AFTER the project — the natural,
# nimsuggest-like invocation form. So don't swallow the rest of the line
# into `arguments`; keep parsing the remaining tokens as switches.
result = false
else:
config.arguments = cmdLineRest(p)
result = true
config.arguments = cmdLineRest(p)
result = true
else:
result = false
inc argsCount

View File

@@ -15,8 +15,7 @@ import options, msgs, lineinfos, pathutils, condsyms,
modulepaths, extccomp, cnif, platform
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
import icmodnames
import icnifcore
import "../dist/nimony/src/gear2" / modnames
type
FilePair = object
@@ -52,24 +51,24 @@ proc parsedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".p.nif"
proc semmedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".s.bif"
getNimcacheDir(c.config).string / f.modname & ".nif"
proc ifaceFile(c: DepContext; f: FilePair): string =
## Interface-cookie sidecar written by `nim m` (ast2nif.writeIfaceCookie,
## OnlyIfChanged). Dependents' nim_m rules use it as their input instead of
## the semmed NIF: a body-only change in a dependency then keeps the sidecar
## mtime and nifmake prunes the whole re-sem cascade behind it.
getNimcacheDir(c.config).string / f.modname & ".iface.bif"
getNimcacheDir(c.config).string / f.modname & ".iface.nif"
proc implFile(c: DepContext; suffix: string): string =
## Implementation-cookie sidecar (ast2nif.writeImplCookie): flips on ANY
## content change of the module (private bodies included; supersedes the
## iface cookie). Used as the edge for dependents that consumed the
## module's bodies at compile time (NeedsImpl edges).
getNimcacheDir(c.config).string / suffix & ".impl.bif"
getNimcacheDir(c.config).string / suffix & ".impl.nif"
proc edgesFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".edges.bif"
getNimcacheDir(c.config).string / f.modname & ".edges.nif"
proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] =
## Reads the module's recorded NeedsImpl edge set (module suffixes whose
@@ -80,10 +79,19 @@ proc readNeedsImpl(c: DepContext; f: FilePair): seq[string] =
## gated input of its rule, so the rule re-fires and re-records.
result = @[]
if fileExists(c.edgesFile(f)):
result = collectBifStrLits(c.edgesFile(f))
var s = nifstreams.open(c.edgesFile(f))
try:
discard processDirectives(s.r)
while true:
let t = next(s)
if t.kind == EofToken: break
if t.kind == StringLit:
result.add pool.strings[t.litId]
finally:
close s
proc semDepsFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".s.deps.bif"
getNimcacheDir(c.config).string / f.modname & ".s.deps.nif"
proc readSemDeps(c: DepContext; f: FilePair): seq[string] =
## The module's REAL direct imports (full source paths) as sem resolved them,
@@ -91,7 +99,16 @@ proc readSemDeps(c: DepContext; f: FilePair): seq[string] =
## (ast2nif.writeSemDeps). Missing file (not yet semmed) -> empty.
result = @[]
if fileExists(c.semDepsFile(f)):
result = collectBifStrLits(c.semDepsFile(f))
var s = nifstreams.open(c.semDepsFile(f))
try:
discard processDirectives(s.r)
while true:
let t = next(s)
if t.kind == EofToken: break
if t.kind == StringLit:
result.add pool.strings[t.litId]
finally:
close s
proc findNifler(): string =
# Look for nifler in common locations
@@ -590,98 +607,6 @@ proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) =
elif t.kind == ParRi: dec depth
t = next(s)
proc collectIncludeNames(depsPath: string; names: var seq[string]) =
## Lightweight scan of a `.deps.nif` prelude: collect the raw path text of
## every entry inside an `(include ...)` node (idents like `semexprs`, string
## literals like `"system/mmdisp"`, and the leaves of `a/b` path infixes).
## Liberal by design — it also picks up entries under a statically-false
## `(when ...)`; that is harmless for the only caller (`includerSbifs`), whose
## over-collection just costs an extra, result-free bif scan downstream.
if not fileExists(depsPath): return
var s = nifstreams.open(depsPath)
defer: nifstreams.close(s)
discard processDirectives(s.r)
var depth = 0
var includeDepth = 0 # the `depth` at which the current `(include` opened; 0 = not inside one
var t = next(s)
while t.kind != EofToken:
case t.kind
of ParLe:
inc depth
if includeDepth == 0 and pool.tags[t.tagId] == "include":
includeDepth = depth
of ParRi:
if includeDepth != 0 and depth == includeDepth:
includeDepth = 0
dec depth
of Ident, StringLit:
if includeDepth != 0:
names.add pool.strings[t.litId]
else: discard
t = next(s)
proc entryStemBase(roots: seq[string]; name: string): (string, string) =
## Resolve include entry `name` to (deps-stem, base-name); ("","") if unfound.
for r in roots:
let p = r / name.addFileExt("nim")
if fileExists(p):
return (moduleSuffix(p, []), splitFile(p).name)
result = ("", "")
proc includerSbifs*(conf: ConfigRef; targetFile: AbsoluteFile): seq[string] =
## For an include file `targetFile`, return the `.s.bif` paths of every module
## that includes it — directly OR transitively (following the include chain
## `module -> incA -> incB -> targetFile`). `nim track` uses this to avoid
## loading and scanning every module bif: an include file has no bif of its
## own, so its type-checked tokens live in the *including* module's bif. Only
## the small `.deps.nif` preludes are read here, never a `.s.bif`.
const depsExt = ".deps.nif"
let nc = getNimcacheDir(conf).string
# Candidate roots for resolving an `(include X)` entry to a real file, so its
# module suffix (== its own deps-file stem) can be computed. Include entries
# carry any sub-path (`system/mmdisp`), so the file's *directory* roots suffice:
# the target's own dir, the project dir, and the search paths cover the
# compiler, the stdlib and typical single-tree projects.
var roots: seq[string] = @[parentDir(targetFile.string)]
if conf.projectPath.string.len > 0: roots.add conf.projectPath.string
for sp in conf.searchPaths: roots.add sp.string
# One pass over every prelude builds the reverse include graph, keyed by base
# file name: `includedBy[b]` = deps stems whose owner directly `include`s a
# file named `b`. `stemBase` maps an include-only file's deps stem back to its
# own base name, so the walk can climb through nested includes.
var includedBy = initTable[string, seq[string]]()
var stemBase = initTable[string, string]()
for depsPath in walkFiles(nc / "*" & depsExt):
let base = extractFilename(depsPath)
if base.endsWith(".p" & depsExt): continue # `.p.deps.nif` twin
let ownerStem = base[0 ..< base.len - depsExt.len]
var names: seq[string] = @[]
collectIncludeNames(depsPath, names)
for n in names:
let (childStem, childBase) = entryStemBase(roots, n)
if childBase.len == 0: continue
includedBy.mgetOrPut(childBase, @[]).add ownerStem
stemBase[childStem] = childBase # this child's stem -> its base name
# Walk UP from the target: a deps stem that includes the current base name is
# either a module (has a `.s.bif` -> collect it) or itself an include file
# (recurse via its own base name).
result = @[]
var seenBase = initHashSet[string]()
var work = @[splitFile(targetFile.string).name]
while work.len > 0:
let b = work.pop()
if seenBase.containsOrIncl(b): continue
for stem in includedBy.getOrDefault(b):
let sbif = nc / stem & ".s.bif"
if fileExists(sbif):
if sbif notin result: result.add sbif # module owner
else:
let ob = stemBase.getOrDefault(stem) # include-only owner: climb higher
if ob.len > 0: work.add ob
proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) =
## Process a module: run nifler and read deps
if not runNifler(c, pair.nimFile):
@@ -741,130 +666,6 @@ proc computeSCCs(c: DepContext): seq[seq[int]] =
if work.len > 0:
lowlink[work[^1].v] = min(lowlink[work[^1].v], lowlink[v])
proc nodeIsStale(c: DepContext; node: Node): bool =
## Driver-time estimate of "this module's `nim m` will (re)run this round",
## matching what nifmake decides from file mtimes. The driver runs BEFORE the
## nifmake pass, so the `.p.nif` parsed file still reflects the *previous* run
## (runNifler even deletes a source-newer one); the stable signal available
## here is the source `.nim` against the last semmed NIF (`.s.bif`).
##
## Only the estimate's *precision* matters, never correctness: nifmake still
## mtime-checks every emitted rule, so an over-estimate produces a batch it
## then skips, and an under-estimate leaves a singleton rule it rebuilds
## anyway (see computeBatches).
for f in node.files: # main file + its includes
let semmed = c.semmedFile(f)
if not fileExists(semmed): return true # never semmed (cold)
if not fileExists(c.parsedFile(f)): return true # parsed dropped by runNifler
let semmedTime = getLastModificationTime(semmed)
if fileExists(f.nimFile) and getLastModificationTime(f.nimFile) > semmedTime:
return true # edited since last sem
result = false
proc computeBatches(c: DepContext): seq[seq[int]] =
## Group SCCs into build batches for the frontend `nim m` rules. A batch is a
## *dirty module together with the transitive closure of its users* (the
## modules that import it, directly or transitively). Rationale: editing a
## module flips its cookies and forces every dependent to re-sem, and that
## re-sem set is a deep import chain that nifmake schedules one depth-level at
## a time — so `--parallel` never speeds it up while every process pays full
## startup + import-NIF-closure load. Compiling the whole closure in one
## `nim m --icGroup` process (source-compiling every member, resolving the
## imports in memory) amortizes that overhead across the affected set.
##
## Safety (why a wrong dirty estimate cannot corrupt output):
## * every rule still declares its real inputs/outputs, so nifmake skips a
## batch whose files are all fresh and rebuilds a missed module through its
## own singleton rule;
## * batches are a partition of the SCC condensation, so each NIF keeps a
## single writer (no two processes mint divergent instance ids into one NIF);
## * the closure is convex in the condensation DAG (any node on an import path
## between two batch members also reaches the seed, so it is in the batch),
## so dropping intra-batch edges cannot create a batch<->external nifmake
## cycle.
##
## `-d:icNoBatch` restores pure per-SCC grouping (the pre-batching behaviour).
let sccs = computeSCCs(c)
if isDefined(c.config, "icNoBatch") or sccs.len == 0:
return sccs
let numSccs = sccs.len
var sccOf = newSeq[int](c.nodes.len)
for sid, comp in sccs:
for nodeIdx in comp: sccOf[nodeIdx] = sid
# SCCs that must never be merged into a batch: system.nim's folded closure and
# the `--import:` implicit modules. They stay their own groups exactly as
# today, so the cmdM bootstrap that NIF-loads `system` is untouched; on a cold
# build the rest of the program forms one big batch that NIF-loads system
# rather than recompiling it in every process.
var pinned = newSeq[bool](numSccs)
if c.systemNodeId >= 0: pinned[sccOf[c.systemNodeId]] = true
for id in c.implicitNodeIds: pinned[sccOf[id]] = true
# Condensation edges. `rev[b]` = SCCs that import b (its users); `fwd` is used
# only to union connected dirty SCCs into one component.
var fwd = newSeq[HashSet[int]](numSccs)
var rev = newSeq[HashSet[int]](numSccs)
for v in 0 ..< c.nodes.len:
for w in c.nodes[v].deps:
let a = sccOf[v]
let b = sccOf[w]
if a != b:
fwd[a].incl b
rev[b].incl a
# Seed = a non-pinned SCC with any stale member (edited/missing NIF).
# mustRecompile = seeds plus every SCC that transitively imports a seed,
# walked over reverse condensation edges (a seed's users).
var must = newSeq[bool](numSccs)
var queue: seq[int] = @[]
for sid, comp in sccs:
if pinned[sid]: continue
for nodeIdx in comp:
if nodeIsStale(c, c.nodes[nodeIdx]):
must[sid] = true
queue.add sid
break
while queue.len > 0:
let s = queue.pop()
for u in rev[s]:
if not must[u] and not pinned[u]:
must[u] = true
queue.add u
# Union-Find the mustRecompile SCCs connected by any condensation edge; each
# connected component becomes one batch, every other SCC stays its own batch.
var parent = newSeq[int](numSccs)
for i in 0 ..< numSccs: parent[i] = i
proc find(x: int): int =
var x = x
while parent[x] != x:
parent[x] = parent[parent[x]] # path halving
x = parent[x]
x
for a in 0 ..< numSccs:
if not must[a]: continue
for b in fwd[a]:
if must[b]:
let ra = find(a)
let rb = find(b)
if ra != rb: parent[ra] = rb
# Assemble: one entry per batch, in first-seen (reverse-topological) SCC order.
# A merged batch is keyed by its union-find root (a `must` sid); a singleton is
# keyed by its own sid (a non-`must` sid) — the two key spaces are disjoint, so
# they never collide.
result = @[]
var batchIndex = initTable[int, int]()
for sid in 0 ..< numSccs:
let key = if must[sid]: find(sid) else: sid
let bi = batchIndex.getOrDefault(key, -1)
if bi == -1:
batchIndex[key] = result.len
result.add sccs[sid]
else:
for nodeIdx in sccs[sid]: result[bi].add nodeIdx
proc computeForwardedArgs(c: DepContext): seq[string] =
## Config/define forwarding shared by the frontend (`nim m`) and backend
## (`nim nifc`) child commands. Depends only on the driver's config, not on
@@ -900,14 +701,6 @@ proc computeForwardedArgs(c: DepContext): seq[string] =
# buckets (and rejects calls as ambiguous that multi-dispatch accepts)
if optMultiMethods in c.config.globalOptions:
result.add "--multimethods:on"
# Forward the debug-info switch: the cg children — not the driver — fill the
# backend C names, and `--debugger:native` selects the Itanium mangling
# scheme (ccgtypes.fillBackendName). A child without it would name routines
# with the plain `_u<disamb>` scheme while a sibling that read the project's
# config.nims (`--debugger:native`) used Itanium, so the same symbol's
# definition and cross-module references would disagree at link.
if optCDebug in c.config.globalOptions:
result.add "--debugger:native"
# the children compile each MODULE as their own project file, which makes
# that module's package the "main package" and unfilters foreign-package
# diagnostics — a vendored package's hintAsError/warningAsError promotions
@@ -992,28 +785,21 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
# Build rules for semantic checking (nim m).
#
# Modules are grouped into BATCHES (computeBatches). A batch is one or more
# strongly-connected components merged together, handed to a single `nim m`
# invocation: the first member is the project file, every member is passed via
# `--icGroup:<path>` so the compiler compiles them all from source in one
# process (resolving imports/recursion in-memory) and writes a NIF for each.
# Only dependencies *outside* the batch become build-graph inputs — intra-batch
# edges are produced by this very rule and listing them would reintroduce a
# cycle nifmake would reject.
#
# Two things drive a multi-module batch:
# * an import CYCLE (A imports B, B imports A) cannot be ordered for separate
# per-module compilation, so its whole SCC is one group (as before); and
# * a DIRTY module together with its transitive USERS — editing a module
# forces every dependent to re-sem, a serial import chain that per-process
# fan-out cannot speed up; batching compiles the whole affected closure in
# one process. A module that is neither in a cycle nor in a dirty closure
# stays its own singleton `nim m <mod>` rule.
let batches = computeBatches(c)
var batchOf = newSeq[int](c.nodes.len)
for batchId, comp in batches:
for nodeIdx in comp: batchOf[nodeIdx] = batchId
for comp in batches:
# Modules are grouped into strongly-connected components: a module that is not
# in an import cycle is its own singleton group and compiles in its own
# `nim m <mod>` invocation as before. A cycle (A imports B, B imports A) cannot
# be ordered for separate per-module compilation, so the whole component is
# handed to a single `nim m` invocation: the first member is the project file,
# every member is passed via `--icGroup:<path>` so the compiler compiles them
# all from source in one process (resolving the recursion in-memory) and writes
# a NIF for each. Only dependencies *outside* the component become build-graph
# inputs — intra-component edges are produced by this very rule and listing
# them would reintroduce the cycle nifmake just rejected.
let sccs = computeSCCs(c)
var sccOf = newSeq[int](c.nodes.len)
for sccId, comp in sccs:
for nodeIdx in comp: sccOf[nodeIdx] = sccId
for comp in sccs:
# Representative (project file for this invocation) = smallest node id, so a
# component containing the root (node 0) is driven by the root.
var members = comp
@@ -1063,7 +849,7 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
var stack: seq[int] = @[]
for m in members:
for depIdx in c.nodes[m].deps:
if batchOf[depIdx] != batchOf[members[0]]: stack.add depIdx
if sccOf[depIdx] != sccOf[members[0]]: stack.add depIdx
var visited = initHashSet[int]()
while stack.len > 0:
let n = stack.pop()
@@ -1077,7 +863,7 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
var directDeps = initHashSet[string]()
for m in members:
for depIdx in c.nodes[m].deps:
if batchOf[depIdx] == batchOf[m]: continue # intra-batch edge
if sccOf[depIdx] == sccOf[m]: continue # intra-component edge
let depName = c.nodes[depIdx].files[0].modname
directDeps.incl depName
let depFile =
@@ -1132,36 +918,6 @@ proc backendCFile(c: DepContext; node: Node): string =
result = changeFileExt(completeCfilePath(c.config,
mangleModuleName(c.config, cfilename).AbsoluteFile), ".nim.c").string
proc computeLiveBackendNodes(c: DepContext): seq[bool] =
## Which nodes the backend must code-generate: the closure reachable from the
## program roots (main + `system` + `--import`ed modules) via the REAL,
## post-sem import edges (`.s.deps`).
##
## The static `.deps` scan over-approximates: it cannot evaluate guards like
## `when defined(windows)` or const-aliased ones (`when useWinVersion`, with
## `const useWinVersion = defined(windows) or defined(nimdoc)`), so it keeps
## the dead branch's import. e.g. on Linux `nativesockets`'s static deps list
## `winlean`; the discovery fixpoint only ever *adds* edges, never prunes, so
## `winlean` stays a node and got a full `lower`/`cg`/`emit`/link pipeline.
## That is harmless for sem (an extra `nim m`) but fatal for codegen:
## `winlean`'s `importc, header: "winsock2.h"` decls emit
## `#include "winsock2.h"` into a C file that cannot compile off-Windows.
## Sem's resolved import set (`.s.deps`) is the real program graph — the
## non-IC compiler would never touch `winlean` here — so restrict the backend
## to it. (`.s.deps` is the same data the discovery loop trusts; it is written
## for every sem'd module, including grouped SCC members.)
result = newSeq[bool](c.nodes.len)
var stack: seq[int] = @[0] # main module
if c.systemNodeId >= 0: stack.add c.systemNodeId
for impId in c.implicitNodeIds: stack.add impId # every module imports these
while stack.len > 0:
let ni = stack.pop()
if ni < 0 or ni >= c.nodes.len or result[ni]: continue
result[ni] = true
for p in readSemDeps(c, c.nodes[ni].files[0]):
let idx = c.processedModules.getOrDefault(c.toPair(p).modname, -1)
if idx >= 0: stack.add idx
proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Per-module backend build file. One `nim_nifc` command template (the actual
## stage/module switches ride in each rule's `(args …)`), then the stages of
@@ -1179,53 +935,15 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
result = nimcache / c.nodes[0].files[0].modname & ".backend.build.nif"
let mainNif = c.nodes[0].files[0].nimFile
# Honor `--out`/`--outdir`: `cmdIc`'s `setOutFile` populated `conf.outFile`
# (the user's `--out`, or the default `<project><exeExt>`), so `absOutFile` is
# the final link target — exactly what a whole-program `nim c` would produce.
# The `link` child computes its own output from its project name, so the path
# is also forwarded to it below.
let exeFile = string(c.config.absOutFile)
let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt)
let mergeFile = nimcache / MergeDecisionFile
# Per-node output paths.
var cnifFiles = newSeq[string](c.nodes.len)
var cFiles = newSeq[string](c.nodes.len)
var tFiles = newSeq[string](c.nodes.len)
# The `lower` stage writes a PROPER module NIF the cg/emit stages load via
# `toNifFilename` (a `.s.bif` sibling), so its `.t.bif` lives at the suffix base
# (mirroring `semmedFile`), not next to the throwaway `.c`.
for i, node in c.nodes:
cFiles[i] = backendCFile(c, node)
cnifFiles[i] = cFiles[i] & ".nif"
tFiles[i] = nimcache / node.files[0].modname & ".t.bif"
# Only code-generate modules the real program actually reaches; statically
# over-approximated nodes (e.g. `winlean` on Linux) are sem'd but not emitted.
let live = computeLiveBackendNodes(c)
# Drop a pruned node's stale backend artifacts: the `merge` stage globs
# `*.c.nif` off disk (not the build-file inputs) and the `link` stage scans
# the loaded closure's `.c`s, so a leftover `.c.nif`/`.c` from a run before
# this module became unreachable (a prior over-approximated build, or an edit
# that removed its last real importer) would still be merged/compiled —
# reintroducing exactly the off-platform `#include` this prune avoids.
var prunedStale = false
for i in 0 ..< c.nodes.len:
if not live[i]:
# `fileExists` before remove so we only force a merge recompute (below)
# when an artifact was actually present — i.e. a build where this module
# WAS emitted, not the steady state where it never is.
if fileExists(cnifFiles[i]) or fileExists(cFiles[i]): prunedStale = true
removeFile(cnifFiles[i])
removeFile(cFiles[i])
# The merge decision is a pure function of the set of `.c.nif`s present; if we
# just removed an over-approximated module's artifacts, a decision computed
# while they were present is stale — it can name a now-absent module as a
# symbol's owner (`asyncdispatch` owning `NTIdomain` here), leaving that symbol
# undefined at link. nifmake will not re-fire `merge` on its own: dropping an
# input makes no remaining input newer than the output. Delete the decision so
# the (now missing) output forces a recompute against the live `.c.nif` set.
if prunedStale:
removeFile(mergeFile)
var b = nifbuilder.open(result)
defer: b.close()
@@ -1247,12 +965,9 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addStrLit a
b.addTree "args"
b.endTree()
# The project file is a fixed command ARGUMENT, not a tracked input: backend
# stages read NIFs (resolved by suffix), never the `.nim` source, so its
# content cannot change any artifact. Passing it as `(input 0)` made its mtime
# an input to every rule, so editing the main module's source re-fired the
# whole backend.
b.addStrLit mainNif
b.addTree "input"
b.addIntLit 0
b.endTree()
b.endTree()
template inputStr(s: string) =
@@ -1264,50 +979,21 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addStrLit s
b.endTree()
# lower: one rule per module. Transforms (eventually) the routines the module
# OWNS once, in the owner's id space, into `<module>.t.nif`, so the `cg` stage
# reads them instead of re-deriving (which makes a closure `:env`'s identity
# diverge across the parallel `cg` processes). Runs per module in parallel.
#
# Input is this module's OWN semmed NIF and nothing else. A module does NOT
# depend on its importers, so listing every semmed NIF (or even the import
# closure) was wrong: it made e.g. `strutils`'s rule depend on the `finish`
# that imports it. nifmake handles the indirect dependency for free — the
# frontend writes `.s.nif`s content-stably, so an interface change to a
# dependency re-sems (and re-emits the `.s.nif` of) every transitive importer;
# a module whose own `.s.nif` is unchanged genuinely needs no re-lowering.
# cg: one rule per module. Inputs are the project (slot 0) and every semmed
# NIF (so the whole program loads and the rule is ordered after the frontend);
# the main module additionally depends on every other `.c.nif` (init metas).
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:lower"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr c.semmedFile(node.files[0])
outputStr tFiles[i]
b.endTree()
# cg: one rule per module. Input is this module's OWN `.t.nif`. cg DOES read
# its dependencies' `.t.nif`s at runtime (loadDepClosure), but ordering is
# guaranteed by nifmake's depth-barriered scheduler: every `lower` is depth 1
# (its `.s.nif` is a leaf) and every `cg` is depth 2, so all lowering finishes
# before any cg starts — no need to list the closure for ordering. For
# invalidation, a dependency's change reaches this module through its own
# `.t.nif` (own `.s.nif` re-sem -> own `lower`); a foreign body this module
# emit-everywhere'd but does not own is dropped by `emit` regardless, so a
# stale copy here is harmless. The main module additionally depends on every
# other `.c.nif` (it reads their init/datInit metas to wire up NimMain).
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:cg"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr tFiles[i]
inputStr mainNif
for n2 in c.nodes:
inputStr c.semmedFile(n2.files[0])
if node.id == 0:
for j in 0 ..< c.nodes.len:
if c.nodes[j].id != 0 and live[j]:
if c.nodes[j].id != 0:
inputStr cnifFiles[j]
outputStr cnifFiles[i]
b.endTree()
@@ -1317,24 +1003,19 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:merge"
for i in 0 ..< c.nodes.len:
if live[i]: inputStr cnifFiles[i]
inputStr mainNif
for cn in cnifFiles: inputStr cn
outputStr mergeFile
b.endTree()
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
for i, node in c.nodes:
if not live[i]: continue
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:emit"
b.addStrLit "--icBackendModule:" & node.files[0].modname
# Inputs: this module's OWN `.c.nif` and the global merge decision. emit also
# loads `.t.nif`s at runtime (getCFile/type resolution), but those are depth 1
# and emit is past the merge barrier, so they always exist — no need to list
# them. (emit still re-fires for every module whenever `merge` rewrites the
# decision file; making that incremental is a separate concern.)
inputStr mainNif
inputStr cnifFiles[i]
inputStr mergeFile
outputStr cFiles[i]
@@ -1345,24 +1026,15 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:link"
# The link child is its own `cmdNifC` process whose project is the main
# module, so it would default the binary to `<maindir>/<main><exeExt>`.
# Forward the resolved target so it writes exactly `exeFile` (`--out`'s
# path splits back into outDir+outFile in the child).
b.addStrLit "--out:" & exeFile
for i in 0 ..< c.nodes.len:
if live[i]: inputStr cFiles[i]
inputStr mainNif
for cf in cFiles: inputStr cf
outputStr exeFile
b.endTree()
b.endTree() # stmts
proc commandIc*(conf: ConfigRef; frontendOnly = false) =
## Main entry point for `nim ic`. With `frontendOnly` (used by `nim track` for
## IDE queries) it runs only Phase 1 — the incremental nifler + `nim m`
## frontend that writes every module's `.s.bif` — and skips the whole-program
## backend (`nim nifc` -> C -> link), which a goto-def / find-usages scan does
## not need.
proc commandIc*(conf: ConfigRef) =
## Main entry point for `nim ic`
when not defined(nimKochBootstrap):
let nifler = findNifler()
if nifler.len == 0:
@@ -1476,22 +1148,10 @@ proc commandIc*(conf: ConfigRef; frontendOnly = false) =
let nifmake = findNifmake()
# Build the per-module rules concurrently: nifmake fans out all commands at
# each DAG depth via execProcesses (defaults to all cores). Cold builds are
# otherwise serial (one child at a time) and leave the machine idle. An
# uncapped fan-out across many cores can exhaust RAM on a large project (each
# `nim m`/`cg` child holds its own module graph), which nifmake's own `-j:N`
# exists to bound. Concurrency is chosen (highest precedence first):
# * `-d:icNoParallel` -> serial (readable, non-interleaved child output)
# * `-d:icJobs:N` -> cap at N (legacy IC-tuning define)
# * `--parallelBuild:N` -> cap at N (the standard Nim build-parallelism
# flag; a no-op for `nim c` under IC, so we give
# it meaning here — lets Nimbus devs pick their
# own value without a `-d:` define)
# * otherwise -> uncapped (all cores)
let parallel =
if isDefined(conf, "icNoParallel"): ""
elif isDefined(conf, "icJobs"): " --parallel:" & conf.symbols["icJobs"]
elif conf.numberOfProcessors > 0: " --parallel:" & $conf.numberOfProcessors
else: " --parallel"
# otherwise serial (one child at a time) and leave the machine idle. Opt out
# with `-d:icNoParallel` (e.g. for readable, non-interleaved child output
# when debugging a build).
let parallel = if isDefined(conf, "icNoParallel"): "" else: " --parallel"
# Phase 1 — frontend (nifler + `nim m`), run to a discovery fixpoint.
var rounds = 0
@@ -1502,12 +1162,10 @@ proc commandIc*(conf: ConfigRef; frontendOnly = false) =
if nifmake.len == 0:
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & buildFile)
# without nifmake we can only print the manual commands; emit the
# backend's too (best effort — discovery cannot run) and stop. An IDE
# query (`frontendOnly`) needs no backend, so skip it there.
if not frontendOnly:
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile)
# backend's too (best effort — discovery cannot run) and stop.
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
rawMessage(conf, hintSuccess, "run:" & " nifmake run" & parallel & " " & backendFile)
return
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(buildFile)
rawMessage(conf, hintExecuting, cmd)
@@ -1551,9 +1209,7 @@ proc commandIc*(conf: ConfigRef; frontendOnly = false) =
# Phase 2 — backend (whole-program `nim nifc`), run once over the now-final
# graph. Kept a separate nifmake run so backend rebuilds are decided purely
# by nifmake's input mtimes, independent of frontend discovery.
# An IDE query (`frontendOnly`) stops after Phase 1: the `.s.bif` it scans
# are all produced by the frontend; codegen + link would be wasted work.
if frontendOk and not frontendOnly:
if frontendOk:
let backendFile = generateBackendBuildFile(c, forwardedArgs)
rawMessage(conf, hintSuccess, "generated: " & backendFile)
let cmd = quoteShell(nifmake) & " run" & parallel & " " & quoteShell(backendFile)

View File

@@ -1473,8 +1473,6 @@ proc genFlags*(s: set[TNodeFlag]; dest: var string) =
of nfSkipFieldChecking: dest.add "s0"
of nfDisabledOpenSym: dest.add "d3"
of nfLazyType: dest.add "l1"
of nfLazyBody: discard # process-local placeholder; never serialized
of nfBroadcast: dest.add "v"
proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] =
@@ -1535,7 +1533,6 @@ proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] =
inc i
else: result.incl nfSem
of 't': result.incl nfTransf
of 'v': result.incl nfBroadcast
of 'w': result.incl nfFirstWrite
else: discard
inc i

View File

@@ -260,26 +260,17 @@ proc ensureIcConfig*(conf: ConfigRef) =
if not fileExists(outPath) or sourcesChanged(outPath):
createDir(cacheDir)
# Re-invoke ourselves as the config producer: reuse this process's command
# line, dropping the command argument (`ic`/`track`) in favour of `icconfig`
# and the explicit output path. Every switch must land BEFORE the project
# file, because anything after the project is swallowed into
# `config.arguments` by `cmdLineRest` (and a non-empty `arguments` without
# `--run` is a hard error). Callers may legitimately put switches after the
# project — `nim track PROJ --def:...` — so we re-order rather than replay
# verbatim: all `-`-prefixed switches first (in encounter order), then the
# non-switch project token(s). The producer re-reads `nim.cfg` itself.
# line, dropping the command argument (`ic`) in favour of `icconfig` and the
# explicit output path, both BEFORE the project file (anything after the
# project is swallowed into `config.arguments` by `cmdLineRest`). The
# producer re-reads `nim.cfg` itself.
var pargs = @["icconfig", "--icConfigOut:" & outPath]
var rest: seq[string] = @[]
var droppedCmd = false
for a in commandLineParams():
if a.len == 0: continue
if a[0] == '-':
pargs.add a
elif not droppedCmd:
droppedCmd = true # drop the original command token (`ic`/`track`)
if not droppedCmd and a.len > 0 and a[0] != '-':
droppedCmd = true # drop the original command token (`ic`)
else:
rest.add a # project file (and any further non-switch tokens) go last
for a in rest: pargs.add a
pargs.add a
let p = startProcess(getAppFilename(), args = pargs,
options = {poStdErrToStdOut})
let outp = p.outputStream.readAll()

View File

@@ -1,55 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Nim's OWN module-suffix, replacing nimony's `gear2/modnames.moduleSuffix`.
##
## nimony's version hashes a path made RELATIVE to `getCurrentDir()` (or the
## shortest search-path-relative form), so the produced suffix depends on the
## current working directory AND the searchPath set. Under `nim ic` the
## DISCOVERY pass (`deps.nim`, in the driver process) and the COMPILE pass
## (`nifgen`/`typekeys`, in a child `nim m` process) can run with different CWDs
## or `--path` sets, so the SAME file hashes to two different suffixes: e.g.
## `std/staticos` became `sta5rk8sn1` at discovery but `sta4c0qxk` at compile, so
## every importer waited forever for a `.s.bif` that was actually written under
## the other name — a cold `nim ic` build (of anything pulling in `std/os`, whose
## `oscommon` does `from std/staticos import PathComponent`) never converged.
##
## Hashing the CANONICAL ABSOLUTE path makes the suffix a pure function of the
## file, identical across every process and call site. The base-name prefix +
## base-36 `uhash` layout is kept byte-for-byte compatible with the old scheme so
## nothing but the hashed string changes.
import std/os
import "../dist/nimony/src/lib" / tinyhashes
const
PrefixLen = 3 # keep it short: the suffix ends up in every mangled C name
Base36 = "0123456789abcdefghijklmnopqrstuvwxyz"
proc moduleSuffix*(path: string; searchPaths: openArray[string]): string =
## `searchPaths` is accepted for signature-compatibility with the replaced
## `modnames.moduleSuffix` but is deliberately IGNORED — the suffix must not
## depend on the search-path set or the CWD (see the module doc).
# Absolute inputs (the norm at every call site: `toFullPath`/`projectFull`)
# pass straight through `normalizedPath` with no `getCurrentDir` involvement;
# a stray relative path is made absolute against the CWD only as a fallback.
var f = path
if not isAbsolute(f):
try: f = absolutePath(f)
except CatchableError: discard
f = normalizedPath(f)
let m = splitFile(f).name
var id = uhash(f)
result = newStringOfCap(10)
for i in 0 ..< min(m.len, PrefixLen):
result.add m[i]
# base-36 of the hash, low digit first (order is irrelevant for identity).
while id > 0'u32:
result.add Base36[int(id mod 36'u32)]
id = id div 36'u32

View File

@@ -1,252 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## nifcore-based IC serialization helpers — Stage 1 of porting the IC backend
## from the old `nifstreams`/`nifcursors` NIF stack to `nifcore` (see
## `doc/ic_nifcore_port.md`).
##
## It hosts:
## * the process-wide shared `Pool`/`TagPool` that stands in for the old global
## `nifstreams.pool`,
## * `writeFileStable`, the content-stable file writer mirroring
## `nifcursors.writeFile(..., OnlyIfChanged)`,
## * the first ported writer (`writeSemDeps`), used as the migration spike.
##
## No `nifstreams`/`nifcursors` types cross this module's boundary: callers pass
## plain Nim values (config, ids, string lists), so it can coexist with the
## still-old-API `ast2nif.nim` during the migration.
import std / [syncio, algorithm]
from std / os import removeFile, moveFile
import options, pathutils, typekeys
import "../dist/nimony/src/lib" / [nifcore, nifcoreparse, nifreader, bif]
# One shared literals pool + tag pool for the whole process — the nifcore
# analogue of the old global `nifstreams.pool`. A single shared pool keeps
# string/symbol/file ids stable across every TokenBuf the IC backend builds,
# preserving the old global-pool semantics during the migration. (Stage 6 may
# move to fresh per-file pools for bif's fast path; see doc/ic_nifcore_port.md.)
let icPool* = newPool()
let icTags* = newTagPool()
proc createIcBuf*(cap = 16): TokenBuf {.inline.} =
## A `TokenBuf` bound to the shared IC pools.
createTokenBuf(cap, icPool, icTags)
proc tagId*(s: string): TagId {.inline.} =
## Intern a tag name in the shared tag pool.
icTags.registerTag(s)
type
IcBuilder* = object
## A thin nifcore `TokenBuf` builder whose surface is *primitive types only*
## (strings/ints/floats). It lets the still-old-API `ast2nif.nim` drive a
## nifcore buffer without any nifcore type crossing the module boundary —
## the bridge that routes IC output onto the nifcore serializer (Stage 2).
buf*: TokenBuf
proc newIcBuilder*(cap = 16): IcBuilder = IcBuilder(buf: createIcBuf(cap))
proc openTag*(b: var IcBuilder; tag: string) {.inline.} = b.buf.openTag(tagId(tag))
proc closeTag*(b: var IcBuilder) {.inline.} = b.buf.closeTag()
proc addSymUse*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymUse(s)
proc addSymDef*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymDef(s)
proc addIdent*(b: var IcBuilder; s: string) {.inline.} = b.buf.addIdent(s)
proc addStrLit*(b: var IcBuilder; s: string) {.inline.} = b.buf.addStrLit(s)
proc addIntLit*(b: var IcBuilder; v: int64) {.inline.} = b.buf.addIntLit(v)
proc addUIntLit*(b: var IcBuilder; v: uint64) {.inline.} = b.buf.addUIntLit(v)
proc addFloatLit*(b: var IcBuilder; v: float64) {.inline.} = b.buf.addFloatLit(v)
proc addCharLit*(b: var IcBuilder; c: char) {.inline.} = b.buf.addCharLit(c)
proc addDotToken*(b: var IcBuilder) {.inline.} = b.buf.addDotToken()
proc lineInfo*(b: var IcBuilder; file: string; line, col: int32; comment = "") =
## Attach line info (+ optional `#comment#`) to the head just emitted. No-op
## when `file` is empty (matches the old "emit only when info is valid").
## Strings are interned in the shared pools; the file/comment ids reproduce
## the old `pool.files`/`pool.strings` entries by string value.
if file.len == 0: return
let fid = icPool.filenames.getOrIncl(file)
let cid = if comment.len > 0: icPool.strings.getOrIncl(comment) else: StrId(0)
b.buf.appendLineInfo(fid, line, col, cid)
proc writeFileStable*(b: var TokenBuf; path: string; onlyIfChanged = false) =
## Serialize `b` to canonical module NIF text and write it. Mirrors
## `nifcursors.writeFile`: the module suffix is derived from `path`
## (`"." & extractModuleSuffix`), and `onlyIfChanged` skips the write when the
## on-disk bytes already match — the content-stability nifmake's incremental
## rebuild depends on.
let content = toModuleString(b, "." & extractModuleSuffix(path))
if onlyIfChanged:
let existing =
try: readFile(path)
except CatchableError: ""
if existing == content: return
writeFile(path, content)
proc writeStable*(b: var IcBuilder; path: string; onlyIfChanged = false) {.inline.} =
writeFileStable(b.buf, path, onlyIfChanged)
proc cursorPool*(c: Cursor): Pool {.inline.} = nifcore.pool(c)
## The literals pool the cursor's buffer was built against. `ast2nif.nim`
## imports `nifcore` with `except pool` (to keep nifstreams' global `pool`
## var the writer uses), so the reader reaches a cursor's pool through here —
## needed once `bif`-loaded buffers carry their OWN fresh pool rather than the
## shared `icPool`.
proc freshModuleCopy(b: var IcBuilder): TokenBuf =
## Re-home `b.buf` into a PRIVATE, module-local pool via `addSubtree` (which
## re-interns only the literals/tags this buffer actually uses). `b.buf` is bound
## to the process-wide shared `icPool`/`icTags`; storing it directly would embed
## the WHOLE shared pool (correct but huge — see `bif.storeToFile`). The copy's
## fresh-pool reload reproduces ids verbatim (the bif fresh-pool INVARIANT).
result = createTokenBuf(b.buf.len, newPool(), newTagPool())
var c = b.buf.beginRead()
while c.hasMore:
addSubtree(result, c)
skip c
proc storeBif*(b: var IcBuilder; path: string; dottedSuffix: string) =
## Persist the buffer as a compact, self-contained binary NIF (`.bif`).
var fresh = freshModuleCopy(b)
bif.store(fresh, path, dottedSuffix)
proc storeBifStable*(b: var IcBuilder; path: string; dottedSuffix: string) =
## Content-stable `bif` write — the binary analogue of `writeFileStable`'s
## `onlyIfChanged`: only replace `path` when the encoded bytes differ, so an
## unchanged sidecar keeps its mtime and nifmake prunes the dependent rebuild
## cascade. Used for the iface/impl cookies + dep sidecars whose byte-stability
## gates incremental builds. (bif encoding is deterministic for a given buffer
## under fresh pools, so equal content ⇒ equal bytes.)
var fresh = freshModuleCopy(b)
let tmp = path & ".tmp"
bif.store(fresh, tmp, dottedSuffix)
let newBytes = readFile(tmp)
let oldBytes =
try: readFile(path)
except CatchableError: ""
if newBytes == oldBytes:
removeFile(tmp)
else:
moveFile(tmp, path)
# --- subtree splicing (shared pool, so a raw subtree copy is exact) ----------
proc addAll*(dest: var IcBuilder; src: var IcBuilder) =
## Append every top-level subtree of `src` into `dest` — the nifcore analogue
## of the old `dest.add wholeBuffer` splice.
var c = src.buf.beginRead()
while c.hasMore:
addSubtree(dest.buf, c)
skip c
proc addStmtsBody*(dest: var IcBuilder; src: var IcBuilder) =
## Append the BODY of a `(stmts . . <body> )` builder into `dest`, dropping the
## wrapper tag and its two leading dot slots (flags/type) — the nifcore
## analogue of the old `for i in 3 ..< content.len-1: dest.add content[i]`.
var c = src.buf.beginRead() # at (stmts
c.into:
skip c # flags dot
skip c # type dot
while c.hasMore:
addSubtree(dest.buf, c)
skip c
# --- cookie input: a line-info-free logical token list of the module ---------
# The cookie hashers (ast2nif) need a flat, ParRi-bearing, index-addressable
# view of the serialized module. nifcore has no ParRi kind and variable-width
# tokens, so we flatten the buffer here (in the clean nifcore world) into a
# neutral `CookieTok` list — no nifcore type crosses into ast2nif.
type
CookieKind* = enum
ckParLe, ckParRi, ckSym, ckSymDef, ckIdent, ckStr, ckInt, ckUInt, ckFloat, ckChar, ckDot
CookieTok* = object
kind*: CookieKind
tag*: string # ckParLe
name*: string # ckSym / ckSymDef
sym*: uint32 # ckSym / ckSymDef id (identity key)
str*: string # ckIdent / ckStr
ival*: int64
uval*: uint64
fval*: float64
cval*: uint32
proc flattenGo(c: var Cursor; b: TokenBuf; acc: var seq[CookieTok]) =
while c.hasMore:
case c.kind
of TagLit:
acc.add CookieTok(kind: ckParLe, tag: b.tags.tagName(c.cursorTagId))
c.into:
flattenGo(c, b, acc)
acc.add CookieTok(kind: ckParRi)
of Symbol:
acc.add CookieTok(kind: ckSym, name: symName(c, b.pool), sym: uint32(symId(c, b.pool)))
skip c
of SymbolDef:
acc.add CookieTok(kind: ckSymDef, name: symName(c, b.pool), sym: uint32(symId(c, b.pool)))
skip c
of Ident:
acc.add CookieTok(kind: ckIdent, str: strVal(c, b.pool)); skip c
of StrLit:
acc.add CookieTok(kind: ckStr, str: strVal(c, b.pool)); skip c
of IntLit:
acc.add CookieTok(kind: ckInt, ival: intVal(c)); skip c
of UIntLit:
acc.add CookieTok(kind: ckUInt, uval: uintVal(c)); skip c
of FloatLit:
acc.add CookieTok(kind: ckFloat, fval: floatVal(c)); skip c
of CharLit:
acc.add CookieTok(kind: ckChar, cval: uint32(ord(charLit(c)))); skip c
of DotToken:
acc.add CookieTok(kind: ckDot); skip c
else:
skip c # LineInfoLit / ExtendedSuffix ride on heads, never standalone
proc flattenForCookie*(b: var IcBuilder): seq[CookieTok] =
## Flatten the nifcore module buffer to the cookie hashers' flat token list.
result = newSeqOfCap[CookieTok](b.buf.len)
var cur = b.buf.beginRead()
flattenGo(cur, b.buf, result)
proc collectBifStrLits*(path: string): seq[string] =
## Read a small `(tag "s" "s" …)` bif sidecar (`semdeps`/`edges`) and return every
## string literal it holds, in order — the binary analogue of the old nifstreams
## scan that collected `StrLit`s. Keeps nifcore types out of `deps.nim`, which
## only needs the recorded string list.
##
## Uses `loadFromFile` (a full read into owned memory) rather than the mmap-backed
## `bif.load`, then CLOSES the handle. `bif.load` intentionally leaves the mapping
## resident for the process lifetime; for the `nim ic` driver that reads these
## sidecars while `nim m` children rewrite them, a lingering read mapping is a
## Windows sharing violation: the child's `open(path, fmWrite)` fails with
## `IOError: cannot open`. These sidecars are tiny, so the zero-copy mmap buys
## nothing here anyway.
result = @[]
var f = open(path, fmRead)
var m = bif.loadFromFile(f)
close(f)
var c = m.buf.beginRead()
while c.hasMore:
if c.kind == StrLit: result.add strVal(c)
inc c
proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) =
## Stage 1 spike: the nifcore port of `ast2nif.writeSemDeps`. Serializes the
## module's resolved direct imports as `(semdeps "path" ...)`. Byte-identical
## to the old writer (verified), so `nim ic` build graphs are unaffected.
let selfSuffix = modname(thisModule, config)
var paths = importPaths
sort paths
var dest = newIcBuilder(4 + 2*paths.len)
dest.openTag "semdeps"
for p in paths:
dest.addStrLit p
dest.closeTag()
let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".s.deps.bif").string
storeBifStable(dest, path, "." & extractModuleSuffix(path))

View File

@@ -1,279 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## NIF-based goto-definition / find-all-usages for `nim track`.
##
## This is the mainline-Nim port of nimony's `idetools.nim`. It answers a
## `--def:FILE,LINE,COL` / `--usages:FILE,LINE,COL` query by *scanning the
## `.s.bif` files* (binary NIF, see `dist/nimony/src/lib/bif.nim`) that the
## preceding `nim ic` frontend (`nim track`) emitted into the nimcache directory
## — NOT by re-running sem. NIF distinguishes a definition (`SymbolDef` token) from a use
## (`Symbol` token) syntactically, so goto-def / find-uses become plain token
## scans over type-checked NIF, which is more reliable than the classic PSym
## engine because generics and macros are type-checked in the NIF too.
##
## Two passes (mirroring nimony's `usages`):
## 1. Load the queried module's `.s.bif` and find the `Symbol`/`SymbolDef`
## token whose line info + identifier length contains `conf.m.trackPos`.
## That yields the mangled symbol NAME and whether it is global (>= 2 dots).
## 2. `--usages`: emit every `Symbol` (use) token; `--def`: every `SymbolDef`.
## A global symbol is scanned across every module `.s.bif`; a local one only
## within the queried module.
##
## IMPORTANT porting note: `bif.load` mints FRESH per-file pools, so a `SymId`
## from module A's buffer is meaningless in module B's. The cross-module match is
## therefore by the mangled NAME string, never by `SymId` (nimony can compare ids
## because it parses every text NIF into one shared global pool; we cannot).
import std / [os, strutils, sets]
import options, msgs, pathutils
import lineinfos as astli
import ast2nif # toNifFilename
from deps import includerSbifs # deps-guided include-file lookup
import "../dist/nimony/src/lib/nifcore"
from "../dist/nimony/src/lib" / bif import load, BifModule, containsSym
proc identLen(name: string): int =
## Length of the displayed identifier: the run before the first `.` of a
## mangled NIF name (`ident.disamb[.moduleSuffix]`). Bounds the column match.
let d = name.find('.')
result = if d < 0: name.len else: d
proc isGlobalName(name: string): bool =
## A global symbol carries `ident.disamb.moduleSuffix` (>= 2 dots); a local at
## most `ident.disamb` (<= 1 dot). `moduleSuffix` is a dot-free hash, so a raw
## dot count is equivalent to nifbuilder's suffix-compressed test for our use.
var dots = 0
for i in 1 ..< name.len:
if name[i] == '.': inc dots
result = dots >= 2
proc posMatch(c: Cursor; conf: ConfigRef; target: TLineInfo; tokenLen: int): bool =
## True when `target` (the queried position) falls within the identifier span
## of the Symbol/SymbolDef token at `c`. Mirrors nimony's `lineInfoMatch`; the
## filename is resolved through the loaded buffer's own pool (fresh per file),
## then mapped to a `FileIndex` exactly like `ast2nif.oldLineInfo`.
let li = rawLineInfo(c)
if not li.isValid: return false
if li.line.int != target.line.int: return false
let f = fileInfoIdx(conf, AbsoluteFile lineInfoFile(c))
if f != target.fileIndex: return false
if target.col.int < li.col.int: return false
if target.col.int > li.col.int + tokenLen: return false
result = true
const sep = '\t'
proc formatSuggest(s: Suggest): string =
## Reproduce `suggest.$Suggest` for the `ideDef`/`ideUse` sections without
## importing `suggest` (which would create an import cycle). Layout:
## `section⭾symkind⭾qualifiedPath⭾forth⭾filePath⭾line⭾column⭾⭾quality`.
## symkind is always `skUnknown` here — the raw NIF scan has no PSym to give a
## real kind (like nimony's `foundSymbol`, which leaves it empty).
result = $s.section
result.add sep
result.add "skUnknown"
result.add sep
if s.qualifiedPath.len != 0:
result.add s.qualifiedPath.join(".")
result.add sep
result.add s.forth
result.add sep
result.add s.filePath
result.add sep
result.add $s.line
result.add sep
result.add $s.column
result.add sep # empty doc field (docgen is off outside nimsuggest)
if s.version == 0 or s.version == 3:
result.add sep
result.add $s.quality
proc emit(conf: ConfigRef; c: Cursor; section: IdeCmd; name: string;
seen: var HashSet[string]) =
## Report one hit as a nimsuggest-compatible result (routed through the
## structured-output hook / `--stdout`). We only have the mangled name + line
## info from the raw NIF, so symkind/type are left empty — like nimony's
## `foundSymbol`. `seen` deduplicates: the same source location can back
## several NIF `Symbol` tokens (e.g. a call argument re-emitted in a lowered
## form), which must surface as one hit.
let li = rawLineInfo(c)
if not li.isValid: return
let key = $section.int & ":" & lineInfoFile(c) & ":" & $li.line.int & ":" & $li.col.int
if seen.containsOrIncl(key):
return # already reported this location for this section
let s = Suggest(section: section,
qualifiedPath: @[name[0 ..< identLen(name)]],
filePath: lineInfoFile(c),
line: li.line.int,
column: li.col.int,
tokenLen: identLen(name),
forth: "",
symkind: 0'u8,
quality: 100,
version: conf.suggestVersion)
if conf.suggestionResultHook != nil:
conf.suggestionResultHook(s)
else:
conf.suggestWriteln(formatSuggest(s))
proc tokenSymId(c: Cursor): SymId {.inline.} =
## SymId (in the cursor's own per-file pool) of a `Symbol`/`SymbolDef` token,
## or `SymId(0)` for an inline-encoded one — which is never our search target:
## a mangled name (`ident.disamb.suffix`) is always longer than
## `StrInlineMaxLen`, so every occurrence of the symbol we look for is stored by
## pool id, decoded here with a shift and no string materialization.
if isInlineLit(c): SymId(0) else: SymId(combinedPayload(c) shr 1)
template symMatches(c: Cursor): bool =
## True when the token at `c` is the searched symbol. The fast path is a pure
## integer compare against `targetSym` (the symbol's id in THIS module's pool,
## resolved once per file by the caller). `targetSym == 0` means the name is not
## representable as a pool id (a rare <=3-byte local): fall back to a string
## compare, correct for both inline and pooled encodings.
(if targetSym != SymId(0): tokenSymId(c) == targetSym else: symName(c) == targetName)
proc scanUses(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
seen: var HashSet[string]) =
## `--usages`: report every `Symbol` (use) occurrence with valid line info.
if m.buf.len == 0: return
var c = m.buf.beginRead()
while c.hasMore:
if c.kind == Symbol and symMatches(c) and rawLineInfo(c).isValid:
emit(conf, c, ideUse, targetName, seen)
inc c
c.endRead()
proc scanDef(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
seen: var HashSet[string]) =
## `--def`: report the declaration of the target symbol if this module owns it
## (has its `SymbolDef`). The `SymbolDef` token itself carries no line info; the
## declaration location lives on the *enclosing tag* (e.g. `(sd @file:line:col`,
## like `bif.buildIndex`'s `mostRecentTagPos`). When that tag has no line info
## either, fall back to the declaration-site `Symbol` occurrence — but only in
## the owning module, so a plain user of the symbol is never reported as a def.
if m.buf.len == 0: return
var c = m.buf.beginRead()
var mostRecentTagPos = 0
var sawDef = false
var emitted = false
var fallbackPos = -1
while c.hasMore:
case c.kind
of TagLit:
mostRecentTagPos = cursorToPosition(m.buf, c)
inc c
of SymbolDef:
if symMatches(c):
sawDef = true
var tc = cursorAt(m.buf, mostRecentTagPos)
if rawLineInfo(tc).isValid:
emit(conf, tc, ideDef, targetName, seen)
emitted = true
tc.endRead()
inc c
of Symbol:
if fallbackPos < 0 and symMatches(c) and rawLineInfo(c).isValid:
fallbackPos = cursorToPosition(m.buf, c)
inc c
else:
inc c
c.endRead()
if sawDef and not emitted and fallbackPos >= 0:
var fc = cursorAt(m.buf, fallbackPos)
emit(conf, fc, ideDef, targetName, seen)
fc.endRead()
proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd;
targetSym: SymId; targetName: string; seen: var HashSet[string]) =
## Emit hits for the target symbol in `m` per the query kind. `ideDus`
## (`--defusages`) reports both the definition and every usage.
if section in {ideDef, ideDus}:
scanDef(conf, m, targetSym, targetName, seen)
if section in {ideUse, ideDus}:
scanUses(conf, m, targetSym, targetName, seen)
proc findPos(conf: ConfigRef; m: var BifModule; target: TLineInfo;
foundName: var string): bool =
## Scan `m` for the `Symbol`/`SymbolDef` token covering the queried position
## `target` and set `foundName` to its mangled name. Returns true on a hit.
if m.buf.len == 0: return false
var c = m.buf.beginRead()
result = false
while c.hasMore:
let k = c.kind
if k == Symbol or k == SymbolDef:
let nm = symName(c)
if posMatch(c, conf, target, identLen(nm)):
foundName = nm
result = true
break
inc c
c.endRead()
proc runIdeQuery*(conf: ConfigRef) =
## Entry point: called from `main.nim` after `commandCheck` when a
## `--def`/`--usages` query is active. Assumes the check just emitted the
## project's `.s.bif` files into `getNimcacheDir(conf)`.
let section = conf.ideCmd
if section notin {ideDef, ideUse, ideDus}: return
let target = conf.m.trackPos
if target.fileIndex.int32 < 0: return
# Pass 1: position -> symbol. Try the queried file's own module bif first (the
# fast path when the position is inside a real module). An include file has no
# module bif of its own — its tokens live in the *including* module's bif with
# include-file line info — so when the direct lookup misses, consult the
# `.deps.nif` preludes (`includerSbifs`) to load only the module(s) that
# include the queried file (directly or transitively), never every bif in the
# nimcache. `ownerFile` is the bif that owns the hit.
let modFile = toNifFilename(conf, target.fileIndex)
var foundName = ""
var ownerFile = ""
if fileExists(modFile):
var qm = load(modFile)
if findPos(conf, qm, target, foundName):
ownerFile = modFile
if foundName.len == 0:
for cand in includerSbifs(conf, toFullPath(conf, target.fileIndex).AbsoluteFile):
if cand == modFile: continue
var m = load(cand)
if findPos(conf, m, target, foundName):
ownerFile = cand
break
if foundName.len == 0: return
# Pass 2: emit definition / usages. `seen` spans every module so a location is
# reported once even when scanned across the whole nimcache.
#
# Cross-file matching is by SymId, not by decoding every token's name. Two
# filters keep it cheap:
# 1. `bif.containsSym` — a sym-table-only probe that reads just the small
# trailing pools, NOT the token block or any `BiTable`. A module that never
# references the symbol is rejected here without a full `load` (no pools
# built, no token block mapped) — so a query whose symbol lives in a few
# modules no longer pays to load the whole nimcache.
# 2. For a module that does contain it, `bif.load` mints a fresh per-file pool,
# so the name is resolved to THIS file's SymId once via `getKeyId`; the scan
# then compares integer ids per token instead of materializing a string for
# each (see `symMatches`).
var seen = initHashSet[string]()
if isGlobalName(foundName):
for f in walkFiles((getNimcacheDir(conf).string) / "*.s.bif"):
if not containsSym(f, foundName): continue
var m = load(f)
let tid = m.buf.pool.syms.getKeyId(foundName)
if tid != SymId(0):
scanBuf(conf, m, section, tid, foundName, seen)
else:
# Local symbol: its mangled name is not unique across modules, so restrict
# the scan to the module it lives in (the one that owns the queried position).
var qm = load(ownerFile)
let tid = qm.buf.pool.syms.getKeyId(foundName)
scanBuf(conf, qm, section, tid, foundName, seen)

View File

@@ -245,18 +245,6 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode
let canon = c.graph.canonTypes.getOrDefault(h)
if canon != nil:
op = getAttachedOp(c.graph, canon, kind)
if op == nil or op.ast.isGenericRoutine:
# IC: injectDestructorCalls is demand-driven and runs HERE (cg), not in the
# `lower` stage, so a structural, env-agnostic op the lower stage never had
# reason to serialize — most often a closure PROC type's `=destroy`/`=sink`
# (which act on the `(ClP_0, ClE_0)` tuple, NOT the concrete env) — must be
# lifted on demand, exactly as the lazy path's cg does. This is safe now:
# closure-env identity resolves via `attachedOps[itemId]`/env-erased typeKey,
# env objects load complete, and atomicRefOp's type-erased path covers any
# still-incomplete env (so the lift never walks a nil field).
excl t.flagsImpl, tfCheckedForDestructor
createTypeBoundOps(c.graph, nil, t, dest.info, c.idgen)
op = getAttachedOp(c.graph, t, kind)
if op == nil:
#echo dest.typ.id
globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &

View File

@@ -34,7 +34,7 @@ import
ropes, wordrecg, renderer,
cgmeth, lowerings, sighashes, modulegraphs, lineinfos,
transf, injectdestructors, sourcemap, astmsgs, pushpoppragmas,
mangleutils, varpartitions
mangleutils
import pipelineutils
@@ -1298,16 +1298,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
xtyp = etySeq
case xtyp
of etySeq:
if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded or
(x.kind == nkSym and sfCursor in x.sym.flags):
if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
else:
useMagic(p, "nimCopy")
lineF(p, "$1 = nimCopy(null, $2, $3);$n",
[a.rdLoc, b.res, genTypeInfo(p, y.typ)])
of etyObject:
if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded or
(x.kind == nkSym and sfCursor in x.sym.flags):
if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
else:
useMagic(p, "nimCopy")
@@ -2094,8 +2092,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
gen(p, n, a)
case mapType(p, v.typ)
of etyObject, etySeq:
if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n) or
sfCursor in v.flags:
if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n):
s = a.res
else:
useMagic(p, "nimCopy")
@@ -2801,11 +2798,6 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, {})
if sfInjectDestructors in prc.flags:
transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
else:
# JS has a GC, so the destructor pass is off; but the cursor (alias) analysis
# is independent of ownership and always memory-safe on a traced target.
# Running it lets last-use `var b = a` aliases skip the deep `nimCopy`.
computeCursors(prc, transformedBody, p.module.graph)
p.nested: genStmt(p, transformedBody)

View File

@@ -176,7 +176,7 @@ proc closureParams(routine: PSym): PNode =
result = routine.typ.n
routine.ast[paramsPos] = result
proc addHiddenParam*(routine: PSym, param: PSym) =
proc addHiddenParam(routine: PSym, param: PSym) =
assert param.kind == skParam
var params = closureParams(routine)
# -1 is correct here as param.position is 0 based but we have at position 0
@@ -309,25 +309,6 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
[s.name.s, owner.name.s, $owner.typ.callConv])
unsealForTransform(owner.typ)
incl(owner.typ, tfCapturesEnv)
# A closure proc type that captures an env owns a REF to it: copying the closure
# value must incref the env and destroying it must decref. That is exactly what
# `tfHasAsgn` signals to `injectDestructorCalls` (so a closure assignment becomes
# `=copy`, not a raw field store).
#
# Set it HERE (closure-type creation) so the flag is DETERMINISTIC and serializes
# with the type — but ONLY under `nim ic`. The per-module `lower` stage is a
# separate process that lowers routines in index order; if a consumer (e.g.
# `workNimAsyncContinue`) was lowered before the closure type's ops were lifted,
# its env store emitted a RAW assign with no incref → freed env → async
# "yielded `nil`". A normal single-process `nim c` build does NOT need this —
# `createTypeBoundOps` sets the flag lazily, in lift order, before it matters
# (the old `liftdestructors ~1498` "XXX Breaks IC!" side effect) — and setting it
# eagerly there REGRESSES codegen: a `=destroy` hook gets generated against the
# bare `void(*)(void)` proc representation but is then called with closure structs
# (`eqdestroy__u2__stdZtypedthreads` type mismatch — broke megatest). So gate on
# `cmdNifC`; normal builds keep the lazy (devel) behavior.
if g.config.cmd == cmdNifC:
incl(owner.typ, tfHasAsgn)
if not isEnv:
owner.typ.callConv = ccClosure

View File

@@ -821,15 +821,13 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
# dynamic Acyclic refs need to use dyn decRef
let useStatic = isFinal(elemType)
let tmp =
if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
declareTempOf(c, body, x)
else:
x
if useStatic:
if isFinal(elemType):
addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
alignOf.typ = getSysType(c.g, c.info, tyInt)
@@ -840,7 +838,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
var cond: PNode
if isCyclic:
if useStatic:
if isFinal(elemType):
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
@@ -875,7 +873,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
of attachedDeepCopy: assert(false, "cannot happen")
of attachedTrace:
if isCyclic:
if useStatic:
if isFinal(elemType):
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)

View File

@@ -210,62 +210,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
if matchesDerivedFieldId(n.sym.itemId, id): result = n.sym
else: discard
proc lookupCapturedField(n: PNode, s: PSym): PSym =
## Find an env field that `addField` would have produced for the captured
## local `s`. Used as a fallback when the derived-itemId match fails because
## `s` is a macro-generated gensym whose process-local id diverges from the
## loaded env field's (see `addField`). `addField` always names a field
## `s.name & $field.position`, so that pair uniquely identifies the field for a
## local of this name without relying on the (unstable) item id.
result = nil
case n.kind
of nkRecList:
for i in 0..<n.len:
result = lookupCapturedField(n[i], s)
if result != nil: return
of nkRecCase:
if n[0].kind != nkSym: return
result = lookupCapturedField(n[0], s)
if result != nil: return
for i in 1..<n.len:
case n[i].kind
of nkOfBranch, nkElse:
result = lookupCapturedField(lastSon(n[i]), s)
if result != nil: return
else: discard
of nkSym:
if n.sym.kind == skField and n.sym.name.s == s.name.s & $n.sym.position:
result = n.sym
else: discard
proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
# Idempotent w.r.t. the captured symbol (mirrors `addUniqueField`): re-lifting
# a LOADED routine re-derives its transformed body (never serialized under IC)
# and re-captures the same locals, but the env object loaded from the NIF
# already carries their fields. Re-adding would duplicate the field and, worse,
# mutate a Sealed loaded type via `propagateToOwner` (the `t.state != Sealed`
# crash). Return the existing field instead.
let existing = lookupInRecord(obj.n, s.itemId)
if existing != nil:
return existing
# Re-lifting a LOADED routine during a VM transform (its transformed body is
# re-derived per process, never serialized) re-captures the same locals, but
# for a macro-generated gensym (e.g. libp2p `p2pProtocolBackendImpl`'s
# `msgVar`) its process-local id diverges from the one baked into the loaded
# env field, so the id match above misses. Reuse the existing same-named field
# rather than appending a divergent duplicate, which keeps the re-derived
# closure consistent (else a stale `:env` access reaches `cannotEval`).
# Confined to a loaded (Sealed) env: in a freshly built env ids are consistent,
# and two distinct same-named captures legitimately get distinct fields there.
if obj.state == Sealed:
let byName = lookupCapturedField(obj.n, s)
if byName != nil:
return byName
# Genuinely new field. Under IC the env may be a loaded Sealed type whose
# transform-time mutation is process-local (the body is discarded after the
# macro runs), so downgrade it to mutable instead of crashing on
# `t.state != Sealed` (mirrors `markAsClosure`).
unsealForTransform(obj)
# because of 'gensym' support, we have to mangle the name with its ID.
# This is hacky but the clean solution is much more complex than it looks.
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
@@ -361,16 +306,6 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
assert t.kind == tyObject
result = lookupInRecord(t.n, v.itemId)
if result != nil: break
# A LOADED (Sealed) env object carries fields baked by the producer process;
# re-lifting a NIF-loaded routine in a consumer (e.g. a macro VM-evaluating an
# imported `p2pProtocolBackendImpl`) re-captures the same local under a
# divergent process-local id, so the derived-itemId match misses. Fall back to
# the name+position identity `addField` uses — SYMMETRIC with `addField`'s
# Sealed by-name reuse — so the access resolves the field `addField` produced
# instead of failing with `not part of closure object type`.
if t.state == Sealed:
result = lookupCapturedField(t.n, v)
if result != nil: break
t = t.baseClass
if t == nil: break
t = t.skipTypes(skipPtrs)

View File

@@ -34,7 +34,6 @@ from icconfig import produceIcConfig
when not defined(nimKochBootstrap):
import nifbackend
import deps
import idetools
when not defined(leanCompiler):
import docgen
@@ -417,20 +416,6 @@ proc mainCommand*(graph: ModuleGraph) =
for it in conf.searchPaths: msgWriteln(conf, it.string)
of cmdCheck:
commandCheck(graph)
of cmdTrack:
# `nim track --def:/--usages:/--track:` — IDE goto-definition / find-usages.
# Runs `nim ic`'s incremental frontend (nifler + per-module `nim m`, so only
# changed modules recompile and each writes a faithful, VM-executed `.s.bif`
# — covering stdlib too), then scans those NIF files (idetools.runIdeQuery).
# Shares the `nim ic` nimcache dir, so a prior `nim ic` build is reused.
setUseIc(true)
wantMainModule(conf)
setOutFile(conf)
when not defined(nimKochBootstrap):
commandIc(conf, frontendOnly = true)
runIdeQuery(conf)
else:
rawMessage(conf, errGenerated, "nim track not available in bootstrap build")
of cmdM:
# cmdM uses NIF files, not ROD files
graph.config.symbolFiles = disabledSf
@@ -451,9 +436,6 @@ proc mainCommand*(graph: ModuleGraph) =
# Generate .build.nif for nifmake
setUseIc(true)
wantMainModule(conf)
# Resolve the output binary path (honoring `--out`) up front, like cmdNifC:
# the backend build file derives the link target from `conf.absOutFile`.
setOutFile(conf)
when not defined(nimKochBootstrap):
commandIc(conf)
else:
@@ -476,7 +458,7 @@ proc mainCommand*(graph: ModuleGraph) =
of cmdJsonscript:
setOutFile(graph.config)
commandJsonScript(graph)
of cmdUnknown, cmdNone:
of cmdUnknown, cmdNone, cmdIdeTools:
rawMessage(conf, errGenerated, "invalid command: " & conf.command)
if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop, cmdM} and

View File

@@ -61,22 +61,12 @@ proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string =
# starts with an EMPTY per-name disamb table, so its `disamb` restarts at 0
# and collides with same-named sem-time symbols loaded from NIFs (two
# `=destroy` hooks both mangling to `_u2` → "conflicting types for ..." in
# the generated C). Most such symbols never cross a process boundary (nifc
# the generated C). These symbols never cross a process boundary (nifc
# lifts, emits and compiles them in one run), so the per-module-unique
# item id is a safe and deterministic discriminator; the `_c` marker keeps
# the namespace disjoint from `_u<disamb>`.
result = "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# EXCEPTION: a backend-minted sym whose `disamb` is content-derived
# (setHookDisamb gave it HookDisambBit) — e.g. the `rttiDestroy` wrapper —
# DOES cross process boundaries: its C name is baked into the type's RTTI
# table, which is emit-everywhere and merge-deduped, so one process's
# `_c<item>` (a per-process backend counter) ends up referenced while the
# wrapper is defined with another's → undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes; use it.
result.addInt s.disamb
else:
result.addInt s.itemId.item
result.addInt s.itemId.item
else:
result = "_u"
# Use `disamb` rather than `itemId.item`: under incremental compilation a

View File

@@ -146,19 +146,11 @@ type
cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
cacheCounters*: Table[string, BiggestInt] # IC: implemented
cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
pendingNifInit*: seq[tuple[module: PSym; topLevel: PNode]]
# EVERY module loaded from a NIF — whether a direct import (moduleFromNifFile)
# or only a dep-of-a-dep (loadTransitiveHooks) — is recorded here with its
# serialized top-level AST. The sem driver drains it once
# (pipelines.finalizeLoadedModules) and applies the module's VM-level load
# effects UNIFORMLY: macro-cache replay (std/macrocache put/inc/add/incl) and
# eager `{.compileTime.}` global init. This is the single place "what a loaded
# module does to global state" lives, so a transitively-reached module — which
# never passes through compilePipelineModule — gets the SAME treatment as a
# direct import instead of silently skipping it (its macrocache state would be
# lost; its CT globals would stay nil and a macro splicing one, e.g.
# chronicles' `chroniclesBlockName`, emits `break nil` / `nil == 0`). To add a
# new per-load VM effect, extend the drain — never a parallel buffer.
transitiveReplayActions*: seq[PNode] # macro-cache replay actions collected from
# the transitive import closure of a NIF-loaded module (loadTransitiveHooks);
# the caller (pipelines) replays them so a dependency's macrocache state — e.g.
# nim-serialization's flavor registration — reaches a module that imports it
# only indirectly. Drained per moduleFromNifFile call.
passes*: seq[TPass]
pipelinePass*: PipelinePass
onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -168,20 +160,12 @@ type
strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
idgen*: IdGenerator
vmTransfIdgen*: IdGenerator # process-local backend idgen for closure envs
# minted while the VM compiles a routine body
# (inVMTransform); see lambdalifting / ast2nif @bk
operators*: Operators
cachedFiles*: StringTableRef
procGlobals*: seq[PNode]
nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF
nifExpansions*: Table[int32, seq[(PSym, TLineInfo)]]
# module position -> (template/macro sym, call-site info) for every expansion
# in that module. Templates/macros leave no trace in the sem'checked AST, so
# this side-channel (written into the `.bif`, see ast2nif) is what lets
# `nim track --usages`/`--def` find them. Populated by `rememberExpansion`.
cachedMods: IntSet
hookClosure: IntSet # modules whose serialized hooks were already registered
@@ -623,14 +607,10 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
when not defined(nimKochBootstrap):
# Try to resolve from NIF for both cmdNifC and cmdM (which uses NIF files)
if g.config.cmd in {cmdNifC, cmdM}:
# First try system module (most compilerprocs are there).
# Only consult the NIF if it actually exists: under nimsuggest's cold
# cache (ideActive) system is compiled from source and has no NIF yet,
# in which case the proc is already registered in-memory and the caller
# found/falls back to it — so degrade to nil instead of asserting.
# First try system module (most compilerprocs are there)
let systemFileIdx = g.config.m.systemFileIdx
if systemFileIdx != InvalidFileIdx and not g.withinSystem and
fileExists(toNifFilename(g.config, systemFileIdx)):
if systemFileIdx != InvalidFileIdx and not g.withinSystem:
# Only try to load from NIF if the file exists (it may not during initial ic build)
result = tryResolveCompilerProc(ast.program, name, systemFileIdx)
if result != nil:
strTableAdd(g.compilerprocs, result)
@@ -642,7 +622,6 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
let module = g.ifaces[moduleIdx].module
if module != nil and module.name.s == "threadpool":
let threadpoolFileIdx = module.position.FileIndex
if not fileExists(toNifFilename(g.config, threadpoolFileIdx)): break
result = tryResolveCompilerProc(ast.program, name, threadpoolFileIdx)
if result != nil:
strTableAdd(g.compilerprocs, result)
@@ -911,14 +890,10 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
result = s.ast[bodyPos]
if result != nil and nfLazyBody in result.flags and forceLazyBodyHook != nil:
# Sanctioned body-access gate (see astdef.bodyPos): materialize the deferred
# IC body so callers may safely touch `.sons` directly, not only via `len`.
forceLazyBodyHook(result)
assert result != nil
when not defined(nimKochBootstrap):
proc registerLoadedHooks*(g: ModuleGraph; logOps: seq[LogEntry]) =
proc registerLoadedHooks(g: ModuleGraph; logOps: seq[LogEntry]) =
let mainSuffix = getMainModuleSuffix(ast.program)
for x in logOps:
# A dependency's NIF may carry hooks whose syms belong to the module we
@@ -974,33 +949,14 @@ when not defined(nimKochBootstrap):
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
registerLoadedHooks(g, precomp.logOps)
# Record this transitively-loaded module so the sem driver applies its
# VM-level load effects (macro-cache replay + `{.compileTime.}` global init)
# exactly as for a direct import — see `pendingNifInit`. A throwaway module
# symbol (same shape as moduleFromNifFile's) gives the drain an idgen/info
# context; it is not registered, so a later direct import still loads fully.
if g.config.cmd == cmdM:
let m = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32),
name: getIdent(g.cache, splitFile(toFullPath(g.config, fileIdx)).name),
infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx))
setOwner(m, getPackage(g.config, g.cache, fileIdx))
g.pendingNifInit.add (m, precomp.topLevel)
# Rebuild generic TYPE- and PROC-instance offers across the WHOLE closure,
# not just direct imports (`moduleFromNifFile`). An instance is frozen at
# the FIRST module to create it (in a scope where its body's symbols
# resolve unambiguously); a consumer many imports away must REUSE it rather
# than re-instantiate in its own scope, which may resolve a body symbol
# differently — a divergent `compiles()`-dependent array bound (SSZ
# `HashArray[8192, Gwei]`, type offer), or an ambiguous unqualified ident
# leaked from an unrelated import (`fromRaw` -> `SkRawPublicKeySize` from
# both `secp` and `secp256k1`, proc offer). Direct-only rebuild left the
# deep offer invisible when the clean instance lives a transitive hop away.
for off in precomp.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
for off in precomp.genericOffers:
g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation(
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
# Collect the dependency's macro-cache replay actions (put/inc/add/incl)
# so the importer being compiled also sees macrocache state registered
# by a transitively-imported module. Pragma replay actions are a backend
# concern and are intentionally not collected here.
for n in precomp.topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
g.transitiveReplayActions.add n
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
@@ -1044,17 +1000,6 @@ when not defined(nimKochBootstrap):
if not fileExists(toNifFilename(g.config, fileIdx)):
return PrecompiledModule(module: nil)
# NOTE: direction-(c) experiment (refuse to NIF-serve include-bearing modules
# under ideActive, forcing a source compile) is disabled — it reproduces the
# known sibling-resolution corruption (system.string -> excpt.nim:746). The
# cold-include *discovery* scan (scanIncludeGraph) stays; the round-trip
# fidelity of included symbols is the separate, still-open loader problem.
when false:
if g.config.ideActive and not g.withinSystem and
fileIdx != g.config.m.systemFileIdx and
nifModuleHasIncludes(g.config, fileIdx):
return PrecompiledModule(module: nil)
# Create module symbol
let filename = AbsoluteFile toFullPath(g.config, fileIdx)
@@ -1076,12 +1021,6 @@ when not defined(nimKochBootstrap):
let ms = materializeReexportedModule(g, mname, msuffix)
if ms != nil:
strTableAdd(g.ifaces[fileIdx.int].interf, ms)
# Re-establish include->module mapping so nimsuggest's `parentModule` can map
# a query in an included file back to this (NIF-loaded) module and recompile
# it, exactly as it does for a from-source module. Without this the include
# relationship is invisible for NIF-served modules.
for incPath in result.includes:
g.addIncludeDep(fileIdx, fileInfoIdx(g.config, AbsoluteFile incPath))
# Rebuild `procInstCache` from this module's generic-instance OFFERS so a
# consumer's `genericCacheGet` finds the instance and SKIPS re-running
@@ -1092,14 +1031,6 @@ when not defined(nimKochBootstrap):
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
# Rebuild `typeInstCache` from this module's generic TYPE-instance OFFERS so a
# consumer's `searchInstTypes` reuses the baked instance (e.g. an SSZ
# `HashArray` whose array bound depends on import-scope-sensitive `compiles()`)
# rather than re-instantiating it with a divergent bound — see ast2nif's
# `(toffer …)`. Keyed by the generic body sym's itemId, as `searchInstTypes`.
for off in result.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
# Mark module as cached
g.cachedMods.incl fileIdx.int
g.hookClosure.incl fileIdx.int
@@ -1131,39 +1062,6 @@ when not defined(nimKochBootstrap):
# walks the closure in nifbackend.loadModuleDependencies.)
if g.config.cmd == cmdM:
loadTransitiveHooks(g, result.deps)
# Record the directly-loaded module for the same VM-level load effects as its
# transitive deps (`pendingNifInit`). AFTER loadTransitiveHooks so the drain
# applies deps before the dependent (macro-cache order).
g.pendingNifInit.add (m, result.topLevel)
proc isModuleFile(g: ModuleGraph; fileIdx: FileIndex): bool =
let i = fileIdx.int32
i >= 0 and i < g.ifaces.len and g.ifaces[i].module != nil
proc registerIncluderFromNif*(g: ModuleGraph; fileIdx: FileIndex): bool =
## Targeted cold-include discovery for nimsuggest: scan the nimcache NIFs
## (`scanIncludeGraph`) for a module whose include-set contains *this* file
## and register only that single include->module edge in `inclToMod`, so a
## query inside the include file resolves its includer via `parentModule`.
##
## Deliberately targeted: registering *every* include relationship (i.e. also
## `system`'s own `include`s) eagerly assigns FileIndexes and pollutes
## `inclToMod`, which perturbs the NIF line-info decode of unrelated modules
## (`system.string` then resolves into `excpt.nim`). Touch nothing but the
## one edge we need.
let target = toFullPath(g.config, fileIdx)
for (includer, includes) in scanIncludeGraph(g.config):
for incFile in includes:
if cmpPaths(incFile, target) == 0:
g.addIncludeDep(fileInfoIdx(g.config, AbsoluteFile includer), fileIdx)
return true
result = false
proc needsIncludeScan*(g: ModuleGraph; fileIdx: FileIndex): bool =
## True when `fileIdx` is neither a known module of its own nor an
## already-known include file — i.e. a cold-opened file whose includer we
## must still discover via `registerIncluderFromNif`.
not g.isModuleFile(fileIdx) and not g.inclToMod.hasKey(fileIdx)
proc configComplete*(g: ModuleGraph) =
#rememberStartupConfig(g.startupPackedConfig, g.config)

View File

@@ -351,7 +351,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
## This is used for 'nim dump' etc. where we don't have nimsuggest
## support.
#if conf.ideActive and optCDebug notin gGlobalOptions: return
#if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
let sep = if msgNoUnitSep notin flags: conf.unitSep else: ""
if not isNil(conf.writelnHook) and msgSkipHook notin flags:
conf.writelnHook(s & sep)
@@ -457,8 +457,8 @@ To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 f
proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string, ignoreMsg: bool) =
if msg in fatalMsgs:
if conf.ideActive: log(s)
if not conf.ideActive or msg != errFatal:
if conf.cmd == cmdIdeTools: log(s)
if conf.cmd != cmdIdeTools or msg != errFatal:
quit(conf, msg)
if msg >= errMin and msg <= errMax or
(msg in warnMin..hintMax and msg in conf.warningAsErrors and not ignoreMsg):
@@ -472,7 +472,7 @@ proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string,
raiseRecoverableError(s)
else:
quit(conf, msg)
elif eh == doAbort and not conf.ideActive:
elif eh == doAbort and conf.cmd != cmdIdeTools:
quit(conf, msg)
elif eh == doRaise:
raiseRecoverableError(s)
@@ -503,7 +503,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
info = context.info
proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
msg >= errGenerated and conf.ideActive and optIdeDebug notin conf.globalOptions
msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
conf.m.fileInfos[fileIdx.int32].lines.add line
@@ -661,7 +661,7 @@ proc warningDeprecated*(conf: ConfigRef, info: TLineInfo = gCmdLineInfo, msg = "
message(conf, info, warnDeprecated, msg)
proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
if (conf.ideActive or conf.cmd == cmdCheck) and conf.structuredErrorHook.isNil: return
if conf.cmd in {cmdIdeTools, cmdCheck} and conf.structuredErrorHook.isNil: return
writeContext(conf, info)
liMessage(conf, info, errInternal, errMsg, doAbort, info2)

View File

@@ -24,22 +24,10 @@ when defined(nimPreviewSlimSystem):
import ast, options, lineinfos, modulegraphs, cgendata, cgen,
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys,
cnif, icmodnames
cnif
from cgmeth import generateIfMethodDispatchers
from transf import transformBody
from injectdestructors import injectDestructorCalls
import ic / replayer
proc systemNifSuffix(conf: ConfigRef): string =
## The system module's NIF suffix, derived from `system.nim`'s path EXACTLY as
## the frontend derives it (deps.nim's `toPair` on `libpath/system.nim`), so the
## backend loads the very `.s.bif` the frontend wrote. It must NOT be a constant:
## `moduleSuffix` (icmodnames) now hashes the absolute path, so the system suffix
## is install-dependent (was hardcoded `sysma2dyk`, valid only for the old
## relative-path scheme where `system.nim` always relativized to `system.nim`).
moduleSuffix((conf.libpath / RelativeFile"system.nim").string,
cast[seq[string]](conf.searchPaths))
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
nifFiles: var seq[string];
depFlags: set[LoadFlag] = {LoadFullAst}): seq[PrecompiledModule] =
@@ -151,74 +139,12 @@ proc signatureHasMetaType(t: PType; depth: int = 0): bool =
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind == tyStatic:
# A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in
# `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise
# fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal
# runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter
# (no bound value, `t.n == nil`) is meta. Without this, every routine whose
# signature touches a `static`-parameterized generic instance (the bulk of
# the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up
# an undefined reference at link (mirrors the tyGenericBody case above).
return t.n == nil
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam,
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyStatic, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
## A concrete, non-generic, runtime routine with a real body, OWNED by the
## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
## routine called only from other modules is still emitted by somebody) and
## the `lower` stage's owned-routine enumeration, so both stages see exactly
## the same set. The exclusions:
## - nested/closure procs (owner is a proc, not a module): emitted via their
## enclosing routine's lambda-lifting, never standalone;
## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge;
## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures:
## not real codegen targets.
## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into
## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never
## per module. A dispatcher is a `copySym` clone of the method that shares the
## method's body sub-tree (incl. its closure iterator); transforming it here
## would lambda-lift that SHARED iterator a SECOND time under a different owner
## identity, baking a conflicting `up` field → "up references do not agree"
## (the divergence is impossible in non-IC, where the dispatcher body is empty
## at lift time). So a dispatcher is never an owned runtime routine.
## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
## iterator, which is expanded at each call site) and must be emitted by its
## owner — else a cross-module `for` over it links to nothing.
##
## Generic INSTANCES (`sfFromGeneric`) are NEVER an owned runtime routine — not
## in `cg` and not in the `lower` stage. They are demanded by the backend's
## emit-everywhere path and deduped by `merge` (content C name); the frontend
## materialises them through the `(offer)` mechanism. The `lower` stage must
## not transform an instance: a not-fully-concrete instance (a closure factory
## over a `static` param, or a `$`/`=` op instance whose body resolves only at
## its further-specialised use sites) still carries unresolved overload choices
## and crashes `transformBody` (empty-`namePos` lambda, nil-typed const-fold).
s.itemId.module == modPos and
(s.kind in {skProc, skFunc, skConverter, skMethod} or
(s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
sfFromGeneric notin s.flags and
sfDispatcher notin s.flags and
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty
# NOTE: an `nkEmpty` body is NOT a disqualifier. A concrete, owned, non-
# forward/-importc/-magic routine whose body folds to nothing is still a real
# definition the owner must emit (`void f(void){}`), exactly as whole-program
# cgen does — else a cross-module caller links to nothing. This bites e.g.
# Nimbus' `extras.incInternalErrors`, a plain `proc` whose sole statement is a
# metrics-counter `.inc()` that the `metrics` library expands to a no-op when
# the importing tool (ncli) builds with `-u:metrics`; the body is then a bare
# `nkEmpty`, but `state_transition_epoch` still calls it. Forward declarations
# (the other empty-body case) carry `sfForward` and are excluded above.
proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
## Generate C code for a single module.
let moduleId = precomp.module.position
@@ -244,7 +170,34 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg":
let modPos = precomp.module.position
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
if s.itemId.module == modPos and
s.kind in {skProc, skFunc, skConverter, skMethod} and
# Only MODULE-level routines: a nested/closure proc (its owner is a
# proc) captures its enclosing scope and cannot be emitted standalone —
# the captured params have no loc → `expr: param not init`. Nested procs
# are emitted via their enclosing routine's lambda-lifting, so seeding
# the enclosing (module-level) routine already covers them.
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
# Skip generic instances: they have no single owning-module top-level
# and are emitted by demand (emit-everywhere, deduped by the merge
# stage). An instance has an empty `genericParamsPos` just like a plain
# concrete proc, so only `sfFromGeneric` tells them apart; seeding one
# would force standalone codegen of an instance body whose `when T is X`
# branches were never folded for this path → `genMagicExpr: mIs`.
sfFromGeneric notin s.flags and
# Every other routine the module owns must be emitted here, exported or
# not: a non-exported helper is still reached from another module when a
# `template`/inline routine expands at a call site there (e.g. msgs'
# `internalErrorImpl` behind the `internalError` template), and that
# caller now only prototypes it. `{.error.}`/`compileTime` sentinels and
# bodyless forward decls are not real codegen targets.
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty and
s.ast[bodyPos].kind != nkEmpty:
# a concrete, non-generic, runtime routine with a real body, owned here
requestProcDef(bmod, s)
proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
@@ -263,20 +216,10 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
## and only needs each module's `(replay ...)` directives, which load anyway.
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile)
let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
if precompSys.module != nil:
# The precompiled-load path does not restore `sfSystemModule` (mirror of the
# `sfMainModule` re-add above). `registerReusedModuleToMain` keys on it to put
# the system module's init right after its datInit AND to emit
# `initStackBottomWith` into `mainDatInit` — so that the main thread's stack
# bottom is set before any module's init runs. Without the flag the system
# init is mis-routed into the regular `otherModsInit` bucket and
# `initStackBottomWith` is never registered, so a GC cycle during a module's
# init (under refc) scans the stack with a nil bottom and crashes.
incl precompSys.module.flagsImpl, sfSystemModule
var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)]
var modules = loadModuleDependencies(g, mainFileIdx, nifFiles, depFlags = {})
# loadModuleDependencies traverses the project's import closure and stops at
@@ -286,7 +229,7 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
# closure here too — otherwise `findTargetModule` cannot resolve their suffix.
block:
var visited = initHashSet[string]()
visited.incl systemNifSuffix(g.config)
visited.incl "sysma2dyk"
for m in modules:
visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position)
var stack: seq[ModuleSuffix] = @[]
@@ -329,14 +272,14 @@ proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
## dispatchers, runs essentially alone since every other `.c.nif` precedes it).
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile)
let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
let precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
var modules: seq[PrecompiledModule] = @[]
var visited = initHashSet[string]()
visited.incl systemNifSuffix(g.config)
visited.incl "sysma2dyk"
# Only the target is codegen'd, so only it needs its full AST; the closure is
# loaded interface-only (demanded bodies come lazily from the kept-open
@@ -380,213 +323,6 @@ proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix:
return precompSys
proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
owner: PSym; seen: var IntSet) =
## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting
## minted, plus any deeper nesting) gets its captured-var→env rewrite produced
## as part of the OWNER's `transformBody`. The nested proc is a module-indexed
## sym whose `.s.nif` sdef carries its PRE-lift body, so without help the whole
## module re-serializer would write that pre-lift body and cg would lose the
## capture mapping (it accesses `x` directly instead of `ClE_0->x0`). Walk the
## owner's transformed body and cache each nested closure's transformed body on
## its sym so `writeSymDef` serializes the lifted body into the routine's
## 2-way-body slot.
if n == nil: return
if n.kind == nkSym:
let s = n.sym
if s != nil and s.kind in routineKinds and s != owner and
s.skipGenericOwner != nil and s.skipGenericOwner.kind != skModule and
not seen.containsOrIncl(s.id):
# Covers ALL nested routines, not only ccClosure ones. A NIMCALL nested proc
# the async transform mints (e.g. workNimAsyncContinue) already has its
# lifted body set by the OWNER's transformBody, but it is NOT in the owned
# loop (owner is a proc, not the module). Without injecting it HERE it is
# serialized transform-only; cg loads it (wasLoaded) and skips injection, so
# a closure-env store stays a raw field assign with no incref -> the env is
# freed before the async callback runs -> "yielded nil". `seen` (shared
# across the owned loop) injects each routine exactly once.
if s.ast != nil and getBody(g, s).kind != nkEmpty:
# Only ccClosure routines are safe to `transformBody` standalone here; a
# nimcall nested proc already has its lifted body from the owner's lift,
# and transforming an arbitrary nested routine with no cached body crashes
# (not in a standalone-transformable state).
let weTransformed = s.transformedBody == nil and
s.typ != nil and s.typ.callConv == ccClosure
if weTransformed:
s.transformedBody = transformBody(g, idgen, s, {})
if s.transformedBody != nil:
# Inject destructors so cg loads a fully-lowered body and never rebuilds
# (mirrors non-IC, which injects every nested proc separately). The
# importer `n2` skField collision this used to trigger is fixed at the
# NIF-naming layer (toNifSymName gives derived env fields a unique
# disamb), so injecting ccClosure nested procs here is safe.
if sfInjectDestructors in s.flags:
s.transformedBody = injectDestructorCalls(g, idgen, s, s.transformedBody)
setNestedClosureBodies(g, idgen, s.transformedBody, s, seen)
else:
for i in 0 ..< n.safeLen:
setNestedClosureBodies(g, idgen, n[i], owner, seen)
proc reownFromTwin(n: PNode; twin, s: PSym) =
## Re-own to `s` every entity the frontend attributed to `s`'s forward-decl
## `twin` (found via the result's owner). lambda-lifting compares owners by
## reference, so a twin-owned `result` is rejected as `illegalCapture`
## ("'result' ... cannot be captured") and, once that is fixed, twin-owned
## locals go missing from `s`'s env ("environment misses: ..."). Both are
## pervasive on chronos `{.async.}` methods. Re-owning to `s` matches the
## single-sym non-IC case. `twin` is ONE specific sym, so only THIS routine's
## result-twin-owned entities match — re-owning entities of OTHER same-name
## twins proved too blunt (it disrupts env construction and reintroduces the
## very capture errors it should fix). `n.sym != s` guards self-ownership.
if n == nil: return
if n.kind == nkSym and n.sym != nil and n.sym != s and n.sym.owner == twin:
setOwner(n.sym, s)
for i in 0 ..< n.safeLen:
reownFromTwin(n[i], twin, s)
proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:<suffix>`):
## enumerate the routines this module OWNS and write them to `<module>.t.nif`.
## Eventually this transforms each owned routine once, in the owner's id space,
## so `cg` reads the result instead of re-deriving it (re-derivation per
## parallel `cg` process is the root of the closure-`:env` identity drift).
## Runs per module in parallel on the shallow backend dep-graph — NOT folded
## into the dense, mostly-serial sem stage.
##
## gate `newSymNode`'s lazy-type marking to the backend (see astdef) — the
## transform builds sym nodes off not-yet-typed stubs, exactly as the `cg`
## stage does.
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: module not found for suffix: " & g.config.icBackendModule)
return
let modPos = target.module.position
let tb = BModuleList(g.backend).mods[modPos]
if tb == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: no backend module for suffix: " & g.config.icBackendModule)
return
# Transform every owned routine ONCE in this single process's id space and
# re-serialize the ENTIRE module as a proper indexed NIF (`writeLoweredModule`)
# with the transformed bodies baked into the routine `(sd)` entries. `cg` loads
# it through the normal module loader, so nested procs (incl. async state
# machines) arrive as real defs with their lifted bodies — no re-derivation.
# This single-writer-per-owner is what keeps closure-`:env` identity stable
# across the parallel `cg` processes (re-derivation per process was the root of
# the `:env` identity drift). `transformBody` with flags {} mirrors the cg call
# (cgen.nim); `injectDestructorCalls` is NOT run here — it stays in `cg` on the
# loaded body.
#
# `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops
# (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly
# the ops THIS stage created (not those loaded from `.s.nif`).
let opsLogStart = g.opsLog.len
# Shared across the owned loop so a nested routine reachable from more than one
# owner is transformed + destructor-injected EXACTLY once (double injection
# would emit two `=destroy`/`=copy` runs).
var seenNested = initIntSet()
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
# REUSE path (`icReuseSemLowering` ON): a routine already transformed during
# sem (CT eval / macro / VM transform) carries its lowered body in the
# `.s.nif` slot (loaded into `transformedBody`) — don't re-transform it.
# Default OFF: the slot is never loaded (see loadSymFromCursor), so
# `transformedBody` is nil here and we always re-derive below. See
# doc/ic_backend_simplify.md §6a/§6b.
if icReuseSemLowering(g.config) and s.transformedBody != nil: continue
# A routine serialized as a forward-decl + impl pair (writeSymDef's
# "separate forward declaration and implementation") loads as TWO syms; the
# impl `s` we transform here can carry body entities (`result`, locals,
# nested routines) owned by its fwd-decl TWIN, not by `s`. lambda-lifting
# compares owners by reference → `illegalCapture` rejects a twin-owned
# `result` and the lifting pass can't find twin-owned locals in `s`'s env.
# Pervasive on chronos `{.async.}` methods. Re-own them to `s`, matching the
# single-sym non-IC case. Backend-only, so frontend effect/exception
# inference is untouched.
if s.ast != nil and s.ast.len > resultPos and
s.ast[resultPos].kind == nkSym and s.ast[resultPos].sym.owner != s:
reownFromTwin(s.ast, s.ast[resultPos].sym.owner, s)
# Retain the transformed body on the sym so `writeSymDef` serializes it in
# the routine's `(sd)` 2-way-body slot.
s.transformedBody = transformBody(g, tb.idgen, s, {})
# Run the destructor injection HERE so the `.t.bif` body is FULLY lowered:
# `injectDestructorCalls` is demand-driven (it decides where destructors go
# by move analysis) and LIFTS the type-bound ops it needs (e.g. a nested
# closure env's `=destroy`) into `g.opsLog` — which the `hooks` collection
# below then serializes. Done in `cg` instead, those ops were lifted per-cg
# process, owned by nobody, and emitted as a prototype-only → undefined at
# link (the `eqdestroy__c<n>` gap). cg must NOT re-inject a loaded body
# (see genProcLvl3's `wasLoaded` gate) so this stays the single injection.
if sfInjectDestructors in s.flags:
s.transformedBody = injectDestructorCalls(g, tb.idgen, s, s.transformedBody)
# Cache the lifted+injected body on nested ccClosure routines too, so a
# module-indexed nested closure serializes its lifted (capture-rewritten,
# destructor-injected) body.
setNestedClosureBodies(g, tb.idgen, s.transformedBody, s, seenNested)
# Collect the hooks this stage lifted, and transform each hook ROUTINE's body
# too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go
# into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves
# the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift
# further hooks (a field's `=destroy`).
var hooks: seq[LogEntry] = @[]
var i = opsLogStart
while i < g.opsLog.len:
let e = g.opsLog[i]
if e.kind == HookEntry and e.sym != nil and e.sym.kind in routineKinds and
e.sym.transformedBody == nil:
hooks.add e
# Transform the hook routine's body and cache it on the sym so `writeSymDef`
# serializes it in the hook's `(sd)` transformed-body slot (`transformBody
# {}` returns the body but does not cache it). Inject the hook's own
# destructors here too (it can destroy fields/temporaries) so cg loads a
# fully-lowered hook and never re-injects.
e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {})
if sfInjectDestructors in e.sym.flags:
e.sym.transformedBody = injectDestructorCalls(g, tb.idgen, e.sym, e.sym.transformedBody)
inc i
# Re-serialize the whole module to its suffix-based `.t.nif` (the path
# `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals
# routines itself.
let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.bif").string
writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " &
$hooks.len & " hooks"
proc visitDep(suffix: string;
suffixToMod: Table[string, PrecompiledModule];
visited: var HashSet[string]; bl: BModuleList;
ordered: var seq[BModule]) =
## Post-order DFS over a module's import closure used to reconstruct the
## dependency (init) order: a dependency's init must be registered before its
## importer's. Appends each reachable non-main module's `BModule` to `ordered`.
if visited.containsOrIncl(suffix): return
let pm = suffixToMod.getOrDefault(suffix)
if pm.module == nil: return
for dep in pm.deps: # dependencies first (post-order)
visitDep(dep.string, suffixToMod, visited, bl, ordered)
if sfMainModule notin pm.module.flags:
let bm = bl.mods[pm.module.position]
if bm != nil: ordered.add bm
proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:<suffix>`):
## generate C for the single module named by `icBackendModule` and write only
@@ -600,8 +336,6 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## module still loads everything (`loadBackendModules`) because NimMain's init
## list and the method dispatchers are whole-program; its `cg` runs essentially
## alone (every other `.c.nif` precedes it), so it does not contend for memory.
# gate `newSymNode`'s lazy-type marking to this stage only (see astdef)
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
@@ -629,10 +363,6 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
"per-module codegen: module not found for suffix: " & g.config.icBackendModule)
return
# The `lower` stage already wrote each module's transformed bodies + lifted
# hooks into its `.t.nif`, which the loaders above read directly (toNifFilename
# resolves the `.t.nif`); transformed bodies arrive via loadSymFromCursor and
# lifted hooks via moduleFromNifFile's registerLoadedHooks. Nothing to apply.
generateCodeForModule(g, target)
let bl = BModuleList(g.backend)
# The main module also owns the whole-program method dispatchers + NimMain.
@@ -643,58 +373,10 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# `cg` processes, so the calls are registered here from each `.c.nif` meta
# head — which is why the main module's `cg` runs last, after every other
# `.c.nif` exists. Modules without init code (no `.c.nif`) register nothing.
#
# The registration order IS the runtime init order, and it must be the
# DEPENDENCY (post-order) order: an imported module's init has to run before
# its importer's. The whole-program backend gets this for free — it iterates
# `modulesClosed`, built in module-FINISH order (a post-order DFS over
# imports). Iterating `bl.mods` by position is WRONG: an importer gets a
# LOWER position than the modules it imports (its file is registered before
# its `import` statements are processed), so position order runs importers
# before their dependencies. That left chronicles' `topics_registry` — whose
# init sets `mainThreadId` — running AFTER a module that calls `registerTopic`
# from its own init, tripping the `getThreadId() == mainThreadId` assert at
# startup. So reconstruct the post-order DFS over the import closure here.
#
# NOTE: this is deliberately a SEPARATE traversal rather than reusing the
# module LOAD order — the per-module backend's C emit is sensitive to load
# order (it determines the main TU's header composition), so the loader must
# keep its existing order and the init order is derived independently here.
var suffixToMod = initTable[string, PrecompiledModule]()
for pm in modules:
if pm.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex pm.module.position)] = pm
if precompSys.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex precompSys.module.position)] = precompSys
var visited = initHashSet[string]()
var ordered: seq[BModule] = @[]
# System (and its include/import closure) must initialize FIRST: its init
# runs `initGC()` (top-level code in `threadimpl`, included into system),
# and every other module's init may allocate — an allocation before the GC
# heap is set up triggers a collection over an uninitialized region and
# crashes (e.g. nim-metrics' `newRegistry` in its init). System is the
# IMPLICIT universal import and appears in no module's explicit `deps`, so a
# DFS rooted at main never reaches it; seed the traversal from system first.
if precompSys.module != nil:
visitDep(cachedModuleSuffix(g.config, FileIndex precompSys.module.position),
suffixToMod, visited, bl, ordered)
# Then order the whole import closure rooted at the main module; main itself
# is excluded above (its init body becomes NimMain).
for pm in modules:
if pm.module != nil and sfMainModule in pm.module.flags:
visitDep(cachedModuleSuffix(g.config, FileIndex pm.module.position),
suffixToMod, visited, bl, ordered)
# Defensive: any loaded module not reachable from main's import closure
# (demand-loaded system internals) keeps its init registered, appended last
# — nothing imports it, so its relative order does not matter.
for m in bl.mods:
if m != nil and sfMainModule notin m.module.flags:
let suffix = cachedModuleSuffix(g.config, FileIndex m.module.position)
if not visited.containsOrIncl(suffix):
ordered.add m
for m in ordered:
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let tb = bl.mods[target.module.position]
if tb != nil:
finishModule(g, tb)
@@ -748,46 +430,34 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
# emit renders a module's final `.c` PURELY from its own `.c.nif` and the merge
# decision (see `renderCFromArtifact` — text filtering, no AST is touched). It
# used to load the target's whole transitive import closure as BModules solely
# to reach `getCFile(bmod)` for the output path. Under the fire-all-every-edit
# merge barrier (every `emit` re-fires whenever `merge` bumps the decision's
# mtime — deliberate insurance so a decision change re-renders all `.c`
# consistently) that per-process `loadDepClosure` was the bulk of a warm
# rebuild's cost: 240 processes each re-parsing a module closure only to filter
# a handful of `.c.nif`s whose bytes are usually unchanged. Derive the `.c`
# path directly instead — the SAME pure computation `deps.nim.backendCFile`
# uses to DECLARE this stage's output (`getCFile` == that formula) — so an emit
# process loads nothing and the fire-all costs process-startup, not a graph load.
let cfilename =
if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx)
else: AbsoluteFile g.config.icBackendModule
let cfile = changeFileExt(completeCfilePath(g.config,
mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string
let artifact = cfile & ".nif"
if not fileExists(artifact):
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module emit: missing .c.nif artifact for suffix: " & g.config.icBackendModule)
"per-module emit: module not found for suffix: " & g.config.icBackendModule)
return
let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
return
let bmod = BModuleList(g.backend).mods[target.module.position]
let cfile = getCFile(bmod).string
let artifact = cfile & ".nif"
var dropped = 0
let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped)
# Write the `.c` content-stably. `merge` re-runs on any edit and bumps the
# decision file's mtime, so nifmake re-fires every `emit` (the filter is cheap);
# but the FILTERED output is usually byte-identical for modules unaffected by
# the edit. Rewriting it unconditionally would bump every `.c`'s mtime and make
# `callCCompiler` recompile every `.o`. Writing only on a real change preserves
# the mtime, so the C compiler recompiles exactly the modules whose `.c` changed
# — the same DCE model as Nimony's. Safe here (unlike a content-stable merge
# decision): a `.c` is a per-module LEAF consumed only by the C compiler's own
# up-to-date check, not a shared prerequisite in nifmake's mtime ordering.
if not fileExists(cfile) or readFile(cfile) != code:
writeFile(cfile, code)
writeFile(cfile, code)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
$dropped & " bodies (" & $code.len & " bytes)"
@@ -813,7 +483,6 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
if precompSys.module != nil:
replayBackendActions(g, precompSys.module, precompSys.topLevel)
let bl = BModuleList(g.backend)
var addedCFiles = initHashSet[string]()
for m in bl.mods:
if m != nil:
let cfile = getCFile(m)
@@ -821,53 +490,17 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# (extra members of system's closure that no build rule targets) had their
# code emit-everywhere'd into the targets, so they have no file to compile.
if not fileExists(cfile.string): continue
addedCFiles.incl extractFilename(cfile.string)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
# `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its
# SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so
# `callCCompiler` skips its compile but still links the existing object. This
# is what makes a localized edit recompile only the handful of `.c`s the
# `emit` stage actually rewrote, instead of every object every time — the
# final piece of per-module backend incrementality after the merge barrier.
addExternalFileToCompile(g.config, cf)
# deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build
# node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)`
# import) that the NIF-`deps` walk above never reaches because the condition is
# off. Such a node still emitted a `.c`, and it can OWN a live generic instance
# that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by
# `strutils.escape`) — so its body must be at link or that reference is
# undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol;
# a node that owns nothing live (a Windows-only winsock node on Linux) is
# correctly skipped.
block:
let nimcache = getNimcacheDir(g.config).string
let decision = readMergeDecision(nimcache / MergeDecisionFile)
if not decision.broken:
var liveOwners = initHashSet[string]()
for cname, owner in decision.owners:
if owner.endsWith(".c.nif") and cname in decision.live:
liveOwners.incl owner
for owner in liveOwners:
let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c"
if addedCFiles.containsOrIncl(cbase): continue
let cfile = AbsoluteFile(nimcache / cbase)
if not fileExists(cfile.string): continue
var cf = Cfile(nimname: cbase, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
addExternalFileToCompile(g.config, cf)
addFileToCompile(g.config, cf)
if g.config.cmd != cmdTcc:
extccomp.callCCompiler(g.config)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
if g.config.icBackendStage == "lower":
generateLowerStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "cg":
if g.config.icBackendStage == "cg":
generateCgStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "merge":
@@ -881,4 +514,4 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
return
else:
rawMessage(g.config, errGenerated,
"the per-module NIF backend requires --icBackendStage:lower|cg|merge|emit|link")
"the per-module NIF backend requires --icBackendStage:cg|merge|emit|link")

View File

@@ -16,7 +16,7 @@ import
import "../dist/nimony/src/lib" / nifbuilder
import "../dist/nimony/src/models" / nifler_tags
import icmodnames
import "../dist/nimony/src/gear2" / modnames
## This was copied from Nifler's bridge.nim. However, this code will evolve
## in a different direction as it needs to translate the semchecked AST which

View File

@@ -14,10 +14,6 @@ define:nimPreviewAsmSemSymbol
define:nimPreviewCStringComparisons
#define:nimPreviewDuplicateModuleError
# Incompatible with Nimony's compat2.nim for now
# NOTE: `-d:virtualParRi` (jump-encoded ParLe + elided ParRi) is NOT yet enabled:
# the IC writer assembles buffers by raw token splicing (`dest.add content[i]`),
# which does not seal scopes the way `addParRi` does, so sealed `(stmts)` get
# jump=0 and serialize empty. Enabling it needs writer buffer-sealing work first.
threads:off

View File

@@ -120,7 +120,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
# driver runs on the exact same config its children will. See icconfig.nim.
when not defined(nimKochBootstrap):
if conf.cmd in {cmdIc, cmdTrack}:
if conf.cmd == cmdIc:
ensureIcConfig(conf)
var graph = newModuleGraph(cache, conf)
@@ -134,7 +134,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
if conf.selectedGC == gcUnselected:
if conf.backend in {backendC, backendCpp, backendObjc} or
(conf.cmd in cmdDocLike and conf.backend != backendJs) or
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM, cmdTrack}:
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM}:
initOrcDefines(conf)
if conf.selectedStrings == stringSso and

View File

@@ -316,7 +316,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen:
if conf.cmd == cmdNimscript:
showHintConf()
conf.configFiles.setLen 0
if not conf.ideActive and conf.cmd notin {cmdCheck, cmdDump}:
if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}:
if conf.cmd == cmdNimscript:
runNimScriptIfExists(conf.projectFull, isMain = true)
else:

View File

@@ -29,7 +29,7 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "30"
icFormatVersion* = "6"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
@@ -140,7 +140,6 @@ type # please make sure we have under 32 options
optDocRaw # for documentation: Don't render markdown for JSON output
optItaniumMangle # mangling follows the Itanium spec
optCompress # turn on AST compression by converting it to NIF
optGenBif # generate semantic BIF alongside ordinary code generation
optWithinConfigSystem # we still compile within the configuration system
TGlobalOptions* = set[TGlobalOption]
@@ -184,6 +183,7 @@ type
cmdCheck # semantic checking for whole project
cmdM # only compile a single
cmdParse # parse a single file (for debugging)
cmdIdeTools # ide tools (e.g. nimsuggest)
cmdNimscript # evaluate nimscript
cmdDoc0
cmdDoc # convert .nim doc comments to HTML
@@ -206,7 +206,6 @@ type
cmdNifC # generate C code from NIF files
cmdIc # generate .build.nif for nifmake
cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif)
cmdTrack # `nim track --def/--usages`: IC frontend build + NIF scan for IDE queries
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
@@ -401,13 +400,6 @@ type
evalMacroCounter*: int
exitcode*: int8
cmd*: Command # raw command parsed as enum
ideActive*: bool # serving IDE tooling (nimsuggest): collect suggestions and
# keep going after errors. Decoupled from `cmd` so the IDE
# server can run under any compilation mode (cmdCheck, cmdM).
ideImportsFromNif*: bool # nimsuggest: load the unchanged import closure from
# precompiled NIF (run under cmdM) instead of recompiling it
# from source (cmdCheck). IC is opt-in: default off (cmdCheck);
# `--ideImports:nif` opts in.
cmdInput*: string # input command
projectIsCmd*: bool # whether we're compiling from a command input
implicitCmd*: bool # whether some flag triggered an implicit `command`
@@ -693,7 +685,6 @@ proc newConfigRef*(): ConfigRef =
command: "", # the main command (e.g. cc, check, scan, etc)
commandArgs: @[], # any arguments after the main command
commandLine: "",
ideImportsFromNif: false, # IC opt-in; see `--ideImports`
implicitImports: @[], # modules that are to be implicitly imported
implicitIncludes: @[], # modules that are to be implicitly included
docSeeSrcUrl: "",
@@ -795,18 +786,7 @@ template quitOrRaise*(conf: ConfigRef, msg = "") =
else:
quit(msg) # quits with QuitFailure
proc icReuseSemLowering*(conf: ConfigRef): bool {.inline.} =
## When ON, the per-module `lower` backend stage REUSES the VM/CT lowering that
## sem cached in the `.s.nif` 2-way-body slot (the non-IC single-lowering
## semantics) instead of re-deriving the transform. Default OFF: the backend
## re-derives every body from the pristine semchecked body (simpler; allowed by
## the 2026-06-27 spec that VM-requested frontend transforms need not influence
## the backend). The switch exists so caching can be restored if a target (e.g.
## Nimbus) depends on the cached lowering being reused, not re-derived. See
## doc/ic_backend_simplify.md §6b.
isDefined(conf, "icReuseSemLowering")
proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.ideActive or conf.cmd in cmdDocLike
proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in cmdDocLike + {cmdIdeTools}
proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc
proc usesSso*(conf: ConfigRef): bool {.inline.} = conf.selectedStrings == stringSso
@@ -932,8 +912,7 @@ proc getOsCacheDir(): string =
proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
proc nimcacheSuffix(conf: ConfigRef): string =
if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c`
elif conf.cmd == cmdCheck: "_check"
if conf.cmd == cmdCheck: "_check"
elif isDefined(conf, "release") or isDefined(conf, "danger"): "_r"
else: "_d"

View File

@@ -167,8 +167,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
s = stream
graph.interactive = stream.kind == llsStdIn
var topLevelStmts =
if {optCompress, optGenBif} * graph.config.globalOptions != {} or
graph.config.cmd == cmdM:
if optCompress in graph.config.globalOptions or graph.config.cmd == cmdM:
newNodeI(nkStmtList, module.info)
else:
nil
@@ -247,20 +246,11 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
# (imported modules should be loaded from existing NIF files). Members of the
# current strongly-connected import group (`--icGroup`) are the exception:
# they are compiled from source here, so each must write its own NIF.
let shouldWriteNif =
if graph.config.ideActive:
# nimsuggest (cmdM): persist NIF for cleanly-compiled, SAVED modules so
# later queries load them instead of recompiling. Never persist the
# actively edited buffer (it may hold unsaved/incomplete code) nor a
# module that failed to compile — that would poison the cache.
graph.config.cmd == cmdM and graph.config.errorCounter == 0 and
graph.config.m.fileInfos[module.position].dirtyFile.isEmpty
else:
({optCompress, optGenBif} * graph.config.globalOptions != {}) or
(graph.config.cmd == cmdM and
(sfMainModule in module.flags or
(graph.config.icGroup.len > 0 and
toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup)))
let shouldWriteNif = (optCompress in graph.config.globalOptions) or
(graph.config.cmd == cmdM and
(sfMainModule in module.flags or
(graph.config.icGroup.len > 0 and
toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup)))
if shouldWriteNif and not graph.config.isDefined("nimscript"):
topLevelStmts.add finalNode
# Collect replay actions from both pragma computations and VM state diff
@@ -299,31 +289,14 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
if not hasNil:
genericOffers.add (inst.sym.instantiatedFrom, inst.sym,
inst.concreteTypes, inst.genericParamsCount)
# Generic TYPE-instance OFFERS: every `tyGenericInst` THIS module created,
# so a consumer reuses its baked structure (array bounds etc.) rather than
# re-instantiating with a scope-divergent bound. See ast2nif.writeNifModule.
var typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[]
for genItemId, instList in graph.typeInstCache:
for inst in instList:
if inst != nil and inst.uniqueId.module == module.position and
inst.kidsLen > 0 and inst[0] != nil and
inst[0].kind == tyGenericBody and inst[0].sym != nil:
typeOffers.add (inst[0].sym, inst)
# The module's REAL resolved direct imports (incl. macro/template-generated
# ones with no surviving syntactic node). Passed to writeNifModule so the
# NIF `deps` section is complete (the backend closure walk needs it), and
# reused below for the `.s.deps` sidecar (frontend graph re-derivation).
let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[])
# The frontend's highest used itemId (max of the sym and type counters):
# the backend seeds its id minting ABOVE this so closure envs / RTTI hooks
# never share a `toId` with a frontend sym/type. See ast2nif `(unusedid)`.
let firstUnusedId = max(idgen.symId, idgen.typeId)
var expansions: seq[(PSym, TLineInfo)] = @[]
discard graph.nifExpansions.take(module.position.int32, expansions)
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId,
expansions)
genericOffers, resolvedImportDeps)
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]
@@ -369,31 +342,6 @@ proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PN
sect.add s.ast
setupCompileTimeVar(module, idgen, graph, sect)
proc finalizeLoadedModules(graph: ModuleGraph) =
## Apply the VM-level load effects of every module just loaded from a NIF —
## direct import OR dep-of-a-dep, both collected in `graph.pendingNifInit` by the
## loader (modulegraphs.moduleFromNifFile / loadTransitiveHooks). This is the ONE
## place that knows what loading a module does to global VM state, so a
## transitively-reached module (which never passes through this proc's caller)
## gets identical treatment. Modules are in dependency order (deps before
## dependents), which is the correct macro-cache replay order.
## 1. macro-cache replay: std/macrocache put/inc/add/incl recorded in the
## module's top level (pragma replay actions are a backend concern, skipped).
## 2. eager `{.compileTime.}` global init (see initLoadedCompileTimeGlobals).
## To add a new per-load effect, extend this proc — do not add a parallel buffer.
if graph.pendingNifInit.len == 0: return
for (m, topLevel) in graph.pendingNifInit:
if topLevel == nil: continue
var replayList = newNodeI(nkStmtList, m.info)
for n in topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
if replayList.len > 0:
replayStateChanges(m, graph, replayList)
initLoadedCompileTimeGlobals(graph, m, topLevel)
graph.pendingNifInit.setLen 0
proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym =
var flags = flags
if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
@@ -432,43 +380,57 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
toFullPath(graph.config, fileIdx) notin graph.config.icGroup):
let precomp = moduleFromNifFile(graph, fileIdx)
if precomp.module == nil:
if graph.config.ideActive:
# nimsuggest bootstrap: this import has no precompiled NIF yet (cold
# cache, or it was invalidated). Don't error — fall through to the
# source-compile path below; the pass-close emits a fresh NIF so the
# next query loads it instead of recompiling.
discard
else:
let nifPath = toNifFilename(graph.config, fileIdx)
# Macro-generated imports (e.g. chronicles' parseStmt("import
# chronicles/textlines") driven by the chronicles_sinks define) are
# invisible to the static scanner, so this module's NIF was never
# built. The importer already recorded this import via
# addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads
# it, re-derives the graph with the missing node + edge, and reruns
# the frontend. We still error — this process cannot finish sem
# without the import — but the discovery is structured data now, not
# a side-channel file.
for importer, deps in graph.importDeps.pairs:
var paths: seq[string] = @[]
for f in deps: paths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, importer.int32, paths)
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
return nil # Don't fall through to compile from source
let nifPath = toNifFilename(graph.config, fileIdx)
# Macro-generated imports (e.g. chronicles' parseStmt("import
# chronicles/textlines") driven by the chronicles_sinks define) are
# invisible to the static scanner, so this module's NIF was never
# built. The importer already recorded this import via
# addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads
# it, re-derives the graph with the missing node + edge, and reruns
# the frontend. We still error — this process cannot finish sem
# without the import — but the discovery is structured data now, not
# a side-channel file.
for importer, deps in graph.importDeps.pairs:
var paths: seq[string] = @[]
for f in deps: paths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, importer.int32, paths)
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
return nil # Don't fall through to compile from source
else:
# Module successfully loaded from NIF file - use it and skip processing
result = precomp.module
if sfSystemModule in flags:
graph.systemModule = result
partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx)))
# Apply the VM-level load effects of this module AND every dep it pulled in
# (moduleFromNifFile recorded them all in graph.pendingNifInit): macro-cache
# replay (else a NIF-loaded module's macro cache is lost — e.g.
# nim-serialization flavor registration) and eager `{.compileTime.}` global
# init. Uniform for direct and transitive deps — see finalizeLoadedModules.
finalizeLoadedModules(graph)
# Replay the module's recorded state changes: macro-cache operations
# (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader
# parsed them into `precomp.topLevel` (mixed with other top-level nodes),
# so filter to the replay actions. A loaded module's `ast` is never
# rebuilt, so this used to be skipped (`result.ast == nil`) and a
# NIF-loaded module's macro cache was lost — e.g. nim-serialization's
# flavor registration became invisible to dependents (`DefaultFlavor:
# automatic serialization is not enabled`).
var replayList = newNodeI(nkStmtList, result.info)
for n in precomp.topLevel:
# Only macro-cache ops (put/inc/add/incl). The pragma replay actions
# (compile/link/passc/hint/...) are a backend/link concern handled by
# the nifc closure, and re-emitting a loaded module's hints/warnings on
# every import would be wrong — so they are deliberately skipped here.
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
# Plus the macro-cache actions of the module's transitive import closure
# (collected by the moduleFromNifFile call above via loadTransitiveHooks),
# so a flavor/type registered in an indirectly-imported module is visible.
for n in graph.transitiveReplayActions: replayList.add n
graph.transitiveReplayActions.setLen 0
if replayList.len > 0:
replayStateChanges(result, graph, replayList)
# Fill the VM slots of the module's `{.compileTime.}` globals now (sem
# would have, but a NIF-loaded module is never semchecked).
initLoadedCompileTimeGlobals(graph, result, precomp.topLevel)
return result # Return early, don't process from source
let path = toFullPath(graph.config, fileIdx)
let filename = AbsoluteFile path
@@ -558,40 +520,14 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
graph.config.libpath / RelativeFile"system.nim")
when not defined(nimKochBootstrap):
# Don't clobber an already-compiled system: nimsuggest's NimScript config
# evaluation compiles `system` into this same graph before we get here.
let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx)
graph.systemModule = precomp.module
if graph.systemModule == nil:
let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx)
graph.systemModule = precomp.module
if graph.systemModule == nil:
if graph.config.ideActive:
# nimsuggest bootstrap: no system NIF yet — compile it from source
# (the pass-close emits it), then continue with the main module.
graph.compilePipelineSystemModule()
else:
let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx)
localError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for system module (expected: " & nifPath & ")")
return
# Apply system's (and its deps') load effects now: the main module is
# compiled from source and never re-enters the moduleFromNifFile drain for
# system, so without this its macro-cache / CT globals would wait until the
# first NIF import is processed. See finalizeLoadedModules.
finalizeLoadedModules(graph)
let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx)
localError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for system module (expected: " & nifPath & ")")
return
discard graph.compilePipelineModule(projectFile, {sfMainModule})
# A batch (`--icGroup`) may hold several independent "top" modules that are
# not all reachable by import from the representative project file: a dirty
# module together with its users forms a DAG, not a cycle, so descending
# from one rep need not touch every member. Compile each remaining member
# explicitly so it writes its NIF. compilePipelineModule is idempotent
# (returns the cached module for one already reached through the rep's
# imports), and an unreached member is in `icGroup` so it is source-compiled
# here rather than NIF-loaded; resolving it pulls in its in-batch deps on
# demand, so no explicit ordering is needed.
for path in graph.config.icGroup:
let memberIdx = fileInfoIdx(graph.config, AbsoluteFile path)
if memberIdx != projectFile:
discard graph.compilePipelineModule(memberIdx, {})
else:
graph.compilePipelineSystemModule()
discard graph.compilePipelineModule(projectFile, {sfMainModule})

View File

@@ -77,7 +77,7 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode;
# templates perform some quick check whether the cursor is actually in
# the generic or template.
when defined(nimsuggest):
if c.config.ideActive and requiresCheck:
if c.config.cmd == cmdIdeTools and requiresCheck:
#if optIdeDebug in gGlobalOptions:
# echo "passing to safeSemExpr: ", renderTree(n)
discard safeSemExpr(c, n)
@@ -89,18 +89,6 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
changeType(c, x, formal, check=true)
result = arg
result = skipHiddenSubConv(result, c.graph, c.idgen)
# Walk through nested statement-list/block expressions to find the innermost
# value node. Empty containers (e.g. `@[]`) inside `nkStmtListExpr` wrappers
# need their type resolved to match the formal type, otherwise the C codegen
# cannot map `tyEmpty` to a concrete type (fixes #25945).
var tail = result
while tail.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkPragmaBlock} and tail.len > 0:
tail = tail.lastSon
if tail.typ != nil and tail.typ.isEmptyContainer and
formal.kind notin {tyUntyped, tyBuiltInTypeClass, tyAnything}:
changeType(c, tail, formal, check=true)
# mark inserted converter as used:
var a = result
if a.kind == nkHiddenDeref: a = a[0]
@@ -288,14 +276,6 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
result = copySym(result)
result.ast = n.sym.ast
put(c.p, n.sym, result)
if result.state == Sealed:
# the symbol was loaded from another module's NIF cache (e.g. a param
# symbol spliced out of an imported proc type by a `typed` macro) and is
# therefore immutable; the caller re-owns it and assigns its type/flags,
# so hand back a fresh, mutable copy owned by the current module instead.
let fresh = copySym(result, c.idgen)
fresh.ast = result.ast
result = fresh
# when there is a nested proc inside a template, semtmpl
# will assign a wrong owner during the first pass over the
# template; we must fix it here: see #909
@@ -584,12 +564,10 @@ const
proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
let info = getCallLineInfo(n)
# the callee identifier's position is the usage site tooling expects (matches
# `markUsed` below), not the whole-call `nOrig.info`.
rememberExpansion(c, info, sym)
rememberExpansion(c, nOrig.info, sym)
pushInfoContext(c.config, nOrig.info, sym.detailedInfo)
let info = getCallLineInfo(n)
markUsed(c, info, sym)
onUse(info, sym)
if sym == c.p.owner:
@@ -896,7 +874,7 @@ proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
result = hloStmt(c, result)
if c.config.cmd == cmdInteractive and not isEmptyType(result.typ):
result = buildEchoStmt(c, result)
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
appendToModule(c.module, result)
trackStmt(c, c.module, result, isTopLevel = true)
if optMultiMethods notin c.config.globalOptions and
@@ -933,7 +911,7 @@ proc semWithPContext*(c: PContext, n: PNode): PNode =
result = nil
else:
result = newNodeI(nkEmpty, n.info)
#if c.config.ideActive: findSuggest(c, n)
#if c.config.cmd == cmdIdeTools: findSuggest(c, n)
proc reportUnusedModules(c: PContext) =
if c.config.cmd == cmdM: return
@@ -942,7 +920,7 @@ proc reportUnusedModules(c: PContext) =
message(c.config, info, warnUnusedImportX, s.name.s)
proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode =
if c.config.ideActive and not c.suggestionsMade:
if c.config.cmd == cmdIdeTools and not c.suggestionsMade:
suggestSentinel(c)
closeScope(c) # close module's scope
rawCloseScope(c) # imported symbols; don't check for unused ones!

View File

@@ -382,9 +382,8 @@ proc addImportFileDep*(c: PContext; f: FileIndex) =
if f notin deps[]: deps[].add f
proc addPragmaComputation*(c: PContext; n: PNode) =
# Also store whenever the semchecked module is serialized to NIF/BIF.
if {optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.cmd == cmdM:
# Also store for NIF-based IC (cmdM mode or optCompress)
if optCompress in c.config.globalOptions or c.config.cmd == cmdM:
addNifReplayAction(c.graph, c.module.position.int32, n)
proc inclSym(sq: var seq[PSym], s: PSym): bool =
@@ -669,15 +668,7 @@ proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) =
## ("find all usages of this template" would not work). We need special
## logic to remember macro/template expansions. This is done here and
## delegated to the "NIF" file mechanism.
##
## We only bother when a NIF file is actually going to be written (IC / `nim m`,
## `--compress`, semantic BIF output, or a running suggestion engine); a plain
## `nim c` throws the record away, so recording it would be pure overhead.
if info.fileIndex == InvalidFileIdx: return
if c.config.cmd == cmdM or
{optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.ideActive:
c.graph.nifExpansions.mgetOrPut(c.module.position.int32, @[]).add (expandedSym, info)
discard "XXX To implement"
const
errVarForOutParamNeededX = "for a 'var' type a variable needs to be passed; but '$1' is immutable"

View File

@@ -26,15 +26,13 @@ const
proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
let info = getCallLineInfo(n)
# `info` (the callee identifier's position, not the whole call node) is what
# tooling wants to see as the usage site — matches `markUsed` below.
rememberExpansion(c, info, s)
rememberExpansion(c, n.info, s)
# IC: this expands `s`'s body into the current module's sem, so the module
# depends on that body — record a NeedsImpl (strong) edge to `s`'s module.
# The iface cookie hashes only signatures now, so a template body edit moves
# only the impl cookie, and just the modules that expanded it re-sem.
recordIcImplDep(c.graph, s)
let info = getCallLineInfo(n)
markUsed(c, info, s)
onUse(info, s)
# Note: This is n.info on purpose. It prevents template from creating an info
@@ -1573,7 +1571,7 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode =
# here at all!
#if isSymChoice(n[1]): return
when defined(nimsuggest):
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
suggestExpr(c, n)
if exactEquals(c.config.m.trackPos, n[1].info): suggestExprNoCheck(c, n)
@@ -3407,7 +3405,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
c.config.expandNodeResult = $n
suggestQuit()
if c.config.ideActive: suggestExpr(c, n)
if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
if nfSem in n.flags: return
case n.kind
of nkIdent, nkAccQuoted:

View File

@@ -476,12 +476,7 @@ proc foldArrayAccess(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNo
#localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n)
of nkBracket:
idx -= toInt64(firstOrd(g.config, x.typ))
if isDefaultBroadcastArray(x, g.config):
# compact default array: any in-bounds index folds to the default element
if idx >= 0 and idx < toInt64(lengthOrd(g.config, x.typ.skipTypes(abstractInst))):
result = copyTree(x[0])
else: result = nil
elif idx >= 0 and idx < x.len: result = x[int(idx)]
if idx >= 0 and idx < x.len: result = x[int(idx)]
else:
result = nil
#localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n)

View File

@@ -273,7 +273,7 @@ proc semGenericStmt(c: PContext, n: PNode,
when defined(nimsuggest):
if withinTypeDesc in flags: inc c.inTypeContext
#if conf.ideActive: suggestStmt(c, n)
#if conf.cmd == cmdIdeTools: suggestStmt(c, n)
semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
case n.kind

View File

@@ -93,7 +93,6 @@ type
graph: ModuleGraph
c: PContext
escapingParams: IntSet
inNimvmBranch: int
PEffects = var TEffects
const
@@ -1118,56 +1117,6 @@ proc trackCall(tracked: PEffects; n: PNode) =
#if canRaise(a):
# echo "this can raise ", tracked.config $ n.info
let op = a.typ
# A routine whose body reaches a compile-time-only magic (`macros.error`,
# `slurp`, `gorge`, `getAst`, …) can never be code-generated — the C/JS
# backends reject those magics (ccgexprs `errXMustBeCompileTime`). Such a
# routine is compile-time-only by construction; mark it `sfCompileTime` so it
# is treated uniformly as such. Non-IC pruned it by demand-driven codegen, but
# the per-module IC backend emits every owned routine (no DCE) and would
# otherwise feed the magic to codegen. Mirrors the `tfTriggersCompileTime ->
# sfCompileTime` path in `semProcAux`.
#
# GATE TO THE IC STAGES ONLY (`cmdM` sem + `cmdNifC` cg). The magic can reach a
# runtime proc's body via an INLINED TEMPLATE (not a macro/template *owner*, so
# the `insideMeta` walk below can't see it) — e.g. confutils' runtime
# `addConfigFile`/json-serialization's `inputFile` expand a serialization
# template that pastes a `getAst`/`quote` magic inline. Under plain `nim c` such
# a proc still code-generates fine (the magic folds / is demand-pruned), so
# marking it `sfCompileTime` there is a pure regression: "request to generate
# code for .compileTime proc". Only the emit-everything IC backend needs the
# mark, so restrict it to `{cmdM, cmdNifC}` (was `!= cmdNimscript`, which
# wrongly swept in `cmdCompileToC`/JS/`cmdCheck`).
if a.kind == nkSym and a.sym.magic in {mNLen..mNError, mSlurp..mQuoteAst} and
tracked.owner != nil and tracked.owner.kind in routineKinds and
tracked.config.cmd in {cmdM, cmdNifC} and tracked.inNimvmBranch == 0:
# ...but NOT under `nim e`: nimscript has no codegen backend to protect, and
# marking a routine `sfCompileTime` makes `semExpr` eagerly fold calls to it
# at sem time (emConst), where module-level globals it reads have no VM slot
# yet — distros' `detectOsWithAllCmd` reaches `gorge` and reads the plain
# global `unameRes` → "cannot evaluate at compile time: unameRes". In the
# normal nimscript run (emRepl) the module's var section runs first and the
# slot exists, so the marking is both unnecessary and harmful here.
#
# ...and NOT if the routine is — or is nested inside — a macro/template:
# those are VM-only (never code-generated), so the per-module IC backend has
# nothing to protect there, while `sfCompileTime` on a macro-internal nested
# closure breaks its captured-variable access in the VM ("cannot evaluate at
# compile time: n" — `tests/macros/tmacros1`'s `innerProc` reading the
# macro-local `n`). Walk the owner chain and bail on the first
# skMacro/skTemplate. NB mark `tracked.owner` (the routine that directly
# reaches the magic), NOT its outermost enclosing: a runtime proc may legally
# nest a compile-time helper — `tests/generics/tunique_type`'s `[]` proc
# contains a nested `buildResult` macro — and marking the proc would wrongly
# make IT compile-time ("request to generate code for .compileTime proc: []").
var encl = tracked.owner
var insideMeta = false
while encl != nil and encl.kind != skModule:
if encl.kind in {skMacro, skTemplate}:
insideMeta = true
break
encl = encl.skipGenericOwner
if not insideMeta:
incl(tracked.owner, sfCompileTime)
if n.typ != nil:
if tracked.owner.kind != skMacro and n.typ.skipTypes(abstractVar).kind != tyOpenArray:
createTypeBoundOps(tracked, n.typ, n.info)
@@ -1209,17 +1158,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
else:
if laxEffects notin tracked.c.config.legacyFeatures and a.kind == nkSym and
a.sym.kind in routineKinds:
# A hook reaching here has no effect list yet, i.e. it has not been
# effect-tracked. Propagating from its (still unset) type flags would
# spuriously mark the caller GC-unsafe/side-effecting: e.g. under
# `nim ic` a concrete `=destroy` reached through a generic
# instantiation is not analyzed before the instance body is tracked
# here. Skip all such hooks (generalizes #25940, which special-cased
# `=asgn`/`=sink`/`=dup`); once analyzed they carry an effect list and
# take the branch below.
let (isHook, _) = findHookKind(a.sym.name.s)
if not isHook:
propagateEffects(tracked, n, a.sym)
propagateEffects(tracked, n, a.sym)
else:
mergeRaises(tracked, effectList[exceptionEffects], n)
mergeTags(tracked, effectList[tagEffects], n)
@@ -1503,9 +1442,7 @@ proc track(tracked: PEffects, n: PNode) =
of nkCaseStmt: trackCase(tracked, n)
of nkWhen: # This should be a "when nimvm" node.
let oldState = tracked.init.len
inc tracked.inNimvmBranch
track(tracked, n[0][1])
dec tracked.inNimvmBranch
tracked.init.setLen(oldState)
track(tracked, n[1][0])
of nkIfStmt, nkIfExpr: trackIf(tracked, n)
@@ -1649,11 +1586,10 @@ proc track(tracked: PEffects, n: PNode) =
message(tracked.config, n.info, warnPtrToCstringConv,
$n[1].typ)
# Check for implicit range conversions. Compile-time constants are already
# fully known here, so only non-constant values need the downsizing warning.
# Check for implicit range conversions
if n.kind == nkHiddenStdConv and (not tracked.isArrayIndexing) and
shouldWarnRangeConversion(tracked.config, n.info, n.typ, n[1].typ) and
getConstExpr(tracked.ownerModule, n[1], tracked.c.idgen, tracked.graph) == nil:
n[1].kind notin {nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit} and
shouldWarnRangeConversion(tracked.config, n.info, n.typ, n[1].typ):
message(tracked.config, n.info, warnImplicitRangeConversion,
typeToString(n[1].typ) & " -> " & typeToString(n.typ))
@@ -1784,18 +1720,13 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode; s: PSym = nil) =
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):
effects[exceptionEffects] = newNodeI(nkArgList, effects.info)
let forbidsSpec = effectSpec(n, wForbids)
let tagsSpec = effectSpec(n, wTags)
if not isNil(tagsSpec):
effects[tagEffects] = tagsSpec
elif not isNil(forbidsSpec):
# `.forbids` without `.tags` still declares a known empty tag set.
# Leaving this as nil would mean "unknown tags", which later widens
# indirect calls to `RootEffect`.
effects[tagEffects] = newNodeI(nkArgList, effects.info)
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):
effects[tagEffects] = newNodeI(nkArgList, effects.info)
let forbidsSpec = effectSpec(n, wForbids)
if not isNil(forbidsSpec):
effects[forbiddenEffects] = forbidsSpec
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):

View File

@@ -531,7 +531,7 @@ proc semUsing(c: PContext; n: PNode): PNode =
if not isTopLevel(c): localError(c.config, n.info, errXOnlyAtModuleScope % "using")
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkIdentDefs, nkVarTuple, nkConstDef}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -838,7 +838,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkIdentDefs, nkVarTuple}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -994,7 +994,7 @@ proc semConst(c: PContext, n: PNode): PNode =
var b: PNode
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkConstDef, nkVarTuple}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -1535,7 +1535,7 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
while i < n.len: # n may grow due to type pragma macros
var a = n[i]
when defined(nimsuggest):
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
inc c.inTypeContext
suggestStmt(c, a)
dec c.inTypeContext
@@ -1812,7 +1812,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
var remainingOwners = initIntSet()
for (owner, _, _) in c.forwardTypeUpdates:
remainingOwners.incl owner.id
while c.forwardTypeUpdates.len > 0:
let pending = move c.forwardTypeUpdates
var madeProgress = false
@@ -1829,7 +1829,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
c.forwardTypeUpdates.add (owner, typ, typeNode)
elif not remainingOwners.missingOrExcl(owner.id):
madeProgress = true
if not madeProgress:
# can't error here unfortunately
break
@@ -2621,17 +2621,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
addParams(c, proto.typ.n, proto.kind)
proto.info = s.info # more accurate line information
proto.options = s.options
# `s` (the impl symbol) is discarded in favour of `proto`. It still carries
# `s.ast == n` (set above) and stays reachable as the owner of body-local
# symbols, so under IC it would be serialized as a SECOND, body-bearing
# `proc` entry — a phantom duplicate of `proto`. The per-module backend then
# codegens that phantom, whose `result` is owned by `proto` (addResult below
# re-parents it), not by the phantom: lambdalifting's capture check
# (`result.skipGenericOwner != owner`) then wrongly classifies `result` as a
# captured outer variable → "'result' … cannot be captured". Drop the
# discarded impl's body so it can never be emitted as a routine (same leak
# class the `miscPos` adoption below guards against for generic params).
let discardedImpl = s
s = proto
n[genericParamsPos] = proto.ast[genericParamsPos]
n[paramsPos] = proto.ast[paramsPos]
@@ -2649,19 +2638,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
if importantComments(c.config) and proto.ast.comment.len > 0:
n.comment = proto.ast.comment
proto.ast = n # needed for code generation
if discardedImpl != proto:
discardedImpl.ast = nil
# The impl symbol is discarded in favour of `proto`, but it stays `Complete`
# in this module, so `ast2nif.shouldWriteSymDef` still serializes it. With
# `sfExported` it would be written importable (`x` marker) and an importer
# would load BOTH it and `proto` into the overload set: "ambiguous call;
# both foo and foo" (identical signatures). Normally a discarded impl is a
# gensym/transient that isn't reached this way, but a `{.async: (raises).}`
# forward-decl + impl reconciles HERE with both syms exported. Strip the
# export so the design's "forward declarations are never importable" holds —
# the def still serializes (other refs may resolve to it) but is invisible
# to importer overload resolution; `proto` carries the export.
excl(discardedImpl, sfExported)
popOwner(c)
pushOwner(c, s)
@@ -2892,8 +2868,7 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult, resolvedIncStmt
proc evalInclude(c: PContext, n: PNode): PNode =
result = newNodeI(nkStmtList, n.info)
var resolvedIncStmt: PNode = nil
if {optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.cmd == cmdM:
if optCompress in c.config.globalOptions:
# New resolve the include filenames to string literals that contain absolute paths,
# nicer for IC:
resolvedIncStmt = newNodeI(nkIncludeStmt, n.info)

View File

@@ -2233,7 +2233,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
result = nil
inc c.inTypeContext
if c.config.ideActive: suggestExpr(c, n)
if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
case n.kind
of nkEmpty: result = n.typ
of nkTypeOfExpr:

View File

@@ -294,22 +294,13 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT
replaceTypeVarsS(cl, n.sym, result.typ)
else:
replaceTypeVarsS(cl, n.sym, replaceTypeVarsT(cl, n.sym.typ))
if result.sym.kind == skField and
if result.sym.kind == skField and result.sym.ast != nil and
(cl.owner == nil or result.sym.owner == cl.owner):
if result.sym.ast != nil:
# instantiate default value of object/tuple field
var n = result.sym.ast
cl.c.fitDefaultNode(cl.c, n, result.sym.typ)
result.sym.ast = n
result.sym.typ = n.typ.skipIntLit(cl.c.idgen)
elif result.typ != nil:
# The field SYM can be SHARED across the branches of an `nkRecWhen` (the
# generic body reuses one `value` PSym, so it carries the LAST branch's
# type), while the resolved field NODE carries the correct branch type.
# Sync the sym to the node so the instantiated field's sym-type and
# node-type agree (else a generic-object instance serializes a field
# whose sym-type diverges from its node-type -> loader/computeSize crash).
result.sym.typ = result.typ
# instantiate default value of object/tuple field
var n = result.sym.ast
cl.c.fitDefaultNode(cl.c, n, result.sym.typ)
result.sym.ast = n
result.sym.typ = n.typ.skipIntLit(cl.c.idgen)
# sym type can be nil if was gensym created by macro, see #24048
if result.sym.typ != nil and result.sym.typ.kind == tyVoid:
# don't add the 'void' field
@@ -489,11 +480,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
else:
header = instCopyType(cl, t)
# The instantiating module owns the instance (and announces it as an offer):
# the generic body's module (`t.genericHead.owner`) has no business owning a
# type that references instantiation-site types — that is the IC parent->child
# heap leak the write-barrier surfaces.
result = newType(tyGenericInst, cl.c.idgen, cl.c.module, son = header.genericHead)
result = newType(tyGenericInst, cl.c.idgen, t.genericHead.owner, son = header.genericHead)
result.flags = header.flags
# be careful not to propagate unnecessary flags here (don't use rawAddSon)
# ugh need another pass for deeply recursive generic types (e.g. PActor)
@@ -847,21 +834,15 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false):
# trough replaceObjBranches in order to resolve any pending nkRecWhen nodes
result = t
# Slow path, we have some work to do. CRUCIAL: only ever mutate a type that
# is LOCAL to the module we are instantiating in (`uniqueId.module ==
# idgen.module`). A type loaded from another module's NIF (foreign) already
# had its object branches resolved when it was originally compiled; mutating
# it in place here is an old→new heap write that re-homes the loaded type to
# the instantiation site (its sym then looks owned by the consumer module and
# loses its `info`, colliding C type names — the libp2p `Message` bug). The
# prior `state != Sealed` guard was insufficient: a freshly-LOADED type is
# `Complete`, not `Sealed` (`Sealed` only means "already re-written to a NIF").
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and
t.elementType.n != nil and t.elementType.uniqueId.module == cl.c.idgen.module.int:
# Slow path, we have some work to do
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil:
discard replaceObjBranches(cl, t.elementType.n)
elif result.n != nil and t.kind == tyObject and result.state != Sealed and
result.uniqueId.module == cl.c.idgen.module.int:
elif result.n != nil and t.kind == tyObject and result.state != Sealed:
# A type loaded from the IC cache already had its object branches
# resolved when it was originally compiled, and must not be mutated in
# place (nor copied, which would break object-inheritance identity), so
# only non-Sealed types are processed here.
# Invalidate the type size as we may alter its structure
result.size = -1
result.n = replaceObjBranches(cl, result.n)

View File

@@ -10,7 +10,6 @@
## Computes hash values for routine (proc, method etc) signatures.
import ast, ropes, modulegraphs, options, msgs, pathutils
from lineinfos import FileIndex
from std/hashes import Hash
import std/tables
import types
@@ -75,19 +74,7 @@ proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) =
c &= ":anon"
else:
var it = s
# The source file path disambiguates same-named object types from different
# modules whose owner-chain names also coincide (e.g. libp2p kademlia/protobuf
# `Message` vs rendezvous/protobuf `Message`, both modules named `protobuf`).
# A type sym that reaches the backend as a `Complete` stub never individually
# loaded carries `unknownLineInfo` (fileIndex -1), which `toFullPath` collapses
# to the `???` placeholder — so the two would hash to ONE mangled C name and the
# wrong struct gets emitted. Fall back to the sym's HOME module file (its
# per-module NIF-suffix path, stable+unique) for the path. Only fires on a -1
# fileIndex; non-IC type syms always have a real `info`, so the fast path is
# taken and the hash is unchanged (koch boot byte-equal).
let infoFi = s.info.fileIndex
let pathFi = if infoFi.int32 >= 0'i32: infoFi else: s.itemId.module.int32.FileIndex
c &= customPath(conf.toFullPath(pathFi))
c &= customPath(conf.toFullPath(s.info))
when defined(icDbgHash):
var ownerSteps = 0
while it != nil:
@@ -203,10 +190,9 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
if CoConsiderOwned in flags:
c &= char(t.kind)
c.hashType t.skipModifier, flags, conf
of tyBool, tyChar, tyPointer, tyCstring, tyInt..tyUInt64:
# no canonicalization for builtin scalar-ish / pointer-like types, so
# that e.g. ``pid_t`` or an imported ``pointer`` alias keep their
# backend spelling instead of collapsing into the generic Nim builtin:
of tyBool, tyChar, tyInt..tyUInt64:
# no canonicalization for integral types, so that e.g. ``pid_t`` is
# produced instead of ``NI``:
c &= char(t.kind)
if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
c.hashSym(t.sym)
@@ -274,7 +260,6 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
c.hashTree(t.n, {}, conf)
of tyTuple:
c &= char(t.kind)
c &= t.len
if t.n != nil and CoType notin flags:
for i in 0..<t.n.len:
assert(t.n[i].kind == nkSym)
@@ -548,3 +533,4 @@ proc idOrSig*(s: PSym, currentModule: string,
if counter != 0:
result.add "_" & rope(counter+1)
sigCollisions.inc(sig)

View File

@@ -911,7 +911,7 @@ proc suggestDecl*(c: PContext, n: PNode; s: PSym) =
defer:
if attached: dec(c.inTypeContext)
# If user is typing out an enum field, then don't provide suggestions
if s.kind == skEnumField and c.config.ideActive and exactEquals(c.config.m.trackPos, n.info):
if s.kind == skEnumField and c.config.cmd == cmdIdeTools and exactEquals(c.config.m.trackPos, n.info):
suggestQuit()
suggestExpr(c, n)

View File

@@ -1386,33 +1386,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = getBody(g, prc)
else:
prc.transformedBody = newNode(nkEmpty) # protects from recursion
# Lambda-lifting a routine body while the VM compiles it (to run a macro
# under `nim ic`) mints a closure `:env` (type + obj + fields + hidden param)
# that the lift welds into the routine's serialized signature. Such an env is
# a PROCESS-LOCAL artifact (its item number is per-process-sequential), so a
# reference to it must never carry a stable cross-module identity — otherwise
# a consumer resolves it against a canonical NIF built by a different process
# that has no matching def ('symbol has no offset', e.g. Nimbus t17.275).
# Lift in the backend (process-local) id space; ast2nif then emits these as
# module-local `@bk` defs (mirrors setAttachedOp's inVMTransform handling).
var liftIdgen = idgen
if g.inVMTransform > 0 and g.config.cmd == cmdM:
if g.vmTransfIdgen == nil:
g.vmTransfIdgen = idGeneratorForBackend(g.systemModule)
liftIdgen = g.vmTransfIdgen
var c = openTransf(g, prc.getModule, "", liftIdgen, flags)
# `liftCapturedVars` rewrites captured locals to `:env.field` IN PLACE on the
# body it is handed; the env-creation prologue lands only in the returned
# wrapper. When the VM drives this transform (running a macro/CT proc), that
# in-place mutation corrupts the routine's PRE-transform `ast[bodyPos]` —
# under IC exactly the node `getBody` serializes to the module's `.s.nif`. So
# snapshot the pristine body before the VM lift and restore `ast[bodyPos]`
# afterwards: the VM still consumes the fully-lifted `result`, but `getBody`
# keeps faithfully returning the pre-transform body for serialization. The
# cg/backend path (`inVMTransform == 0`) is untouched.
let vmPristineBody =
if g.inVMTransform > 0: copyTree(getBody(g, prc))
else: nil
var c = openTransf(g, prc.getModule, "", idgen, flags)
result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, flags)
result = processTransf(c, result, prc)
liftDefer(c, result)
@@ -1422,8 +1396,6 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = g.transformClosureIterator(c.idgen, prc, result)
incl(result.flags, nfTransf)
if vmPristineBody != nil:
prc.ast[bodyPos] = vmPristineBody
if useCache in flags or prc.typ.callConv == ccInline:
# genProc for inline procs will be called multiple times from different modules,

View File

@@ -13,7 +13,7 @@
import std/[assertions, sets]
import "../dist/nimony/src/lib" / [treemangler]
import icmodnames
import "../dist/nimony/src/gear2" / modnames
import astdef, idents, options, lineinfos, msgs
import ic / [enum2nif]
@@ -47,11 +47,6 @@ type
CoConsiderOwned
CoDistinct
CoHashTypeInsideNode
CoPrecise # produce a FRONTEND-faithful, unique key (for the
# stable NIF *name*, not hook dedup): keep distinctions
# sem makes that the backend identity collapses — e.g.
# an `int literal(x)` type carries its value so it does
# not merge with `int`. See `getTypeKey`/`typeKeyHook`.
TypeLoader* = proc (t: PType) {.nimcall.}
SymLoader* = proc (s: PSym) {.nimcall.}
@@ -144,39 +139,6 @@ proc maybeImported(c: var Context; s: PSym; conf: ConfigRef) {.inline.} =
if s != nil and {sfImportc, sfExportc} * s.flagsImpl != {}:
c.symKey(s, conf)
proc emitPreciseFlags(c: var Context; t: PType; flags: set[ConsiderFlag]) {.inline.} =
## Under CoPrecise the NIF *name* must be as fine as `types.sameType`, which
## compares `eqTypeFlags * flags` (see `sameFlags`). Without this a
## `proc() {.gcsafe.}` keyed identically to `proc()`, and a `ref X not nil`
## identically to `ref X` — the loader would then merge two frontend-distinct
## types onto one NIF name. Emitted only for the naming path (CoPrecise); the
## `setAttachedOp` hook key (no CoPrecise) is unaffected, so its byte layout is
## unchanged.
if CoPrecise in flags:
let ef = eqTypeFlags * t.flagsImpl
if ef != {}:
withTree c.m, "´tflags":
for f in ef: c.m.addIntLit ord(f)
proc backendTypeName(t: PType; conf: ConfigRef): string =
## Stable cross-module identity of a backend-minted (lower-stage) type: its
## serialized `@bk` NIF name (mirrors ast2nif.nifTypeName). A closure-env
## object/ref minted by the `lower` stage has NO stable STRUCTURAL key — its
## captured-field types re-resolve to different modules in the producing vs the
## consuming process (e.g. field `x0` → `int` in the producer, → the consumer's
## alias in the consumer) — but this name (kind + item + home-module suffix) is
## identical in both, because the consumer loads the producer's name verbatim.
## Keying hooks by it makes producer `setAttachedOp` and consumer `getAttachedOp`
## agree. The trailing `@bk` (= ast2nif.BackendLocalMarker) keeps it disjoint
## from any normal type's structural key.
result = "`t"
result.addInt ord(t.kind)
result.add '.'
result.addInt t.uniqueId.item
result.add '.'
result.add modname(t.uniqueId.module, conf)
result.add "@bk"
proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) =
if t == nil:
c.m.addEmpty()
@@ -186,14 +148,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
assert c.tl != nil
c.tl(t)
if t.uniqueId.isBackendMinted:
# Backend-minted (lower-stage) closure-env types key by their stable NIF name,
# never by structure (which diverges across the NIF boundary). An env `ref`
# that is itself NOT backend-minted still keys stably: it recurses here and
# reaches its `@bk` object, which short-circuits to a stable name.
c.m.addSymbol backendTypeName(t, conf)
return
case t.kind
of tyGenericInvocation:
for a in t.sonsImpl:
@@ -248,10 +202,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
of tyInt:
withTree c.m, "i":
c.m.addIntLit -1
# An `int literal(x)` type (nImpl holds the value) must stay distinct from
# plain `int` for the NIF name, else overload resolution breaks on reload.
if CoPrecise in flags and t.nImpl != nil:
c.m.addIntLit t.nImpl.intVal
maybeImported(c, t.symImpl, conf)
of tyInt8:
withTree c.m, "i":
@@ -289,14 +239,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
withTree c.m, "u":
c.m.addIntLit 64
maybeImported(c, t.symImpl, conf)
of tyFloat:
withTree c.m, "f":
c.m.addIntLit -1
# A `float literal(x)` type (nImpl = nkFloatLit) must stay distinct from
# plain `float`, just like the `int literal(x)` case above.
if CoPrecise in flags and t.nImpl != nil and t.nImpl.kind in {nkFloatLit..nkFloat64Lit}:
c.m.addFloatLit t.nImpl.floatVal
maybeImported(c, t.symImpl, conf)
of tyObject, tyEnum:
if t.typeInstImpl != nil:
# prevent against infinite recursions here, see bug #8883:
@@ -412,9 +354,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.m.addIdent toNifTag(t.callConvImpl)
if tfVarargs in t.flagsImpl: c.m.addIdent "´varargs"
# `.gcsafe`/`.noSideEffect` (in eqTypeFlags) distinguish proc types under
# sameType, so they must distinguish the NIF name too.
emitPreciseFlags(c, t, flags)
of tyArray:
withTree c.m, toNifTag(t.kind):
if t.sonsImpl.len == 0:
@@ -429,9 +368,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.typeKey t.sonsImpl[i], flags, conf
if tfNotNil in t.flagsImpl and CoType notin flags:
c.m.addIdent "´notnil"
# tfNotNil/tfVarIsPtr/tfIsOutParam (eqTypeFlags) part of sameType identity
# for ref/ptr/var/lent/sink; the hook path (no CoPrecise) keeps its layout.
emitPreciseFlags(c, t, flags)
proc typeKey*(t: PType; conf: ConfigRef; tl: TypeLoader; sl: SymLoader): string =
var c: Context = Context(m: createMangler(30, -1), tl: tl, sl: sl,

View File

@@ -633,34 +633,6 @@ proc lengthOrd*(conf: ConfigRef; t: PType): Int128 =
let first = firstOrd(conf, t)
result = last - first + One
const broadcastArrayThreshold* = 32
## `getNullValue` represents the default of an `array[N, T]` with `N` above this
## as a single *broadcast* element — a one-son `nkBracket` standing for `N`
## identical zero copies — instead of materialising `N` zero nodes. This keeps
## huge zeroed arrays (e.g. SSZ byte buffers in nimbus) compact in the IC caches
## (`.s.bif`/`.t.bif`), in the VM, and in the generated C (`{0}` zero-fills).
proc isDefaultBroadcastArray*(n: PNode; conf: ConfigRef): bool =
## True iff `n` is a broadcast default array: a single son standing for
## `lengthOrd` identical zero copies. Identified by the explicit `nfBroadcast`
## marker (set by `getNullValue`), NOT by `len == 1 < lengthOrd` — the latter
## also matches an ordinary 1-element collection that happens to be an
## `nkBracket` carrying an array type, e.g. a `@[a, b, c]` seq value shrunk to
## length 1 by `setLen`/`delete` (its VM node keeps the array-literal type).
result = n != nil and n.kind == nkBracket and nfBroadcast in n.flags
proc expandBroadcastArray*(n: PNode; conf: ConfigRef) =
## Materialise a broadcast default array (see `isDefaultBroadcastArray`) into a
## full `lengthOrd`-son `nkBracket`, each son a copy of the single default
## element. Used by VM ops that index-address, mutate, or measure such a node;
## the common read-only paths leave it compact. Clears `nfBroadcast` since the
## node is now a fully materialised literal.
if isDefaultBroadcastArray(n, conf):
let total = toInt(lengthOrd(conf, n.typ.skipTypes(abstractInst)))
let elem = n[0]
for i in 1 ..< total: n.add copyTree(elem)
n.flags.excl nfBroadcast
# -------------- type equality -----------------------------------------------
type

View File

@@ -677,13 +677,9 @@ proc deps(c: var Partitions; dest, src: PNode) =
else:
let srcid = variableId(c, s)
if srcid >= 0:
if s.kind notin {skResult, skParam} and
c.s[srcid].aliveEnd < c.s[vid].aliveEnd and
c.g.config.backend != backendJs:
# you cannot borrow from a local that lives shorter than 'vid'.
# On a traced (JS/GC) target the source object stays alive as long
# as the alias references it, so this lifetime rule does not apply;
# value-semantics safety is enforced by `dangerousMutation` instead.
if s.kind notin {skResult, skParam} and (
c.s[srcid].aliveEnd < c.s[vid].aliveEnd):
# you cannot borrow from a local that lives shorter than 'vid':
when explainCursors: echo "B not a cursor ", d.sym, " ", c.s[srcid].aliveEnd, " ", c.s[vid].aliveEnd
c.s[vid].flags.incl preventCursor
elif {isReassigned, preventCursor} * c.s[srcid].flags != {}:
@@ -1007,22 +1003,13 @@ proc checkBorrowedLocations*(par: var Partitions; body: PNode; config: ConfigRef
#if par.s[rid].con.kind == isRootOf and dangerousMutation(par.graphs[par.s[rid].con.graphIndex], par.s[i]):
# cannotBorrow(config, s, par.graphs[par.s[rid].con.graphIndex])
proc jsDeepCopied(t: PType): bool =
## On the JS backend `nimCopy` deep-copies these type classes on every
## assignment, so eliding the copy for a safe alias is worthwhile even when
## the type has no C-style destructor.
t.skipTypes({tyGenericInst, tyAlias, tyDistinct, tyVar, tyLent}).kind in
{tyObject, tyTuple, tyArray, tySequence, tyString}
proc computeCursors*(s: PSym; n: PNode; g: ModuleGraph) =
let jsCursors = g.config.backend == backendJs
var par = computeGraphPartitions(s, n, g, {cursorInference})
for i in 0 ..< par.s.len:
let v = addr(par.s[i])
if v.flags * {ownsData, preventCursor, isConditionallyReassigned} == {} and
v.sym.kind notin {skParam, skResult} and
v.sym.flags * {sfThread, sfGlobal} == {} and
(hasDestructor(v.sym.typ) or (jsCursors and jsDeepCopied(v.sym.typ))) and
v.sym.flags * {sfThread, sfGlobal} == {} and hasDestructor(v.sym.typ) and
v.sym.typ.skipTypes({tyGenericInst, tyAlias}).kind != tyOwned and
(getAttachedOp(g, v.sym.typ, attachedAsgn) == nil or
sfError notin getAttachedOp(g, v.sym.typ, attachedAsgn).flags):

View File

@@ -702,10 +702,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
# A bodge, but this takes in `toOpenArray(rb, rc, rc)` and emits
# nkTupleConstr(x, y, z) into the `regs[ra]`. These can later be used for calculating the slice we have taken.
decodeBC(rkNode)
# Slicing/openArray needs the real length and per-element nodes, so a
# compact default array must be materialised first.
if isDefaultBroadcastArray(regs[ra].node, c.config):
expandBroadcastArray(regs[ra].node, c.config)
let
collection = regs[ra].node
leftInd = regs[rb].intVal
@@ -774,15 +770,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
regs[ra].node.intVal = src.strVal[idx].ord
else:
stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.strVal.len-1))
elif isDefaultBroadcastArray(src, c.config):
# `a[i]` on a compact default array yields `default(T)` directly, without
# ever materialising the (potentially huge) array — the point of the
# broadcast form. See `getNullValue`/`isDefaultBroadcastArray`.
let total = toInt(lengthOrd(c.config, src.typ.skipTypes(abstractInst)))
if idx <% total:
regs[ra].node = copyTree(src[0])
else:
stackTrace(c, tos, pc, formatErrorIndexBound(idx, total-1))
elif src.kind notin {nkEmpty..nkFloat128Lit} and idx <% src.len:
regs[ra].node = src[idx]
else:
@@ -794,9 +781,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
stackTrace(c, tos, pc, formatErrorIndexBound(regs[rc].intVal, high(int)))
let idx = regs[rc].intVal.int
let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[]
# Taking the address of an element needs distinct, stable per-slot nodes, so
# a compact default array must be materialised first.
if isDefaultBroadcastArray(src, c.config): expandBroadcastArray(src, c.config)
case src.kind
of nkTupleConstr:
let
@@ -845,8 +829,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
let idx = regs[rb].intVal.int
assert regs[ra].kind == rkNode
let arr = regs[ra].node
# Writing a slot materialises a compact default array into a full literal.
if isDefaultBroadcastArray(arr, c.config): expandBroadcastArray(arr, c.config)
case arr.kind
of nkTupleConstr: # refer to `opcSlice`
let
@@ -1049,8 +1031,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
case node.kind
of nkTupleConstr: # refer to `of opcSlice`
regs[ra].intVal = node[2].intVal - node[1].intVal + 1 - high
elif isDefaultBroadcastArray(node, c.config):
regs[ra].intVal = toInt(lengthOrd(c.config, node.typ.skipTypes(abstractInst))) - high
else:
# safeArrLen also return string node len
# used when string is passed as openArray in VM
@@ -1333,41 +1313,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
# a macro observed this symbol's implementation: NeedsImpl edge to
# its home module under IC.
recordIcImplDep(c.graph, a.sym)
if a.sym.ast.isNil:
regs[ra].node = newNode(nkNilLit)
else:
let tree = copyTree(a.sym.ast)
# A NIF-loaded routine's `ast[paramsPos]` is an `nkEmpty` placeholder:
# ast2nif strips the formal params (recoverable from `typ.n`, see
# writeNode's `skipParams`). A macro that reads `fn.getImpl[paramsPos]`
# — e.g. taskpools `spawn` reads the return type via `getImpl[3][0]` —
# needs them, so reconstruct a read-only formalParams from the proc
# type. The synthesized type-expression nodes carry the resolved
# `PType`, which is all a macro can query for a loaded routine.
if tree.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
nkConverterDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo} and
tree.safeLen > paramsPos and tree[paramsPos].kind == nkEmpty and
a.sym.typ != nil and a.sym.typ.n != nil and
a.sym.typ.n.kind == nkFormalParams:
let t = a.sym.typ
let fp = newNodeI(nkFormalParams, a.sym.info)
let rt = t.returnType
# `opMapTypeInstToAst` (inst=true) reproduces a source-like type
# declaration — crucially it renders an array's range bound as
# `range 0..N` (the `inst=false` form emits `range[0, N]`, which
# re-sems to "'range' expects one type parameter").
fp.add(if rt != nil: opMapTypeInstToAst(c.cache, rt, a.sym.info, c.idgen)
else: newNodeI(nkEmpty, a.sym.info))
for i in 1 ..< t.n.len:
if t.n[i].kind == nkSym:
let p = t.n[i].sym
let def = newNodeI(nkIdentDefs, p.info)
def.add newIdentNode(p.name, p.info)
def.add opMapTypeInstToAst(c.cache, p.typ, p.info, c.idgen)
def.add newNodeI(nkEmpty, p.info)
fp.add def
tree[paramsPos] = fp
regs[ra].node = tree
regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
else: copyTree(a.sym.ast)
regs[ra].node.flags.incl nfIsRef
else:
stackTrace(c, tos, pc, "node is not a symbol")

View File

@@ -851,26 +851,14 @@ proc genBinaryStmt(c: PCtx; n: PNode; opc: TOpcode) =
c.freeTemp(tmp)
c.freeTemp(dest)
proc genMutatingValue(c: PCtx; n: PNode): TRegister =
## Loads the value of an in-place mutation target while keeping it attached to
## its original storage. Compound lvalues must be resolved through their
## address: a normal value load can return a detached copy (for example, when
## indexing a broadcast default array).
if needsAsgnPatch(n):
let address = c.genx(n, {gfNodeAddr})
result = c.getTemp(n.typ)
c.gABC(n, opcLdDeref, result, address)
c.freeTemp(address)
else:
result = c.genx(n)
proc genBinaryStmtVar(c: PCtx; n: PNode; opc: TOpcode) =
var x = n[1]
if x.kind in {nkAddr, nkHiddenAddr}: x = x[0]
let
dest = c.genMutatingValue(x)
dest = c.genx(x)
tmp = c.genx(n[2])
c.gABC(n, opc, dest, tmp, 0)
#c.genAsgnPatch(n[1], dest)
c.freeTemp(tmp)
c.freeTemp(dest)
@@ -1174,7 +1162,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag
of mIncl, mExcl:
unused(c, n, dest)
var d = c.genMutatingValue(n[1])
var d = c.genx(n[1])
var tmp = c.genx(n[2])
c.genSetType(n[1], d)
c.gABC(n, if m == mIncl: opcIncl else: opcExcl, d, tmp)
@@ -1933,11 +1921,7 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags
let strType = getSysType(c.graph, n.info, tyString)
var msgReg: TDest = c.getTemp(strType)
let fieldName = $accessExpr[1]
# Re-navigate the discriminant in the object type: under `nim ic` `disc.sym` is a
# field-use stub with a nil `owner`, which `genFieldDefect` dereferences. Look up the
# canonical discriminant field by name. Byte-neutral for non-IC (returns the same sym).
let dfield = lookupFieldAgain(accessExpr[0].typ, disc.sym)
let msg = genFieldDefect(c.config, fieldName, dfield)
let msg = genFieldDefect(c.config, fieldName, disc.sym)
let strLit = newStrNode(msg, accessExpr[1].info)
strLit.typ = strType
c.genLit(strLit, msgReg)
@@ -2041,20 +2025,8 @@ proc getNullValue(c: PCtx; typ: PType, info: TLineInfo; conf: ConfigRef): PNode
getNullValueAux(c, t, t.n, result, conf, currPosition)
of tyArray:
result = newNodeIT(nkBracket, info, t)
let n = toInt(lengthOrd(conf, t))
if n > 0:
for i in 0..<toInt(lengthOrd(conf, t)):
result.add getNullValue(c, elemType(t), info, conf)
# For a large array, keep a single broadcast element (the default of every
# slot is identical) instead of `n` copies; `isDefaultBroadcastArray`
# consumers expand on demand. Small arrays stay fully materialised so the
# well-trodden paths are untouched. See `broadcastArrayThreshold`.
if n <= broadcastArrayThreshold:
for i in 1..<n:
result.add getNullValue(c, elemType(t), info, conf)
else:
# Broadcast form: mark the single-son node so `isDefaultBroadcastArray`
# recognises it unambiguously (see `nfBroadcast`).
result.flags.incl nfBroadcast
of tyTuple:
result = newNodeIT(nkTupleConstr, info, t)
for a in t.kids:

View File

@@ -134,11 +134,11 @@ nimblepath="$home/.nimble/pkgs/"
# BSD got posix_spawn only recently, so we deactivate it for osproc:
define:useFork
@elif haiku:
gcc.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
gcc.cpp.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
clang.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
clang.cpp.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
tcc.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
gcc.options.linker = "-Wl,--as-needed -lnetwork"
gcc.cpp.options.linker = "-Wl,--as-needed -lnetwork"
clang.options.linker = "-Wl,--as-needed -lnetwork"
clang.cpp.options.linker = "-Wl,--as-needed -lnetwork"
tcc.options.linker = "-Wl,--as-needed -lnetwork"
@elif not genode:
# -fopenmp
gcc.options.linker = "-ldl"

View File

@@ -21,7 +21,6 @@ Advanced commands:
see also: --dump.format:json (useful with: `| jq`)
//check checks the project for syntax and semantics
(can be combined with --defusages)
//track goto-definition / find-usages via `nim ic`
Runtime checks (see -x):
--objChecks:on|off turn obj conversion checks on|off
@@ -34,8 +33,6 @@ Runtime checks (see -x):
--infChecks:on|off turn Inf checks on|off
Advanced options:
--def:FILE,LINE,COL find the definition of the symbol at the position
--usages:FILE,LINE,COL find all usages of the symbol at the position
--defusages:FILE,LINE,COL
find the definition and all usages of a symbol
-o:FILE, --out:FILE set the output filename
@@ -122,7 +119,6 @@ Advanced options:
--lineDir:on|off generation of #line directive on|off
--embedsrc:on|off embeds the original source code as comments
in the generated output
--genBif:on|off generate per-module semantic BIF metadata in nimcache
--tlsEmulation:on|off turn thread local storage emulation on|off
--implicitStatic:on|off turn implicit compile time evaluation on|off
--trmacros:on|off turn term rewriting macros on|off

View File

@@ -39,16 +39,6 @@ debugging a build).
Artifacts (the NIF zoo)
=======================
Semantic BIF from regular builds
--------------------------------
``--genBif:on`` makes a regular compiler invocation write each semantically
checked module as ``<suffix>.s.bif`` under the build's nimcache directory. This
reuses the semantic artifact format used by IC without enabling incremental
compilation or changing how the program is generated and linked. Tools such as
language servers, debuggers, and binding generators can request these artifacts
when they need resolved symbols and types from an ordinary build.
Per module ``<suffix>`` (a content hash of the path; see *NIF symbols* below),
under the nimcache directory:
@@ -298,75 +288,6 @@ Validation bar (held on every change): `koch bootic` must reach its byte-identic
fixed point, and binary size must not regress (DCE parity), across the
external-package CI set.
Further possible improvements
=============================
A warm-edit profiling pass (2026-07-02, self-compiling the compiler into a
dedicated `--nimcache`, editing one private proc body — `internalErrorImpl` — in
the hub module `compiler/msgs.nim`) surfaced where a **hub-module** warm rebuild
actually spends its time. The result refines the "a body-only edit re-fires one
module" claim above: that holds for the *backend*, but the *frontend* can still
cascade.
Measured: no-op `0.05s`; hub body edit `~15s`, split **~13s frontend / ~1.6s
backend**. Editing a body in a leaf (few importers) is fast; editing a body in a
widely-imported module is not, and the cost is almost entirely frontend re-sem.
- **Frontend over-invalidation (the dominant hub-edit cost).** Editing *any* body
in a module — even a private routine that is only ever *called* — flips that
module's whole-module **impl cookie** (`writeImplCookie` hashes the entire
serialized module). Every module carrying a **NeedsImpl** edge on it then
re-sems, even though the symbol it actually consumed is unchanged (e.g. a
dependent that expanded the `internalError` *template* needs the template body,
which is untouched; it does **not** need `internalErrorImpl`'s body). In the
msgs edit this re-fires **57** `nim m` processes. A `.s.bif` mtime diff *hides*
this — `.s.bif` is content-stable, so a re-semmed-but-identical module keeps its
timestamp; count actual `nim m` PIDs to see the fan-out.
The precise fix is **per-symbol NeedsImpl gating**: record which *symbols'*
bodies a dependent consumed (the recording site `modulegraphs.recordIcImplDep`
already receives the `PSym`; it currently coarsens to `module(s.itemId)`) and
gate the dependent
on only those. The obstacle is that `nifmake` gates on file mtimes, so
per-symbol granularity needs either many cookie files or a bucketing scheme, and
"which bodies are compile-time-consumable" is entangled with `getImpl` and the
CT call graph (a macro that runs a private helper at CT *does* consume its body).
A conservative narrowing — keep template/generic/macro/`sfCompileTime` bodies
(plus `getImpl` targets) in the impl cookie but drop ordinary runtime routine
bodies — captures the common "edit a private implementation proc" case, at the
cost of proving the exclusion is complete.
- **Serial re-sem chains.** The 57 re-sems above run essentially **one at a time**
despite `--parallel`, because the core modules they belong to form a deep import
*chain* and `nifmake`'s depth-barriered scheduler runs one depth level at a time
(≈1 node per level). This is independent of the invalidation problem: even
perfect per-symbol precision leaves a serial tail whenever the re-sem set is a
chain. Mitigations live in the scheduler (content-stability already stops the
cascade at one level, but does not flatten the chain).
- **Emit stage need not load the module graph (done).** `generateEmitStage` used
to `loadDepClosure`/`loadBackendModules` — materializing a module's whole
transitive import closure as `BModule`s — solely to reach `getCFile(bmod)` for
the output path. `renderCFromArtifact` is pure text filtering over the `.c.nif`
plus the merge decision; it needs none of that. Deriving the `.c` path directly
from the suffix (the same pure computation `deps.backendCFile` uses to *declare*
the stage's output) lets an `emit` process load nothing. Under the
fire-all-every-edit `emit` barrier (see below) this halved backend CPU
(user-time `51s → 24s` on the msgs edit); wall-clock barely moved because the
frontend dominates, but the reduced CPU/RAM contention matters when an editor is
running alongside. `koch ic` stays byte-identical.
- **Do NOT make the merge decision content-stable.** A tempting frontend to the
above: `emit` re-fires for *every* live module whenever `merge` rewrites the
decision file's mtime (deliberate — a decision change must re-render every `.c`
consistently). Writing the decision `OnlyIfChanged` (with a stamp output so the
`merge` rule is not perpetually stale) makes a warm no-op instant, but a real
edit then fires `emit` only for the modules whose `.c.nif` changed — and that
produces **multiple-definition link errors** even when the decision is
byte-identical. Fire-all `emit` is a correctness invariant, not just insurance
(see the comment at `generateEmitStage`): partial `emit` leaves inconsistent
ownership across the `.c` set. This path was tried and reverted; do not retry.
Code, logic & debugging
========================

View File

@@ -11,16 +11,16 @@
const
# examples of possible values for repos: Head, ea82b54
NimbleStableCommit = "a399f502dec7ffcd905c1cf54b13274ad990bada" # 0.24.1
AtlasStableCommit = "aa6fb162006f3015aa84c4305e15cb4d230f5ad6" # 0.14.7
ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2
SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936"
NimbleStableCommit = "aa03f886e4a111d6af9090c6a1f1271d64b66f7b" # 0.22.2
AtlasStableCommit = "ff1f4289482dce94ba9f95b3b0ae16d16e21eb3d" # 0.10.1
ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1
SatStableCommit = "e63eaea8baf00bed8bcd5a29ffd8823abb265b39"
NimonyStableCommit = "f831b953d7c21d9a4b11d0042039e7f84d7c8dc9" # unversioned \
NimonyStableCommit = "5fa72628a6867f8ca09f8955a493749cf65f006a" # unversioned \
# Note that Nimony uses Nim as a git submodule but we don't want to install
# Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
# is **required** here.
# Commit from 2026-07-10 -- stable .bif file format
# Commit from 2026-06-14
# examples of possible values for fusion: #head, #ea82b54, 1.2.3
FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"
@@ -618,8 +618,7 @@ proc runIcTestFile(inp: string) =
# which exercises the NIF import/load path the single-file tests do not.
const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils",
"tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer",
"tconverterreexport", "ttypeoffer", "ttransitiveoffer",
"tmodsymref", "tmethupref", "temit", "ttraitparam"]
"tconverterreexport"]
proc icTest(args: string) =
temp("")
@@ -669,16 +668,7 @@ proc runCI(cmd: string) =
# boot without -d:nimHasLibFFI to make sure this still works
# `--lib:lib` is needed for bootstrap on openbsd, for reasons described in
# https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd).
#
# Bootstrap exactly once per platform. The refc-mm bootstrap is a
# platform-independent compiler-correctness check, so Linux uses it as its sole
# boot (and then runs the whole suite against the refc-built compiler), while
# the other platforms cover the default ORC bootstrap. `koch` is rebuilt
# per-runner, so `when defined(linux)` selects the Linux job at compile time.
when defined(linux):
kochExecFold("Boot Nim refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")
else:
kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib")
kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib")
when false: # debugging: when you need to run only 1 test in CI, use something like this:
execFold("debugging test", "nim r tests/stdlib/tosproc.nim")
@@ -732,6 +722,8 @@ proc runCI(cmd: string) =
execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest")
execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")
proc testUnixInstall(cmdLineRest: string) =
csource("-d:danger" & cmdLineRest)

View File

@@ -28,15 +28,6 @@ elif defined(netbsd):
EVFILT_PROC* = 4 ## attached to struct proc
EVFILT_SIGNAL* = 5 ## attached to struct proc
EVFILT_TIMER* = 6 ## timers (in ms)
elif defined(haiku):
const
EVFILT_READ* = -1
EVFILT_WRITE* = -2
EVFILT_AIO* = -3 ## attached to aio requests
EVFILT_VNODE* = -4 ## attached to vnodes
EVFILT_PROC* = -5 ## attached to struct proc
EVFILT_SIGNAL* = -6 ## attached to struct proc
EVFILT_TIMER* = -7 ## timers
when defined(macosx):
const
EVFILT_MACHPORT* = -8 ## Mach portsets

View File

@@ -121,13 +121,6 @@ template unCheckedInc(x) =
inc(x)
{.pop.}
template newSeqForOverwrite(T: typedesc; len: int): untyped =
## Allocates a fixed-length seq whose elements will be assigned by index.
when supportsCopyMem(T) and declared(newSeqUninit):
newSeqUninit[T](len)
else: # TODO: use `newSeqUnsafe` when that's available
newSeq[T](len)
func concat*[T](seqs: varargs[seq[T]]): seq[T] =
## Takes several sequences' items and returns them inside a new sequence.
## All sequences must be of the same type.
@@ -1112,7 +1105,7 @@ template mapIt*(s: typed, op: untyped): untyped =
evalOnceAs(s2, s, compiles((let _ = s)))
var i = 0
var result = newSeqForOverwrite(OutType, s2.len)
var result = newSeq[OutType](s2.len)
for it {.inject.} in s2:
result[i] = op
i += 1
@@ -1178,7 +1171,10 @@ template newSeqWith*(len: int, init: untyped): untyped =
assert seqRand[0] != seqRand[1]
type T = typeof(init)
let newLen = len
var result = newSeqForOverwrite(T, newLen)
when supportsCopyMem(T) and declared(newSeqUninit):
var result = newSeqUninit[T](newLen)
else: # TODO: use `newSeqUnsafe` when that's available
var result = newSeq[T](newLen)
for i in 0 ..< newLen:
result[i] = init
move(result) # refs bug #7295

View File

@@ -11,119 +11,17 @@ when defined(nimPreviewSlimSystem):
when weirdTarget:
discard
elif defined(windows):
import std/winlean
from std/strutils import toHex, toLowerAscii
const
reparseHeaderSize = 8
substituteNameOffsetField = 8
substituteNameLengthField = 10
symlinkFlagsField = 16
mountPointPathBufferOffset = 16
symlinkPathBufferOffset = 20
type
ReparseBuffer = array[MAXIMUM_REPARSE_DATA_BUFFER_SIZE, byte]
ReparseLinkInfo = object
tag: int32
flags: int32
pathBufOffset: int
flagsField: int
substituteNameOffset: int
substituteNameLength: int
template readU16(buf: ReparseBuffer; off: int): uint16 =
uint16(buf[off]) or (uint16(buf[off + 1]) shl 8)
template readI32(buf: ReparseBuffer; off: int): int32 =
cast[int32](
uint32(buf[off]) or (uint32(buf[off + 1]) shl 8) or
(uint32(buf[off + 2]) shl 16) or (uint32(buf[off + 3]) shl 24))
func startsWithAsciiIgnoreCase(wide: openArray[Utf16Char]; prefix: openArray[char]): bool =
## Matches an ASCII prefix against UTF-16 code units.
##
## This is only correct for ASCII prefixes.
## It is not a valid general case-insensitive Unicode comparison
## and must not be used for arbitrary UTF-16 text.
if prefix.len > wide.len:
return false
var i = 0
while i < prefix.len:
let rune = ord(wide[i])
if rune > 0x7F or toLowerAscii(char(rune)) != toLowerAscii(prefix[i]):
return false
inc i
true
proc decodeWinTarget(wide: openArray[Utf16Char]): string =
if wide.startsWithAsciiIgnoreCase(r"\??\unc\"):
r"\\" & $(wide.toOpenArray(8, wide.len - 1))
elif wide.startsWithAsciiIgnoreCase(r"\??\"):
$(wide.toOpenArray(4, wide.len - 1))
else:
$wide
template invalidReparseData(path, details: string) =
raise newException(OSError,
"expandSymlink: invalid reparse data for " & path & " (" & details & ")")
proc parseReparseLinkInfo(buf: ReparseBuffer; bytesReturned: int;
symlinkPath: string): ReparseLinkInfo =
if bytesReturned < reparseHeaderSize:
invalidReparseData(symlinkPath, "truncated header")
let
reparseDataLen = int(readU16(buf, 4))
wholeDataLen = reparseHeaderSize + reparseDataLen
if wholeDataLen > bytesReturned:
invalidReparseData(symlinkPath, "payload exceeds returned size")
result.tag = readI32(buf, 0)
case result.tag
of IO_REPARSE_TAG_SYMLINK:
result.pathBufOffset = symlinkPathBufferOffset
result.flagsField = symlinkFlagsField
of IO_REPARSE_TAG_MOUNT_POINT:
result.pathBufOffset = mountPointPathBufferOffset
result.flagsField = -1
else:
raise newException(OSError,
"expandSymlink: unsupported reparse tag for " & symlinkPath &
" (ReparseTag=0x" & toHex(result.tag) & ")")
if result.pathBufOffset > wholeDataLen:
invalidReparseData(symlinkPath, "missing path buffer")
result.substituteNameOffset = int(readU16(buf, substituteNameOffsetField))
result.substituteNameLength = int(readU16(buf, substituteNameLengthField))
if result.substituteNameLength <= 0:
invalidReparseData(symlinkPath, "empty substitute name")
if (result.substituteNameOffset and 1) != 0 or
(result.substituteNameLength and 1) != 0:
invalidReparseData(symlinkPath, "unaligned UTF-16 substitute name")
let startByte = result.pathBufOffset + result.substituteNameOffset
let endByte = startByte + result.substituteNameLength
if startByte < result.pathBufOffset or endByte < startByte or
endByte > wholeDataLen:
invalidReparseData(symlinkPath, "substitute name out of bounds")
result.flags =
if result.flagsField >= 0:
readI32(buf, result.flagsField)
else:
0
import std/[winlean, times]
elif defined(posix):
import std/posix
when weirdTarget:
{.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".}
else:
{.pragma: noWeirdTarget.}
when defined(nimscript):
# for procs already defined in scriptconfig.nim
template noNimJs(body): untyped = discard
@@ -158,65 +56,13 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} =
raiseOSError(osLastError(), $(src, dest))
proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} =
## Returns the stored target of the symbolic link `symlinkPath`.
## Returns a string representing the path to which the symbolic link points.
##
## This expands exactly one level of indirection, like POSIX `readlink`.
## If the target is itself a symbolic link, it is returned as-is rather than
## being expanded further.
##
## On POSIX, raises `OSError` if `symlinkPath` is not a symbolic link or if
## the target cannot be read.
##
## On Windows, this supports symbolic links and junctions by reading the
## reparse point payload directly. Unsupported reparse tags raise `OSError`.
##
## On Nintendo Switch this is currently a noop: `symlinkPath` is simply
## returned, without checking whether it is actually a symbolic link.
## On Windows this is a noop, `symlinkPath` is simply returned.
##
## See also:
## * `createSymlink proc`_
when defined(windows):
let handle = createFileW(
newWideCString(symlinkPath),
0'i32,
FILE_SHARE_READ or FILE_SHARE_WRITE or FILE_SHARE_DELETE,
nil,
OPEN_EXISTING,
FILE_FLAG_OPEN_REPARSE_POINT or FILE_FLAG_BACKUP_SEMANTICS,
Handle(0)
)
if handle == INVALID_HANDLE_VALUE:
raiseOSError(osLastError(), "expandSymlink: cannot open " & symlinkPath)
defer:
discard closeHandle(handle)
var buf: ReparseBuffer
var bytesReturned: DWORD
if deviceIoControl(
handle,
FSCTL_GET_REPARSE_POINT,
nil, 0'i32,
addr buf[0], DWORD(buf.len),
bytesReturned,
nil
) == 0:
raiseOSError(osLastError(),
"expandSymlink: DeviceIoControl failed for " & symlinkPath)
let
info = parseReparseLinkInfo(buf, int(bytesReturned), symlinkPath)
startByte = info.pathBufOffset + info.substituteNameOffset
runeLen = info.substituteNameLength shr 1
wideSlicePtr = cast[ptr UncheckedArray[Utf16Char]](addr buf[startByte])
if info.tag == IO_REPARSE_TAG_SYMLINK and
(info.flags and SYMLINK_FLAG_RELATIVE) != 0:
return $(wideSlicePtr.toOpenArray(0, runeLen - 1))
decodeWinTarget(wideSlicePtr.toOpenArray(0, runeLen - 1))
elif defined(nintendoswitch):
when defined(windows) or defined(nintendoswitch):
result = symlinkPath
else:
var bufLen = 1024

View File

@@ -24,20 +24,9 @@ proc createSymlink*(src, dest: Path) {.inline.} =
createSymlink(src.string, dest.string)
proc expandSymlink*(symlinkPath: Path): Path {.inline.} =
## Returns the stored target of the symbolic link `symlinkPath`.
## Returns a string representing the path to which the symbolic link points.
##
## This expands exactly one level of indirection, like POSIX `readlink`.
## If the target is itself a symbolic link, it is returned as-is rather than
## being expanded further.
##
## On POSIX, raises `OSError` if `symlinkPath` is not a symbolic link or if
## the target cannot be read.
##
## On Windows, this supports symbolic links and junctions by reading the
## reparse point payload directly. Unsupported reparse tags raise `OSError`.
##
## On Nintendo Switch this is currently a noop: `symlinkPath` is simply
## returned, without checking whether it is actually a symbolic link.
## On Windows this is a noop, `symlinkPath` is simply returned.
##
## See also:
## * `createSymlink proc`_

View File

@@ -185,88 +185,47 @@ when not (defined(cpu16) or defined(cpu8)):
proc newWideCString*(s: string): WideCStringObj =
result = newWideCString(cstring s, s.len)
iterator decodeUtf16(w: WideCString; replacement: int): int =
## Looks for a terminating NUL for length
proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
result = newStringOfCap(estimate + estimate shr 2)
var i = 0
while w[i].int16 != 0'i16:
var ch = ord(w[i])
inc i
if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
# If the 16 bits following the high surrogate are NOT in the source...
if w[i].int16 == 0'i16:
ch = replacement #invalid UTF-16
# If the 16 bits following the high surrogate are in the source buffer...
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
ch = replacement #invalid UTF-16
#invalid UTF-16
ch = replacement
elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
ch = replacement #invalid UTF-16
yield ch
#invalid UTF-16
ch = replacement
iterator decodeUtf16(w: openArray[Utf16Char]; replacement: int): int =
## Doesn't look for terminating NUL for length, trusts `w.len`
var i = 0
while i < w.len:
var ch = ord(w[i])
inc i
if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
# If the 16 bits following the high surrogate are NOT in the source...
if i >= w.len:
ch = replacement #invalid UTF-16
else:
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
ch = replacement #invalid UTF-16
elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
ch = replacement #invalid UTF-16
yield ch
proc addUtf8(dest: var string; rune: int) =
if rune < 0x80:
dest.add chr(rune)
elif rune < 0x800:
dest.add chr((rune shr 6) or 0xc0)
dest.add chr((rune and 0x3f) or 0x80)
elif rune < 0x10000:
dest.add chr((rune shr 12) or 0xe0)
dest.add chr(((rune shr 6) and 0x3f) or 0x80)
dest.add chr((rune and 0x3f) or 0x80)
elif rune <= 0x10FFFF:
dest.add chr((rune shr 18) or 0xf0)
dest.add chr(((rune shr 12) and 0x3f) or 0x80)
dest.add chr(((rune shr 6) and 0x3f) or 0x80)
dest.add chr((rune and 0x3f) or 0x80)
else:
# replacement char (in case user give very large number):
dest.add chr(0xFFFD shr 12 or 0b1110_0000)
dest.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
dest.add chr(0xFFFD and ones(6) or 0b10_0000_00)
proc `$`*(w: openArray[Utf16Char]; replacement: int = 0xFFFD): string =
## Decodes a length-delimited UTF-16 slice to UTF-8.
##
## Unlike the `WideCString` overloads, this preserves the provided length
## and does not search for a terminating NUL.
if w.len == 0:
result = ""
else:
result = newStringOfCap(w.len + w.len shr 2)
for rune in w.decodeUtf16(replacement):
result.addUtf8(rune)
proc `$`*(w: WideCString; estimate: int; replacement: int = 0xFFFD): string =
result = newStringOfCap(estimate + estimate shr 2)
for rune in w.decodeUtf16(replacement):
result.addUtf8(rune)
if ch < 0x80:
result.add chr(ch)
elif ch < 0x800:
result.add chr((ch shr 6) or 0xc0)
result.add chr((ch and 0x3f) or 0x80)
elif ch < 0x10000:
result.add chr((ch shr 12) or 0xe0)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
elif ch <= 0x10FFFF:
result.add chr((ch shr 18) or 0xf0)
result.add chr(((ch shr 12) and 0x3f) or 0x80)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
else:
# replacement char(in case user give very large number):
result.add chr(0xFFFD shr 12 or 0b1110_0000)
result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
result.add chr(0xFFFD and ones(6) or 0b10_0000_00)
proc `$`*(s: WideCString): string =
result = s $ 80

View File

@@ -804,24 +804,6 @@ when defined(gcDestructors):
sysAssert c.next == nil, "c.next pointer must be nil"
atomicPrepend a.sharedFreeListBigChunks, c
proc takeFromSharedFreeListBigChunks(a: var MemRegion): PBigChunk {.inline.} =
when hasThreadSupport:
while true:
result = atomicLoadN(addr a.sharedFreeListBigChunks, ATOMIC_ACQUIRE)
if result == nil:
break
let next = result.next.loada
var expected = result
if atomicCompareExchangeN(addr a.sharedFreeListBigChunks, addr expected, next,
weak = true, ATOMIC_ACQUIRE, ATOMIC_RELAXED):
result.next.storea nil
break
else:
result = a.sharedFreeListBigChunks
if result != nil:
a.sharedFreeListBigChunks = result.next
result.next = nil
proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} =
atomicPrepend c.owner.sharedFreeLists[size], f
@@ -845,14 +827,21 @@ when defined(gcDestructors):
inc(c.free, total)
dec(a.occ, total)
proc freeDeferredObjects(a: var MemRegion) =
# Pop only as many nodes as we can process. Detaching the entire list and
# re-enqueuing its unprocessed tail through atomicPrepend would overwrite
# that tail's next pointer and lose the rest of the list.
for _ in 0..MaxSteps:
let it = takeFromSharedFreeListBigChunks(a)
proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) =
var it = root
var maxIters = MaxSteps # make it time-bounded
while true:
let rest = it.next.loada
it.next.storea nil
deallocBigChunk(a, cast[PBigChunk](it))
if maxIters == 0:
if rest != nil:
addToSharedFreeListBigChunks(a, rest)
sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed"
break
it = rest
dec maxIters
if it == nil: break
deallocBigChunk(a, it)
when defined(heaptrack):
const heaptrackLib =
@@ -980,7 +969,13 @@ proc rawAlloc(a: var MemRegion, requestedSize: int, alignment: int = 0): pointer
trackSize(c.size)
else:
when defined(gcDestructors):
freeDeferredObjects(a)
when hasThreadSupport:
let deferredFrees = atomicExchangeN(addr a.sharedFreeListBigChunks, nil, ATOMIC_RELAXED)
else:
let deferredFrees = a.sharedFreeListBigChunks
a.sharedFreeListBigChunks = nil
if deferredFrees != nil:
freeDeferredObjects(a, deferredFrees)
# For big chunks with custom alignment, allocate extra space.
# Since chunks are page-aligned, the needed padding is a compile-time
@@ -1402,4 +1397,4 @@ template instantiateForRegion(allocator: untyped) {.dirty.} =
#sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem)
{.pop.}
{.pop.}
{.pop.}

View File

@@ -207,10 +207,10 @@ when defined(nativeStacktrace) and nativeStackTraceSupported:
if enabled:
if dlresult != 0:
var oldLen = s.len
add(s, cstrToStrBuiltin(tempDlInfo.dli_fname))
add(s, tempDlInfo.dli_fname)
if tempDlInfo.dli_sname != nil:
for k in 1..max(1, 25-(s.len-oldLen)): add(s, ' ')
add(s, cstrToStrBuiltin(tempDlInfo.dli_sname))
add(s, tempDlInfo.dli_sname)
else:
add(s, '?')
add(s, "\n")

View File

@@ -892,6 +892,9 @@ when not defined(useNimRtl):
"API usage error: GC_enable called but GC is already enabled")
dec(gch.recGcLock)
proc GC_setStrategy(strategy: GC_Strategy) =
discard
proc GC_enableMarkAndSweep() =
gch.cycleThreshold = InitialCycleThreshold

View File

@@ -3,6 +3,14 @@
when not usesDestructors:
{.pragma: nodestroy.}
when hasAlloc:
type
GC_Strategy* = enum ## The strategy the GC should use for the application.
gcThroughput, ## optimize for throughput
gcResponsiveness, ## optimize for responsiveness (default)
gcOptimizeTime, ## optimize for speed
gcOptimizeSpace ## optimize for memory footprint
when hasAlloc and not defined(js) and not usesDestructors:
proc GC_disable*() {.rtl, inl, gcsafe, raises: [].}
## Disables the GC. If called `n` times, `n` calls to `GC_enable`
@@ -58,6 +66,9 @@ when hasAlloc and defined(js):
template GC_fullCollect* =
{.warning: "GC_fullCollect is a no-op in JavaScript".}
template GC_setStrategy* =
{.warning: "GC_setStrategy is a no-op in JavaScript".}
template GC_enableMarkAndSweep* =
{.warning: "GC_enableMarkAndSweep is a no-op in JavaScript".}

View File

@@ -491,6 +491,8 @@ when not defined(useNimRtl):
"API usage error: GC_enable called but GC is already enabled")
dec(gch.recGcLock)
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() =
gch.cycleThreshold = InitialThreshold

View File

@@ -415,6 +415,7 @@ when hasThreadSupport:
proc GC_disable() = discard
proc GC_enable() = discard
proc GC_fullCollect() = discard
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""

View File

@@ -76,6 +76,7 @@ when not defined(useNimRtl):
proc GC_disable() = boehmGC_disable()
proc GC_enable() = boehmGC_enable()
proc GC_fullCollect() = boehmGCfullCollect()
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""

View File

@@ -12,6 +12,7 @@ proc GC_disable() = discard
proc GC_enable() = discard
proc go_gc() {.importc: "go_gc", dynlib: goLib.}
proc GC_fullCollect() = go_gc()
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard

View File

@@ -55,6 +55,8 @@ when not defined(gcOrc) and not defined(gcYrc):
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc getOccupiedMem(): int = discard
proc getFreeMem(): int = discard
proc getTotalMem(): int = discard

View File

@@ -8,6 +8,7 @@ proc initGC() = discard
proc GC_disable() = discard
proc GC_enable() = discard
proc GC_fullCollect() = discard
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""

View File

@@ -89,7 +89,7 @@ elif defined(emscripten) and not defined(StandaloneHeapSize):
var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock)
var mmapDescr = cast[PEmscriptenMMapBlock](mmapDescrPos)
var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos)
mmapDescr.realSize = realSize
mmapDescr.realPointer = realPointer
@@ -99,7 +99,7 @@ elif defined(emscripten) and not defined(StandaloneHeapSize):
proc osDeallocPages(p: pointer, size: int) {.inline.} =
var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock)
var mmapDescr = cast[PEmscriptenMMapBlock](mmapDescrPos)
var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos)
munmap(mmapDescr.realPointer, mmapDescr.realSize)
elif defined(genode) and not defined(StandaloneHeapSize):

View File

@@ -25,55 +25,31 @@ when defined(gcYrc):
HasCollectorLock
Collecting
AlignedCounter = object
## one counter per cache line to avoid false sharing between stripes
c {.align: 64.}: int
AlignedRwLock = object
## One RwLock per cache line. {.align: 64.} causes the compiler to round
## the struct size up to 64 bytes, so consecutive array elements never
## share a cache line (sizeof(RwLock) = 56 on Linux x86_64 → 8 byte pad).
lock {.align: 64.}: RwLock
# Asymmetric two-class exclusion: seq structure mutations and collections
# exclude each other, but seq ops run concurrently with seq ops and
# collections run concurrently with collections. This replaces the old
# RwLock scheme (which allowed only ONE collector, serializing parallel
# collection) and also sidesteps POSIX's requirement that a rwlock be
# unlocked by its acquiring thread.
var
gSeqActive: array[NumLockStripes, AlignedCounter] # in-flight seq ops
gGcActive: int # active collections
gYrcLocks: array[NumLockStripes, AlignedRwLock]
var
lockState {.threadvar.}: YrcLockState
proc getYrcStripe(): int {.inline.} =
## Map this thread to one of the NumLockStripes counter stripes.
## Map this thread to one of the NumLockStripes RwLock stripes.
## getThreadId() is already cached thread-locally in threadids.nim.
getThreadId() and (NumLockStripes - 1)
proc acquireMutatorLock() {.compilerRtl, inl.} =
if lockState == HasNoLock:
let s = getYrcStripe()
while true:
# SEQ_CST inc-then-check pairs with the collector's SEQ_CST
# inc-then-drain (Dekker-style store/load ordering)
discard atomicFetchAdd(addr gSeqActive[s].c, 1, ATOMIC_SEQ_CST)
if atomicLoadN(addr gGcActive, ATOMIC_SEQ_CST) == 0: break
discard atomicFetchSub(addr gSeqActive[s].c, 1, ATOMIC_SEQ_CST)
while atomicLoadN(addr gGcActive, ATOMIC_ACQUIRE) != 0:
discard
acquireRead gYrcLocks[getYrcStripe()].lock
lockState = HasMutatorLock
proc releaseMutatorLock() {.compilerRtl, inl.} =
if lockState == HasMutatorLock:
lockState = HasNoLock
discard atomicFetchSub(addr gSeqActive[getYrcStripe()].c, 1, ATOMIC_SEQ_CST)
proc yrcGcFenceEnter() =
## A collection announces itself and waits for in-flight seq structure
## mutations to drain. Multiple collections may hold the fence at once.
discard atomicFetchAdd(addr gGcActive, 1, ATOMIC_SEQ_CST)
for s in 0 ..< NumLockStripes:
while atomicLoadN(addr gSeqActive[s].c, ATOMIC_SEQ_CST) > 0:
discard
proc yrcGcFenceExit() =
discard atomicFetchSub(addr gGcActive, 1, ATOMIC_SEQ_CST)
releaseRead gYrcLocks[getYrcStripe()].lock
template yrcMutatorLock*(t: typedesc; body: untyped) =
{.noSideEffect.}:
@@ -95,6 +71,23 @@ when defined(gcYrc):
{.noSideEffect.}:
releaseMutatorLock()
template yrcCollectorLock(body: untyped) =
if lockState == HasMutatorLock: releaseMutatorLock()
let prevState = lockState
let hadToAcquire = prevState < HasCollectorLock
if hadToAcquire:
# Acquire all stripes in ascending order — the only thread ever holding
# multiple write locks is the collector, so there is no lock-order cycle.
for yrcI in 0..<NumLockStripes:
acquireWrite(gYrcLocks[yrcI].lock)
lockState = HasCollectorLock
try:
body
finally:
if hadToAcquire:
for yrcI in 0..<NumLockStripes:
releaseWrite(gYrcLocks[yrcI].lock)
lockState = prevState
else:
template yrcMutatorLock*(t: typedesc; body: untyped) =

View File

@@ -514,25 +514,10 @@ proc setLengthStr(s: var SmallString; newLen: int; zeroing: bool) =
s.more.fullLen = newLen
s.more.data[newLen] = '\0'
else:
# shared or static block: detach from the shared/static buffer.
# shared or static block: detach and go back to inline
if newLen <= 0:
nimDestroyStrV1(s)
s.bytes = 0
elif newLen > PayloadSize:
# Still too long for inline: detach into a fresh unique heap block
# rather than overflowing the inline overlay.
let old = s.more
let p = cast[ptr LongString](alloc(LongStringDataOffset + newLen + 1))
p.rc = 1
p.fullLen = newLen
p.capImpl = newLen
copyMem(addr p.data[0], addr old.data[0], newLen)
p.data[newLen] = '\0'
if slen == HeapSlen and atomicSubFetch(old.rc, 1) == 0:
dealloc(old)
s.more = p
setSSLen(s, HeapSlen)
copyMem(inlinePtr(s), addr p.data[0], AlwaysAvail) # sync hot prefix
else:
let old = s.more
let inl = inlinePtr(s)

View File

@@ -223,9 +223,8 @@ proc addChar(s: NimString, c: char): NimString =
proc appendString(dest, src: NimString) {.compilerproc, inline.} =
## Raw, does not prepare `dest` space for copying
if src != nil:
copyMem(addr(dest.data[dest.len]), addr(src.data), src.len)
copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1)
inc(dest.len, src.len)
dest.data[dest.len] = '\0'
proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} =
## Sets the `s` length to `newLen` zeroing memory on growth.

View File

@@ -27,10 +27,6 @@ else:
template afterThreadRuns() =
for i in countdown(nimThreadDestructionHandlers.len-1, 0):
nimThreadDestructionHandlers[i]()
when declared(nimYrcThreadTeardown):
# YRC: spill this thread's candidate roots so its garbage remains
# collectible after the thread is gone
nimYrcThreadTeardown()
proc onThreadDestruction*(handler: proc () {.closure, gcsafe, raises: [].}) =
## Registers a *thread local* handler that is called at the thread's

File diff suppressed because it is too large Load Diff

View File

@@ -1,83 +1,56 @@
/-
YRC Safety Proof — lock-free SATB collector with parallel collections
=====================================================================
Self-contained, no Mathlib. Checked with Lean 4 (v4.32.0).
YRC Safety Proof (self-contained, no Mathlib)
==============================================
Formal model of YRC's key invariant: the cycle collector never frees
an object that any mutator thread can reach.
Formal model of the safety arguments behind lib/system/yrc.nim in its
current form: lock-free write barrier, optimistic capture / validate /
commit, and up to `MaxPar` concurrent collections over disjoint
CAS-claimed partitions.
## Model overview
## What the implementation does (the things we model)
We model the heap as a set of objects with directed edges (ref fields).
Each thread owns a set of *stack roots* — objects reachable from local variables.
The write barrier (nimAsgnYrc) does:
1. atomic store dest ← src (graph is immediately current)
2. buffer inc(src) (deferred)
3. buffer dec(old) (deferred)
Write barrier `nimAsgnYrc(dest, src)`:
1. direct ATOMIC incRef of src (rc word mutation, visible to all)
2. atomicExchange dest ← src (graph is immediately current;
old value read atomically)
3. buffer dec(old) in a striped queue (deferred — this queue IS the
snapshot-at-the-beginning log)
A collection (any mutator thread can become a collector):
1. merge queues into rc words, steal candidate roots (under gMergeLock)
2. CAPTURE: Tarjan SCC traversal; each visited cell is claimed by
CAS-ing a collection tag into its spare header word (claimCell);
cells claimed by another ACTIVE collection are not traversed
(claimCell → -1, deferred via crossPend)
3. compute deadness per SCC: ext(S) = sumRefs internal deadIn
4. VALIDATE at commit: an SCC is freed only if no queue entry mentions
a member (dirty check) and every member's rc word is unchanged
since capture (recheck) — validateDead
5. COMMIT: nil all slots of dead cells, trialDec edges to survivors,
wait out concurrent captures (grace period), then free — commitDead
## Proof structure
§1 Heap model, reachability, the core safety theorem.
§2 Write barrier: no lost objects.
§3 Mutator operational semantics and GARBAGE STABILITY: a closed
(externally unreferenced) set stays closed under every mutator step,
allocation, and foreign frees. This is why optimistic
capture/validate/commit is sound and why aborts cost nothing.
§4 Commit validation arithmetic: validated ext(D) = 0 implies D is
closed; corollary CROSS-TARGET LIVENESS — a cell referenced from
outside a collection's partition is never freed by that collection.
§5 Tag uniqueness and partition disjointness for parallel collections.
§6 Grace period: no capture ever dereferences a freed cell.
§7 The asymmetric seq/GC fence (seqs_v2.nim): Dekker-style mutual
exclusion between seq structure mutations and collections.
§8 Deadlock freedom for the remaining locks (gMergeLock + stripes) and
the spin-wait ordering argument.
The collector (under global lock) does:
1. Merge all buffered inc/dec into merged RCs
2. Trial deletion (markGray): subtract internal edges from merged RCs
3. scan: objects with RC ≥ 0 after trial deletion are rescued (scanBlack)
4. Free objects that remain white (closed cycles with zero external refs)
-/
-- Objects and threads are just natural numbers for simplicity.
abbrev Obj := Nat
abbrev Thread := Nat
/-! ## §1 Heap model and reachability -/
/-! ### State -/
/-- The state of the heap at a point in time. -/
/-- The state of the heap and collector at a point in time. -/
structure State where
/-- Physical heap edges: `edges x y` means object `x` has a ref field
pointing to `y`. Always up-to-date (atomic stores/exchanges). -/
/-- Physical heap edges: `edges x y` means object `x` has a ref field pointing to `y`.
Always up-to-date (atomic stores). -/
edges : Obj Obj Prop
/-- Stack roots per thread: local variables and the shared candidate
roots buffer (both are "external" to any captured subgraph). -/
/-- Stack roots per thread. `roots t x` means thread `t` has a local variable pointing to `x`. -/
roots : Thread Obj Prop
/-- Live allocations. The allocator hands out only unallocated objects;
captured cells stay allocated until their collection frees them. -/
allocated : Obj Prop
/-- Pending buffered increments (not yet merged). -/
pendingInc : Obj Nat
/-- Pending buffered decrements (not yet merged). -/
pendingDec : Obj Nat
/-- An object is *reachable* if some thread can reach it via stack roots
plus heap edges. -/
/-! ### Reachability -/
/-- An object is *reachable* if some thread can reach it via stack roots + heap edges. -/
inductive Reachable (s : State) : Obj Prop where
| root (t : Thread) (x : Obj) : s.roots t x Reachable s x
| step (x y : Obj) : Reachable s x s.edges x y Reachable s y
/-- Directed reachability following physical heap edges only. -/
/-- Directed reachability between heap objects (following physical edges only). -/
inductive HeapReachable (s : State) : Obj Obj Prop where
| refl (x : Obj) : HeapReachable s x x
| step (x y z : Obj) : HeapReachable s x y s.edges y z HeapReachable s x z
/-- If a root reaches `r` and `r` heap-reaches `x`, then `x` is Reachable. -/
theorem heapReachable_of_reachable (s : State) (r x : Obj)
(hr : Reachable s r) (hp : HeapReachable s r x) :
Reachable s x := by
@@ -85,62 +58,70 @@ theorem heapReachable_of_reachable (s : State) (r x : Obj)
| refl => exact hr
| step _ _ _ hedge ih => exact Reachable.step _ _ ih hedge
/-- An object has an *external reference* if some thread points to it. -/
/-! ### What the collector frees -/
/-- An object has an *external reference* if some thread's stack roots point to it. -/
def hasExternalRef (s : State) (x : Obj) : Prop :=
t, s.roots t x
/-- Externally anchored: heap-reachable from an externally referenced
object. This is what deadness computation + survivor rescue computes. -/
/-- An object is *externally anchored* if it is heap-reachable from some
object that has an external reference. This is what scanBlack computes:
it starts from objects with trialRC ≥ 0 (= has external refs) and traces
the current physical graph. -/
def anchored (s : State) (x : Obj) : Prop :=
r, hasExternalRef s r HeapReachable s r x
/-- The collector frees `x` only if `x` is not anchored. -/
/-- The collector frees `x` only if `x` is *not anchored*:
no external ref, and not reachable from any externally-referenced object.
This models: after trial deletion, x remained white, and scanBlack
didn't rescue it. -/
def collectorFrees (s : State) (x : Obj) : Prop :=
¬ anchored s x
/-- Every reachable object is anchored. -/
/-! ### Main safety theorem -/
/-- **Lemma**: Every reachable object is anchored.
If thread `t` reaches `x`, then there is a chain from a stack root
(which has an external ref) through heap edges to `x`. -/
theorem reachable_is_anchored (s : State) (x : Obj)
(h : Reachable s x) : anchored s x := by
induction h with
| root t x hroot =>
exact x, t, hroot, HeapReachable.refl x
| step a b _ h_edge ih =>
| step a b h_reach_a h_edge ih =>
obtain r, h_ext_r, h_path_r_a := ih
exact r, h_ext_r, HeapReachable.step r a b h_path_r_a h_edge
/-- **Core Safety Theorem**: freed objects are unreachable. -/
/-- **Main Safety Theorem**: If the collector frees `x`, then no thread
can reach `x`. Freed objects are unreachable.
This is the contrapositive of `reachable_is_anchored`. -/
theorem yrc_safety (s : State) (x : Obj)
(h_freed : collectorFrees s x) : ¬ Reachable s x := by
intro h_reach
exact h_freed (reachable_is_anchored s x h_reach)
/-! ## §2 The write barrier
/-! ### The write barrier preserves reachability -/
`nimAsgnYrc` performs the atomic inc of `src` BEFORE the exchange, so
there is no instant at which the edge `a → src` exists without src's rc
accounting for it; and the exchange reads `old` atomically, so two
racing barriers on the same slot can never both dec the same old value.
The dec of `old` is deferred: until the next merge, old's rc is merely
inflated — always conservative. -/
/-- Model of `nimAsgnYrc(field a, src)`: `a`'s field pointed to `old`,
now points to `src`. The graph update is immediate (atomicExchange). -/
/-- Model of `nimAsgnYrc(dest_field_of_a, src)`:
Object `a` had a field pointing to `old`, now points to `src`.
Graph update is immediate. The new edge takes priority (handles src = old). -/
def writeBarrier (s : State) (a old src : Obj) : State :=
{ s with
edges := fun x y =>
if x = a y = src then True
else if x = a y = old then False
else s.edges x y }
else s.edges x y
pendingInc := fun x => if x = src then s.pendingInc x + 1 else s.pendingInc x
pendingDec := fun x => if x = old then s.pendingDec x + 1 else s.pendingDec x }
/-- Overwriting a slot with nil: only removes an edge. -/
def storeNil (s : State) (a old : Obj) : State :=
{ s with
edges := fun x y =>
if x = a y = old then False else s.edges x y }
/-- **No Lost Object Theorem**: If thread `t` holds a stack ref to `a` and
executes `a.field = b` (replacing old), then `b` is reachable afterward.
/-- **No Lost Object**: if thread `t` holds `a` and stores `a.f = b`,
then `b` is reachable afterwards — the exchange publishes the edge
atomically, so a concurrent collection's survivor rescue traces it. -/
This is why the "lost object" problem from concurrent GC literature
doesn't arise in YRC: the atomic store makes `a→b` visible immediately,
and `a` is anchored (thread `t` holds it), so scanBlack traces `a→b`
and rescues `b`. -/
theorem no_lost_object (s : State) (t : Thread) (a old b : Obj)
(h_root_a : s.roots t a) :
Reachable (writeBarrier s a old b) b := by
@@ -148,597 +129,225 @@ theorem no_lost_object (s : State) (t : Thread) (a old b : Obj)
· exact Reachable.root t a h_root_a
· simp [writeBarrier]
/-! ## §3 Mutator semantics and garbage stability
/-! ### Non-atomic write barrier window safety
The heart of optimistic capture/validate/commit is the *garbage
stability theorem*: a set with no external references cannot acquire
one later, because mutators can only copy references they can reach.
Hence a dead set that VALIDATES at commit time stays dead through the
grace window and until the actual `free` calls — no re-validation is
needed, and an aborted (dirty) capture merely wasted its own work.
The write barrier does three steps non-atomically:
1. atomicStore(dest, src) — graph update
2. buffer inc(src) — deferred
3. buffer dec(old) — deferred
Every constructor's precondition encodes the fundamental capability
restriction: to use a reference you must hold it. `P` is the set of
cells protected from foreign frees (in yrc: cells stamped with an
active tag are never freed by another collection — §5). -/
If the collector runs between steps 1 and 2 (inc not yet buffered):
- src has a new incoming heap edge not yet reflected in RCs
- But src is reachable from the mutator's stack (mutator held a ref to store it)
- So src has an external ref → trialRC ≥ 1 → scanBlack rescues src ✓
def addRoot (s : State) (t : Thread) (x : Obj) : State :=
{ s with roots := fun t' y => (t' = t y = x) s.roots t' y }
If the collector runs between steps 2 and 3 (dec not yet buffered):
- old's RC is inflated by 1 (the dec hasn't arrived)
- This is conservative: old appears to have more refs than it does
- Trial deletion won't spuriously free it ✓
-/
def delRoot (s : State) (t : Thread) (x : Obj) : State :=
{ s with roots := fun t' y => if t' = t y = x then False else s.roots t' y }
def allocObj (s : State) (t : Thread) (x : Obj) : State :=
/-- Model the state between steps 1-2: graph updated, inc not yet buffered.
`src` has new edge but RC doesn't reflect it yet. -/
def stateAfterStore (s : State) (a old src : Obj) : State :=
{ s with
roots := fun t' y => (t' = t y = x) s.roots t' y
allocated := fun y => y = x s.allocated y }
edges := fun x y =>
if x = a y = src then True
else if x = a y = old then False
else s.edges x y }
def freeObj (s : State) (x : Obj) : State :=
{ s with
edges := fun u v => if u = x v = x then False else s.edges u v
allocated := fun y => if y = x then False else s.allocated y }
/-- Even in the window between atomic store and buffered inc,
src is still reachable (from the mutator's stack via a→src). -/
theorem src_reachable_in_window (s : State) (t : Thread) (a old src : Obj)
(h_root_a : s.roots t a) :
Reachable (stateAfterStore s a old src) src := by
apply Reachable.step a src
· exact Reachable.root t a h_root_a
· simp [stateAfterStore]
/-- One step of the concurrent system, as seen by a fixed observer
protecting the cell set `P`. -/
inductive MutStep (P : Obj Prop) (s : State) : State Prop where
/-- `a.f = src`: the mutator must hold refs to `a` and `src`. -/
| write (a old src : Obj)
(ha : Reachable s a) (hsrc : Reachable s src) :
MutStep P s (writeBarrier s a old src)
/-- `a.f = nil`. -/
| writeNil (a old : Obj) (ha : Reachable s a) :
MutStep P s (storeNil s a old)
/-- Copy a reachable ref into a local / the roots buffer. -/
| rootCopy (t : Thread) (x : Obj) (hx : Reachable s x) :
MutStep P s (addRoot s t x)
/-- Drop a local ref (scope exit, roots-buffer unregistration). -/
| rootDrop (t : Thread) (x : Obj) :
MutStep P s (delRoot s t x)
/-- Allocate: the allocator returns only unallocated addresses. -/
| alloc (t : Thread) (x : Obj) (hfresh : ¬ s.allocated x) :
MutStep P s (allocObj s t x)
/-- A DIFFERENT collection frees one of its own dead cells: it is
unreachable (its own §1 safety) and not protected (§5 partition
disjointness: it carries the other collection's tag, not ours). -/
| foreignFree (x : Obj) (hunreach : ¬ Reachable s x) (hprot : ¬ P x) :
MutStep P s (freeObj s x)
/-- Therefore src is anchored in the window → collector won't free it. -/
theorem src_safe_in_window (s : State) (t : Thread) (a old src : Obj)
(h_root_a : s.roots t a) :
¬ collectorFrees (stateAfterStore s a old src) src := by
intro h_freed
exact h_freed (reachable_is_anchored _ _ (src_reachable_in_window s t a old src h_root_a))
/-- Reflexive-transitive closure: an arbitrary interleaving of steps by
all mutators and all other collections. -/
inductive MutSteps (P : Obj Prop) (s : State) : State Prop where
| refl : MutSteps P s s
| tail {s' s'' : State} :
MutSteps P s s' MutStep P s' s'' MutSteps P s s''
/-! ### Deadlock freedom
/-- `S` is *closed*: no thread points into it and no heap edge enters it
from outside. This is exactly "validated dead set" (§4). -/
def closed (s : State) (S : Obj Prop) : Prop :=
( t x, S x ¬ s.roots t x)
( u v, S v s.edges u v S u)
YRC uses three classes of locks:
• gYrcGlobalLock (level 0)
• stripes[i].lockInc (level 2*i + 1, for i in 0..N-1)
• stripes[i].lockDec (level 2*i + 2, for i in 0..N-1)
/-- Members of a closed set are unreachable. -/
theorem closed_unreachable (s : State) (S : Obj Prop)
(h : closed s S) : x, Reachable s x ¬ S x := by
intro x hr
induction hr with
| root t x hroot => exact fun hS => h.1 t x hS hroot
| step a b _ hedge ih => exact fun hS => ih (h.2 a b hS hedge)
Total order: global < lockInc[0] < lockDec[0] < lockInc[1] < lockDec[1] < ...
/-- The invariant carried through the grace window: `S` closed and all
members still allocated (their memory has not been reused). -/
def DeadInv (s : State) (S : Obj Prop) : Prop :=
closed s S x, S x s.allocated x
Every code path in yrc.nim acquires locks in strictly ascending level order:
/-- **One-step stability**: no single action of any mutator, allocator or
other collection can break the invariant of a closed set. -/
theorem step_preserves_deadInv (s s' : State) (S : Obj Prop)
(hinv : DeadInv s S) (hstep : MutStep S s s') : DeadInv s' S := by
obtain hcl, halloc := hinv
cases hstep with
| write a old src ha hsrc =>
refine fun t x hS hroot => hcl.1 t x hS hroot, ?_, fun x hS => halloc x hS
intro u v hSv hedge
simp only [writeBarrier] at hedge
by_cases h1 : u = a v = src
· exact absurd (h1.2 hSv) (closed_unreachable s S hcl src hsrc)
· by_cases h2 : u = a v = old
· -- corner case old = src: the "remove old" branch is overridden
-- by the "add src" branch, so the edge survives — but then
-- v = old = src is reachable, hence not in S
simp [h2] at hedge
have hSsrc : S src := by rw [ hedge, h2.2]; exact hSv
exact absurd hSsrc (closed_unreachable s S hcl src hsrc)
· simp [h1, h2] at hedge
exact hcl.2 u v hSv hedge
| writeNil a old ha =>
refine fun t x hS hroot => hcl.1 t x hS hroot, ?_, fun x hS => halloc x hS
intro u v hSv hedge
simp only [storeNil] at hedge
by_cases h2 : u = a v = old
· simp [h2] at hedge
· simp [h2] at hedge
exact hcl.2 u v hSv hedge
| rootCopy t x hx =>
refine ?_, fun u v hSv hedge => hcl.2 u v hSv hedge, fun y hS => halloc y hS
intro t' y hSy hroot
simp only [addRoot] at hroot
cases hroot with
| inl h => exact absurd (h.2 hSy) (closed_unreachable s S hcl x hx)
| inr h => exact hcl.1 t' y hSy h
| rootDrop t x =>
refine ?_, fun u v hSv hedge => hcl.2 u v hSv hedge, fun y hS => halloc y hS
intro t' y hSy hroot
simp only [delRoot] at hroot
by_cases h : t' = t y = x
· simp [h] at hroot
· simp [h] at hroot
exact hcl.1 t' y hSy hroot
| alloc t x hfresh =>
refine ?_, fun u v hSv hedge => hcl.2 u v hSv hedge, ?_
· intro t' y hSy hroot
simp only [allocObj] at hroot
cases hroot with
| inl h => exact hfresh (h.2 halloc y hSy)
| inr h => exact hcl.1 t' y hSy h
· intro y hS
simp only [allocObj]
exact Or.inr (halloc y hS)
| foreignFree x hunreach hprot =>
refine fun t y hSy hroot => hcl.1 t y hSy hroot, ?_, ?_
· intro u v hSv hedge
simp only [freeObj] at hedge
by_cases h : u = x v = x
· simp [h] at hedge
· simp [h] at hedge
exact hcl.2 u v hSv hedge
· intro y hSy
simp only [freeObj]
have hyx : ¬ y = x := fun he => hprot (he hSy)
simp [hyx]
exact halloc y hSy
**nimIncRefCyclic** (mutator fast path):
acquire lockInc[myStripe] → release → done.
Holds exactly one lock. ✓
/-- **Garbage Stability Theorem**: once a set is closed, it stays closed
(and unreusable) under any interleaving of concurrent activity. -/
theorem deadInv_stable (s s' : State) (S : Obj Prop)
(hinv : DeadInv s S) (hsteps : MutSteps S s s') : DeadInv s' S := by
induction hsteps with
| refl => exact hinv
| tail _ hstep ih => exact step_preserves_deadInv _ _ S ih hstep
**nimIncRefCyclic** (overflow path):
acquire gYrcGlobalLock (level 0), then for i=0..N-1: acquire lockInc[i] → release.
Ascending: 0 < 1 < 3 < 5 < ... ✓
/-- Snapshot garbage cannot be resurrected: members of a set that was
closed at commit time are unreachable at every later point. -/
theorem garbage_stability (s s' : State) (S : Obj Prop)
(hinv : DeadInv s S) (hsteps : MutSteps S s s') :
x, S x ¬ Reachable s' x := by
intro x hS hr
exact closed_unreachable s' S (deadInv_stable s s' S hinv hsteps).1 x hr hS
**nimDecRefIsLastCyclic{Dyn,Static}** (fast path):
acquire lockDec[myStripe] → release → done.
Holds exactly one lock. ✓
/-- **Commit-then-free safety**: if the dead set validated (was closed)
at commit time, then freeing its members after ANY amount of further
concurrent activity (the grace window, other collections' frees,
destructor-driven mutations) satisfies the §1 free condition. -/
theorem commit_free_safe (s s' : State) (S : Obj Prop)
(hinv : DeadInv s S) (hsteps : MutSteps S s s') :
x, S x collectorFrees s' x := by
intro x hS hanch
obtain r, t, hroot, hpath := hanch
have hr : Reachable s' x :=
heapReachable_of_reachable s' r x (Reachable.root t r hroot) hpath
exact closed_unreachable s' S (deadInv_stable s s' S hinv hsteps).1 x hr hS
**nimDecRefIsLastCyclic{Dyn,Static}** (overflow path):
calls collectCycles → acquire gYrcGlobalLock (level 0),
then mergePendingRoots which for i=0..N-1:
acquire lockInc[i] → release, acquire lockDec[i] → release.
Ascending: 0 < 1 < 2 < 3 < 4 < ... ✓
/-! ## §4 Commit validation arithmetic
**collectCycles / GC_runOrc** (collector):
acquire gYrcGlobalLock (level 0),
then mergePendingRoots (same ascending pattern as above). ✓
`computeDeadness` marks an SCC dead when
ext(S) = sumRefs(S) internal(S) deadIn(S) = 0,
i.e. summed over the whole dead set D (union of dead SCCs):
Σ_{c∈D} rc(c) = #(edges within D).
`validateDead` then establishes that the captured rc words are the
COMMIT-TIME rc values (rc recheck) and that no unmerged queue entry
mentions a member (dirty check via markDirtyFromQueues — the deferred
dec queues double as the SATB log; direct incs are atomic rc mutations
caught by the recheck). Under yrc's invariant "rc counts every
reference: heap slots, stack refs, and the roots-buffer flag" (the
roots-buffer refs are excluded by clearing inRootsFlag on the slice
BEFORE computeDeadness — collectCyclesImpl), we get: every member's rc
splits into internal references (from D) and external ones, and the
totals matching forces every external count to zero. -/
**nimAsgnYrc / nimSinkYrc** (write barrier):
Calls nimIncRefCyclic then nimDecRefIsLastCyclic*.
Each call acquires and releases its lock independently.
No nesting between the two calls. ✓
theorem sum_map_split (l : List Obj) (f g h : Obj Nat)
(hp : c, c l f c = g c + h c) :
(l.map f).sum = (l.map g).sum + (l.map h).sum := by
induction l with
| nil => simp
| cons a l ih =>
have ha : f a = g a + h a := hp a (by simp)
have ih' := ih (fun c hc => hp c (List.mem_cons_of_mem a hc))
simp only [List.map_cons, List.sum_cons]
omega
Since every path follows the total order, deadlock is impossible.
-/
theorem sum_zero_all (l : List Nat) (h : l.sum = 0) :
x, x l x = 0 := by
induction l with
| nil => intro x hx; cases hx
| cons a l ih =>
simp only [List.sum_cons] at h
intro x hx
cases List.mem_cons.mp hx with
| inl he => subst he; omega
| inr hm => exact ih (by omega) x hm
/-- **Validation soundness (arithmetic)**: if every member's commit-time
rc splits as internal + external, and the collector's check
Σ rc = Σ internal passed, then no member has any external ref. -/
theorem validated_no_external
(members : List Obj) (rc inD extIn : Obj Nat)
(h_exact : c, c members rc c = inD c + extIn c)
(h_check : (members.map rc).sum = (members.map inD).sum) :
c, c members extIn c = 0 := by
have hsplit := sum_map_split members rc inD extIn h_exact
have hzero : (members.map extIn).sum = 0 := by omega
intro c hc
exact sum_zero_all _ hzero (extIn c) (List.mem_map_of_mem hc)
/-- **Validated implies closed**: bridging the counts to the graph. The
two counting premises say what `extIn` MEANS: any stack/root ref and
any heap edge from a non-member contributes at least one external
count (this is the rc-exactness established by merge + validate). -/
theorem validated_closed (s : State) (D : Obj Prop)
(members : List Obj) (extIn : Obj Nat)
(hmem : x, D x x members)
(h_roots_counted : t c, D c s.roots t c 1 extIn c)
(h_edges_counted : u c, D c ¬ D u s.edges u c 1 extIn c)
(h_zero : c, c members extIn c = 0) :
closed s D := by
constructor
· intro t x hD hroot
have h1 := h_roots_counted t x hD hroot
have h2 := h_zero x (hmem x hD)
omega
· intro u v hD hedge
by_cases hu : D u
· exact hu
· have h1 := h_edges_counted u v hD hu hedge
have h2 := h_zero v (hmem v hD)
omega
/-! ## §5 Parallel collections: tags, partitions, cross-target liveness -/
/-- Tags are issued from a monotonic counter under gMergeLock
(startCollection). Distinct issue times give distinct tags, so a
stale stamp from a finished collection can never be mistaken for a
different active collection's tag. (The implementation wraps the
counter at 2³¹; the model assumes no wrap-around while a tag is
active — an ABA that would need 2³¹ collections to complete during
one collection's lifetime.) -/
theorem tags_distinct (issue : Nat Nat)
(hmono : i j, i < j issue i < issue j) :
i j, issue i = issue j i = j := by
intro i j heq
cases Nat.lt_trichotomy i j with
| inl h => have := hmono i j h; omega
| inr h =>
cases h with
| inl h => exact h
| inr h => have := hmono j i h; omega
/-- Each cell's header stores ONE stamp (claimCell CASes the whole
word), so two active collections with distinct tags claim disjoint
partitions. -/
theorem partitions_disjoint (stamp : Obj Nat) (tagA tagB : Nat)
(hne : tagA tagB) :
x, stamp x = tagA stamp x = tagB False := by
intro x hA hB
exact hne (hA hB)
/-- **Cross-target liveness**: a cell claimed by collection B but
referenced from OUTSIDE B's partition is never in B's dead set.
B's internal count for the cell only includes edges from B's dead
members; the foreign edge contributes an external count, and
validation forces external counts to zero — so the cell's SCC fails
the deadness check (equivalently: it is demoted). This is why
claimCell may simply refuse foreign-claimed cells (return -1) and
crossPend defer them: their owner provably keeps them alive this
round, and re-registration makes them candidates for the next. -/
theorem cross_target_live (D : Obj Prop) (claimedB : Obj Prop)
(members : List Obj) (extIn : Obj Nat)
(s : State)
(hDsub : x, D x claimedB x)
(hmem : x, D x x members)
(h_edges_counted : u c, D c ¬ D u s.edges u c 1 extIn c)
(h_zero : c, c members extIn c = 0)
(u c : Obj) (hedge : s.edges u c) (hu : ¬ claimedB u) :
¬ D c := by
intro hDc
have hDu : ¬ D u := fun h => hu (hDsub u h)
have h1 := h_edges_counted u c hDc hDu hedge
have h2 := h_zero c (hmem c hDc)
omega
/-! ## §6 The grace period
A concurrent capture holds raw `(slot, value)` snapshots (TraceEntry);
the value pointer is dereferenced later (header read in claimCell). A
capture that overlapped our validation may have snapshotted a slot
that USED to point into our dead set. commitDead therefore waits, for
every other slot that is in capture phase (gSlotPhase == 1), until
that capture ends — captures never wait on anyone, so this is bounded.
Two obligations:
(a) captures that started BEFORE our commit are waited out — temporal
argument below (`grace_no_use_after_free`);
(b) captures that start AT/AFTER our commit never snapshot a dead
cell in the first place (`post_commit_snap_misses_dead`): they
only read slots of cells they claim; our dead cells carry our
still-active tag, so claimCell refuses them (never traversed),
and no slot OUTSIDE the dead set points into it (closedness, held
through the window by §3 stability). -/
/-- Any snapshot value read by a post-commit capture comes from a slot
of a cell that capture claimed; claimed cells are never dead cells
of another active collection (§5), and the dead set is closed. -/
theorem post_commit_snap_misses_dead (s : State)
(D claimedC snap : Obj Prop)
(hdisj : x, claimedC x ¬ D x)
(hclosed : closed s D)
(hsnap : v, snap v u, claimedC u s.edges u v) :
v, D v ¬ snap v := by
intro v hD hs
obtain u, hu, he := hsnap v hs
exact hdisj u hu (hclosed.2 u v hD he)
/-- One concurrent capture, with its interval in a global time order and
the set of values it ever snapshots. `derefs x t` = the capture
reads x's header at time t (always within its interval, always on a
snapshotted value). -/
structure CaptureWindow where
start : Nat
finish : Nat
snap : Obj Prop
derefs : Obj Nat Prop
/-- **Grace safety**: no capture dereferences a dead cell at or after
its free time. `commitT` is when the dead set validated; `freeT` is
when commitDead's free loop runs. The premises are exactly the
protocol: (grace) commitDead's spin means any capture that started
before commit has finished before we free; (miss) §6(b) above. -/
theorem grace_no_use_after_free
(C : CaptureWindow) (D : Obj Prop) (commitT freeT : Nat)
(h_deref : x t, C.derefs x t C.start t t C.finish C.snap x)
(h_grace : C.start < commitT C.finish < freeT)
(h_miss : commitT C.start x, D x ¬ C.snap x) :
x t, D x C.derefs x t t < freeT := by
intro x t hD hd
obtain h1, h2, h3 := h_deref x t hd
cases Nat.lt_or_ge C.start commitT with
| inl h => have := h_grace h; omega
| inr h => exact absurd h3 (h_miss h x hD)
/-! ## §7 The asymmetric seq/GC fence (seqs_v2.nim)
Seq structure mutations (which may FREE the old buffer on realloc)
must not overlap a collection, but seq-vs-seq and collection-vs-
collection may run concurrently. The committed fence:
mutator (acquireMutatorLock): collector (yrcGcFenceEnter):
1. FetchAdd gSeqActive[s] SC 1. FetchAdd gGcActive SC
2. Load gGcActive SC 2. Load gSeqActive[s] SC (each s)
proceed iff it read 0 proceed when all read 0
(else back off: FetchSub, spin, retry)
Under sequential consistency all four operations occupy positions in
one total order. Suppose both sides are in their critical sections
simultaneously (neither has executed its matching FetchSub). The
mutator read gGcActive = 0 AFTER its own inc: since the collector's
inc precedes its critical section and no dec intervened, the
collector's inc must be ordered after the mutator's read — and
symmetrically for the collector's read. That yields a cycle in the
total order: -/
theorem fence_mutual_exclusion
(mutInc mutChk gcInc gcChk : Nat) -- positions in the SC total order
(h_mut_po : mutInc < mutChk) -- program order, mutator
(h_gc_po : gcInc < gcChk) -- program order, collector
(h_mut_read0 : mutChk < gcInc) -- mutator read gGcActive = 0
(h_gc_read0 : gcChk < mutInc) : -- collector read counter = 0
False := by omega
/-! ## §8 Deadlock freedom
### Locks
The queue producers went lock-free (reserve a slot by fetch-add, then
publish it: incs with an RMW exchange — the validation peek must see
every completed inc barrier — decs with a release store of the desc,
self-protecting via the unexplained rc surplus). The validation peek
is lock-free too. What remains:
• gMergeLock (level 0: tag-slot claim + orphan roots)
• stripes[i].consumerLock (level i + 1, i in 0..N-1: excludes
DRAINS of the same stripe against each
other — drains wait out the two-store
publication window and close each batch
with a CAS, so no producer coordination
is needed)
• gWaitLock (leaf: pairs gWaitCond's wait/broadcast;
only ever held around a predicate check,
a wait(), or a broadcast() — never while
acquiring any other lock, and no other
lock is held when it is taken)
Total order: gMergeLock < consumerLock[0] < consumerLock[1] < ...
In fact the current paths never HOLD two of these at once — strictly
stronger than the ascending-order requirement the theorem needs:
**nimIncRefCyclic / nimAsgnYrc / nimSinkYrc / enqueueDec /
registerLocal / markDirtyFromQueues**: lock-free, no locks at all;
enqueueDec's overflow calls collectCycles with nothing held. ✓
**drainStripe**: consumerLock[i] alone; the processing inside
(trialDec, registerLocal into the thread-local buffer) takes no
lock. drainAllStripes: consumerLock[i] ascending, released between
stripes. ✓
**startCollection / nimYrcThreadTeardown**: drain first (consumerLock,
released), THEN gMergeLock for the slot claim / orphan spill —
sequential, never nested. adoptOrphans: gMergeLock alone. ✓
**validateDead / commitDead**: no locks (candidate re-registration is
the lock-free registerLocal). ✓
### Blocking waits (parked on gWaitCond after a bounded spin)
W1 backpressure (startCollection): waits for a free tag slot —
gMergeLock is RELEASED first; slots free when collections finish
(finishCollection broadcasts).
W2 solo gate (runCollection): a non-solo collection waits for
gSoloCapture = 0 — cleared when the solo collection's CAPTURE
ends (collectCyclesImpl broadcasts), before its commit.
W3 grace (commitDead): waits for other slots to leave capture phase
(broadcast at the phase 1→2 transition and at finish).
W4 fence (yrcGcFenceEnter): spins for in-flight seq ops — each is a
short critical section that never blocks (releaseMutatorLock is
a plain FetchSub).
W1W3 park on gWaitCond: the waiter re-checks its predicate under
gWaitLock before sleeping, and every state transition that can make a
predicate true (capture-end, collection-finish) broadcasts under the
same lock — so a transition either happens before the re-check (the
waiter never sleeps) or after it (the waiter is inside wait() and is
woken). No missed wakeups, and the wait-for structure is unchanged.
No wait cycle exists: order the blocking conditions by what they wait
FOR. A capture phase terminates unconditionally (finite traversal, no
waits inside — claimCell returns -1 immediately on contention). W2
waits only on a capture; W3 waits only on captures; a collection
executes W2 BEFORE its own capture and W3 AFTER it, so "X waits (W2)
on S's capture" and "S waits (W3) on X's capture" cannot hold
simultaneously: S clears gSoloCapture before entering commit, so by
the time S is in W3, X has passed W2. W1 waits on full collections,
which terminate because W2/W3/W4 do. Formally, the lock part is the
same ascending-order argument as before: -/
/-- Lock levels in YRC. -/
/-- Lock levels in YRC. Each lock maps to a unique natural number. -/
inductive LockId (n : Nat) where
| mergeLock : LockId n
| consumerLock (i : Nat) (h : i < n) : LockId n
| global : LockId n
| lockInc (i : Nat) (h : i < n) : LockId n
| lockDec (i : Nat) (h : i < n) : LockId n
/-- The level (priority) of each lock in the total order. -/
def lockLevel {n : Nat} : LockId n Nat
| .mergeLock => 0
| .consumerLock i _ => i + 1
| .global => 0
| .lockInc i _ => 2 * i + 1
| .lockDec i _ => 2 * i + 2
/-- All lock levels are distinct (the order is total and well-defined). -/
/-- All lock levels are distinct (the level function is injective). -/
theorem lockLevel_injective {n : Nat} (a b : LockId n)
(h : lockLevel a = lockLevel b) : a = b := by
cases a with
| mergeLock =>
| global =>
cases b with
| mergeLock => rfl
| consumerLock j hj => simp [lockLevel] at h
| consumerLock i hi =>
| global => rfl
| lockInc j hj => simp [lockLevel] at h
| lockDec j hj => simp [lockLevel] at h
| lockInc i hi =>
cases b with
| mergeLock => simp [lockLevel] at h
| consumerLock j hj =>
have : i = j := by simp [lockLevel] at h; exact h
| global => simp [lockLevel] at h
| lockInc j hj =>
have : i = j := by simp [lockLevel] at h; omega
subst this; rfl
| lockDec j hj => simp [lockLevel] at h; omega
| lockDec i hi =>
cases b with
| global => simp [lockLevel] at h
| lockInc j hj => simp [lockLevel] at h; omega
| lockDec j hj =>
have : i = j := by simp [lockLevel] at h; omega
subst this; rfl
/-- gMergeLock has the lowest level. -/
theorem mergeLock_level_min {n : Nat} (l : LockId n) (h : l .mergeLock) :
lockLevel (.mergeLock : LockId n) < lockLevel l := by
cases l with
| mergeLock => exact absurd rfl h
| consumerLock i hi => simp [lockLevel]
/-- Helper: stripe lock levels are strictly ascending across stripes. -/
theorem stripe_levels_ascending (i : Nat) :
2 * i + 1 < 2 * i + 2 2 * i + 2 < 2 * (i + 1) + 1 := by
constructor <;> omega
/-- **Deadlock Freedom** (2-thread wait cycle; N-thread follows by the
same transitivity on the wait-for chain): impossible when every
thread acquires locks in strictly ascending level order. -/
/-- lockInc levels are strictly ascending with index. -/
theorem lockInc_level_strict_mono {n : Nat} (i j : Nat) (hi : i < n) (hj : j < n)
(hij : i < j) : lockLevel (.lockInc i hi : LockId n) < lockLevel (.lockInc j hj) := by
simp [lockLevel]; omega
/-- lockDec levels are strictly ascending with index. -/
theorem lockDec_level_strict_mono {n : Nat} (i j : Nat) (hi : i < n) (hj : j < n)
(hij : i < j) : lockLevel (.lockDec i hi : LockId n) < lockLevel (.lockDec j hj) := by
simp [lockLevel]; omega
/-- Global lock has the lowest level (level 0). -/
theorem global_level_min {n : Nat} (l : LockId n) (h : l .global) :
lockLevel (.global : LockId n) < lockLevel l := by
cases l with
| global => exact absurd rfl h
| lockInc i hi => simp [lockLevel]
| lockDec i hi => simp [lockLevel]
/-- **Deadlock Freedom**: Any sequence of lock acquisitions that follows the
"acquire in ascending level order" discipline cannot deadlock.
This is a standard result: a total order on locks with the invariant that
every thread acquires locks in strictly ascending order prevents cycles
in the wait-for graph, which is necessary and sufficient for deadlock.
We prove the 2-thread case (the general N-thread case follows by the
same transitivity argument on the wait-for cycle). -/
theorem no_deadlock_from_total_order {n : Nat}
-- Two threads each hold a lock and wait for another
(held₁ waited₁ held₂ waited₂ : LockId n)
-- Thread 1 holds held₁ and wants waited₁ (ascending order)
(h1 : lockLevel held₁ < lockLevel waited₁)
-- Thread 2 holds held₂ and wants waited₂ (ascending order)
(h2 : lockLevel held₂ < lockLevel waited₂)
-- Deadlock requires: thread 1 waits for what thread 2 holds,
-- and thread 2 waits for what thread 1 holds
(h_wait1 : waited₁ = held₂)
(h_wait2 : waited₂ = held₁) :
False := by
subst h_wait1; subst h_wait2
omega
/-! ## Summary of verified properties (all QED, no sorry)
/-! ### Summary of verified properties (all QED, no sorry)
§1 `yrc_safety` — the collector frees only unanchored objects, which
no thread can reach. No use-after-free at the graph level.
§2 `no_lost_object` — the atomically published edge is traced.
§3 `step_preserves_deadInv`, `deadInv_stable`, `garbage_stability` —
a closed set stays closed under every mutator write, root
copy/drop, allocation, and foreign free: snapshot garbage cannot
be resurrected. `commit_free_safe` — freeing a commit-validated
dead set after ANY further concurrent activity is safe.
§4 `validated_no_external`, `validated_closed` — the Σrc = Σinternal
check plus rc-exactness forces zero external references, i.e. the
dead set is closed at commit time (feeding §3).
§5 `tags_distinct`, `partitions_disjoint`, `cross_target_live` —
concurrent collections own disjoint partitions, and a cell
referenced across a partition boundary is never freed by its
owner this round (soundness of claimCell's -1 + crossPend).
§6 `post_commit_snap_misses_dead`, `grace_no_use_after_free` — with
commitDead's grace spin, no capture ever dereferences freed
memory.
§7 `fence_mutual_exclusion` — the SEQ_CST Dekker pairing in
seqs_v2.nim excludes seq structure mutation during collection.
§8 `lockLevel_injective`, `mergeLock_level_min`,
`no_deadlock_from_total_order` — the remaining locks form a total
order acquired ascending; spin-waits form an acyclic wait-for
structure (prose above).
1. `reachable_is_anchored`: Every reachable object is anchored
(has a path from an externally-referenced object via heap edges).
## What is NOT proved
2. `yrc_safety`: The collector only frees unanchored objects,
which are unreachable by all threads. **No use-after-free.**
• Tarjan/SCC implementation correctness: that `capture` computes the
actual SCCs and that computeDeadness's per-SCC sums equal the model's
Σrc/Σinternal for the emitted dead set (condensation, sinks-first
order, deadIn accounting). §4 takes the counts as given.
• rc-exactness mechanics: that merge + the dirty check + the rc-word
recheck really imply "commit-time rc = internal + external" (§4's
h_exact). The argument: rc is only mutated by atomic direct incs
(caught by the recheck), merged queue entries (queues drained at
merge; later entries caught by the dirty peek), and the collector's
own inRootsFlag toggles (excluded from the compared word — see the
comment above claimCell).
• The C11 memory model: §7 assumes sequential consistency for the
SEQ_CST operations (sound: SEQ_CST ops do form a total order) and
the acquire/release reasoning elsewhere is informal.
• Liveness/completeness: every dead cycle is EVENTUALLY freed.
Aborted (dirty) SCCs and crossPend targets are re-registered as
candidates, so they are re-examined; termination of that loop under
adversarial mutators is not formalized. Also unproved: termination
bounds for the four spin-waits (prose in §8).
• Tag wrap-around: 2³¹ collections completing during one collection's
lifetime could forge a stale stamp (noted at `tags_distinct`).
3. `no_lost_object`: After `a.field = b`, `b` is reachable
(atomic store makes the edge visible immediately).
## Epoch stamps (generational pruning)
4. `src_safe_in_window`: Even between the atomic store and
the buffered inc, the collector cannot free src.
Commit re-stamps proven-live cells with (epochBase|epoch, survivalAge)
in the claim word; a capture treats a current-epoch stamp of age ≥
YrcPromoteAge on a DESCENDANT as an opaque live external and does not
descend. Soundness needs no new lemmas: a pruned cell is simply an
uncaptured cell, so the captured set shrinks and every §3§6 statement
quantifies over a smaller S. Pruning can only ADD unexplained external
refs to captured SCCs (a pruned predecessor's refs are never explained
by internal/deadIn), so it can force a false "live", never a false
"dead" — the conservative direction. Completeness (bounded float,
≤ ~2 epochs) rests on four hooks, each keeping a dec-witness
registered:
E1 roots never prune: a registered candidate is always fully
root-scanned, stamps notwithstanding;
E2 an SCC that pruned an out-edge and survives keeps one member
registered (flagPruned) — its "live" verdict may lean on a stamp
that went stale within the epoch;
E3 a commit-time dec into a stamped cell re-registers the target —
the dec may be the death blow to a cell no collection analyzed;
E4 explicit full collects advance the epoch first, so all stamps
are stale and nothing is pruned.
Not formalized. Also noted: the epoch clock counts collections, and
short epochs (≲ 4) resonate with the adaptive threshold — pruned
collections are cheap, so collections and hence epoch turns speed up,
re-tracing MORE than with no stamps; a work-based clock would fix it.
5. `lockLevel_injective`: All lock levels are distinct (well-defined total order).
6. `global_level_min`: The global lock has the lowest level.
7. `lockInc_level_strict_mono`, `lockDec_level_strict_mono`:
Stripe locks are strictly ordered by index.
8. `no_deadlock_from_total_order`: A 2-thread deadlock cycle is impossible
when both threads acquire locks in ascending level order.
Together these establish that YRC's write barrier protocol
(atomic store → buffer inc → buffer dec) is safe under concurrent
collection, and the locking discipline prevents deadlock.
## What is NOT proved: Completeness (liveness)
This proof covers **safety** (no use-after-free) and **deadlock-freedom**,
but does NOT prove **completeness** — that all garbage cycles are eventually
collected.
Completeness depends on the trial deletion algorithm (Bacon 2001) correctly
identifying closed cycles. Specifically it requires proving:
1. After `mergePendingRoots`, merged RCs equal logical RCs
(buffered inc/dec exactly compensate graph changes since last merge).
2. `markGray` subtracts exactly the internal (heap→heap) edge count from
each node's merged RC, yielding `trialRC(x) = externalRefCount(x)`.
3. `scan` correctly partitions: nodes with `trialRC ≥ 0` are rescued by
`scanBlack`; nodes with `trialRC < 0` remain white.
4. White nodes form closed subgraphs with zero external refs → garbage.
These properties follow from the well-known Bacon trial-deletion algorithm
and are assumed here rather than re-proved. The YRC-specific contribution
(buffered RCs, striped queues, concurrent mutators) is what our safety
proof covers — showing that concurrency does not break the preconditions
that trial deletion relies on (physical graph consistency, eventual RC
consistency after merge).
Reference: D.F. Bacon and V.T. Rajan, "Concurrent Cycle Collection in
Reference Counted Systems", ECOOP 2001 — the deadness arithmetic is
the condensation form of their trial deletion; the capture/validate/
commit structure and the SATB use of the deferred-dec queues are
yrc-specific.
Reference Counted Systems", ECOOP 2001.
-/

View File

@@ -1,594 +0,0 @@
/-
Tarjan-based deadness computation — correctness proof
=====================================================
Self-contained, no Mathlib. Checked with Lean 4 (v4.32.0).
Companion to yrc_proof.lean; models the NOVEL part of yrc.nim's
collector: cycle detection via a single Tarjan SCC traversal plus one
linear reverse scan over the condensation, replacing Bacon-style trial
deletion (three traversals: markGray / scan / collectWhite).
## The algorithm (capture / computeDeadness in yrc.nim)
`capture` runs an iterative Tarjan DFS from the candidate roots. Each
visited cell is claimed (dense index in the header), its rc word is
snapshotted, and every traversed slot contributes one edge record.
SCCs are numbered 0, 1, 2, … in POP (completion) order. Tarjan's
invariant: when an SCC is completed, every SCC it points to was
completed earlier — so every condensation cross edge goes from a
HIGHER SCC id to a LOWER one ("sinks first").
`computeDeadness` then makes ONE pass s = nScc1 … 0 (sources before
sinks, since in-edges come from higher ids):
ext(s) = sumRefs(s) internal(s) deadIn(s)
if not forcedLive(s) and ext(s) == 0:
s is DEAD; for each cross edge s → t: deadIn(t) += 1
else:
s is LIVE; for each cross edge s → t: forcedLive(t) := true
where sumRefs(s) = Σ rc over members, internal(s) = # captured edges
within s, and forcedLive is seeded from cells still registered in the
roots buffer (inRootsFlag).
## What we prove
Fix the SPEC of liveness on the condensation: an SCC is live iff it
has an external reference, a roots-buffer seed, or a captured cross
edge from a live SCC (`LiveScc`, an inductive definition).
1. `scan_dead_iff_not_live` — any deadness assignment satisfying the
scan's per-SCC equation (well-defined thanks to the sinks-first
edge order) marks an SCC dead IFF it is not live. Soundness AND
completeness in one theorem: the single reverse scan computes the
garbage set EXACTLY on the captured snapshot.
2. `impl_fixpoint_is_spec` — the implementation's ARITHMETIC form
(ext = sumRefs internal deadIn with forcedLive propagation) is
the same equation, given rc-exactness (sumRefs = external +
internal + cross-in; established by merge + commit validation, see
yrc_proof.lean §4).
3. Cell-level bridge: `tarjan_sound` — cells of dead SCCs are
unreachable in the snapshot; `tarjan_complete` — every captured
garbage cell IS marked dead (this needs strong connectivity of the
SCCs and exactness of the external counts; Bacon needs his second
and third traversals for the same guarantee).
4. `demotion_closure_sound` — validate-time demotion (an SCC dropped
from the dead set because a mutator dirtied it) must PROPAGATE
along captured cross edges: the freed set stays closed only if the
demoted set is successor-closed within the dead set. A demoted SCC
survives with its out-edges intact, so any still-dead target would
be freed while a surviving cell points at it.
## What is assumed (and where it is discharged)
• The sinks-first edge order (`horder`) — Tarjan's classical
invariant; the DFS itself is not modeled.
• rc-exactness (`hcount`) — discharged operationally by yrc_proof §4
(merge + dirty check + rc-word recheck).
• That `capture` records exactly the heap edges among captured cells
and that SCC members are mutually reachable (`h_edge_resp`,
`h_conn`, `h_cross_real`) — properties of the traversal + Tarjan.
-/
abbrev Obj := Nat
/-! ## §1 Descending induction
The scan processes higher SCC ids first; every recursive dependency
of `dead s` is on some `u > s`. This induction principle is the
well-definedness of the whole scheme. -/
theorem descending_induction {n : Nat} (P : Fin n Prop)
(step : s : Fin n, ( u : Fin n, s < u P u) P s) :
s, P s := by
have key : k, s : Fin n, n - s.val k P s := by
intro k
induction k with
| zero =>
intro s hs
have := s.isLt
omega
| succ k ih =>
intro s _
apply step
intro u hu
apply ih
have h1 := u.isLt
have h2 : s.val < u.val := hu
omega
intro s
exact key n s (by omega)
/-! ## §2 The condensation and the liveness spec
`edges` are the captured condensation cross edges (with multiplicity:
one entry per traversed slot, exactly like cap.edges bucketed into
crossTgt). `extRefs s` counts references into SCC `s` from OUTSIDE
the capture: stack refs, uncaptured heap cells, other collections'
partitions — everything in Σrc not explained by captured edges.
`seed s` is the inRootsFlag forcedLive seeding. -/
section Condensation
variable {n : Nat}
variable (edges : List (Fin n × Fin n))
variable (extRefs : Fin n Nat)
variable (seed : Fin n Bool)
/-- The SPEC: an SCC is live iff something external anchors it —
directly or through a chain of captured cross edges. -/
inductive LiveScc : Fin n Prop where
| ext (s : Fin n) : 0 < extRefs s LiveScc s
| root (s : Fin n) : seed s = true LiveScc s
| pred (u s : Fin n) : (u, s) edges LiveScc u LiveScc s
/-- The per-SCC equation the reverse scan establishes: dead iff no
external refs, no seed, and ALL cross predecessors dead. (The
sinks-first order makes this a valid definition: every predecessor
has a higher id and is decided first — see `descending_induction`;
without that order the "definition" would be circular.) -/
def ScanEq (dead : Fin n Bool) : Prop :=
s, dead s = true
(extRefs s = 0 seed s = false
e edges, e.2 = s dead e.1 = true)
/-- Live SCCs are never marked dead (soundness direction). -/
theorem live_not_dead (dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead) :
s, LiveScc edges extRefs seed s dead s true := by
intro s hl
induction hl with
| ext s h =>
intro hd
have := ((hfix s).mp hd).1
omega
| root s h =>
intro hd
have := ((hfix s).mp hd).2.1
rw [h] at this
cases this
| pred u s hmem _ ih =>
intro hd
exact ih (((hfix s).mp hd).2.2 (u, s) hmem rfl)
/-- Non-live SCCs are always marked dead (completeness direction) —
by descending induction along the scan order. -/
theorem not_live_dead
(horder : e edges, e.2 < e.1)
(dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead) :
s, ¬ LiveScc edges extRefs seed s dead s = true := by
refine descending_induction
(fun s => ¬ LiveScc edges extRefs seed s dead s = true) ?_
intro s ihs hnl
rw [hfix]
refine ?_, ?_, ?_
· cases Nat.eq_zero_or_pos (extRefs s) with
| inl h => exact h
| inr h => exact absurd (LiveScc.ext s h) hnl
· cases hsd : seed s with
| false => rfl
| true => exact absurd (LiveScc.root s hsd) hnl
· intro e he hes
have hlt : s < e.1 := by
have := horder e he
rw [hes] at this
exact this
apply ihs e.1 hlt
intro hlu
have hmem : (e.1, s) edges := by
rw [ hes]
simpa using he
exact hnl (LiveScc.pred e.1 s hmem hlu)
/-- **Main condensation theorem**: the single reverse scan computes
EXACTLY the non-live SCCs. One Tarjan DFS + one linear scan replace
Bacon's three graph traversals, with no loss of precision on the
snapshot. -/
theorem scan_dead_iff_not_live
(horder : e edges, e.2 < e.1)
(dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead) :
s, dead s = true ¬ LiveScc edges extRefs seed s := by
intro s
constructor
· intro hd hl
exact live_not_dead edges extRefs seed dead hfix s hl hd
· exact not_live_dead edges extRefs seed horder dead hfix s
/-! ## §3 The implementation's arithmetic form
computeDeadness does not test "all predecessors dead" directly; it
maintains ext(s) = sumRefs(s) internal(s) deadIn(s) and a
forcedLive flag pushed along cross edges of live SCCs. We show this
is the same equation, given rc-exactness:
sumRefs s = extRefs s + internal s + (# cross edges into s).
ext(s) = 0 then says extRefs s = 0 AND every cross in-edge came from
a dead predecessor; ¬forcedLive says no seed and no LIVE predecessor
pushed the flag — together exactly `ScanEq`. -/
def inCount (s : Fin n) : Nat :=
edges.countP (fun e => e.2 == s)
def deadInCount (dead : Fin n Bool) (s : Fin n) : Nat :=
edges.countP (fun e => e.2 == s && dead e.1)
/-- countP is monotone under pointwise implication. -/
theorem countP_le_of_imp {α : Type} (l : List α) (p q : α Bool)
(himp : x l, p x = true q x = true) :
l.countP p l.countP q := by
induction l with
| nil => simp
| cons a l ih =>
have iht := ih (fun x hx => himp x (List.mem_cons_of_mem a hx))
by_cases hpa : p a = true
· have hqa := himp a (by simp) hpa
simp [hpa, hqa]
omega
· simp only [List.countP_cons]
have : p a = false := by
cases h : p a
· rfl
· exact absurd h hpa
simp [this]
omega
/-- If a stronger predicate matches as often as a weaker one, they
agree on every element. -/
theorem countP_eq_forces_all {α : Type} (l : List α) (p q : α Bool)
(himp : x l, q x = true p x = true)
(heq : l.countP p = l.countP q) :
x l, p x = true q x = true := by
induction l with
| nil => intro x hx; cases hx
| cons a l ih =>
have himpt : x l, q x = true p x = true :=
fun x hx => himp x (List.mem_cons_of_mem a hx)
have hmono := countP_le_of_imp l q p himpt
intro x hx hpx
simp only [List.countP_cons] at heq
cases List.mem_cons.mp hx with
| inl hxa =>
subst hxa
cases hqx : q x with
| true => rfl
| false =>
exfalso
simp [hpx, hqx] at heq
omega
| inr hxl =>
have hqa_pa : (if q a = true then 1 else 0) (if p a = true then 1 else 0) := by
by_cases hq : q a = true
· simp [hq, himp a (by simp) hq]
· simp [hq]
have heqt : l.countP p = l.countP q := by
by_cases hq : q a = true
· simp [hq, himp a (by simp) hq] at heq
omega
· have hqf : q a = false := by
cases h : q a
· rfl
· exact absurd h hq
by_cases hp : p a = true
· simp [hp, hqf] at heq
omega
· have hpf : p a = false := by
cases h : p a
· rfl
· exact absurd h hp
simp [hpf, hqf] at heq
omega
exact ih himpt heqt x hxl hpx
/-- If all cross predecessors of `s` are dead, deadIn equals the full
in-count (and vice versa). -/
theorem deadIn_eq_inCount_iff (dead : Fin n Bool) (s : Fin n) :
deadInCount edges dead s = inCount edges s
( e edges, e.2 = s dead e.1 = true) := by
constructor
· intro heq e he hes
have himp : x edges, (fun e => e.2 == s && dead e.1) x = true
(fun e => e.2 == s) x = true := by
intro x _ hx
simp only [Bool.and_eq_true] at hx
exact hx.1
have := countP_eq_forces_all edges
(fun e => e.2 == s) (fun e => e.2 == s && dead e.1)
himp heq.symm e he
have hbeq : (e.2 == s) = true := by
simp [hes]
have := this hbeq
simp only [Bool.and_eq_true] at this
exact this.2
· intro hall
unfold deadInCount inCount
apply List.countP_congr
intro e he
by_cases hes : e.2 = s
· simp [hes, hall e he hes]
· have : (e.2 == s) = false := by
simp [hes]
simp [this]
/-- The implementation's per-SCC decision, verbatim from
computeDeadness: NOT forced (no seed, no live predecessor pushed
the flag) and ext = sumRefs internal deadIn = 0 (stated
subtraction-free). -/
def ImplEq (sumRefs internal : Fin n Nat) (dead : Fin n Bool) : Prop :=
s, dead s = true
(¬ (seed s = true e edges, e.2 = s dead e.1 = false)
sumRefs s = internal s + deadInCount edges dead s)
/-- **The arithmetic is the spec**: under rc-exactness, the
implementation's equation is `ScanEq`, so `scan_dead_iff_not_live`
applies to computeDeadness as written. -/
theorem impl_fixpoint_is_spec
(sumRefs internal : Fin n Nat) (dead : Fin n Bool)
(hcount : s, sumRefs s = extRefs s + internal s + inCount edges s)
(himpl : ImplEq edges seed sumRefs internal dead) :
ScanEq edges extRefs seed dead := by
intro s
rw [himpl s]
constructor
· rintro hnf, harith
have hor := hnf
rw [not_or] at hor
obtain hseed, hnopred := hor
have hseedf : seed s = false := by
cases h : seed s
· rfl
· exact absurd h hseed
have hall : e edges, e.2 = s dead e.1 = true := by
intro e he hes
cases h : dead e.1 with
| true => rfl
| false => exact absurd e, he, hes, h hnopred
have hdc := (deadIn_eq_inCount_iff edges dead s).mpr hall
have hc := hcount s
refine by omega, hseedf, hall
· rintro hext, hseedf, hall
have hdc := (deadIn_eq_inCount_iff edges dead s).mpr hall
refine ?_, ?_
· rw [not_or]
refine by simp [hseedf], ?_
rintro e, he, hes, hdf
rw [hall e he hes] at hdf
cases hdf
· have hc := hcount s
omega
end Condensation
/-! ## §4 Cell-level correctness
Bridge from the condensation to the actual heap snapshot. `extRef`
covers every reference source outside the capture: mutator stacks,
the roots buffer, uncaptured heap cells' slots that the arithmetic
cannot explain, and other collections' partitions (cross-collection
edges — this is the SCC-side view of `cross_target_live` in
yrc_proof.lean §5). -/
structure CellGraph where
edge : Obj Obj Prop
extRef : Obj Prop
/-- A cell is live iff an external reference anchors it through heap
edges (the cell-level ground truth; `anchored` of yrc_proof.lean). -/
inductive CellLive (g : CellGraph) : Obj Prop where
| ext (x : Obj) : g.extRef x CellLive g x
| step (x y : Obj) : CellLive g x g.edge x y CellLive g y
/-- Paths through heap edges, used to move liveness around inside an
SCC (Tarjan guarantees SCC members are mutually reachable). -/
inductive EdgePath (g : CellGraph) : Obj Obj Prop where
| refl (x : Obj) : EdgePath g x x
| step (x y z : Obj) : EdgePath g x y g.edge y z EdgePath g x z
theorem cellLive_along_path (g : CellGraph) (u v : Obj)
(hl : CellLive g u) (hp : EdgePath g u v) : CellLive g v := by
induction hp with
| refl => exact hl
| step _ _ _ hedge ih => exact CellLive.step _ _ ih hedge
section CellBridge
variable {n : Nat}
variable (g : CellGraph)
variable (edges : List (Fin n × Fin n))
variable (extRefs : Fin n Nat)
variable (seed : Fin n Bool)
variable (captured : Obj Prop)
variable (scc : Obj Fin n)
/-- Any live captured cell sits in a live SCC.
Premises are properties of `capture`:
* `h_edge_resp` — every heap edge between captured cells was
recorded (same SCC → internal; different → cross edge);
* `h_closed` — an edge from an UNCAPTURED cell is unexplained by
the captured arithmetic, so it lands in extRefs;
* `h_ext` — direct external refs (stacks, roots buffer, foreign
partitions) are counted in extRefs. -/
theorem captured_live_scc
(h_edge_resp : u v, captured u captured v g.edge u v
scc u = scc v (scc u, scc v) edges)
(h_closed : u v, captured v g.edge u v ¬ captured u
0 < extRefs (scc v))
(h_ext : v, captured v g.extRef v 0 < extRefs (scc v)) :
x, CellLive g x captured x
LiveScc edges extRefs seed (scc x) := by
intro x hl
induction hl with
| ext x h =>
intro hc
exact LiveScc.ext _ (h_ext x hc h)
| step u v hu hedge ih =>
intro hcv
by_cases hcu : captured u
· cases h_edge_resp u v hcu hcv hedge with
| inl heq => rw [ heq]; exact ih hcu
| inr hmem => exact LiveScc.pred _ _ hmem (ih hcu)
· exact LiveScc.ext _ (h_closed u v hcv hedge hcu)
/-- **Soundness**: every cell of a dead SCC is unanchored in the
snapshot — freeing it is justified by yrc_proof.lean §1
(`yrc_safety`) + §3 (stability through the commit window). -/
theorem tarjan_sound
(dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead)
(h_edge_resp : u v, captured u captured v g.edge u v
scc u = scc v (scc u, scc v) edges)
(h_closed : u v, captured v g.edge u v ¬ captured u
0 < extRefs (scc v))
(h_ext : v, captured v g.extRef v 0 < extRefs (scc v)) :
x, captured x dead (scc x) = true ¬ CellLive g x := by
intro x hc hd hl
exact live_not_dead edges extRefs seed dead hfix (scc x)
(captured_live_scc g edges extRefs seed captured scc
h_edge_resp h_closed h_ext x hl hc) hd
/-- Every cell of a live SCC is genuinely live. Needs the converse
premises: external counts are EXACT (no phantom refs — deferred
decs inflate rc, so in the running system this holds only after
the merge; overcounts delay collection by a round, they never
cause a wrong free), cross edges are real edges, and SCC members
are mutually reachable (Tarjan). -/
theorem live_scc_cells_live
(h_ext_exact : s : Fin n, 0 < extRefs s
v, captured v scc v = s g.extRef v)
(h_seed_exact : s : Fin n, seed s = true
v, captured v scc v = s g.extRef v)
(h_cross_real : (u s : Fin n), (u, s) edges
cu cv, captured cu captured cv scc cu = u scc cv = s
g.edge cu cv)
(h_conn : u v, captured u captured v scc u = scc v
EdgePath g u v) :
s, LiveScc edges extRefs seed s
x, captured x scc x = s CellLive g x := by
intro s hl
induction hl with
| ext s h =>
intro x hc hs
obtain v, hcv, hsv, hev := h_ext_exact s h
exact cellLive_along_path g v x (CellLive.ext v hev)
(h_conn v x hcv hc (by rw [hsv, hs]))
| root s h =>
intro x hc hs
obtain v, hcv, hsv, hev := h_seed_exact s h
exact cellLive_along_path g v x (CellLive.ext v hev)
(h_conn v x hcv hc (by rw [hsv, hs]))
| pred u s hmem _ ih =>
intro x hc hs
obtain cu, cv, hccu, hccv, hscu, hscv, he := h_cross_real u s hmem
have hculive : CellLive g cu := ih cu hccu hscu
exact cellLive_along_path g cv x (CellLive.step cu cv hculive he)
(h_conn cv x hccv hc (by rw [hscv, hs]))
/-- **Completeness**: every captured garbage cell is marked dead — the
scan collects ALL cycles reachable from the candidate set in one
round (on the snapshot; concurrent inflation only defers). -/
theorem tarjan_complete
(horder : e edges, e.2 < e.1)
(dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead)
(h_ext_exact : s : Fin n, 0 < extRefs s
v, captured v scc v = s g.extRef v)
(h_seed_exact : s : Fin n, seed s = true
v, captured v scc v = s g.extRef v)
(h_cross_real : (u s : Fin n), (u, s) edges
cu cv, captured cu captured cv scc cu = u scc cv = s
g.edge cu cv)
(h_conn : u v, captured u captured v scc u = scc v
EdgePath g u v) :
x, captured x ¬ CellLive g x dead (scc x) = true := by
intro x hc hnl
apply not_live_dead edges extRefs seed horder dead hfix
intro hl
exact hnl (live_scc_cells_live g edges extRefs seed captured scc
h_ext_exact h_seed_exact h_cross_real h_conn (scc x) hl x hc rfl)
end CellBridge
/-! ## §5 Validate-time demotion must propagate
validateDead demotes a dead SCC when a mutator dirtied it (queue
entry or changed rc word). A demoted SCC becomes a survivor: its
slots are NOT nil'd at commit, so its captured out-edges remain in
the heap. If a cross target of a demoted SCC stayed in the dead set,
the commit would free a cell that a surviving cell still points to —
deadIn had explained that edge away under the assumption that the
predecessor dies too.
Minimal instance of the hazard: two SCCs, one edge 1 → 0, both
computed dead (ext = 0 for both; SCC 0's only reference comes from
SCC 1, subtracted as deadIn). Demote SCC 1 alone, and the freed set
{0} has a live in-edge from the surviving SCC 1.
The theorem below states the repair: if the demoted set `K` is
successor-closed within the dead set (demoting s also demotes every
dead t with a captured edge s → t, transitively — one countdown pass
suffices because edges go from higher to lower ids), then the freed
set F = dead K is predecessor-closed: every captured edge into F
comes from F. Combined with extRefs = 0 and no seed (ScanEq) this
makes F closed in the sense of yrc_proof.lean §3, so freeing F is
covered by `commit_free_safe` there. -/
theorem demotion_closure_sound {n : Nat}
(edges : List (Fin n × Fin n))
(extRefs : Fin n Nat) (seed : Fin n Bool)
(dead : Fin n Bool)
(hfix : ScanEq edges extRefs seed dead)
(K : Fin n Prop) -- the demoted SCCs
(hK_closed : e edges, K e.1 dead e.2 = true K e.2) :
-- every captured edge into the freed set comes from the freed set
e edges, (dead e.2 = true ¬ K e.2)
(dead e.1 = true ¬ K e.1) := by
intro e he hd2, hk2
have hd1 : dead e.1 = true :=
((hfix e.2).mp hd2).2.2 e he rfl
refine hd1, ?_
intro hk1
exact hk2 (hK_closed e he hk1 hd2)
/-- Without successor-closure the guarantee genuinely fails: in the
two-SCC instance above, demoting only SCC 1 leaves the freed set
{0} with an in-edge from a survivor. (Concrete witness, checked by
`decide`-style evaluation.) -/
example :
let edges : List (Fin 2 × Fin 2) := [(1, 0)]
let dead : Fin 2 Bool := fun _ => true
let K : Fin 2 Prop := fun s => s = 1 -- demote only SCC 1
-- ScanEq holds for `dead` (both SCCs legitimately computed dead) …
ScanEq edges (fun _ => 0) (fun _ => false) dead
-- … yet the freed set {0} has an in-edge from surviving SCC 1:
((1, 0) edges dead 0 = true ¬ K 0 K 1) := by
refine ?_, ?_
· intro s
simp
· refine by simp, rfl, by simp, rfl
/-! ## Summary (all QED, no sorry)
* `descending_induction` — the sinks-first SCC numbering makes the
reverse scan a well-founded definition.
* `scan_dead_iff_not_live` — the scan marks an SCC dead iff it is
not externally anchored: exact garbage identification in ONE
linear pass over the condensation.
* `impl_fixpoint_is_spec` — the implementation's arithmetic
(ext = sumRefs internal deadIn, forcedLive propagation) is
that same equation under rc-exactness.
* `tarjan_sound` / `tarjan_complete` — at the cell level: dead cells
are unanchored (frees are safe) and unanchored captured cells are
freed (nothing is missed on the snapshot).
* `demotion_closure_sound` + counterexample — demotion is sound iff
it propagates along captured cross edges to still-dead targets;
a lone demotion can leave the freed set with a surviving
predecessor.
Not modeled: the Tarjan DFS itself (its two classical invariants —
SCC partition and sinks-first emission — enter as premises), the
iterative traceStack encoding, crossPend (cross-collection edges are
folded into `extRefs`, justified by yrc_proof.lean §5), and the
temporal validity of rc-exactness (yrc_proof.lean §4).
-/

View File

@@ -261,11 +261,6 @@ const
FILE_ATTRIBUTE_OFFLINE* = 0x00001000'i32
FILE_ATTRIBUTE_NOT_CONTENT_INDEXED* = 0x00002000'i32
IO_REPARSE_TAG_MOUNT_POINT* = 0xA0000003'i32
IO_REPARSE_TAG_SYMLINK* = 0xA000000C'i32
MAXIMUM_REPARSE_DATA_BUFFER_SIZE* = 16 * 1024
SYMLINK_FLAG_RELATIVE* = 0x1'i32
FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'i32
FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'i32
FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32
@@ -287,10 +282,6 @@ const
MOVEFILE_REPLACE_EXISTING* = 0x1'i32
MOVEFILE_WRITE_THROUGH* = 0x8'i32
# CTL_CODE(FILE_DEVICE_FILE_SYSTEM = 9, func = 42, METHOD_BUFFERED = 0,
# FILE_ANY_ACCESS = 0)
FSCTL_GET_REPARSE_POINT* = 0x000900A8'i32
type
WIN32_FIND_DATA* {.pure.} = object
dwFileAttributes*: int32
@@ -663,12 +654,6 @@ proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD,
dwCreationDisposition, dwFlagsAndAttributes: DWORD,
hTemplateFile: Handle): Handle {.
stdcall, dynlib: "kernel32", importc: "CreateFileW".}
proc deviceIoControl*(hDevice: Handle, dwIoControlCode: DWORD,
lpInBuffer: pointer, nInBufferSize: DWORD,
lpOutBuffer: pointer, nOutBufferSize: DWORD,
lpBytesReturned: var DWORD,
lpOverlapped: pointer): WINBOOL {.
stdcall, dynlib: "kernel32", importc: "DeviceIoControl".}
proc deleteFileW*(pathName: WideCString): int32 {.
importc: "DeleteFileW", dynlib: "kernel32", stdcall.}
proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD,

View File

@@ -35,7 +35,7 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp
# suggestionResultHook, because suggest.nim is included by sigmatch.
# So we import that one instead.
import compiler / [options, commands, modules,
passes, passaux, msgs, pipelines,
passes, passaux, msgs,
sigmatch, ast,
idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper,
pathutils, condsyms, syntaxes, suggestsymdb]
@@ -261,14 +261,6 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int,
var isKnownFile = true
let dirtyIdx = fileInfoIdx(conf, file, isKnownFile)
# Cold-opened include file: nothing has been compiled/loaded yet, so its
# includer is unknown. Scan the nimcache NIFs for the module that `include`s
# this exact file and register that one edge; `parentModule(dirtyIdx)` then
# resolves to the includer, which we (re)compile below.
if conf.ideImportsFromNif and graph.needsIncludeScan(dirtyIdx):
discard graph.registerIncluderFromNif(dirtyIdx)
let isInclude = graph.inclToMod.hasKey(dirtyIdx)
if not dirtyfile.isEmpty: msgs.setDirtyFile(conf, dirtyIdx, dirtyfile)
else: msgs.setDirtyFile(conf, dirtyIdx, AbsoluteFile"")
@@ -277,9 +269,9 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int,
conf.errorCounter = 0
if conf.suggestVersion == 1:
graph.usageSym = nil
if not isKnownFile and not isInclude:
if not isKnownFile:
graph.clearInstCache(dirtyIdx)
graph.compilePipelineProject(dirtyIdx)
graph.compileProject(dirtyIdx)
if conf.suggestVersion == 0 and conf.ideCmd in {ideUse, ideDus} and
dirtyfile.isEmpty:
discard "no need to recompile anything"
@@ -287,19 +279,10 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int,
let modIdx = graph.parentModule(dirtyIdx)
graph.markDirty dirtyIdx
graph.markClientsDirty dirtyIdx
# For an include-file query the includer must be re-sem'd so the include body
# (where trackPos sits) is re-checked. An already-loaded includer is only
# recompiled when dirty (pipelines.compilePipelineModule), and the include
# edge isn't always in `g.deps` for markClientsDirty to catch (notably on the
# EPC path), so mark the includer dirty explicitly.
if isInclude:
graph.markDirty modIdx
if conf.ideCmd != ideMod:
# `isInclude`: a freshly discovered include file is not "known" yet, but we
# still must (source-)compile its includer to serve the query.
if isKnownFile or isInclude:
if isKnownFile:
graph.clearInstCache(modIdx)
graph.compilePipelineProject(modIdx)
graph.compileProject(modIdx)
if conf.ideCmd in {ideUse, ideDus}:
let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym
if u != nil:
@@ -594,7 +577,7 @@ proc recompileFullProject(graph: ModuleGraph) =
graph.vm = nil
graph.resetAllModules()
GC_fullCollect()
graph.compilePipelineProject()
graph.compileProject()
proc mainThread(graph: ModuleGraph) =
let conf = graph.config
@@ -638,14 +621,10 @@ var
proc mainCommand(graph: ModuleGraph) =
let conf = graph.config
# Use the pipeline driver (same as `nim check`): it is where IC/NIF loading
# and emission live. The legacy `passes.compileProject` path has no NIF support.
setPipeLinePass(graph, SemPass)
# cmdM loads the unchanged import closure from precompiled NIF; cmdCheck
# recompiles everything from source. `ideActive` keeps suggestion collection
# and error-resilience regardless of which mode we run under.
conf.setCmd(if conf.ideImportsFromNif: cmdM else: cmdCheck)
conf.ideActive = true
clearPasses(graph)
registerPass graph, verbosePass
registerPass graph, semPass
conf.setCmd cmdIdeTools
defineSymbol(conf.symbols, $conf.backend)
wantMainModule(conf)
@@ -667,7 +646,7 @@ proc mainCommand(graph: ModuleGraph) =
# compile the project before showing any input so that we already
# can answer questions right away:
benchmark "Initial compilation":
compilePipelineProject(graph)
compileProject(graph)
open(requests)
open(results)
@@ -819,7 +798,7 @@ proc recompilePartially(graph: ModuleGraph, projectFileIdx = InvalidFileIdx) =
try:
benchmark "Recompilation":
graph.compilePipelineProject(projectFileIdx)
graph.compileProject(projectFileIdx)
except Exception as e:
myLog fmt "Failed to recompile partially with the following error:\n {e.msg} \n\n {e.getStackTrace()}"
try:
@@ -1099,22 +1078,9 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}, tag: {tag}"
var fileIndex: FileIndex = default(FileIndex)
# The module to (re)compile for this query. For a normal file it is the file
# itself; for an `include` file it is the module that includes it — the include
# body is only sem'd as part of its includer, and the includer compiles as the
# main module (source, never NIF-loaded), so the include statements at the
# cursor get re-checked. The query position stays in the include file.
var moduleToCompile: FileIndex = default(FileIndex)
var isIncludeQuery = false
if not (cmd in {ideRecompile, ideGlobalSymbols}):
fileIndex = fileInfoIdx(conf, file)
# Discover an include file's includer from the NIF include graph (cold query)
# so `parentModule` can map it; see registerIncluderFromNif.
if conf.ideImportsFromNif and graph.needsIncludeScan(fileIndex):
discard graph.registerIncluderFromNif(fileIndex)
isIncludeQuery = graph.inclToMod.hasKey(fileIndex)
moduleToCompile = if isIncludeQuery: graph.parentModule(fileIndex) else: fileIndex
msgs.setDirtyFile(
conf,
fileIndex,
@@ -1134,20 +1100,14 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
# these commands require partially compiled project
elif cmd in {ideSug, ideCon, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration, ideExpand} and
(graph.needsCompilation(fileIndex) or cmd in {ideSug, ideCon} or isIncludeQuery):
(graph.needsCompilation(fileIndex) or cmd in {ideSug, ideCon}):
# for ideSug use v2 implementation
if cmd in {ideSug, ideCon}:
conf.m.trackPos = newLineInfo(fileIndex, line, col)
conf.m.trackPosAttached = false
else:
conf.m.trackPos = default(TLineInfo)
# An include file's includer must be (re)compiled from source so the
# include body is re-sem'd; force it dirty since the include file itself
# is not a module the dirty machinery tracks.
if isIncludeQuery:
graph.markDirty moduleToCompile
graph.markClientsDirty moduleToCompile
graph.recompilePartially(moduleToCompile)
graph.recompilePartially(fileIndex)
case cmd
of ideDef:
@@ -1156,14 +1116,13 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
graph.suggestResult(s.sym, s.sym.info)
of ideType:
let s = graph.findSymData(file, line, col)
if not s.isNil and s.sym.typ != nil:
if not s.isNil:
let typeSym = s.sym.typ.sym
if typeSym != nil:
graph.suggestResult(typeSym, typeSym.info, ideType)
elif s.sym.typ.len != 0 and s.sym.typ[0] != nil:
elif s.sym.typ.len != 0:
let genericType = s.sym.typ[0].sym
if genericType != nil:
graph.suggestResult(genericType, genericType.info, ideType)
graph.suggestResult(genericType, genericType.info, ideType)
of ideUse, ideDus:
let symbol = graph.findSymData(file, line, col)
if not symbol.isNil:
@@ -1190,15 +1149,11 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile,
graph.markDirtyIfNeeded(file.string, fileIndex)
of ideSug, ideCon:
# ideSug/ideCon performs partial build of the file, thus mark it dirty for the
# future calls. For an include file, drive everything off its includer module
# (the include file has no module of its own — getModule would be nil).
# future calls.
graph.markDirtyIfNeeded(file.string, fileIndex)
if isIncludeQuery:
graph.markClientsDirty fileIndex
graph.recompilePartially(moduleToCompile)
let m = graph.getModule moduleToCompile
if m != nil:
incl m, sfDirty
graph.recompilePartially(fileIndex)
let m = graph.getModule fileIndex
incl m, sfDirty
of ideOutline:
let n = parseFile(fileIndex, graph.cache, graph.config)
graph.iterateOutlineNodes(n, graph.fileSymbols(fileIndex).deduplicateSymInfoPair(false))
@@ -1356,10 +1311,11 @@ else:
proc mockCommand(graph: ModuleGraph) =
retval = graph
let conf = graph.config
conf.setCmd(if conf.ideImportsFromNif: cmdM else: cmdCheck)
conf.ideActive = true
conf.setCmd cmdIdeTools
defineSymbol(conf.symbols, $conf.backend)
setPipeLinePass(graph, SemPass)
clearPasses(graph)
registerPass graph, verbosePass
registerPass graph, semPass
wantMainModule(conf)
@@ -1375,7 +1331,7 @@ else:
# compile the project before showing any input so that we already
# can answer questions right away:
compilePipelineProject(graph)
compileProject(graph)
proc mockCmdLine(pass: TCmdLinePass, cmd: string; conf: ConfigRef) =

View File

@@ -22,14 +22,6 @@ const
import std/compilesettings
# nimsuggest's incremental (NIF/IC) mode is opt-in via `--ideImports:nif`. By default
# nimsuggest recompiles the import closure from source (cmdCheck), which is fast and
# stable, so the whole suite runs that path. Only the tests listed below exercise the
# IC path; running every test under IC dominated the suite's wall-clock (each test
# recompiles `system` cold into NIF, plus a few warm-cache-only ordering/highlight
# quirks) and blew CI's job timeout.
const icTests = ["tic.nim", "tv3_import.nim"]
proc parseTest(filename: string; epcMode=false): Test =
const cursorMarker = "#[!]#"
let nimsug = "bin" / addFileExt("nimsuggest_testing", ExeExt)
@@ -82,14 +74,6 @@ proc parseTest(filename: string; epcMode=false): Test =
# else: ignore empty lines for better readability of the specs
inc i
tmp.close()
# The IC tests opt into the NIF path and get their own private cache. The stdio
# variant (epcMode=false) starts cold and writes the NIFs; the EPC variant reuses
# them warm, so a single test exercises both the NIF write and the NIF read path.
if extractFilename(filename) in icTests:
let nimcache = getTempDir() / ("nimsuggest_ic_" &
extractFilename(result.dest).changeFileExt(""))
if not epcMode: removeDir(nimcache)
result.cmd.add " --ideImports:nif --nimcache:" & nimcache
# now that we know the markers, substitute them:
for a in mitems(result.script):
a[0] = a[0] % markers

View File

@@ -1,7 +1,7 @@
doAssert true#[!]#
discard """
$nimsuggest --tester $file
$nimsuggest --tester $1
>highlight $1
highlight;;skTemplate;;1;;0;;8
highlight;;skTemplate;;1;;0;;8

View File

@@ -20,13 +20,6 @@ echo fo#[!]#oGeneric.bar
# bad type
echo unde#[!]#fined
# type of a void proc: typ[0] (return type) is nil, must not crash
var s = ""
s.a#[!]#dd('x')
# type of a module symbol: typ is nil, must not crash
import std/str#[!]#utils
discard """
$nimsuggest --v3 --tester $file
>type $1
@@ -36,6 +29,4 @@ type skType tv3_typeDefinition.Foo2 Foo2 $file 11 2 "" 100
>type $3
type skType tv3_typeDefinition.FooGeneric FooGeneric $file 14 2 "" 100
>type $4
>type $5
>type $6
"""

View File

@@ -2,6 +2,7 @@
[![Build Status](https://dev.azure.com/nim-lang/Nim/_apis/build/status/nim-lang.Nim?branchName=devel)](https://dev.azure.com/nim-lang/Nim/_build/latest?definitionId=1&branchName=devel)
This repository contains the Nim compiler, Nim's stdlib, tools, and documentation.
For more information about Nim, including downloads and documentation for
the latest release, check out [Nim's website][nim-site] or [bleeding edge docs](https://nim-lang.github.io/Nim/).

View File

@@ -13,7 +13,7 @@
# included from testament.nim
import important_packages
import std/[strformat, strutils, tables]
import std/[strformat, strutils]
from std/sequtils import filterIt
const
@@ -116,52 +116,56 @@ proc dllTests(r: var TResults, cat: Category, options: string) =
# ------------------------------ GC tests -------------------------------------
proc gcTests(r: var TResults, cat: Category, options: string) =
template run(filename, extraOptions: untyped) =
testSpec r, makeTest("tests/gc" / filename, options & extraOptions, cat)
template testWithoutMs(filename: untyped) =
testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat)
testSpec r, makeTest("tests/gc" / filename, options &
" -d:release -d:useRealtimeGC --mm:refc", cat)
when filename != "gctest":
testSpec r, makeTest("tests/gc" / filename, options &
" --gc:orc", cat)
testSpec r, makeTest("tests/gc" / filename, options &
" --gc:orc -d:release", cat)
template testWithoutBoehm(filename: untyped) =
testWithoutMs filename
testSpec r, makeTest("tests/gc" / filename, options &
" --gc:markAndSweep", cat)
testSpec r, makeTest("tests/gc" / filename, options &
" -d:release --gc:markAndSweep", cat)
# The matrix every gc test file goes through: refc (debug + realtime-release)
# and orc (debug + release). This is the coverage we actually rely on today.
template test(filename: untyped) =
run filename, " --mm:refc"
run filename, " -d:release -d:useRealtimeGC --mm:refc"
run filename, " --gc:orc"
run filename, " --gc:orc -d:release"
# markAndSweep and boehm are legacy collectors. Exercising them for every gc
# test file tripled this category's CI cost for little added signal, so only
# `gctest` keeps them alive. `gctest` does not build under orc.
template testLegacyGc(filename: untyped) =
run filename, " --mm:refc"
run filename, " -d:release -d:useRealtimeGC --mm:refc"
run filename, " --gc:markAndSweep"
run filename, " -d:release --gc:markAndSweep"
testWithoutBoehm filename
when not defined(windows) and not defined(android) and not defined(osx):
# boehm linking is broken on macOS 13 and there is no usable boehm.dll for
# Windows, so those platforms skip it.
run filename, " --gc:boehm"
run filename, " -d:release --gc:boehm"
# boehm library linking broken on macos 13
# AR: cannot find any boehm.dll on the net, right now, so disabled
# for windows:
testSpec r, makeTest("tests/gc" / filename, options &
" --gc:boehm", cat)
testSpec r, makeTest("tests/gc" / filename, options &
" -d:release --gc:boehm", cat)
testLegacyGc "gctest"
test "foreign_thr"
testWithoutBoehm "foreign_thr"
test "gcemscripten"
test "growobjcrash"
test "gcbench"
test "gcleak"
test "gcleak2"
testWithoutBoehm "gctest"
test "gcleak3"
test "gcleak4"
# Disabled because it works and takes too long to run:
#test "gcleak5"
test "weakrefs"
testWithoutBoehm "weakrefs"
test "cycleleak"
test "closureleak"
test "refarrayleak"
test "tlists"
test "thavlak"
testWithoutBoehm "closureleak"
testWithoutMs "refarrayleak"
testWithoutBoehm "tlists"
testWithoutBoehm "thavlak"
test "stackrefleak"
test "cyclecollector"
test "trace_globals"
testWithoutBoehm "trace_globals"
test "tfinalizers"
# ------------------------- threading tests -----------------------------------
@@ -406,13 +410,16 @@ proc listPackages(packageFilter: string): seq[NimblePackage] =
# at least should be a regex; a substring match makes no sense.
result = pkgs.filterIt(packageFilter in it.name)
else:
if testamentData0.testamentNumBatch == 0:
if testamentData0.batchArg == "allowed_failures":
result = pkgs.filterIt(it.allowFailure)
elif testamentData0.testamentNumBatch == 0:
result = pkgs
else:
result = @[]
for i in 0..<pkgs.len:
let pkgs2 = pkgs.filterIt(not it.allowFailure)
for i in 0..<pkgs2.len:
if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch:
result.add pkgs[i]
result.add pkgs2[i]
proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest =
result = TTest(cat: cat, name: test, options: options, debugInfo: debugInfo,
@@ -441,7 +448,10 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
(outp, status) = execCmdEx(cmd, workingDir = workingDir2)
status == QuitSuccess
if not ok:
r.finishTest(test, targetC, "", "", cmd & "\n" & outp, reFailed)
if pkg.allowFailure:
inc r.passed
inc r.failedButAllowed
r.finishTest(test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure)
continue
outp
@@ -457,7 +467,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
discard tryCommand(cmds[i], maxRetries = 3)
discard tryCommand(cmds[^1], reFailed = reBuildFailed)
inc r.passed
r.finishTest(test, targetC, "", "", "", reSuccess)
r.finishTest(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure)
errors = r.total - r.passed
if errors == 0:
@@ -478,236 +488,6 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
# ---------------- IC tests ---------------------------------------------
# ---- Metamorphic IC tests --------------------------------------------------
#
# A metamorphic IC test drives a *sequence of edits across several modules*
# through `nim ic` in a fixed build directory (same absolute paths throughout,
# which is what keeps the cache content-stable) and asserts the invariants the
# incremental backend is supposed to guarantee — see doc/ic_ideas.md:
#
# * clean build == incremental build (a fresh in-place rebuild of the
# final sources is byte-identical to
# the binary and full cache set the
# incremental edits converged to)
# * a no-op edit changes no artifact (`noop`)
# * a body-only edit touches no interface (`body-edit`: no `*.iface.bif`
# cookie changes -> no importer re-sem)
# * an interface edit propagates to (`iface-edit`: an `*.iface.bif`
# importers cookie changes and >= 2 modules'
# `*.s.bif` codegen is rebuilt)
#
# File format (a `tests/ic/t*.nim` whose body, after the spec header, contains a
# line `#? metamorphic`):
#
# #? metamorphic
# #!FILE a.nim
# proc greet*(): string = "hi"
# #!FILE main.nim # `main.nim` is always the build root
# import a
# echo greet()
# #!STEP expect: hi
# #!FILE a.nim # re-emit a module to "edit" it
# proc greet*(): string = "hi" # identical content
# #!STEP expect: hi; noop
#
# `#!FILE <name>` blocks (re)write a module in the virtual file system; the
# accumulated file set is materialised before each `#!STEP`. A `#!STEP`'s
# attributes are `;`-separated, each either `key: value` or a bare flag:
# expect: <stdout> noop body-edit iface-edit modules: <n> clean
# The last step always also runs the clean==incremental check.
type MetamorphicError = object of CatchableError
resultKind: TResultEnum
expected, given: string
proc mmRaise(kind: TResultEnum, expected, given: string) =
var e = newException(MetamorphicError, given)
e.resultKind = kind
e.expected = expected
e.given = given
raise e
proc isMetamorphicIcTest(content: string): bool =
for line in content.splitLines:
if line.strip == "#? metamorphic": return true
proc snapshotDir(dir: string): Table[string, string] =
## relative path -> raw file contents, for every file under `dir`.
result = initTable[string, string]()
if dirExists(dir):
for it in walkDirRec(dir):
result[it.relativePath(dir)] = readFile(it)
proc changedPaths(prev, cur: Table[string, string]): seq[string] =
result = @[]
for k, v in cur:
if prev.getOrDefault(k) != v: result.add k
for k in prev.keys:
if k notin cur: result.add k
proc isProvenance(path: string): bool =
## Build-provenance sidecars that legitimately differ between a fresh build and
## an edit-accumulated one (they record build history, not codegen). Excluded
## only from the cross-build clean==incremental comparison — a *no-op* edit must
## still leave even these untouched.
path.endsWith(".frontend.build.nif")
proc stableBinary(path: string): string =
## Contents of a linked executable past its header region, for comparing whether
## two builds produced the same *code*. Linkers embed build-time-volatile fields
## in the header (e.g. the mingw PE `TimeDateStamp` and its derived `CheckSum`),
## so two builds seconds apart differ there even with identical codegen. Skipping
## a generous fixed window keeps the clean-vs-incremental check about codegen.
const headerSkip = 4096
var f: File
if not open(f, path, fmRead):
raise newException(IOError, "cannot open: " & path)
defer: close(f)
if getFileSize(f) > headerSkip:
setFilePos(f, headerSkip)
result = readAll(f)
proc changedModuleCount(changed: seq[string]): int =
## distinct modules whose codegen (`*.s.bif`) was rebuilt.
var mods: seq[string] = @[]
for p in changed:
if p.endsWith(".s.bif"):
let key = p.extractFilename.split('.')[0]
if key notin mods: mods.add key
result = mods.len
proc runMetamorphicIcTest(r: var TResults; file: string; cat: Category; options: string) =
var test = TTest(cat: cat, name: file, options: options,
spec: initSpec(file), startTime: epochTime())
test.spec.targets = {targetC}
inc r.total
# Absolute paths: `nim ic` runs with `workingDir = buildDir`, so a relative
# `--nimcache` would resolve against the build dir, not where we read it back.
let buildDir = (file.changeFileExt("") & "_mm").absolutePath
let nc = buildDir / "nc"
let bin = buildDir / "prog".addFileExt(ExeExt)
removeDir(buildDir)
createDir(buildDir)
template compileIc(): untyped =
execCmdEx2(compilerPrefix, ["ic", "--hint:Conf:off", "--warnings:off",
"--nimcache:" & nc, "--out:" & bin, "main.nim"],
workingDir = buildDir)
# Parse the source into a flat op list: ("file", name, content) | ("step", attrs, "").
type OpKind = enum opFile, opStep
type Op = object
kind: OpKind
a, b: string
var ops: seq[Op] = @[]
block parse:
var curName = ""
var buf = ""
template flushFile() =
if curName.len > 0: ops.add Op(kind: opFile, a: curName, b: buf)
curName = ""; buf = ""
for raw in readFile(file).splitLines:
let s = raw.strip
if s.startsWith("#!FILE"):
flushFile()
curName = s["#!FILE".len .. ^1].strip
elif s.startsWith("#!STEP"):
flushFile()
ops.add Op(kind: opStep, a: s["#!STEP".len .. ^1].strip)
elif curName.len > 0:
buf.add raw; buf.add "\n"
let lastStep = block:
var n = 0
for o in ops:
if o.kind == opStep: inc n
n
var vfs = initTable[string, string]()
var prevSnap = initTable[string, string]()
var prevBin = ""
var stepIdx = 0
try:
for o in ops:
if o.kind == opFile:
vfs[o.a] = o.b
continue
inc stepIdx
let where = "step " & $stepIdx
# Parse step attributes.
var attrs = initTable[string, string]()
for part in o.a.split(';'):
let p = part.strip
if p.len == 0: continue
let c = p.find(':')
if c >= 0: attrs[p[0 ..< c].strip] = p[c+1 .. ^1].strip
else: attrs[p] = ""
for fn, content in vfs: writeFile(buildDir / fn, content)
let (_, cout, ccode) = compileIc()
if ccode != 0:
mmRaise(reBuildFailed, "", where & ": `nim ic` failed:\n" & cout)
let (_, rout, rcode) = execCmdEx2(bin.absolutePath, [], workingDir = buildDir)
if rcode != 0:
mmRaise(reBuildFailed, "", where & ": program exited with " & $rcode & ":\n" & rout)
if "expect" in attrs:
let want = attrs["expect"].replace("\\n", "\n")
if rout.strip == want.strip: discard
else: mmRaise(reOutputsDiffer, want, where & " output:\n" & rout.strip)
let snap = snapshotDir(nc)
let binBytes = stableBinary(bin)
if stepIdx > 1:
let changed = changedPaths(prevSnap, snap)
if "noop" in attrs and (changed.len != 0 or binBytes != prevBin):
mmRaise(reOutputsDiffer, "no artifact change",
where & ": no-op edit changed " & $changed.len & " cache file(s): " & changed.join(", "))
if "body-edit" in attrs:
for p in changed:
if p.endsWith(".iface.bif"):
mmRaise(reOutputsDiffer, "no interface change",
where & ": body-only edit changed an interface cookie: " & p)
if "iface-edit" in attrs:
var sawIface = false
for p in changed:
if p.endsWith(".iface.bif"): sawIface = true
if not sawIface:
mmRaise(reOutputsDiffer, "interface change", where & ": interface edit changed no `*.iface.bif` cookie")
if changedModuleCount(changed) < 2:
mmRaise(reOutputsDiffer, "propagation to importer",
where & ": interface edit did not propagate (only " & $changedModuleCount(changed) & " module rebuilt)")
if "modules" in attrs:
let want = parseInt(attrs["modules"])
let got = changedModuleCount(changed)
if got != want:
mmRaise(reOutputsDiffer, $want & " modules rebuilt", where & ": " & $got & " module(s) rebuilt")
prevSnap = snap
prevBin = binBytes
if "clean" in attrs or stepIdx == lastStep:
removeDir(nc)
let (_, cout2, ccode2) = compileIc()
if ccode2 != 0:
mmRaise(reBuildFailed, "", where & ": clean rebuild failed:\n" & cout2)
let cleanSnap = snapshotDir(nc)
let cleanBin = stableBinary(bin)
if cleanBin != binBytes:
mmRaise(reOutputsDiffer, "clean binary == incremental binary",
where & ": clean rebuild produced a different binary")
var diff: seq[string] = @[]
for p in changedPaths(snap, cleanSnap):
if not isProvenance(p): diff.add p
if diff.len != 0:
mmRaise(reOutputsDiffer, "clean cache == incremental cache",
where & ": clean rebuild differs in " & $diff.len & " cache file(s): " & diff.join(", "))
prevSnap = cleanSnap
prevBin = cleanBin
finishTest(r, test, targetC, "", "", "", reSuccess)
inc r.passed
except MetamorphicError:
let e = (ref MetamorphicError)(getCurrentException())
finishTest(r, test, targetC, "", e.expected, e.given, e.resultKind)
proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
isNavigatorTest: bool) =
template editedTest() =
@@ -718,18 +498,11 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
const tempExt = "_temp.nim"
for it in walkDirRec(testsDir):
# `_mm` directories hold materialised modules + nimcache for metamorphic
# tests; never collect their files as tests in their own right.
if "_mm" in it: continue
if isTestFile(it) and not it.endsWith(tempExt):
let content = readFile(it)
if isMetamorphicIcTest(content):
runMetamorphicIcTest(r, it, cat, options)
continue
let nimcache = nimcacheDir(it, options, targetC)
removeDir(nimcache)
let content = readFile(it)
for fragment in content.split("#!EDIT!#"):
let file = it.replace(".nim", tempExt)
writeFile(file, fragment)

View File

@@ -22,11 +22,16 @@ When this is the case, a workaround is to test this package here by adding `--pa
type NimblePackage* = object
name*, cmd*, url*: string
useHead*: bool
allowFailure*: bool
## When true, we still run the test but the test is allowed to fail.
## This is useful for packages that currently fail but that we still want to
## run in CI, e.g. so that we can monitor when they start working again and
## are reminded about those failures without making CI fail for unrelated PRs.
var packages*: seq[NimblePackage]
proc pkg(name: string; cmd = "nimble test -l"; url = "", useHead = true) =
packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead)
proc pkg(name: string; cmd = "nimble test -l"; url = "", useHead = true, allowFailure = false) =
packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure)
pkg "alea"
pkg "argparse"
@@ -49,7 +54,7 @@ pkg "chroma"
pkg "chronicles", "nim c -o:chr -r chronicles.nim"
pkg "chronos", "nim c -r -d:release tests/testall"
pkg "cligen", "nim c --path:. -r cligen.nim"
pkg "combparser", "nimble test"
pkg "combparser", "nimble test --mm:orc"
pkg "compactdict"
pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension"
pkg "confutils", "nimble install -y toml_serialization json_serialization unittest2; nimble test"
@@ -66,6 +71,7 @@ pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/eas
pkg "elvis", url = "https://github.com/nim-lang/elvis"
pkg "eth", "nim c -o:common -r tests/common/all_tests"
pkg "faststreams"
pkg "fidget"
pkg "fusion"
pkg "gara"
pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim"
@@ -107,7 +113,7 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need
pkg "NimData", "nim c -o:nimdataa src/nimdata.nim"
pkg "nimes", "nim c src/nimes.nim"
pkg "nimfp", "nim c -o:nfp -r src/fp.nim"
pkg "nimgame2", "nim c nimgame2/nimgame.nim"
pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim"
pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
pkg "nimib"
pkg "nimlsp"
@@ -156,12 +162,14 @@ pkg "ssz_serialization", "nim c -r tests/test_all.nim"
pkg "stew"
pkg "stint", "nimble test_internal"
pkg "strslice"
pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim"
pkg "supersnappy"
pkg "synthesis"
pkg "taskpools"
pkg "telebot", "nim c -o:tbot -r src/telebot.nim"
pkg "tempdir"
pkg "templates"
pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git"
pkg "terminaltables", "nim c src/terminaltables.nim"
pkg "termstyle", "nim c -r termstyle.nim"
pkg "testutils"
@@ -176,7 +184,7 @@ pkg "unittest2"
pkg "unpack"
when not defined(arm64):
pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true
pkg "websock", "nim c -d:chronicles_log_level=INFO tests/all_tests.nim"
pkg "websock", "nim c -d:chronosStrictException -d:chronicles_log_level=INFO --mm:refc tests/all_tests.nim"
pkg "websocket", "nim c websocket.nim"
pkg "with"
pkg "yaml"

View File

@@ -86,7 +86,8 @@ proc isNimRepoTests(): bool =
type
Category = distinct string
TResults = object
total, passed, skipped: int
total, passed, failedButAllowed, skipped: int
## xxx rename passed to passedOrAllowedFailure
data: string
TTest = object
name: string
@@ -231,6 +232,7 @@ proc initResults: TResults =
result = TResults(
total: 0,
passed: 0,
failedButAllowed: 0,
skipped: 0,
data: ""
)
@@ -260,25 +262,28 @@ template maybeStyledEcho(args: varargs[untyped]): untyped =
proc `$`(x: TResults): string =
result = """
Tests passed: $2 / $1 <br />
Tests skipped: $3 / $1 <br />
""" % [$x.total, $x.passed, $x.skipped]
Tests passed or allowed to fail: $2 / $1 <br />
Tests failed and allowed to fail: $3 / $1 <br />
Tests skipped: $4 / $1 <br />
""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped]
proc testName(test: TTest, target: TTarget, extraOptions: string): string =
proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string =
var name = test.name.replace(DirSep, '/')
name.add ' ' & $target
if allowFailure:
name.add " (allowed to fail) "
if test.options.len > 0: name.add ' ' & test.options
if extraOptions.len > 0: name.add ' ' & extraOptions
name.strip()
proc addResult(r: var TResults, test: TTest, target: TTarget,
extraOptions, expected, given: string, success: TResultEnum, duration: float,
givenSpec: ptr TSpec = nil) =
allowFailure = false, givenSpec: ptr TSpec = nil) =
# instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
# instead of having to pass individual fields, or abusing existing ones like expected vs given.
# test.name is easier to find than test.name.extractFilename
# A bit hacky but simple and works with tests/testament/tshould_not_work.nim
let name = testName(test, target, extraOptions)
let name = testName(test, target, extraOptions, allowFailure)
let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5)
if backendLogging:
@@ -341,22 +346,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
proc finishTest(r: var TResults, test: TTest, target: TTarget,
extraOptions, expected, given: string, successOrig: TResultEnum,
givenSpec: ptr TSpec = nil) =
allowFailure = false, givenSpec: ptr TSpec = nil) =
## calculates duration of test, reports result
## `retries` option in the test is ignored
let duration = epochTime() - test.startTime
let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
else: successOrig
addResult(r, test, target, extraOptions, expected, given, success, duration, givenSpec)
addResult(r, test, target, extraOptions, expected, given, success, duration, allowFailure, givenSpec)
proc finishTestRetryable(r: var TResults, test: TTest, target: TTarget,
extraOptions, expected, given: string, successOrig: TResultEnum,
givenSpec: ptr TSpec = nil): bool =
allowFailure = false, givenSpec: ptr TSpec = nil): bool =
## if test failed and has remaining retries, return `true`,
## otherwise calculate duration and report result
##
##
## warning: if `true` is returned, then the result is not reported,
## it has to be retried or `finishTest` should be called instead
## it has to be retried or `finishTest` should be called instead
result = false
let duration = epochTime() - test.startTime
let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
@@ -364,7 +369,7 @@ proc finishTestRetryable(r: var TResults, test: TTest, target: TTarget,
if test.spec.retries > 0 and success notin {reSuccess, reDisabled, reJoined, reInvalidSpec}:
return true
else:
addResult(r, test, target, extraOptions, expected, given, success, duration, givenSpec)
addResult(r, test, target, extraOptions, expected, given, success, duration, allowFailure, givenSpec)
proc toString(inlineError: InlineError, filename: string): string =
result = "$file($line, $col) $kind: $msg" % [
@@ -492,7 +497,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
return
if test.spec.err != reRetry:
test.startTime = epochTime()
if testName(test, target, extraOptions) in skips:
if testName(test, target, extraOptions, false) in skips:
test.spec.err = reDisabled
if test.spec.err in {reDisabled, reJoined}:
@@ -777,12 +782,7 @@ proc main() =
if kind == pcDir and cat notin ["testdata", "nimcache"]:
cats.add cat
if isNimRepoTests():
# `ic` and `navigator` are real `tests/` dirs (already collected above) *and*
# listed in AdditionalCategories; without this guard they'd run twice, which
# doubled the (expensive) `ic` category on every `all` run. `debugger`,
# `examples` and `lib` have no matching `tests/` dir, so they are still added.
for cat in AdditionalCategories:
if cat notin cats: cats.add cat
cats.add AdditionalCategories
if useMegatest: cats.add MegaTestCat
var cmds: seq[string] = @[]

View File

@@ -1,12 +0,0 @@
# Issue: genMagicExpr: mAsgn internal error when using Isolated[T] with primitive types
# in tuple assignment to pointer dereference
import std/isolation
proc main() =
var x: ptr Isolated[float]
x = cast[ptr Isolated[float]](alloc0(sizeof(Isolated[float])))
x[] = isolate(42.0)
dealloc(x)
main()

View File

@@ -1,28 +0,0 @@
discard """
output: '''a
b
c
a
b
c'''
"""
# A `var T` loop variable (here from `mitems`) declared inside an enclosing
# loop must not be reset by dereferencing: at module scope it is emitted as a
# global, and the in-loop reset path used `resetLoc`, which dereferenced the
# still-uninitialized borrowed pointer and crashed with a SIGSEGV.
for p in @["abc", "123"]:
var testA = @["a", "b", "c"]
for l in testA.mitems:
echo(l)
# also exercise actual mutation through the borrowed reference
block:
var ok = true
for p in @["x", "y"]:
var s = @[1, 2, 3]
for l in s.mitems:
l += 10
if s != @[11, 12, 13]: ok = false
doAssert ok

View File

@@ -36,4 +36,4 @@ proc main() =
main()
GC_fullCollect()
when not defined(useMalloc):
echo getOccupiedMem() < 10 * 1024 * 1024, " peak memory: ", getMaxMem() < 12 * 1024 * 1024
echo getOccupiedMem() < 10 * 1024 * 1024, " peak memory: ", getMaxMem() < 10 * 1024 * 1024

View File

@@ -13,7 +13,7 @@ import asyncdispatch, times
var done = false
proc somethingAsync() {.async.} =
yield sleepAsync 1000
yield sleepAsync 5000
echo "async done"
done = true
@@ -21,5 +21,5 @@ asyncCheck somethingAsync()
var count = 0
while not done:
count += 1
drain 200
drain 1000
echo "iteration: ", count

View File

@@ -13,7 +13,7 @@ else:
# This reproduces a case where a socket remains stuck waiting for writes
# even when the socket is closed.
const
timeout = 2000
timeout = 8000
var port = Port(0)
var sent = 0

View File

@@ -30,13 +30,3 @@ block:
var x = M(x: 1)
doAssert(x.x == 1)
block: # bug #25931
type
N {.importc: "const void *".} = pointer
S = object
f: proc (_: N) {.cdecl.}
#var _: proc (_: pointer) {.cdecl.}
proc d(_: proc (_: pointer) {.cdecl.}) = discard
discard S()
d(proc (_: pointer) {.cdecl.} = discard)

View File

@@ -1,45 +0,0 @@
discard """
targets: "c cpp"
output: "13"
"""
# bug #25883: C codegen assigns same type hash to tuples with different nesting
# but identical flattened content.
# ((Int[1], Int[2]), Int[13], Int[14]) and ((Int[1], Int[2], Int[13]), Int[14])
# must get distinct C type names.
type
Int[V: static int] = object
proc main() =
var b = ((1, 2), 13, 14)
var c = ((1, 2, 13), 14)
echo c[0][2]
main()
block:
type
Int[V: static int] = object
Layout[Sh, St] = object
shape: Sh
stride: St
func makeB(): auto =
Layout[((Int[2], Int[3]), Int[5], Int[7]), ((Int[1], Int[2]), Int[6], Int[30])](
shape: ((Int[2](), Int[3]()), Int[5](), Int[7]()),
stride: ((Int[1](), Int[2]()), Int[6](), Int[30]())
)
func makeC(): auto =
Layout[((Int[2], Int[3], Int[5]), Int[7]), ((Int[1], Int[2], Int[6]), Int[30])](
shape: ((Int[2](), Int[3](), Int[5]()), Int[7]()),
stride: ((Int[1](), Int[2](), Int[6]()), Int[30]())
)
proc main() =
let b = makeB()
let c = makeC()
main()

View File

@@ -75,15 +75,3 @@ block: # importc type inheritance
doAssert(cast[cint](b) == 123)
var c = foo(b)
doAssert(cast[cint](c) == 123)
block: # bug #25945
var stateRefund = 0
let authCode =
if true:
if false:
stateRefund += 0
@[]
else:
@([1.byte])
discard (if true: (discard; @[]) else: @[0])

Some files were not shown because too many files have changed in this diff Show More