mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-09-01 19:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

IC: grind the Cursor vocabulary against the PNode loader, and fix what it found

Browse Source 
The `BNode` accessors had no oracle. The self-test in `bnode.nim` checks them
against each other and against the raw token stream, which a uniformly wrong
vocabulary satisfies — it passed 5.2M assertions on an off-by-two model. This
adds the missing oracle and lets it drive the next migration step.

`cgen.grindLockstep` (opt-in, `NIM_IC_BNODE_GRIND=1` on an `--ic:on` build)
walks the `.bif` cursor and the materialised `PNode` for the SAME body side by
side and requires `kind`, `len`, `info`, `flags`, the literal payloads, `sym`
and `typ` to agree at every node of every routine body in the dependency
closure. The `PNode` is the oracle, so it compares everything rather than what
someone thought to check. It found two bugs, both silent:

* `BNode.typ` answered `nil` for every bare `Symbol`. `ast.typ` does not: it
  falls back to `n.sym.typ` when the loader set `nfLazyType`, which it does for
  exactly that shape. The consequence is not less information but a DIFFERENT
  answer — `canRaise` asks `fn.typ.kind == tyProc` about a call's callee, so
  nil turns "this call can raise" into "it cannot" and drops the
  goto-exception check after the call.

* `(ht . <sym>)` — an explicitly nil node type — made `n.typ` LOAD-ORDER
  DEPENDENT in the loader itself. `newSymNode` marks the node lazy only if the
  symbol was still an unloaded stub at that moment, so the same `.bif` node
  answered `sym.typ` or `nil` depending on what happened to touch that symbol
  first. Pinned to the lazy reading, so the answer is a property of the file
  rather than of the traversal order.

With `typ` correct, the blocker recorded in `allPathsAsgnResult` is gone.
`ast.canRaise`/`canRaiseConservative` cannot become `AnyNode` procs where they
live — `BNode` is defined in `bnode.nim`, which imports `ast` — so their bodies
move into templates that `bnode` instantiates for its own node type. One source
of truth, no cycle, no second copy. `ccgcalls.canRaiseDisp` and
`cgen.allPathsAsgnResult` follow.

Adds the leaf accessors (`intVal`, `floatVal`, `strVal`, `ident`, `flags`)
because nothing in `ccgexprs` can migrate without them, and `rawDesc` for
diagnosing a disagreement in terms of what the token stream literally says.

`compiler/bodynav.nim` replaces the `BodyScope` snapshot with a scope chain the
traversal maintains — `openScope`/`closeScope`/`registerDefHere`, lookup
falling through to the decoder — ported from Nimony's `typenav`. The scope
becomes a product of the walk, so nothing is copied ahead of time and nothing
can be stale. How much of a live problem the snapshot was is measured rather
than assumed: `-d:icLocalSymStats` reports `localHit=0 fieldStub=2 miss=0
sdReg=5902 extractReg=45` over the stdlib closure, i.e. definitions register
constantly and not one use ever resolved through the table, because
`isLocalSym` is a hardwired `false`. The hazard was latent; this keeps it
latent once that stops being true.

Verified: 0 disagreements over the whole `--ic:on` closure with the walk
driving the nav; deliberately breaking `intVal`, `flags` and the nav key each
make the grinder fire on the first bodies it reaches, so the clean run is not
vacuous; the default path emits 215/215 byte-identical `.c` against the
pre-change compiler and does not compile `bodynav` at all; `tests/ic` 39/39.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XEF7FJvUkGKvG9LSGuEaNR
This commit is contained in:
Araq
2026-08-30 08:07:38 +02:00
parent 2447dfdc7d
commit 5799220c98
10 changed files with 1225 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
compiler/ast.nim
View File

@@ -1723,36 +1723,56 @@ proc addParam*(procType: PType; param: PSym) =
procType.n.add newSymNode(param)
rawAddSon(procType, param.typ)
const magicsThatCanRaise = {
const magicsThatCanRaise* = {
mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mEcho}
proc canRaiseConservative*(fn: PNode): bool =
if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
result = false
else:
result = true
# `canRaise` and `canRaiseConservative` are asked by the C backend, which is
# migrating to reading routine bodies straight off a `.bif` `Cursor` rather than
# off a materialised `PNode` tree (see `compiler/bnode.nim`). Both predicates
# only ever look at a node's `kind`, `sym` and `typ`, so ONE body serves either
# spelling -- but `BNode` is defined in `bnode.nim`, which imports this module,
# so the `BNode` overloads cannot live here. The bodies therefore live in
# templates and `bnode.nim` instantiates them for its own node type: one source
# of truth, no import cycle, and no second copy to keep in sync.
#
# `fn.typ.n` below is a *type's* formal-params node, not a routine body: it is
# always fully materialised, so indexing it is not the hazard that indexing a
# body node is.
proc canRaise*(fn: PNode): bool =
if fn.kind == nkSym and (fn.sym.magic notin magicsThatCanRaise or
{sfImportc, sfInfixCall} * fn.sym.flags == {sfImportc} or
sfGeneratedOp in fn.sym.flags):
result = false
elif fn.kind == nkSym and fn.sym.magic == mEcho:
result = true
elif fn.typ != nil and fn.typ.kind == tyProc and fn.typ.n != nil:
# TODO check for n having sons? or just return false for now if not
if fn.typ.n[0].kind == nkSym:
result = false
template canRaiseConservativeImpl*(fnArg: typed): bool =
block:
let fn = fnArg
not (fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise)
template canRaiseImpl*(fnArg: typed): bool =
block:
let fn = fnArg
var res: bool
if fn.kind == nkSym and (fn.sym.magic notin magicsThatCanRaise or
{sfImportc, sfInfixCall} * fn.sym.flags == {sfImportc} or
sfGeneratedOp in fn.sym.flags):
res = false
elif fn.kind == nkSym and fn.sym.magic == mEcho:
res = true
elif fn.typ != nil and fn.typ.kind == tyProc and fn.typ.n != nil:
# TODO check for n having sons? or just return false for now if not
if fn.typ.n[0].kind == nkSym:
res = false
else:
# A proc-typed value with no explicit raises slot still has
# unspecified effects, which sempass2 treats conservatively.
# Codegen needs to do the same in order to keep goto-exception
# checks after indirect/closure calls.
res = ((fn.typ.n[0].len < effectListLen) or
fn.typ.n[0][exceptionEffects] == nil or
fn.typ.n[0][exceptionEffects].safeLen > 0)
else:
# A proc-typed value with no explicit raises slot still has
# unspecified effects, which sempass2 treats conservatively.
# Codegen needs to do the same in order to keep goto-exception
# checks after indirect/closure calls.
result = ((fn.typ.n[0].len < effectListLen) or
fn.typ.n[0][exceptionEffects] == nil or
fn.typ.n[0][exceptionEffects].safeLen > 0)
else:
result = false
res = false
res
proc canRaiseConservative*(fn: PNode): bool = canRaiseConservativeImpl(fn)
proc canRaise*(fn: PNode): bool = canRaiseImpl(fn)
proc toHumanStrImpl[T](kind: T, num: static int): string =
result = $kind

114
compiler/ast2nif.nim
View File

@@ -205,9 +205,9 @@ will tell us the precise offsets anyway.
]#
const
hiddenTypeTagName = "ht"
symDefTagName = "sd"
typeDefTagName = "td"
hiddenTypeTagName* = "ht"
symDefTagName* = "sd"
typeDefTagName* = "td"
bindingIdTagName = "bid"
var
@@ -241,6 +241,19 @@ type
emittedCanonTypes: Table[string, int32] # canonical type name -> itemId.item of the def
when defined(icLocalSymStats):
# TEMPORARY instrumentation: how is `localSyms` actually populated? The
# snapshot-vs-shared-table question only matters if body-local NIF names exist
# at all, and `isLocalSym` below returns a hardwired `false`.
import std / exitprocs
var lsLocalHit, lsFieldStub, lsMiss, lsSdReg, lsExtractReg: int
addExitProc proc () =
if lsLocalHit + lsFieldStub + lsMiss + lsSdReg + lsExtractReg > 0:
stderr.writeLine "LOCALSYM localHit=" & $lsLocalHit &
" fieldStub=" & $lsFieldStub & " miss=" & $lsMiss &
" sdReg=" & $lsSdReg & " extractReg=" & $lsExtractReg
proc isLocalSym(sym: PSym): bool {.inline.} =
## Every symbol is emitted as a *global* (module-suffixed) name so that its
## `sdef` gets an index entry and is resolvable by index lookup even when
@@ -394,7 +407,7 @@ proc stripFieldMarker(rawName: string): string {.inline.} =
else:
rawName[0 ..< rawName.len - FieldMarker.len]
proc isFieldNifName(name: string): bool {.inline.} =
proc isFieldNifName*(name: string): bool {.inline.} =
## True for an object field's local NIF name `<ident>`f.<disamb>` (see
## `FieldMarker`): no module suffix, marker on the ident.
let sn = parseSymName(name)
@@ -1363,7 +1376,7 @@ var modFlagsTag = registerTag("modflags")
# instead of a plain construction, and no read was ever recognised as a move.
# Only wrap when there is something to say, so the common sym use stays a bare
# token.
const symNodeFlagsTagName = "nflags"
const symNodeFlagsTagName* = "nflags"
var symNodeFlagsTag = registerTag(symNodeFlagsTagName)
const PersistedSymNodeFlags = PersistentNodeFlags - {nfLazyType, nfHasComment}
@@ -2833,6 +2846,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s
let sym = PSym(itemId: id, kindImpl: skStub, name: stubName,
disamb: sn.count.int32, state: Complete)
localSyms[symName] = sym
when defined(icLocalSymStats): inc lsExtractReg
# `loadSymFromCursor` enters the `(sd` and consumes the whole block,
# leaving n positioned after the closing `)`.
loadSymFromCursor(c, sym, n, thisModule, localSyms)
@@ -2898,12 +2912,15 @@ proc loadSymStub(c: var DecodeContext; symAsStr: string; thisModule: string;
if sn.module.len == 0:
result = localSyms.getOrDefault(symAsStr)
if result != nil:
when defined(icLocalSymStats): inc lsLocalHit
return result
elif isFieldMarked(sn.name):
when defined(icLocalSymStats): inc lsFieldStub
# A cross-context object-field reference reaching a non-dotExpr slot (e.g. a
# `{.guard.}` field, an owner): stub it like any other field use.
return c.loadFieldStub(symAsStr, thisModule, localSyms)
else:
when defined(icLocalSymStats): inc lsMiss
raiseAssert "local symbol '" & symAsStr & "' not found in localSyms."
# Global symbol - look up in index for lazy loading
result = c.syms.getOrDefault(symAsStr)[0]
@@ -3263,6 +3280,18 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
s = c.loadSymStub(n, thisModule, localSyms)
result = newSymNode(s, info)
result.typField = typ
if typ == nil:
# `(ht . <sym>)` — an EXPLICITLY nil node type. Without this the
# node's type is LOAD-ORDER DEPENDENT: `newSymNode` above marks the
# node lazy only when the symbol was still an unloaded stub at this
# moment, so the very same `.bif` node answers `sym.typ` or `nil`
# for `n.typ` depending on whether something else happened to touch
# that symbol first. Pin it to the lazy reading — the one `newSymNode`
# exists to provide (see `nifcBackendActive` in astdef: snapshotting
# a nil leaves the node permanently typeless and the backend then
# reads `t.flags` off it) — so the answer is a property of the file,
# not of the traversal order.
result.flags.incl nfLazyType
elif tagIs(n, symDefTagName):
let info = c.infos.oldLineInfo(n.info, cursorPool(n))
let name = n.firstSon
@@ -3288,6 +3317,7 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
sym = PSym(itemId: id, kindImpl: skStub, name: stubName,
disamb: sn.count.int32, state: Complete)
localSyms[symName] = sym # register for later references
when defined(icLocalSymStats): inc lsSdReg
# Now fully load the symbol from the sdef
loadSymFromCursor(c, sym, n, thisModule, localSyms)
sym.state = c.loadedState # mark as fully loaded
@@ -3466,6 +3496,80 @@ proc materializeLazyBody*(c: var DecodeContext; node: PNode) =
node.typField = real.typField
node.flags = real.flags
# ---------------------------------------------------------------------------
# Cursor-native backend seam (see `bnode.nim`)
#
# The three things a `.bif` `Cursor` cannot answer on its own — what symbol a
# `Symbol` token names, what type a node's type slot denotes, and what
# `TLineInfo` its packed line info maps to — all need the decoder's state. They
# are exposed here rather than reimplemented in `bnode` so that the Cursor
# backend and the `PNode` loader resolve names through exactly the same code.
# ---------------------------------------------------------------------------
type
BodyScope* = object
## Resolution scope for reading ONE routine body straight off a cursor.
## `thisModule` is the owning module's NIF suffix (a `Symbol` token with no
## module suffix is body-local and appears in no index) and `localSyms` is
## the enclosing sym def's local symbols, so a param/local reference
## resolves to the SAME `PSym` the signature already created.
thisModule*: string
localSyms*: Table[string, PSym]
proc lazyBodyCursor*(c: var DecodeContext; node: PNode; scope: var BodyScope;
body: var Cursor): bool =
## Non-destructive lookup of a deferred routine body: the cursor at its
## `(stmtlist ...)` plus the scope its symbol references resolve in. Unlike
## `materializeLazyBody` this does NOT consume the pending entry, so the
## `PNode` path still works afterwards and the two representations of the same
## body can be walked side by side and compared — which is how a proc migrated
## to `BNode` is checked against the one it replaces.
let key = cast[int](node)
if not c.pendingBodies.hasKey(key): return false
let pb = c.pendingBodies[key]
body = pb.cursor
scope = BodyScope(thisModule: pb.thisModule, localSyms: pb.localSyms)
result = true
proc symFromCursor*(c: var DecodeContext; n: Cursor; scope: var BodyScope): PSym =
## The `PSym` a `Symbol` / `SymbolDef` / `(sd ...)` token names. Non-consuming
## (`loadSymStub` advances a `var Cursor`; this one works on a copy).
##
## The bare `SymbolDef` case goes through the by-name overload: the cursor
## overload of `loadSymStub` deliberately rejects it, because inside the
## loader a def token is always reached through its `(sd ...)` wrapper and a
## bare one means a malformed stream. A reader that starts at an arbitrary
## token has no such guarantee, and the def NAMES the same symbol the use
## does.
var cur = n
if cur.kind == SymbolDef:
result = loadSymStub(c, symName(cur), scope.thisModule, scope.localSyms)
else:
result = loadSymStub(c, cur, scope.thisModule, scope.localSyms)
proc typeFromCursor*(c: var DecodeContext; n: Cursor; scope: var BodyScope): PType =
## The `PType` a node's type slot denotes — a `Symbol`, an inline `(td ...)`,
## or a `DotToken` for "no type of its own". Non-consuming.
var cur = n
result = loadTypeStub(c, cur, scope.localSyms)
proc nodeFlagsFromCursor*(n: Cursor): TNodeFlags =
## The node-flags slot: an `Ident` naming the set, or a `DotToken` for empty.
## Non-consuming.
var cur = n
result = loadAtom(TNodeFlags, cur)
proc identFromCursor*(c: var DecodeContext; n: Cursor): PIdent =
## The `PIdent` an `Ident` token names, interned in the SAME cache the loader
## uses — `nkIdent` nodes compare by identity in places.
result = c.cache.getIdent(strVal(n))
proc lineInfoFromCursor*(c: var DecodeContext; n: Cursor): TLineInfo =
## The `TLineInfo` for a token's packed line info. The `FileId` inside belongs
## to the `.bif`'s OWN filename pool, so the mapping needs both the pool and
## the `ConfigRef` the `LineInfoWriter` holds.
result = c.infos.oldLineInfo(n.info, cursorPool(n))
forceLazyBodyHook = proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.} =
# `len` (the sole caller path) MUST stay effect-free, so this hook is typed
# `raises: []`. The underlying `loadNode` chain infers `raises: [KeyError]`

5
compiler/astdef.nim
View File

@@ -957,7 +957,10 @@ iterator items*(n: PNode): PNode =
iterator sons*(n: PNode): PNode =
## Iterates over the children of `n`. Preferred over `for i in 0..<n.len: n[i]`
## as it does not rely on random indexed access (see doc/ic_backend_nif_native.md).
## as it does not rely on random indexed access, and over `for x in n.sons`,
## which reads the raw FIELD and so skips the `len` hook that materialises a
## deferred `nfLazyBody` body — over such a body that loop silently visits
## nothing. See `compiler/bnode.nim` for the backend vocabulary this feeds.
for i in 0..<n.safeLen: yield n[i]
iterator isons*(n: PNode; start = 0): tuple[i: int, n: PNode] =

735
compiler/bnode.nim
View File

@@ -15,23 +15,55 @@
## compiles (measured: a 370-byte module costs 0.20s/0.16s in lower/cg, the main
## module 3.40s/3.16s, and the two are ~85% of the serial backend critical path).
##
## With `-d:newIcBackend` `BNode` is a `Cursor`; without it a plain `PNode`,
## which is what every build does today. Codegen migrates to the vocabulary
## below one area at a time and the compiler keeps building throughout, because
## on the `PNode` side the vocabulary is what `ast`/`astdef` already provide —
## `kind`, `len`, `safeLen`, `sym`, `typ`, `info`, `firstSon`, `secondSon`,
## `lastSon` and the `sons`/`isons`/`sonsFrom`/`sonsButLast`/`isonsButLast`
## iterators all exist. This module deliberately does NOT redefine them for
## `PNode`: an identical second overload would make every call site ambiguous.
## It adds only what the AST lacks (`son`, `hasSons`), and supplies the whole
## vocabulary on the `Cursor` side.
## With `-d:newIcBackend` `BNode` is a `distinct Cursor`; without it a plain
## `PNode`, which is what every build does today. Codegen migrates to the
## vocabulary below one area at a time and the compiler keeps building
## throughout, because on the `PNode` side the vocabulary is what `ast`/`astdef`
## already provide — `kind`, `len`, `safeLen`, `sym`, `typ`, `info`, `firstSon`,
## `secondSon`, `lastSon` and the `sons`/`isons`/`sonsFrom`/`sonsButLast`/
## `isonsButLast` iterators all exist. This module does not redefine those for
## `PNode`; it adds only what the AST lacks (`son`, `hasSons`, `isNilNode`) and
## supplies the whole vocabulary on the `Cursor` side.
##
## THE COST MODEL DIFFERS, and that is what the vocabulary is shaped around. A
## `Cursor` is a position in a token stream, and a child is reached by stepping
## over each preceding sibling. Stepping is cheap — a `TagLit` token stores the
## width of its whole subtree, so `nifcore.skip` is a single pointer add
## regardless of how big that subtree is — but there is no random access and no
## way to walk backwards:
## THE NODE ENCODING. This is the part that a "a `Cursor` is just a tree
## position" reading gets wrong, and getting it wrong is silent. `ast2nif`'s
## writer emits every node through `withNode`, which is
##
## (<kind-tag> <flags> <type> <child 0> <child 1> ...)
##
## so a node's own flags and its `typ` occupy the FIRST TWO raw children and the
## AST's child 0 is the THIRD. `nifcore.childCursor` — and `ast2nif`'s own
## `firstSon(n: Cursor)`, which is a RAW structural accessor used to reach the
## name inside an `(sd ...)`/`(td ...)` — return the flags slot, not child 0.
## Every accessor below therefore steps over that two-token prefix. The
## exceptions are exactly:
##
## * `(none)` and a type-less `(empty)`, written bare with NO prefix and no
## children — so "zero raw children" means "no prefix", never "prefix but no
## AST children", which is what makes the two cases distinguishable at all;
## * an `nkSym`, which is not a `TagLit` node: it is a bare `Symbol` token, or
## `(ht <type> <sym>)` when the node's type differs from the symbol's, or
## `(nflags <flags> <symnode>)` when persisted node flags need saying, or an
## `(sd <name> ...)` definition;
## * a nil child, written as a `DotToken` — `isNilNode` is the `n == nil` of the
## `PNode` world, which has no `Cursor` counterpart.
##
## `len` follows `safeLen`'s rule and answers 0 for `nkNone..nkNilLit`: the
## payload token of an `nkIntLit`/`nkStrLit`/`nkIdent` sits where a child would,
## and is not one.
##
## `BNode` is DISTINCT from `Cursor` for that same reason. The two accessor sets
## are both spelled `firstSon` and differ by two positions, so a raw structural
## cursor reaching a call site that wants AST children must not silently
## typecheck — and the distinction also keeps `ast2nif.firstSon(n: Cursor)` from
## colliding with this module's.
##
## THE COST MODEL DIFFERS from a `PNode`'s, and that is what the vocabulary is
## shaped around. A `Cursor` is a position in a token stream, and a child is
## reached by stepping over each preceding sibling. Stepping is cheap — a
## `TagLit` token stores the width of its whole subtree, so `nifcore.skip` is a
## single pointer add regardless of how big that subtree is — but there is no
## random access and no way to walk backwards:
##
## * `firstSon` / `secondSon` / `son(n, k)` — O(k) in the NUMBER of preceding
## siblings, not in their size. Cheap for the small constant `k` that nearly
@@ -51,18 +83,75 @@
## OWN tag pool, so a tag id means nothing outside the file it came from and
## there can be no process-global id -> `TNodeKind` table; the answer is
## memoized per pool instead, and the memo is dropped when the pool changes.
## `when isMainModule` at the bottom of this file checks that — run it over
## several `.bif` files at once, since one file alone cannot catch a memo that
## fails to notice the switch.
##
## RESOLUTION CONTEXT. `sym`, `typ` and `info` cannot be answered by the cursor
## alone: a `Symbol` token holds only a NAME, a type slot only a type's name,
## and a packed line info a `FileId` in the `.bif`'s own filename pool. All
## three need the decoder's state, and the decision recorded here is that they
## take it from AMBIENT STATE rather than from a parameter:
##
## * the `DecodeContext` is the process-wide `ast.program`, which already exists
## and is already what `ast.loadSym`/`loadType` resolve through;
## * the per-body half is a `bodynav.BodyNav` on a STACK, pushed by
## `withBodyScope` for the span of one routine body. It has to be a stack and
## not a single slot because generating one routine can pull in another
## (`genProcNoForward`) before the first is finished.
##
## Threading a `DecodeContext` parameter through the ~230 `PNode`-taking procs
## in the cgen files instead would be exactly the churn the `BNode` seam exists
## to avoid, and codegen is already single-threaded within a stage process
## (`--icBackendStage:cg --icBackendModule:X` compiles one module per process).
##
## The nav is not merely a renamed snapshot: it is a SCOPE CHAIN THE TRAVERSAL
## MAINTAINS (`openScope` / `closeScope` / `registerDefHere`), ported from
## Nimony's `typenav`, so a reader descending a body always has exactly the
## definitions it has already walked past and nothing is copied ahead of time.
## `bodynav`'s module doc has the measurement that says how much of a live
## problem the old snapshot was — the honest answer is "none yet" — and why the
## mechanism is still the right shape. `sym` goes through it; `typ` does not,
## because `ast2nif` already materialises types lazily from the module's type
## index and a frame has nothing to add.
##
## HOW THIS IS VERIFIED. Two oracles, neither of them a hand-written
## expectation:
##
## * the `isMainModule` self-test at the bottom of this file checks the
## accessors against the raw token stream of real `.bif` files. Necessary but
## NOT sufficient, and the reason is worth remembering: a vocabulary that is
## uniformly wrong — every accessor off by the same two positions — satisfies
## every accessor-against-accessor check there is. This test passed 5.2M
## assertions on exactly that wrong model.
## * `cgen.grindBNode` (opt-in, `NIM_IC_BNODE_GRIND=1` on an `--ic:on` build)
## walks the cursor and the materialised `PNode` for the SAME body in
## LOCKSTEP and requires `kind`, `len`, `info`, `flags`, the literal payloads,
## `sym` and `typ` to agree at every node of every routine body in the
## dependency closure — and, separately, requires each migrated `AnyNode` proc
## to return the same answer for the whole body. The `PNode` is the oracle, so
## it compares everything rather than what someone thought to check. That walk
## also DRIVES the nav — it opens a scope per node and offers each child to
## `registerDefHere` before descending — so what is graded is the vocabulary as
## a cursor-native pass would actually use it, not a random-access shortcut.
##
## The second oracle is what found the `typ` bug (a bare `Symbol` answered `nil`
## where `ast.typ` falls back to the symbol's type — which silently turns
## `canRaise` into "cannot raise" and drops the goto-exception check after a
## call) and the `.sons` bug named below. A clean run means nothing until you
## have broken an accessor on purpose and watched the grinder fire.
##
## The cgen files hold to one invariant, which is what makes the eventual flip
## mechanical: NO `[]` ON A `PNode` OUTSIDE OF TREE CONSTRUCTION. Every read is
## mechanical: NO `[]` AND NO `.sons` ON A `PNode` OUTSIDE OF TREE
## CONSTRUCTION. `.sons` is not merely the un-portable spelling of the walk: it
## is the raw FIELD, so it bypasses the `len` hook that materializes a deferred
## `nfLazyBody` body, and `for x in n.sons` over a routine body that is still a
## placeholder silently visits NOTHING. `for x in sons(n)` goes through
## `safeLen` and materializes. (This is not hypothetical — it is what the
## differential grinder in `cgen.grindBNode` reported the first time it ran, and
## it had been sitting in `containsResult` itself.) Every read is
## `firstSon`/`secondSon`/`lastSon`/`son(n, k)` or one of the iterators; the
## remaining subscripts are writes that build a fresh `nkProcDef`
## (`theProc[namePos] = ...`), which a `Cursor` backend will not do at all, and
## accesses to a `PType`, a `string`, a `seq` or a `Table`, none of which are
## `BNode`s. The `firstSon`/`secondSon`/`lastSon`/`son` family is defined for
## `PNode` only, so a mistaken base does not compile.
## `BNode`s.
##
## A `PType` has its own vocabulary and its own reason for preferring it: `t[0]`
## is the return type, the base class, the index type or the generic head
@@ -79,11 +168,17 @@
## materializes them lazily from the module's type index, which is the seam
## that matters on that side.
import ast, lineinfos
import ast, lineinfos, idents
when defined(nimPreviewSlimSystem):
import std / assertions
when defined(newIcBackend):
import "../dist/nimony/src/lib/nifcore" except pool
import ic / enum2nif
import ast2nif, icnifcore, bodynav
export ast2nif.BodyScope
export bodynav
# Imported only under the define: `cgen` is compiled during the koch
# bootstrap, where the nimony libs are unavailable (`ast2nif` is guarded the
# same way). `nifcore` — NOT `nifcursors` — is the reader half of NIF: `bif`
@@ -91,7 +186,17 @@ when defined(newIcBackend):
# `nifcore.Cursor`. `nifcursors` is the writer/builder cursor over
# `PackedToken`s and is a different type entirely.
type BNode* = Cursor
type BNode* = distinct Cursor
## A cursor at an AST NODE, as distinct from a raw structural cursor: see
## "THE NODE ENCODING" above. The conversion is deliberately not implicit.
template raw*(n: BNode): Cursor = Cursor(n)
## Escape hatch to the underlying structural cursor. Only this module and
## the decoder should need it.
proc toBNode*(c: Cursor): BNode {.inline.} = BNode(c)
## A raw cursor known to sit on an AST node — e.g. the body cursor
## `ast2nif.lazyBodyCursor` hands back.
# `nifcore` also has a `kind(c: Cursor): NifKind` — the TOKEN kind (TagLit,
# SymUse, IntLit, ...). It and `kind(n: BNode): TNodeKind` below differ only
@@ -99,6 +204,12 @@ when defined(newIcBackend):
# nifcore one is always spelled `nifcore.kind`. Modules that import `bnode`
# do not import `nifcore`, so they see only the `TNodeKind` one.
const
LeafKinds* = {nkNone..nkNilLit}
## Exactly `astdef.safeLen`'s set: these have no children, and any token
## sitting after their flags/type prefix is a PAYLOAD (an int, a string,
## an ident), not a child.
var kindCachePool: TagPool = nil
var kindCache: seq[int16] = @[]
## `TagId -> TNodeKind` for ONE tag pool, -1 where not yet resolved.
@@ -107,151 +218,401 @@ when defined(newIcBackend):
## works through one module at a time, so a single-entry memo is enough;
## a pool switch just drops the cache.
proc kind*(n: BNode): TNodeKind =
## The `TNodeKind` a `.bif` tag encodes — the inverse of `toNifTag`, which
proc tagKind(c: Cursor): TNodeKind =
## The `TNodeKind` a `.bif` TAG encodes — the inverse of `toNifTag`, which
## is what wrote it (`ast2nif`: `pool.tags.getOrIncl(toNifTag(n.kind))`).
## `parse` is a compare against ~180 strings, far too much per node, so the
## answer is memoized per tag id.
if nifcore.kind(n) != TagLit: return nkEmpty
let pool = n.tags
## answer is memoized per tag id. The three wrapper tags that encode an
## `nkSym` are folded into the memo; `parse` answers `nkNone` for them (and
## for every non-AST tag, such as the module-level `(unusedid ...)`).
let pool = c.tags
if pool != kindCachePool:
kindCachePool = pool
kindCache = @[]
let id = int(uint32(cursorTagId(n)))
let id = int(uint32(cursorTagId(c)))
if id >= kindCache.len:
let oldLen = kindCache.len
kindCache.setLen(id + 1)
for i in oldLen ..< kindCache.len: kindCache[i] = -1'i16
if kindCache[id] < 0:
kindCache[id] = int16(ord(parse(TNodeKind, pool.tagName(cursorTagId(n)))))
let name = pool.tagName(cursorTagId(c))
let k = if name == hiddenTypeTagName or name == symDefTagName or
name == symNodeFlagsTagName: nkSym
else: parse(TNodeKind, name)
kindCache[id] = int16(ord(k))
result = TNodeKind(kindCache[id])
proc hasSons*(n: BNode): bool {.inline.} =
nifcore.kind(n) == TagLit and n.cursorJump > 0
proc kind*(n: BNode): TNodeKind =
## The node kind. A bare `Symbol`/`SymbolDef` token IS an `nkSym` node — it
## is how the common symbol use is written — and a `DotToken` is the nil
## child, which has no kind at all and answers `nkNone`; test it with
## `isNilNode` rather than comparing kinds.
case nifcore.kind(n.raw)
of TagLit: tagKind(n.raw)
of Symbol, SymbolDef: nkSym
else: nkNone
proc firstSon*(n: BNode): BNode {.inline.} = childCursor(n)
proc isNilNode*(n: BNode): bool {.inline.} =
## The `n == nil` of the `PNode` world: `ast2nif` writes a nil child as a
## `DotToken`, and a cursor is never itself nil.
nifcore.kind(n.raw) == DotToken
proc hasPrefix(c: Cursor): bool {.inline.} =
## Whether this `TagLit` carries the `withNode` flags/type prefix. `(none)`
## and a type-less `(empty)` are written bare, and they are the only nodes
## with zero raw children, so the test is exact.
cursorJump(c) > 0
proc astChildren(n: BNode): Cursor =
## A cursor at AST child 0, or an exhausted cursor when there is none.
## Steps over the two-token flags/type prefix; see "THE NODE ENCODING".
result = childCursor(n.raw)
if result.hasMore: skip result # flags
if result.hasMore: skip result # type
template walkChildren(n: BNode; c, body: untyped) =
## Shared guard for every child accessor: only a `TagLit` node that is not a
## leaf kind has children at all.
if nifcore.kind(n.raw) == TagLit and kind(n) notin LeafKinds:
var c = astChildren(n)
body
proc hasSons*(n: BNode): bool =
walkChildren(n, c):
return c.hasMore
result = false
proc son*(n: BNode; i: int): BNode =
## Child `i`. O(i) — `skip` is a single pointer add, because a `TagLit`
## token carries the width of its whole subtree.
result = childCursor(n)
for _ in 0 ..< i: skip result
walkChildren(n, c):
for _ in 0 ..< i:
doAssert c.hasMore, "son: index out of range"
skip c
doAssert c.hasMore, "son: index out of range"
return BNode(c)
raiseAssert "son: node has no children"
proc firstSon*(n: BNode): BNode {.inline.} = son(n, 0)
proc secondSon*(n: BNode): BNode {.inline.} = son(n, 1)
proc len*(n: BNode): int =
## Counts the children — O(len). Never put this in a loop condition; the
## iterators below and `hasSons` exist so it is not needed there.
## iterators below and `hasSons` exist so it is not needed there. Follows
## `safeLen`: a leaf kind answers 0 even though its payload token is there.
result = 0
if nifcore.kind(n) != TagLit: return 0
var c = childCursor(n)
while c.hasMore:
inc result
skip c
walkChildren(n, c):
while c.hasMore:
inc result
skip c
proc safeLen*(n: BNode): int {.inline.} = len(n)
## Same as `len`: a non-`TagLit` token has no children and answers 0, so the
## `PNode` distinction (`len` faults on a literal, `safeLen` does not) has
## nothing to guard here.
## Same as `len`: `len` already answers 0 for a leaf, so the `PNode`
## distinction (`len` faults on a literal, `safeLen` does not) has nothing
## to guard here.
proc lastSon*(n: BNode): BNode =
## O(len) — the token stream has no back pointer. Fine once per node, a
## trap inside a loop; `sonsButLast` is the loop form.
var c = childCursor(n)
while true:
result = c
skip c
if not c.hasMore: break
walkChildren(n, c):
while c.hasMore:
result = BNode(c)
skip c
return result
raiseAssert "lastSon: node has no children"
iterator sons*(n: BNode): BNode =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
walkChildren(n, c):
while c.hasMore:
yield c
yield BNode(c)
skip c
iterator sonsFrom*(n: BNode; start: int): BNode =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
walkChildren(n, c):
for _ in 0 ..< start:
if not c.hasMore: break
skip c
while c.hasMore:
yield c
yield BNode(c)
skip c
iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
walkChildren(n, c):
var i = 0
while i < start and c.hasMore:
skip c
inc i
while c.hasMore:
yield (i, c)
yield (i, BNode(c))
skip c
inc i
iterator sonsButLast*(n: BNode; count = 1): BNode =
## One pass with `count` nodes of lookahead — the token stream cannot be
## walked backwards, so the tail is held back instead of subtracted.
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
var pending: seq[BNode] = @[]
walkChildren(n, c):
var pending: seq[Cursor] = @[]
while c.hasMore:
pending.add c
skip c
if pending.len > count:
yield pending[0]
yield BNode(pending[0])
pending.delete(0)
iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
var pending: seq[BNode] = @[]
walkChildren(n, c):
var pending: seq[Cursor] = @[]
var i = 0
while c.hasMore:
pending.add c
skip c
if pending.len > count:
yield (i, pending[0])
yield (i, BNode(pending[0]))
inc i
pending.delete(0)
# The three that need MORE THAN THE CURSOR, which is the real boundary this
# migration now sits at: none of them can stay a unary accessor.
# ---- resolution: the three that need more than the cursor -----------------
var navStack {.threadvar.}: seq[BodyNav]
## Stack, not a single slot: generating one routine can pull in another
## before the first is finished. See "RESOLUTION CONTEXT" above.
proc pushBodyNav*(nav: sink BodyNav) =
navStack.add nav
proc popBodyNav*(): BodyNav {.discardable.} =
doAssert navStack.len > 0, "popBodyNav without a matching push"
result = navStack.pop()
template withBodyScope*(scope: BodyScope; body: untyped) =
## Runs `body` with a fresh `BodyNav` over `scope` current, so `sym`/`typ`
## inside it resolve the body's names. Pops even if `body` raises, because a
## codegen error is reported and compilation continues.
pushBodyNav(initBodyNav(scope))
try:
body
finally:
popBodyNav()
proc currentNav*(): ptr BodyNav =
## The nav for the body being read. A `ptr` because the accessors below
## MUTATE it (the frame cache), and because copying a nav per access would
## defeat the point. Valid only until the next push — nothing between taking
## it and using it pushes, and nothing may: `symAt` bottoms out in the
## decoder, which never re-enters codegen.
doAssert navStack.len > 0,
"BNode.sym/typ outside withBodyScope: a Symbol token is a NAME and needs " &
"the owning module plus the body's scope to resolve"
result = addr navStack[^1]
proc openScope*(kind = nsBlock) {.inline.} = openScope(currentNav()[], kind)
proc closeScope*() {.inline.} = closeScope(currentNav()[])
proc registerDefs*(n: BNode) {.inline.} = registerDefs(currentNav()[], n.raw)
proc registerDefHere*(n: BNode) {.inline.} =
discard registerDefHere(currentNav()[], n.raw)
proc navStats*(): tuple[hits, fallbacks, registered: int] =
let nav = currentNav()
result = (nav.hits, nav.fallbacks, nav.registered)
template withNodeScope*(kind: NavScopeKind; body: untyped) =
## The traversal-side half of the nav: a walk brackets each scope-bearing
## construct with this, and the definitions it passes are registered as it
## goes. See `bodynav`.
openScope(kind)
try:
body
finally:
closeScope()
proc sym*(n: BNode): PSym =
## The `PSym` an `nkSym` node names, resolved through the current nav: its
## scope chain first, the decoder second. The wrapper forms
## (`(nflags <flags> <symnode>)`, `(ht <type> <symnode>)`) are peeled by
## `bodynav.symToken`, beside the code that derives the lookup key from them,
## so the two cannot drift apart.
result = symAt(currentNav()[], n.raw)
proc symTyp(n: BNode): PType =
## The type of the symbol a sym-shaped node names, or nil.
let s = sym(n)
result = if s == nil: nil else: s.typ
proc typ*(n: BNode): PType =
## The node's type, INCLUDING the lazy fallback `ast.typ` performs.
##
## A sym node whose node type equals its symbol's is written as a bare
## `Symbol` with no type slot at all (`writeSymNode`), and the `PNode`
## loader marks such a node `nfLazyType` so `ast.typ` answers `n.sym.typ`.
## Answering `nil` here instead is not a *smaller* answer, it is a DIFFERENT
## one, and silently: `ast.canRaise` asks `fn.typ.kind == tyProc` about a
## call's callee, so a nil type turns "this call can raise" into "it cannot"
## and the goto-exception check after the call is dropped. The
## `.bif`-vs-`PNode` grinder found exactly that.
let c = n.raw
case nifcore.kind(c)
of Symbol, SymbolDef:
result = symTyp(n)
of DotToken:
result = nil
of TagLit:
let name = c.tags.tagName(cursorTagId(c))
if name == hiddenTypeTagName:
# `(ht <type> <sym>)`: the node type is spelled out because it differed
# from the symbol's at write time.
result = typeAt(currentNav()[], childCursor(c))
if result == nil:
# `(ht . <sym>)` — an EXPLICITLY nil node type, which the writer tries
# hard not to emit but does for a sym whose own type was nil. The
# loader does not read it as nil either: `newSymNode(sym, info)` runs
# first and sets `nfLazyType` whenever the symbol was still an
# unloaded stub (`typImpl == nil`), and only then is `typField`
# overwritten with this nil — so `ast.typ` falls back to `sym.typ`.
# Mirror that. (The AST's answer is strictly speaking load-order
# dependent — a sym already loaded at that moment would leave
# `nfLazyType` clear and yield nil — which is a fragility of the
# loader, not of this mirror. The grinder walks every body in the
# dependency closure and this is the branch it lands on.)
result = symTyp(n)
elif name == symNodeFlagsTagName:
var inner = childCursor(c)
skip inner
result = typ(BNode(inner))
elif name == symDefTagName:
result = symTyp(n)
elif not hasPrefix(c):
result = nil
else:
var t = childCursor(c)
skip t # the flags slot
result = typeAt(currentNav()[], t)
else:
result = nil
proc rawDesc*(n: BNode): string =
## What the token stream literally says here — the NIF token kind and, for a
## tag, its name. Diagnostics only: the vocabulary above is the interface,
## this is the thing it is an interface TO, and a disagreement between the
## two spellings is almost always explained by the raw shape.
let c = n.raw
result = $nifcore.kind(c)
case nifcore.kind(c)
of TagLit: result.add "/" & c.tags.tagName(cursorTagId(c))
of Symbol, SymbolDef: result.add "/" & symName(c)
else: discard
# ---- leaf payloads --------------------------------------------------------
#
# `sym` — a `Symbol` token holds only a NAME. `ast2nif.loadSymStub` turns one
# into a `PSym` from a `DecodeContext` plus the OWNING MODULE's name
# plus that routine body's `localSyms` table, because a name with no
# module suffix is body-local and is not in any index.
# `typ` — same shape: `ast2nif.createTypeStub(c, symName(n))`, so also a
# `DecodeContext`.
# `info` — `rawLineInfo(n)` gives a `NifLineInfo` whose `FileId` belongs to the
# `.bif`'s own pool; `ast2nif.oldLineInfo` maps it to a `TLineInfo`
# through a `LineInfoWriter`, which needs the `ConfigRef`.
# A literal is `(<kind> <flags> <type> <atom>)`: the value is the single token
# after the prefix. These are the accessors `ccgexprs` reaches for on nearly
# every expression node, so nothing in the expression codegen can migrate
# until they exist.
proc atom(n: BNode): Cursor {.inline.} =
## The payload token of a leaf node.
result = astChildren(n)
proc intVal*(n: BNode): BiggestInt =
let c = atom(n)
case nifcore.kind(c)
of IntLit: result = BiggestInt(nifcore.intVal(c))
of UIntLit: result = cast[BiggestInt](nifcore.uintVal(c))
of CharLit: result = BiggestInt(ord(charLit(c)))
else: raiseAssert "intVal on " & rawDesc(n)
proc floatVal*(n: BNode): BiggestFloat =
let c = atom(n)
doAssert nifcore.kind(c) == FloatLit, "floatVal on " & rawDesc(n)
result = BiggestFloat(nifcore.floatVal(c))
proc strVal*(n: BNode): string =
let c = atom(n)
doAssert nifcore.kind(c) == StrLit, "strVal on " & rawDesc(n)
result = nifcore.strVal(c)
proc ident*(n: BNode): PIdent =
let c = atom(n)
doAssert nifcore.kind(c) == Ident, "ident on " & rawDesc(n)
result = identFromCursor(program, c)
proc flags*(n: BNode): TNodeFlags =
## The node's own flags, MINUS the two the `PNode` side owns rather than the
## file: `nfHasComment` is never written (comment text lives in a process-
## local side channel) and `nfLazyType` is a marker the loader adds to say
## "ask the symbol for my type" — which is what `typ` above does here
## unconditionally, so on a cursor the flag has nothing to mark.
let c = n.raw
case nifcore.kind(c)
of TagLit:
let name = c.tags.tagName(cursorTagId(c))
if name == symNodeFlagsTagName:
# `(nflags <flags> <symuse>)`: the wrapper carries the flags the bare
# `Symbol` token had nowhere to put.
var inner = childCursor(c)
result = nodeFlagsFromCursor(inner)
skip inner
result = result + flags(BNode(inner))
elif name == hiddenTypeTagName or name == symDefTagName:
result = {}
elif not hasPrefix(c):
result = {}
else:
result = nodeFlagsFromCursor(childCursor(c))
else:
result = {}
proc lazyBodyBNode*(node: PNode; scope: var BodyScope; body: var BNode): bool =
## The cursor for a routine body that is still a deferred `nfLazyBody`
## placeholder, plus the scope its names resolve in. This is where a `BNode`
## comes FROM: until codegen is driven off `.bif` cursors end to end, it is
## the only supply, and it is what lets a migrated proc be run against the
## `PNode` proc it replaces on the same input. Non-destructive — the
## placeholder is still materializable afterwards.
var c: Cursor = default(Cursor)
result = lazyBodyCursor(program, node, scope, c)
if result: body = BNode(c)
proc info*(n: BNode): TLineInfo =
## No body scope needed: the packed line info resolves through the
## `.bif`'s own filename pool plus the `ConfigRef`.
result = lineInfoFromCursor(program, n.raw)
# ---- predicates shared with the `PNode` spelling ---------------------------
#
# So the next step is not "implement these three" but deciding where codegen
# gets that context from — a parameter, or module-global state for the span of
# one module's `cg` stage, the way `ast2nif` already keeps its writer state.
# Until then the `{.error.}` stubs report the exact missing piece AT ITS CALL
# SITE rather than collapsing into a cascade of unrelated type errors.
proc sym*(n: BNode): PSym {.error:
"BNode.sym: needs a resolution context, not just the cursor — a Symbol " &
"token is a NAME, and ast2nif.loadSymStub resolves it from a DecodeContext " &
"plus the owning module plus the body's localSyms.".} = discard
proc typ*(n: BNode): PType {.error:
"BNode.typ: needs a resolution context — ast2nif.createTypeStub takes a " &
"DecodeContext.".} = discard
proc info*(n: BNode): TLineInfo {.error:
"BNode.info: needs a resolution context — rawLineInfo(n) is a NifLineInfo " &
"in the .bif's own file pool; ast2nif.oldLineInfo maps it through a " &
"LineInfoWriter, which holds the ConfigRef.".} = discard
# `ast.canRaise` / `ast.canRaiseConservative` only ever look at a node's
# `kind`, `sym` and `typ`, all three of which this module now answers off a
# `Cursor`. They cannot be written as `AnyNode` procs in `ast.nim` because
# `BNode` is defined HERE and this module imports `ast`; so `ast.nim` keeps
# the body in a template and these two instantiate it. There is no second
# copy of the logic — change the template and both spellings change.
proc canRaiseConservative*(fn: BNode): bool = canRaiseConservativeImpl(fn)
proc canRaise*(fn: BNode): bool = canRaiseImpl(fn)
# ---- mixed-mode plumbing --------------------------------------------------
#
# `son` and `hasSons` are the two vocabulary members the AST does not already
# have, so during the migration they must exist for BOTH node types: the cgen
# files are full of call sites that read a `PSym.ast`, a `PType.n` or a
# freshly built tree, and those stay `PNode`s no matter how far codegen has
# moved. Without these the define does not compile at all and no proc can be
# migrated incrementally.
template son*(n: PNode; i: int): PNode = n[i]
template hasSons*(n: PNode): bool = n.safeLen > 0
template isNilNode*(n: PNode): bool = n == nil
type AnyNode* = PNode | BNode
## The migration vehicle. A proc written against the vocabulary and typed
## `AnyNode` serves BOTH representations from one body, which means it can
## be migrated without a flag day and — more usefully — that the two
## instantiations can be run against each other on real input.
else:
type BNode* = PNode
type AnyNode* = PNode
# Only the two the AST does not already have. Everything else in the
# Only the three the AST does not already have. Everything else in the
# vocabulary is `ast`/`astdef`'s own `PNode` API — see the module doc.
template son*(n: BNode; i: int): BNode =
## Named indexed access. Exists so a call site states "child i" in a form
@@ -263,75 +624,152 @@ else:
## `Cursor`.
n.safeLen > 0
template isNilNode*(n: BNode): bool =
## `n == nil`, in the form that survives the flip: a `Cursor` is never nil,
## and a nil child is a `DotToken` in the token stream.
n == nil
when isMainModule and defined(newIcBackend):
## Self-test for the `Cursor` half, which nothing else can reach yet: the
## backend still runs on `PNode`s, so these accessors have no call sites that
## a normal build type-checks, let alone executes. Run it against real `.bif`
## files:
## Self-test for the `Cursor` half. The backend still runs on `PNode`s, so
## these accessors have no call sites that a normal build type-checks, let
## alone executes. Run it against real `.bif` files:
##
## nim c -d:newIcBackend compiler/bnode.nim
## ./compiler/bnode <nimcache>/*.s.bif
## ./compiler/bnode <nimcache>/*.bif
##
## Pass SEVERAL files — each `.bif` carries its own tag pool, so one file
## alone cannot catch a `kind` cache that fails to notice the pool changed.
##
## What this checks that the previous version could not: the accessors agree
## with the ENCODING, not merely with each other. Checking `sons` against
## `len` and `firstSon` is satisfied just as well by a vocabulary that is
## uniformly off by two — which is what it was, and this is what caught it.
##
## `sym`/`typ` are NOT exercised here: they resolve through `ast.program` and
## a `BodyScope`, i.e. a live compiler, so their checks belong to the backend
## and not to a standalone binary. What IS checked is every shape assumption
## they rest on — that `(ht ...)`/`(nflags ...)` have exactly two raw children
## with the symbol second, and that `(sd ...)` opens with a `SymbolDef`.
import std / [os, syncio, assertions]
from "../dist/nimony/src/lib" / bif import load, BifModule
var nodes = 0
var checks = 0
var astNodes = 0
proc walk(n: BNode; base: TokenBuf) =
proc walk(c: Cursor; base: TokenBuf) =
inc nodes
if nifcore.kind(n) != TagLit: return
template pos(c: BNode): int = cursorToPosition(base, c)
let n = BNode(c)
template pos(x: Cursor): int = cursorToPosition(base, x)
var listed: seq[int] = @[]
for ch in sons(n): listed.add pos(ch)
doAssert listed.len == len(n), "sons/len disagree"
doAssert (listed.len > 0) == hasSons(n), "hasSons/len disagree"
inc checks, 2
# Every RAW child, which is what the traversal follows: the AST model below
# deliberately does not see the module-level metadata tags, and a walk that
# only followed `sons` would never reach most of the file.
var rawKids: seq[Cursor] = @[]
if nifcore.kind(c) == TagLit:
var ch = childCursor(c)
while ch.hasMore:
rawKids.add ch
skip ch
if listed.len > 0:
doAssert pos(firstSon(n)) == listed[0], "firstSon"
doAssert pos(lastSon(n)) == listed[^1], "lastSon"
inc checks, 2
if listed.len > 1:
doAssert pos(secondSon(n)) == listed[1], "secondSon"
inc checks
for i in 0 ..< listed.len:
doAssert pos(son(n, i)) == listed[i], "son " & $i
inc checks, listed.len
for start in 0 .. min(3, listed.len):
var got: seq[int] = @[]
for ch in sonsFrom(n, start): got.add pos(ch)
doAssert got == listed[start .. ^1], "sonsFrom " & $start
var gotI: seq[int] = @[]
for i, ch in isons(n, start):
doAssert i == start + gotI.len, "isons index"
gotI.add pos(ch)
doAssert gotI == listed[start .. ^1], "isons " & $start
inc checks, 2
for count in 1 .. 2:
let want = if listed.len > count: listed[0 ..< listed.len - count]
else: newSeq[int]()
var got: seq[int] = @[]
for ch in sonsButLast(n, count): got.add pos(ch)
doAssert got == want, "sonsButLast " & $count
var gotI: seq[int] = @[]
for i, ch in isonsButLast(n, count):
doAssert i == gotI.len, "isonsButLast index"
gotI.add pos(ch)
doAssert gotI == want, "isonsButLast " & $count
inc checks, 2
# `kind` against an uncached lookup — this is what catches a stale cache
# when the tag pool changes from one file to the next.
doAssert kind(n) == parse(TNodeKind, n.tags.tagName(cursorTagId(n))), "kind"
doAssert isNilNode(n) == (nifcore.kind(c) == DotToken), "isNilNode"
inc checks
for ch in sons(n): walk(ch, base)
if nifcore.kind(c) in {Symbol, SymbolDef}:
doAssert kind(n) == nkSym, "a Symbol token is an nkSym node"
doAssert not hasSons(n), "a Symbol node has no children"
doAssert len(n) == 0, "a Symbol node has length 0"
inc checks, 3
if nifcore.kind(c) == TagLit:
let k = kind(n)
let name = c.tags.tagName(cursorTagId(c))
# The encoding invariants the whole vocabulary rests on.
if name == symDefTagName:
doAssert k == nkSym, "(sd ...) is an nkSym node"
doAssert rawKids.len > 0 and nifcore.kind(rawKids[0]) == SymbolDef,
"(sd ...) opens with a SymbolDef"
inc checks, 2
elif name == symNodeFlagsTagName or name == hiddenTypeTagName:
doAssert k == nkSym, name & " wraps an nkSym"
doAssert rawKids.len == 2, name & " is exactly (payload, symnode)"
doAssert nifcore.kind(rawKids[1]) in {Symbol, SymbolDef, TagLit},
name & "'s second child is the symbol"
inc checks, 3
elif k != nkNone:
# A real AST node written through `withNode`: either bare (no prefix and
# no children) or prefix + children. "Exactly one raw child" is
# impossible, and that is what pins the prefix down.
doAssert rawKids.len != 1,
"AST node " & name & " has a flags/type prefix or nothing at all"
inc checks
if rawKids.len == 0:
doAssert not hasSons(n), "bare " & name & " has no children"
doAssert len(n) == 0, "bare " & name & " has length 0"
inc checks, 2
else:
inc astNodes
let want =
if k in LeafKinds: newSeq[int]()
else: (block:
var s: seq[int] = @[]
for i in 2 ..< rawKids.len: s.add pos(rawKids[i])
s)
var listed: seq[int] = @[]
for ch in sons(n): listed.add pos(ch.raw)
doAssert listed == want,
"sons of " & name & " must start AFTER the flags/type prefix"
doAssert listed.len == len(n), "sons/len disagree"
doAssert (listed.len > 0) == hasSons(n), "hasSons/len disagree"
inc checks, 3
if listed.len > 0:
doAssert pos(firstSon(n).raw) == listed[0], "firstSon"
doAssert pos(lastSon(n).raw) == listed[^1], "lastSon"
inc checks, 2
if listed.len > 1:
doAssert pos(secondSon(n).raw) == listed[1], "secondSon"
inc checks
for i in 0 ..< listed.len:
doAssert pos(son(n, i).raw) == listed[i], "son " & $i
inc checks, listed.len
for start in 0 .. min(3, listed.len):
var got: seq[int] = @[]
for ch in sonsFrom(n, start): got.add pos(ch.raw)
doAssert got == listed[start .. ^1], "sonsFrom " & $start
var gotI: seq[int] = @[]
for i, ch in isons(n, start):
doAssert i == start + gotI.len, "isons index"
gotI.add pos(ch.raw)
doAssert gotI == listed[start .. ^1], "isons " & $start
inc checks, 2
for count in 1 .. 2:
let wantB = if listed.len > count: listed[0 ..< listed.len - count]
else: newSeq[int]()
var got: seq[int] = @[]
for ch in sonsButLast(n, count): got.add pos(ch.raw)
doAssert got == wantB, "sonsButLast " & $count
var gotI: seq[int] = @[]
for i, ch in isonsButLast(n, count):
doAssert i == gotI.len, "isonsButLast index"
gotI.add pos(ch.raw)
doAssert gotI == wantB, "isonsButLast " & $count
inc checks, 2
# `kind` against an uncached lookup — this is what catches a stale cache
# when the tag pool changes from one file to the next.
let direct =
if name == hiddenTypeTagName or name == symDefTagName or
name == symNodeFlagsTagName: nkSym
else: parse(TNodeKind, name)
doAssert k == direct, "kind"
inc checks
for ch in rawKids: walk(ch, base)
let files = commandLineParams()
if files.len == 0:
@@ -341,4 +779,5 @@ when isMainModule and defined(newIcBackend):
var c = beginRead(m.buf)
walk(c, m.buf)
endRead c
echo "bnode: files=", files.len, " nodes=", nodes, " checks=", checks, " OK"
echo "bnode: files=", files.len, " nodes=", nodes, " astNodes=", astNodes,
" checks=", checks, " OK"

268
compiler/bodynav.nim Normal file
View File

@@ -0,0 +1,268 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## `BodyNav` — a scope-chained navigator over a `.bif` routine body.
##
## Ported from Nimony's `nimony/typenav.nim` (`TypeCache` / `TypeScope`). The
## idea being stolen is not the type algebra — we do not need it, `typ` returns
## a fully materialized `PType` — but the SHAPE of the resolution context:
##
## * a chain of scope frames, each a small table, linked to its parent;
## * `openScope` / `closeScope` / `registerLocal`, called BY THE TRAVERSAL as it
## descends and as it walks past each definition;
## * a lookup that consults the chain and, on a miss, falls through to the
## module index (`typenav`'s `tryLoadSym`; here the decoder's own
## `symFromCursor`).
##
## The consequence is the point: the scope is a PRODUCT OF THE WALK. Nothing is
## snapshotted, so nothing can be stale, and a reader that starts at the top of
## a body and descends always has exactly the definitions it has already passed.
##
## WHAT THIS REPLACES. `ast2nif.PendingBody` stashes `localSyms` — a COPY of the
## enclosing sym def's local symbols, taken when the body was deferred — and
## `bnode`'s `BodyScope` then copies it again. `materializeLazyBody` loads the
## body with its own `var pb`, so every definition the load creates lands in a
## table that is discarded on return. A cursor-side reader holding the earlier
## copy therefore cannot see them, and would mint its own `PSym` for the same
## name: two objects, one symbol.
##
## HOW BIG THAT PROBLEM ACTUALLY IS, measured rather than assumed. Build with
## `-d:icLocalSymStats` and every process reports its `localSyms` traffic on
## exit. Over a full `--ic:on` build of the standard-library closure (104
## backend processes):
##
## localHit=0 fieldStub=2 miss=0 sdReg=5902 extractReg=45
##
## Definitions register constantly and NOT ONE use ever resolves through the
## table. The reason is `ast2nif.isLocalSym`, which returns a hardwired `false`:
## every symbol is emitted with a module suffix and resolves through the
## decoder's global `syms` memo, so both spellings get the same `PSym` whatever
## either one has cached. The 5902 registrations are object FIELDS, whose uses
## deliberately go to `loadFieldStub` instead.
##
## So the stale snapshot is a LATENT hazard, not a live bug, and this module is
## not a bug fix — it is the mechanism that keeps it latent once `isLocalSym`
## stops being `false`, or once a body-local name appears for any other reason.
## Said plainly so nobody has to re-derive it: today the nav changes no answers,
## and the grinder in `cgen` proves that by requiring the navigated symbol to be
## the same object the `PNode` loader produced, at every node of every body.
##
## It is not decorative either, and that also has a number. Over the same build,
## the grinder's traversal reports `navHits=42236 navFallbacks=12658
## navRegistered=311`: the chain answers 77% of lookups, and 311 definitions are
## registered by the walk rather than read from a table someone filled in
## earlier. Sabotaging the key (truncating it to three characters, so
## `c_fwrite` and `c_fflush` collide) makes the grinder fail on the first body
## it reaches — so a clean run means the resolution is right, not that the
## lookup never happened.
##
## FIELDS ARE NOT REGISTERED, and that is deliberate. `loadFieldStub` mints a
## fresh stub per use because two distinct fields can share a name (and a
## position) across types — `a.x` and `b.x` in one body are two different
## symbols. Caching a field by its bare name would hand the second use the first
## one's stub, and its type. The nav skips field names entirely and leaves that
## path exactly as it was.
import std / tables
import ast, ast2nif
when defined(nimPreviewSlimSystem):
import std / assertions
import "../dist/nimony/src/lib/nifcore" except pool
type
NavScopeKind* = enum
nsBlock, ## an ordinary nested scope
nsRoutine ## a routine boundary — see `crossedRoutines`
NavScope {.acyclic.} = ref object
locals: Table[string, PSym]
parent: NavScope
kind: NavScopeKind
BodyNav* = object
## The resolution context for ONE routine body. `base` is what the decoder
## itself needs (the owning module plus a table `loadSymStub` can write
## into); the frame chain on top of it is this module's contribution.
base*: BodyScope
current: NavScope
hits*: int ## resolved from the chain
fallbacks*: int ## resolved through the decoder
registered*: int ## definitions the walk registered
proc initBodyNav*(base: sink BodyScope): BodyNav =
## A nav over a body, seeded with whatever resolution context the decoder
## handed out. The root frame is a routine frame: a body IS one.
result = BodyNav(base: base,
current: NavScope(locals: initTable[string, PSym](),
parent: nil, kind: nsRoutine))
proc openScope*(nav: var BodyNav; kind = nsBlock) {.inline.} =
nav.current = NavScope(locals: initTable[string, PSym](),
parent: nav.current, kind: kind)
proc closeScope*(nav: var BodyNav) {.inline.} =
doAssert nav.current.parent != nil, "closeScope past the root frame"
nav.current = nav.current.parent
template withScope*(nav: var BodyNav; kind: NavScopeKind; body: untyped) =
openScope(nav, kind)
try:
body
finally:
closeScope(nav)
proc registerLocal*(nav: var BodyNav; name: string; s: PSym) {.inline.} =
## Record a definition the walk has just passed, in the innermost frame.
nav.current.locals[name] = s
inc nav.registered
proc lookupLocal*(nav: BodyNav; name: string): PSym =
## The chain only. `nil` when nothing in scope carries this name.
var it {.cursor.} = nav.current
while it != nil:
let s = it.locals.getOrDefault(name)
if s != nil: return s
it = it.parent
result = nil
proc crossedRoutines*(nav: BodyNav; name: string): int =
## How many routine frames separate the use from the definition — 0 when the
## definition is in the current routine. `typenav` computes the same thing as
## `LocalInfo.crossedProc`, and it is what tells a closure pass that a name is
## captured rather than local. Nothing consumes it here yet; it is the reason
## the frames carry a kind at all, and dropping the kind would make it
## unrecoverable later.
var it {.cursor.} = nav.current
var crossed = 0
while it != nil:
if it.locals.getOrDefault(name) != nil: return crossed
if it.kind == nsRoutine: inc crossed
it = it.parent
result = -1
# ---------------------------------------------------------------------------
# Names
#
# A symbol reaches the reader in four shapes and they all NAME the same thing;
# `navName` is the one place that knows which token holds the name, so the
# lookup key is derived identically no matter which wrapper the writer chose.
proc navName*(n: Cursor): string =
## The NIF name a token denotes, or `""` when the token names no symbol.
case nifcore.kind(n)
of Symbol, SymbolDef:
result = symName(n)
of TagLit:
let tag = n.tags.tagName(cursorTagId(n))
if tag == symDefTagName:
let name = childCursor(n)
result = if nifcore.kind(name) in {Symbol, SymbolDef}: symName(name) else: ""
elif tag == hiddenTypeTagName:
# `(ht <type> <sym>)`
var inner = childCursor(n)
skip inner
result = navName(inner)
elif tag == symNodeFlagsTagName:
# `(nflags <flags> <symnode>)`
var inner = childCursor(n)
skip inner
result = navName(inner)
else:
result = ""
else:
result = ""
proc symToken*(n: Cursor): Cursor =
## The token that actually NAMES the symbol, with the wrappers stripped.
## `loadSymStub` accepts a `Symbol`, a `SymbolDef` or an `(sd ...)` and
## rejects everything else, so the `(ht ...)` / `(nflags ...)` forms have to be
## peeled here rather than at each call site — the same peeling `navName` does
## for the key, kept beside it so the two cannot drift apart.
result = n
while nifcore.kind(result) == TagLit:
let tag = result.tags.tagName(cursorTagId(result))
if tag == hiddenTypeTagName or tag == symNodeFlagsTagName:
var inner = childCursor(result)
skip inner # the explicit type / the node flags
result = inner
else:
break
proc cacheFrame(nav: var BodyNav): NavScope =
## Where a decoder-resolved name is remembered: the nearest ROUTINE frame.
## Not the innermost frame — a `.bif` name is unique within its module (see
## `isLocalSym`), so its meaning cannot change between frames, and caching it
## deeper would only throw it away sooner. Not the root either, so that a
## nested routine's names die with the nested routine.
result = nav.current
while result.kind != nsRoutine and result.parent != nil:
result = result.parent
proc symAt*(nav: var BodyNav; n: Cursor): PSym =
## The symbol a token names: the chain first, the decoder second.
let name = navName(n)
if name.len > 0:
let cached = lookupLocal(nav, name)
if cached != nil:
inc nav.hits
return cached
inc nav.fallbacks
result = symFromCursor(program, symToken(n), nav.base)
if result != nil and name.len > 0 and not isFieldNifName(name):
cacheFrame(nav).locals[name] = result
proc typeAt*(nav: var BodyNav; n: Cursor): PType {.inline.} =
## Types are not navigated: `ast2nif` already materializes them lazily from
## the module's type index, keyed by name, so there is no per-body state to
## keep and nothing a frame could cache that the decoder does not already.
result = typeFromCursor(program, n, nav.base)
# ---------------------------------------------------------------------------
# Registration during a walk
proc registerDefHere*(nav: var BodyNav; n: Cursor): bool {.discardable.} =
## Register `n` if `n` ITSELF is a definition; do not descend. This is the
## incremental half: a walk calls it on each child before recursing into it,
## so a use can only resolve from the chain to a definition the walk has
## already passed. A use that precedes its definition simply misses and falls
## through to the decoder, which is the behaviour there was before — the nav
## degrades to the old path rather than answering wrongly.
result = false
if nifcore.kind(n) == TagLit and
n.tags.tagName(cursorTagId(n)) == symDefTagName:
let name = navName(n)
if name.len > 0 and not isFieldNifName(name):
let s = symFromCursor(program, n, nav.base)
if s != nil:
registerLocal(nav, name, s)
result = true
proc registerDefs*(nav: var BodyNav; n: Cursor) =
## Register every definition in the SUBTREE at `n` — `typenav.registerLocals`
## with the recursion left in, because a Nim body puts `nkIdentDefs` under an
## `nkVarSection` under the statement list rather than declaring at one level.
##
## Call it on entering a scope to get the eager behaviour (every definition
## known before any use is resolved, which is what a RANDOM-ACCESS reader
## needs), or per statement to get the incremental one (only definitions
## already walked past are visible, which is what a real pass wants and what
## makes use-before-def detectable rather than silently working).
if nifcore.kind(n) == TagLit and
n.tags.tagName(cursorTagId(n)) == symDefTagName:
let name = navName(n)
if name.len > 0 and not isFieldNifName(name):
let s = symFromCursor(program, n, nav.base)
if s != nil: registerLocal(nav, name, s) # `(sd ...)` needs no peeling
return
var c = childCursor(n)
while c.hasMore:
registerDefs(nav, c)
skip c

2
compiler/ccgcalls.nim
View File

@@ -9,7 +9,7 @@
#
# included from cgen.nim
proc canRaiseDisp(p: BProc; n: PNode): bool =
proc canRaiseDisp(p: BProc; n: AnyNode): bool =
# we assume things like sysFatal cannot raise themselves
if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
result = false

16
compiler/ccgexprs.nim
View File

@@ -1441,7 +1441,7 @@ proc genEcho(p: BProc, n: PNode) =
var logCall: CallBuilder
p.s(cpsStmts).addStmt():
p.s(cpsStmts).addCall(logCall, logName):
for it in n.sons:
for it in sons(n):
if it.skipConv.kind == nkNilLit:
p.s(cpsStmts).addArgument(logCall):
p.s(cpsStmts).add("\"\"")
@@ -3173,7 +3173,7 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimZeroMem"),
rdLoc(d),
cSizeof(getTypeDesc(p.module, e.typ)))
for it in e.sons:
for it in sons(e):
if it.kind == nkRange:
idx = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter
a = initLocExpr(p, it.firstSon)
@@ -3202,7 +3202,7 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
# small set
var ts = cUintType(size * 8)
p.s(cpsStmts).addAssignment(rdLoc(d), cIntValue(0))
for it in e.sons:
for it in sons(e):
if it.kind == nkRange:
idx = getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt)) # our counter
a = initLocExpr(p, it.firstSon)
@@ -3853,7 +3853,7 @@ proc containsOpaqueImportcFieldAux(t: PType; n: PNode): bool =
if n == nil: return false
case n.kind
of nkRecList:
for child in n.sons:
for child in sons(n):
if containsOpaqueImportcFieldAux(t, child):
return true
of nkRecCase:
@@ -3990,7 +3990,7 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
case obj.kind
of nkRecList:
let isUnion = tfUnion in t.flags
for it in obj.sons:
for it in sons(obj):
getNullValueAux(p, t, it, constOrNil, result, init, isConst, info)
if isUnion:
# generate only 1 field for default value of union
@@ -4100,7 +4100,7 @@ proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Builder)
if p.vccAndC and not n.hasSons and n.typ.kind == tyArray:
result.addField(arrInit, name = ""):
getDefaultValue(p, n.typ.elementType, n.info, result)
for it in n.sons:
for it in sons(n):
var ind, val: PNode
if it.kind == nkExprColonExpr:
ind = it.firstSon
@@ -4152,7 +4152,7 @@ proc genConstSeq(p: BProc, n: PNode, t: PType; isConst: bool; result: var Builde
def.addField(structInit, name = "data"):
var arrInit: StructInitializer
def.addStructInitializer(arrInit, kind = siArray):
for ni in n.sons:
for ni in sons(n):
def.addField(arrInit, name = ""):
genBracedInit(p, ni, isConst, base, def)
p.module.s[cfsStrData].add extract(def)
@@ -4180,7 +4180,7 @@ proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool; result: var Buil
def.addField(structInit, name = "data"):
var arrInit: StructInitializer
def.addStructInitializer(arrInit, kind = siArray):
for ni in n.sons:
for ni in sons(n):
def.addField(arrInit, name = ""):
genBracedInit(p, ni, isConst, base, def)
p.module.s[cfsStrData].add extract(def)

2
compiler/ccgstmts.nim
View File

@@ -522,7 +522,7 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
d = getTemp(p, n.typ)
genLineDir(p, n)
let lend = getLabel(p)
for it in n.sons:
for it in sons(n):
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
if it.len == 2:

4
compiler/ccgtypes.nim
View File

@@ -767,7 +767,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
check: var IntSet; result: var Builder; unionPrefix = "") =
case n.kind
of nkRecList:
for ni in n.sons:
for ni in sons(n):
genRecordFieldsAux(m, ni, rectype, check, result, unionPrefix)
of nkRecCase:
if n.firstSon.kind != nkSym: internalError(m.config, n.info, "genRecordFieldsAux")
@@ -1075,7 +1075,7 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes
let owner = hashOwner(t.sym)
if not gDebugInfo.hasEnum(t.sym.name.s, t.sym.info.line, owner):
var vals: seq[(string, int)] = @[]
for son in t.n.sons:
for son in sons(t.n):
assert(son.kind == nkSym)
let field = son.sym
vals.add((field.name.s, field.position.int))

212
compiler/cgen.nim
View File

@@ -1344,20 +1344,20 @@ const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTempl
nkMacroDef, nkMixinStmt, nkBindStmt, nkFormalParams} +
declarativeDefs
proc containsResult(n: BNode): bool =
proc containsResult(n: AnyNode): bool =
result = false
case n.kind
of succ(nkEmpty)..pred(nkSym), succ(nkSym)..nkNilLit, harmless:
discard
of nkReturnStmt:
for ni in n.sons:
for ni in sons(n):
if containsResult(ni): return true
result = n.hasSons and n.firstSon.kind == nkEmpty
of nkSym:
if n.sym.kind == skResult:
result = true
else:
for ni in n.sons:
for ni in sons(n):
if containsResult(ni): return true
proc easyResultAsgn(n: PNode): PNode =
@@ -1380,10 +1380,7 @@ proc easyResultAsgn(n: PNode): PNode =
type
InitResultEnum = enum Unknown, InitSkippable, InitRequired
proc allPathsAsgnResult(p: BProc; n: BNode): InitResultEnum =
## Migrated to `BNode` (see bnode.nim). With `newIcBackend` off this is
## `PNode` and nothing changes; with it on, this body is where the Cursor
## vocabulary has to exist, and its `{.error.}` stubs name what is missing.
proc allPathsAsgnResult(p: BProc; n: AnyNode): InitResultEnum =
# Exceptions coming from calls don't have not be considered here:
#
# proc bar(): string = raise newException(...)
@@ -1409,7 +1406,7 @@ proc allPathsAsgnResult(p: BProc; n: BNode): InitResultEnum =
result = Unknown
case n.kind
of nkStmtList, nkStmtListExpr:
for it in n:
for it in sons(n):
result = allPathsAsgnResult(p, it)
if result != Unknown: return result
of nkAsgn, nkFastAsgn, nkSinkAsgn:
@@ -1436,7 +1433,7 @@ proc allPathsAsgnResult(p: BProc; n: BNode): InitResultEnum =
of nkIfStmt, nkIfExpr:
var exhaustive = false
result = InitSkippable
for it in n:
for it in sons(n):
# Every condition must not use 'result':
if it.len == 2 and containsResult(it.firstSon):
return InitRequired
@@ -1498,8 +1495,8 @@ proc allPathsAsgnResult(p: BProc; n: BNode): InitResultEnum =
# arithmetic operations may raise exceptions
result = InitRequired
else:
for i in 0..<n.safeLen:
allPathsInBranch(n[i])
for it in sons(n):
allPathsInBranch(it)
of nkRaiseStmt:
result = InitRequired
of nkChckRangeF, nkChckRange64, nkChckRange:
@@ -1507,8 +1504,195 @@ proc allPathsAsgnResult(p: BProc; n: BNode): InitResultEnum =
# bug #22852
result = InitRequired
else:
for i in 0..<n.safeLen:
allPathsInBranch(n[i])
for it in sons(n):
allPathsInBranch(it)
when defined(newIcBackend):
import std / [exitprocs, syncio]
var bnodeGrind = -1
# Whether the scope chain is load-bearing or decorative is a question with a
# number for an answer, so it gets counted rather than asserted. Reported per
# process on exit; a run in which `navHits` is 0 means every lookup fell
# through to the decoder and the chain is doing nothing.
var navHits, navFallbacks, navRegistered: int
proc grindLockstep(m: BModule; prc: PSym; c: BNode; a: PNode; path: string) =
## Walk the `.bif` cursor and the materialised `PNode` for the SAME body in
## lockstep and require every vocabulary member to answer identically at
## every node. This grades the VOCABULARY rather than any one migrated proc,
## which is the difference that matters: a proc-level oracle only sees an
## accessor that the proc happens to reach on that body, so a wrong accessor
## stays invisible until some later proc migrates and quietly miscompiles.
## `typ` was exactly that — it answered `nil` for every bare `Symbol`, which
## no `containsResult` body could notice.
##
## Must run AFTER the proc-level comparisons: reading `a.kind`/`a.len` fires
## the lazy-body hook and materialises the body, which is fine here (the
## cursor is unaffected) but would spoil their cursor-answer-first ordering.
template bail(what, cur, ast: string) =
internalError(m.config, prc.info,
"BNode/PNode disagree on " & what & " at <body>" & path & " in " &
prc.name.s & ": cursor=" & cur & " ast=" & ast)
if a == nil:
if not c.isNilNode: bail("nil-ness", "not-nil", "nil")
return
if c.isNilNode: bail("nil-ness", "nil", "not-nil")
if c.kind != a.kind: bail("kind", $c.kind, $a.kind)
let here = path & "." & $a.kind
if c.safeLen != a.safeLen: bail("len", $c.safeLen, $a.safeLen)
if c.info != a.info:
bail("info", $(m.config, c.info), $(m.config, a.info))
# Symbols first: a wrong symbol shows up as a wrong TYPE two lines below,
# and "cursor=nil ast=tyProc" is a much worse bug report than "these are
# different symbols".
if a.kind == nkSym:
let cs = c.sym
let asym = a.sym
template describe(x: PSym): string =
(if x == nil: "nil"
else: x.name.s & "/" & $x.kind & "/" & $x.itemId & "/" & $x.state)
if cs == nil or asym == nil:
if cs != asym: bail("sym nil-ness", describe(cs), describe(asym))
elif cs != asym:
# A cross-context object-field reference is stubbed FRESH at every use
# (`loadFieldStub`: two distinct fields can share a local name and
# position across types, so ONE shared stub would mistype one of them).
# Pointer identity is therefore not part of the contract for fields —
# what codegen consumes is the name it re-navigates the reclist with
# (`lookupFieldAgain`) and, for tuples, the position.
if cs.kind == skField and asym.kind == skField:
if cs.name.s != asym.name.s or cs.position != asym.position:
bail("field sym", describe(cs) & "@" & $cs.position,
describe(asym) & "@" & $asym.position)
else:
bail("sym identity", describe(cs), describe(asym))
# `nfHasComment` is never serialised and `nfLazyType` is a `PNode`-side
# marker (see `bnode.flags`); everything else must round-trip exactly.
const ownedByTheAst = {nfHasComment, nfLazyType}
if c.flags - ownedByTheAst != a.flags - ownedByTheAst:
bail("flags", $(c.flags - ownedByTheAst), $(a.flags - ownedByTheAst))
case a.kind
of nkCharLit..nkUInt64Lit:
if c.intVal != a.intVal: bail("intVal", $c.intVal, $a.intVal)
of nkFloatLit..nkFloat128Lit:
# Compare the BITS: two NaNs are never `==`, and a float that survives the
# round trip must be the same float, not merely an equal one.
if cast[uint64](c.floatVal) != cast[uint64](a.floatVal):
bail("floatVal bits", $cast[uint64](c.floatVal),
$cast[uint64](a.floatVal))
of nkStrLit..nkTripleStrLit:
if c.strVal != a.strVal: bail("strVal", c.strVal, a.strVal)
of nkIdent:
if c.ident != a.ident: bail("ident", c.ident.s, a.ident.s)
else: discard
let ct = c.typ
let at = a.typ
if (ct == nil) != (at == nil):
bail("typ nil-ness",
(if ct == nil: "nil" else: $ct.kind) & " raw=" & c.rawDesc,
(if at == nil: "nil" else: $at.kind) & " kind=" & $a.kind &
" typField=" & (if a.typField == nil: "nil" else: $a.typField.kind) &
" lazy=" & $(nfLazyType in a.flags) &
(if a.kind != nkSym: "" else:
" sym=" & a.sym.name.s & "/" & $a.sym.kind & "/" & $a.sym.state &
" symTypImpl=" & (if a.sym.typImpl == nil: "nil" else: $a.sym.typImpl.kind)))
elif ct != nil and ct != at:
# Fields carry their own stub type, so a tolerated field-sym difference
# brings a tolerated type difference with it; compare by kind there.
if a.kind == nkSym and a.sym.kind == skField:
if ct.kind != at.kind:
bail("field typ", $ct.kind, $at.kind)
else:
bail("typ identity", $ct.kind & "/" & $ct.itemId, $at.kind & "/" & $at.itemId)
# `safeLen` already matched, so indexed access stays in range on both sides.
# `son` rescans from the first child each time, which is quadratic — fine for
# a debug-only oracle over routine bodies, and it keeps the walk honest by
# exercising the same accessor migrated code will use.
#
# The descent is bracketed by a nav scope and each child is offered to
# `registerDefHere` BEFORE it is entered, so this walk maintains the scope
# chain exactly the way a cursor-native pass would have to (see `bodynav`).
# That is the part being graded here: not just that the accessors agree, but
# that they still agree when the resolution context is built by the
# traversal instead of handed to it.
if a.safeLen > 0:
withNodeScope(nsBlock):
var i = 0
for child in sons(a):
let cc = son(c, i)
registerDefHere(cc)
grindLockstep(m, prc, cc, child, here & "[" & $i & "]")
inc i
proc grindBNode(m: BModule; p: BProc; prc: PSym) =
## Differential grinding for the migrating vocabulary, opt-in via
## `NIM_IC_BNODE_GRIND`: run every proc that has moved to `AnyNode` over
## BOTH representations of the SAME body and require the same answer. This
## is the only thing that executes the `Cursor` accessors — codegen itself
## is still driven off `PNode`s — and it is deliberately the same technique
## that found the IC bugs earlier on this branch: an oracle beats a
## hand-written expectation, because it compares everything, not what
## someone thought to check.
##
## Order matters. The `PNode` walk calls `len`, which fires the lazy-body
## hook and MATERIALIZES the deferred body; the cursor answer is therefore
## taken first. `lazyBodyBNode` itself does not consume the pending entry.
##
## `allPathsAsgnResult` is graded here too, and it is the more valuable of
## the two: it reaches `typ` (via `skipTypes` on a case selector) and
## `canRaiseDisp` (via `sym`), so a disagreement exercises the resolution
## path — `symFromCursor` / `typeFromCursor` against the body's `localSyms`
## — and not just the child walk.
##
## `grindLockstep` runs last and grades the vocabulary itself rather than
## these two procs; it is the check that actually covers accessors no
## migrated proc happens to call yet.
if bnodeGrind < 0:
bnodeGrind = ord(existsEnv("NIM_IC_BNODE_GRIND"))
if bnodeGrind == 1:
addExitProc proc () =
stderr.writeLine "BNODEGRIND navHits=" & $navHits &
" navFallbacks=" & $navFallbacks & " navRegistered=" & $navRegistered
if bnodeGrind == 0: return
let ast = prc.ast
if ast == nil or ast.safeLen <= bodyPos: return
let body = son(ast, bodyPos)
if body == nil: return
var scope = default(BodyScope)
var viaCursor = default(BNode)
if not lazyBodyBNode(body, scope, viaCursor): return
var curResult = false
var curPaths = Unknown
withBodyScope(scope):
curResult = containsResult(viaCursor)
curPaths = allPathsAsgnResult(p, viaCursor)
let astResult = containsResult(body)
if curResult != astResult:
internalError(m.config, prc.info,
"BNode/PNode disagree on containsResult for " & prc.name.s &
": cursor=" & $curResult & " ast=" & $astResult)
let astPaths = allPathsAsgnResult(p, body)
if curPaths != astPaths:
internalError(m.config, prc.info,
"BNode/PNode disagree on allPathsAsgnResult for " & prc.name.s &
": cursor=" & $curPaths & " ast=" & $astPaths)
withBodyScope(scope):
grindLockstep(m, prc, viaCursor, body, "")
let (hits, fallbacks, registered) = navStats()
navHits += hits
navFallbacks += fallbacks
navRegistered += registered
proc getProcTypeCast(m: BModule, prc: PSym): Rope =
result = getTypeDesc(m, prc.loc.t)
@@ -1581,6 +1765,8 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
# CT-evaluated or earlier-referenced routine), NOT a `.t.bif` load — gating on
# it there would WRONGLY skip destructor injection and miscompile (orc
# decref-on-freed). The `.t.bif`-loaded-body concept exists only under cmdNifC.
when defined(newIcBackend):
grindBNode(m, p, prc)
let wasLoaded = m.config.cmd == cmdNifC and prc.transformedBody != nil
var procBody = transformBody(m.g.graph, m.idgen, prc, {})
if sfInjectDestructors in prc.flags and not wasLoaded: