From 47144c7962dad558e47faeb4416d006d85966c6f Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 30 Aug 2026 12:15:36 +0200 Subject: [PATCH] IC: a `PNode` <-> `TokenBuf` bridge, so rewrites stay on `PNode` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator does not migrate to a `Cursor` and should not: transf, destructor injection, closure lifting and the tree codegen builds as it goes all CONSTRUCT nodes, and a cursor is a read pointer into a shared token buffer. `nodebridge` is the seam instead — a rewriting pass keeps producing a `PNode`, and anything that only reads is handed a `TokenBuf`, from which a `Cursor` (and so a `BNode`) is a pointer. The bridge is NOT the `.bif` format, and the difference is the point. A `.bif` is read by a different process, so every symbol and type has to be written as a NAME to look up again. A bridged buffer is read by the process that built it, so a symbol reference is `(bsym )` into a side table holding the very `PSym` the encoder was handed, and the type slot is `(btyp )` the same way. The node shape is otherwise identical to the file format, so `bnode` reads a bridged buffer with the accessors it already has; `bodynav` grows one branch each in `symAt`/`typeAt`, and `bnode` learns that `bsym` is an `nkSym`. That buys the property this branch has been blocked on: **`sym` IS IDEMPOTENT ON A BRIDGED BUFFER, FIELDS INCLUDED.** On the file path it cannot be — `loadFieldStub` mints a fresh stub per use because two distinct fields can share a name and a position across types — which is what stops `aliases.isPartOf` moving to the seam. A bridge hands back the object it was given. The grinder now asserts exactly that: field syms are excluded from the idempotence check on the file path and INCLUDED on a bridged one. Verification is the existing oracle pointed at a harder target. `grindBNode` compares two decodings of one file and has to excuse two differences; the bridge is compared against its own live input and must excuse NEITHER, so both tolerances are counted and the bridge asserts it took neither. `toPNode` is covered without a hand-written comparator that could share the encoder's bugs: decode, RE-ENCODE, and grade the second buffer against the ORIGINAL tree, so anything the decoder drops shows up as a disagreement. Reach, measured: 1431 bodies and 260_431 nodes, against 782 and 67_857 for the file path — the bridge sees every body, including the one-line `nkAsgn` ones `ast2nif` never defers and the grinder therefore never saw. 0 disagreements. Not vacuous: dropping node flags, perturbing the sym index and dropping the type slot each make it fail immediately (`flags`, `sym identity`, `typ nil-ness`). A bridged buffer must never be written to a file — `(bsym …)` means nothing without the tables beside it. `ast2nif` remains the only serializer. Verified: 215/215 byte-identical `.c` against HEAD on the default path; all four build configurations compile, plus `nodebridge` standalone. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01XEF7FJvUkGKvG9LSGuEaNR --- compiler/ast2nif.nim | 12 ++ compiler/bnode.nim | 23 ++- compiler/bodynav.nim | 56 +++++++- compiler/cgen.nim | 67 ++++++++- compiler/nodebridge.nim | 304 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 455 insertions(+), 7 deletions(-) create mode 100644 compiler/nodebridge.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index dae74b2c87..01fee4fc64 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -210,6 +210,18 @@ const typeDefTagName* = "td" bindingIdTagName = "bid" + bridgeSymTagName* = "bsym" + ## `(bsym )` — a symbol reference in the IN-PROCESS bridge format + ## (`nodebridge.nim`), where the payload is an INDEX into the bridge's own + ## `seq[PSym]` rather than a NIF name. Never written to a file: a `.bif` has + ## to name symbols because the reader is a different process, but a bridged + ## buffer is read by the process that built it, so it can hand back the very + ## same `PSym` object. That is what makes the bridge lossless, and + ## incidentally what makes `sym` idempotent for FIELDS on a bridged buffer — + ## the file path cannot be, because `loadFieldStub` mints per use. + bridgeTypeTagName* = "btyp" + ## `(btyp )` — the same for a node's type slot. + var sdefTag = registerTag(symDefTagName) tdefTag = registerTag(typeDefTagName) diff --git a/compiler/bnode.nim b/compiler/bnode.nim index 1847f5cdb5..eeafa8fb77 100644 --- a/compiler/bnode.nim +++ b/compiler/bnode.nim @@ -266,7 +266,7 @@ when defined(newIcBackend): if kindCache[id] < 0: let name = pool.tagName(cursorTagId(c)) let k = if name == hiddenTypeTagName or name == symDefTagName or - name == symNodeFlagsTagName: nkSym + name == symNodeFlagsTagName or name == bridgeSymTagName: nkSym else: parse(TNodeKind, name) kindCache[id] = int16(ord(k)) result = TNodeKind(kindCache[id]) @@ -423,6 +423,18 @@ when defined(newIcBackend): finally: popBodyNav() + template withBridge*(tables: BridgeTables; body: untyped) = + ## Read a bridged buffer: the nav resolves `(bsym …)` / `(btyp …)` straight + ## out of `tables`. `base` stays empty on purpose — a bridged buffer names + ## nothing, so there is nothing for the decoder to resolve, and leaving a + ## fallback in place would turn a corrupt index into a confusing name + ## lookup instead of the assertion it should be. + pushBodyNav(initBridgeNav(tables)) + try: + body + finally: + popBodyNav() + proc currentNav*(): ptr BodyNav = ## The nav for the body being read. A `ptr` because the accessors below ## MUTATE it (the frame cache), and because copying a nav per access would @@ -528,7 +540,11 @@ when defined(newIcBackend): var inner = childCursor(c) skip inner result = typ(BNode(inner)) - elif name == symDefTagName: + elif name == symDefTagName or name == bridgeSymTagName: + # A bare `(bsym …)` is the bridge's spelling of a bare `Symbol`, so it + # answers the same thing: the symbol's own type. The bridge encoder + # always wraps a sym node in `(ht …)`, so this is the belt to that + # braces rather than a path it relies on. result = symTyp(n) elif not hasPrefix(c): result = nil @@ -620,7 +636,8 @@ when defined(newIcBackend): result = nodeFlagsFromCursor(inner) skip inner result = result + flags(BNode(inner)) - elif name == hiddenTypeTagName or name == symDefTagName: + elif name == hiddenTypeTagName or name == symDefTagName or + name == bridgeSymTagName: result = {} elif not hasPrefix(c): result = {} diff --git a/compiler/bodynav.nim b/compiler/bodynav.nim index d2aa430fa7..a47e320712 100644 --- a/compiler/bodynav.nim +++ b/compiler/bodynav.nim @@ -87,11 +87,29 @@ type parent: NavScope kind: NavScopeKind + BridgeTables* = ref object + ## The side tables of an IN-PROCESS bridged buffer (`nodebridge.nim`). + ## A `.bif` names its symbols because the reader is a different process; a + ## buffer built and read inside ONE process does not have to, and paying the + ## name round trip anyway would be worse than pointless — it is what makes + ## the file path unable to give a field a stable identity (`loadFieldStub` + ## mints per use). Here a symbol reference is an index and resolution hands + ## back the very same object, so `symAt` is exact and idempotent for every + ## symbol kind, fields included. + syms*: seq[PSym] + types*: seq[PType] + BodyNav* = object ## The resolution context for ONE routine body. `base` is what the decoder ## itself needs (the owning module plus a table `loadSymStub` can write ## into); the frame chain on top of it is this module's contribution. + ## + ## `bridge` is non-nil only while reading a bridged buffer. It is consulted + ## FIRST and, when it answers, it answers exactly — there is no fallback, + ## because a `(bsym …)` index that the tables cannot resolve is a corrupt + ## buffer, not a cache miss. base*: BodyScope + bridge*: BridgeTables current: NavScope hits*: int ## resolved from the chain fallbacks*: int ## resolved through the decoder @@ -104,6 +122,16 @@ proc initBodyNav*(base: sink BodyScope): BodyNav = current: NavScope(locals: initTable[string, PSym](), parent: nil, kind: nsRoutine)) +proc initBridgeNav*(tables: BridgeTables): BodyNav = + ## A nav over an in-process bridged buffer. `base` stays empty — a bridged + ## buffer names nothing, so there is nothing for the decoder to resolve — but + ## the ROOT FRAME still has to exist: a walk brackets its descent with + ## `openScope`/`closeScope`, and a nav without a root frame makes the first + ## `closeScope` pop past the bottom. + result = BodyNav(bridge: tables, + current: NavScope(locals: initTable[string, PSym](), + parent: nil, kind: nsRoutine)) + proc openScope*(nav: var BodyNav; kind = nsBlock) {.inline.} = nav.current = NavScope(locals: initTable[string, PSym](), parent: nav.current, kind: kind) @@ -206,8 +234,25 @@ proc cacheFrame(nav: var BodyNav): NavScope = while result.kind != nsRoutine and result.parent != nil: result = result.parent +proc bridgeIndex(n: Cursor; tag: string): int = + ## The `` payload of a `(bsym …)` / `(btyp …)` token, or -1 when `n` + ## is not that shape. + result = -1 + if nifcore.kind(n) == TagLit and n.tags.tagName(cursorTagId(n)) == tag: + let payload = childCursor(n) + if nifcore.kind(payload) == IntLit: + result = int(nifcore.intVal(payload)) + proc symAt*(nav: var BodyNav; n: Cursor): PSym = - ## The symbol a token names: the chain first, the decoder second. + ## The symbol a token names: the bridge first (exact), then the chain, then + ## the decoder. + if nav.bridge != nil: + let idx = bridgeIndex(symToken(n), bridgeSymTagName) + if idx >= 0: + doAssert idx < nav.bridge.syms.len, + "bridged sym index out of range: " & $idx + inc nav.hits + return nav.bridge.syms[idx] let name = navName(n) if name.len > 0: let cached = lookupLocal(nav, name) @@ -219,10 +264,17 @@ proc symAt*(nav: var BodyNav; n: Cursor): PSym = if result != nil and name.len > 0 and not isFieldNifName(name): cacheFrame(nav).locals[name] = result -proc typeAt*(nav: var BodyNav; n: Cursor): PType {.inline.} = +proc typeAt*(nav: var BodyNav; n: Cursor): PType = ## Types are not navigated: `ast2nif` already materializes them lazily from ## the module's type index, keyed by name, so there is no per-body state to ## keep and nothing a frame could cache that the decoder does not already. + if nav.bridge != nil: + if nifcore.kind(n) == DotToken: return nil + let idx = bridgeIndex(n, bridgeTypeTagName) + if idx >= 0: + doAssert idx < nav.bridge.types.len, + "bridged type index out of range: " & $idx + return nav.bridge.types[idx] result = typeFromCursor(program, n, nav.base) # --------------------------------------------------------------------------- diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cd07d4dc34..0a7229fd4b 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1525,6 +1525,7 @@ proc allPathsAsgnResult(p: BProc; n: AnyNode): InitResultEnum = when defined(newIcBackend): import std / [exitprocs, syncio] + import nodebridge var bnodeGrind = -1 # Whether the scope chain is load-bearing or decorative is a question with a @@ -1536,6 +1537,12 @@ when defined(newIcBackend): # something next to how many nodes were actually graded and how many were # excused, so all three are counted and reported together. var gradeGraded, gradeSkipDecl, gradeSkipTyp: int + # The two differences `grindLockstep` EXCUSES on the file path. Counted so the + # bridge can assert it needed neither: a bridged buffer hands back the very + # objects it was given, so any tolerance firing there is a bug in the bridge, + # not a property of the format. + var tolHtNil, tolFieldSym: int + var bridgeGraded: int const nkIntLits = {nkCharLit..nkUInt64Lit} @@ -1636,7 +1643,14 @@ when defined(newIcBackend): # to migrate `aliases.isPartOf` failed, and it failed LOUDLY only because # this grinder existed. Left as a live check so the day it starts holding # is visible. - if a.kind == nkSym and a.sym != nil and a.sym.kind != skField: + # On the FILE path fields are excluded: `loadFieldStub` mints per use, so + # two reads of one token give two stubs. On a BRIDGED buffer they are NOT + # excluded, because the bridge hands back the object it was given — that is + # the property that makes field-comparing code (`aliases.isPartOf`) correct + # on a bridge and wrong on a file, and it is asserted here rather than + # merely claimed in `nodebridge`'s doc. + let bridged = currentNav().bridge != nil + if a.kind == nkSym and a.sym != nil and (bridged or a.sym.kind != skField): if c.sym != c.sym: bail("sym is not idempotent", "two different PSyms", "one PSym") @@ -1756,6 +1770,7 @@ when defined(newIcBackend): # what codegen consumes is the name it re-navigates the reclist with # (`lookupFieldAgain`) and, for tuples, the position. if cs.kind == skField and asym.kind == skField: + inc tolFieldSym if cs.name.s != asym.name.s or cs.position != asym.position: bail("field sym", describe(cs) & "@" & $cs.position, describe(asym) & "@" & $asym.position) @@ -1794,6 +1809,7 @@ when defined(newIcBackend): a.typField == nil and a.sym != nil and at == a.sym.typ and c.hasExplicitNilType if htNilTyp: + inc tolHtNil result = false elif (ct == nil) != (at == nil): bail("typ nil-ness", @@ -1851,6 +1867,48 @@ when defined(newIcBackend): inc gradeGraded grindPredicates(m, p, prc, c, a, here) + proc grindBridge(m: BModule; p: BProc; prc: PSym; body: PNode) = + ## Grade the `PNode` -> `TokenBuf` bridge against its own input. + ## + ## This is a strictly harder test than the file path gets, and deliberately. + ## `grindBNode` compares a cursor loaded from a `.bif` against a `PNode` + ## loaded from the same `.bif` — two decodings of one file, which is why it + ## has to excuse two differences (a field symbol is stubbed per use, and + ## `(ht . )`'s nil is load-order dependent). The bridge is handed a live + ## tree and hands the same objects back, so it must need NEITHER excuse, and + ## the counters are checked to make sure the run did not quietly take one. + ## + ## Then the same buffer is decoded and RE-ENCODED, and the second buffer is + ## graded against the ORIGINAL tree. That is what covers `toPNode`: anything + ## the decoder drops is missing from the re-encoding and shows up as a + ## disagreement with the original, so both directions are checked by the one + ## oracle rather than by a hand-written comparator that could agree with the + ## bug. + if bnodeGrind == 0 or body == nil: return + let htBefore = tolHtNil + let fieldBefore = tolFieldSym + + var enc = toTokenBuf(body, m.config) + withBridge(enc.tables): + grindLockstep(m, p, prc, BNode(rootCursor(enc)), body, "", + gradeable = true) + + var rt = toPNode(enc) + var enc2 = toTokenBuf(rt, m.config) + withBridge(enc2.tables): + grindLockstep(m, p, prc, BNode(rootCursor(enc2)), body, "", + gradeable = true) + + if tolHtNil != htBefore: + internalError(m.config, prc.info, + "bridge needed the `(ht . )` tolerance in " & prc.name.s & + " — it encodes the node's own type explicitly, so it cannot legitimately") + if tolFieldSym != fieldBefore: + internalError(m.config, prc.info, + "bridge needed the field-symbol tolerance in " & prc.name.s & + " — it hands back the same PSym, so identity must already match") + inc bridgeGraded + proc grindBNode(m: BModule; p: BProc; prc: PSym) = ## Differential grinding for the migrating vocabulary, opt-in via ## `NIM_IC_BNODE_GRIND`: run every proc that has moved to `AnyNode` over @@ -1891,7 +1949,7 @@ when defined(newIcBackend): stderr.writeLine "BNODEGRIND navHits=" & $navHits & " navFallbacks=" & $navFallbacks & " navRegistered=" & $navRegistered & " graded=" & $gradeGraded & " skipDecl=" & $gradeSkipDecl & - " skipTyp=" & $gradeSkipTyp + " skipTyp=" & $gradeSkipTyp & " bridged=" & $bridgeGraded if bnodeGrind == 0: return let ast = prc.ast if ast == nil or ast.safeLen <= bodyPos: return @@ -2003,6 +2061,11 @@ proc genProcLvl3*(m: BModule, prc: PSym) = var procBody = transformBody(m.g.graph, m.idgen, prc, {}) if sfInjectDestructors in prc.flags and not wasLoaded: procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody) + when defined(newIcBackend): + # AFTER every rewrite this routine's body gets, which is the tree the bridge + # actually has to carry: transformed, and destructor-injected when this + # process did the injecting. + grindBridge(m, p, prc, procBody) let tmpInfo = prc.info discard freshLineInfo(p, prc.info) diff --git a/compiler/nodebridge.nim b/compiler/nodebridge.nim new file mode 100644 index 0000000000..6b62232f7e --- /dev/null +++ b/compiler/nodebridge.nim @@ -0,0 +1,304 @@ +# +# +# The Nim Compiler +# (c) Copyright 2026 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `PNode` <-> `TokenBuf`, in one process. +## +## WHY THIS EXISTS. The backend splits in two along a line that is not the one +## the migration to `BNode` was drawn along. Passes that REWRITE — transf, +## destructor injection, closure lifting, the tree the code generator builds as +## it goes — construct new nodes, and a `Cursor` is a read cursor into a shared +## token buffer, so they cannot be expressed against it and there is no reason +## to try. Passes that READ want the cursor. The bridge is the seam between +## them: a rewriting pass keeps producing a `PNode`, and anything that only +## reads gets a `TokenBuf`, from which a `Cursor` — and so a `BNode` — is a +## pointer. +## +## HOW IT DIFFERS FROM THE `.bif` FORMAT, and why that is the point. A `.bif` +## is read by a DIFFERENT PROCESS, so every symbol and type has to be written as +## a NAME the reader can look up again. A bridged buffer is read by the process +## that built it, so it does not: a symbol reference is `(bsym )`, an index +## into a side table holding the very `PSym` the encoder was handed, and the +## type slot is `(btyp )` the same way. +## +## Three consequences, and the middle one is the reason to prefer this over +## routing rewrites back through the file format: +## +## * It is LOSSLESS. No name mangling, no module index, no stubs, so nothing can +## be lost or renamed on the way through. `toPNode(toTokenBuf(n))` is `n` +## again, and `cgen`'s grinder checks the stronger property — that the cursor +## answers identically to the ORIGINAL `PNode` at every node, with no +## tolerated differences at all, unlike the file path which needs two. +## * `sym` IS IDEMPOTENT HERE, FIELDS INCLUDED. On the file path it is not, and +## cannot be: a cross-context field reference has no index entry, so +## `loadFieldStub` mints a fresh `skField` stub per use because two distinct +## fields can share a name and a position across types. That is what blocks +## `aliases.isPartOf` from moving to the seam (see `bnode.sym`). A bridged +## buffer hands back the same object every time, so code that compares field +## identity is correct on it. +## * It is cheap. No string formatting, no pool lookups for names, no index +## seeks — the encoder is a tree walk and two `seq.add`s. +## +## WHAT IT IS NOT. The buffer is transient and process-local: `(bsym …)` means +## nothing without the tables beside it, so a bridged buffer must never be +## written to a file. The `.bif` writer in `ast2nif` is still the only thing +## that serializes, and it is a different job — it has to name things precisely +## because the reader cannot see this process's heap. +## +## USE: +## +## var b = toTokenBuf(n, conf) +## withBridge(b.tables): +## let root = BNode(b.rootCursor) # read it like any other `BNode` +## ... +## let back = toPNode(b) # a fresh `PNode` tree, if a rewrite needs one +## +## `withBridge` and `BNode` live in `bnode.nim` and exist only under +## `-d:newIcBackend`; this module is below that seam and does not depend on it, +## so the encoder and the round trip are usable either way. + +import std / tables + +import ast, astdef, idents, options, msgs, lineinfos +import icnifcore, ast2nif +import ic / enum2nif + +import "../dist/nimony/src/lib/nifcore" except pool + +import bodynav + +when defined(nimPreviewSlimSystem): + import std / assertions + +type + BridgeBuf* = object + ## An encoded tree plus everything needed to read it back. Not copyable — + ## it owns a `TokenBuf`. + bld*: IcBuilder + tables*: BridgeTables + conf: ConfigRef + symIdx: Table[int, int] ## PSym identity -> index into `tables.syms` + typeIdx: Table[int, int] ## PType identity -> index into `tables.types` + +proc initBridgeBuf*(conf: ConfigRef; cap = 64): BridgeBuf = + BridgeBuf(bld: newIcBuilder(cap), tables: BridgeTables(), conf: conf, + symIdx: initTable[int, int](), typeIdx: initTable[int, int]()) + +# --------------------------------------------------------------------------- +# Encode +# +# The shape mirrors the `.bif` node encoding exactly — `( +# …)` — so `bnode` reads a bridged buffer with the accessors it +# already has. Only the two leaves that would have been NAMES differ. + +proc symIndex(b: var BridgeBuf; s: PSym): int = + ## Symbols are deduplicated by identity, so the same `PSym` referenced twenty + ## times costs one table slot and twenty equal indices — which is also what + ## makes `sym` idempotent on the way back. + let key = cast[int](s) + result = b.symIdx.getOrDefault(key, -1) + if result < 0: + result = b.tables.syms.len + b.tables.syms.add s + b.symIdx[key] = result + +proc typeIndex(b: var BridgeBuf; t: PType): int = + let key = cast[int](t) + result = b.typeIdx.getOrDefault(key, -1) + if result < 0: + result = b.tables.types.len + b.tables.types.add t + b.typeIdx[key] = result + +proc emitInfo(b: var BridgeBuf; info: TLineInfo) = + ## Line info goes through the SAME filename pool the `.bif` writer uses + ## (`icPool.filenames`, keyed by full path), so `bnode.info` — which resolves + ## through the decoder's `oldLineInfo` — needs no bridge-specific path. + if info == unknownLineInfo: return + b.bld.lineInfo(msgs.toFullPath(b.conf, info.fileIndex), + info.line.int32, info.col.int32) + +proc emitFlags(b: var BridgeBuf; flags: TNodeFlags) = + var asIdent = "" + genFlags(flags, asIdent) + if asIdent.len > 0: b.bld.addIdent asIdent + else: b.bld.addDotToken() + +proc emitTypeSlot(b: var BridgeBuf; t: PType) = + if t == nil: + b.bld.addDotToken() + else: + b.bld.openTag bridgeTypeTagName + b.bld.addIntLit typeIndex(b, t).int64 + b.bld.closeTag() + +proc encodeNode(b: var BridgeBuf; n: PNode) + +proc encodeSym(b: var BridgeBuf; n: PNode) = + ## `(nflags (ht (bsym )))`, always the full chain. + ## + ## The wrappers are unconditional on purpose. The `.bif` writer emits them + ## only when the node differs from its symbol, which is what creates the + ## `(ht . )` shape whose nil is load-bearing and whose meaning depends on + ## whether the symbol was loaded yet — a real ambiguity that cost a reverted + ## commit on this branch. A bridge has no reason to inherit it: spelling the + ## node's own type and flags out every time costs four tokens and makes the + ## answer exact by construction. + b.bld.openTag symNodeFlagsTagName + b.emitInfo(n.info) + b.emitFlags(n.flags) + b.bld.openTag hiddenTypeTagName + b.emitTypeSlot(n.typ) # the LAZY-AWARE accessor: what `ast.typ` says + b.bld.openTag bridgeSymTagName + b.bld.addIntLit symIndex(b, n.sym).int64 + b.bld.closeTag() # bsym + b.bld.closeTag() # ht + b.bld.closeTag() # nflags + +proc encodeNode(b: var BridgeBuf; n: PNode) = + if n == nil: + b.bld.addDotToken() + return + if n.kind == nkSym and n.sym != nil: + encodeSym(b, n) + return + b.bld.openTag toNifTag(n.kind) + b.emitInfo(n.info) + b.emitFlags(n.flags) + b.emitTypeSlot(n.typ) + case n.kind + of nkCharLit: + b.bld.addCharLit char(n.intVal) + of nkIntLit..nkInt64Lit: + b.bld.addIntLit n.intVal + of nkUIntLit..nkUInt64Lit: + b.bld.addUIntLit cast[uint64](n.intVal) + of nkFloatLit..nkFloat128Lit: + b.bld.addFloatLit n.floatVal + of nkStrLit..nkTripleStrLit: + b.bld.addStrLit n.strVal + of nkIdent: + b.bld.addIdent n.ident.s + of nkSym: + # `n.sym == nil`, which `encodeSym` cannot express. It is a broken node + # either way; encode it as a childless `nkSym` so the walk stays total. + discard + of nkNone, nkEmpty, nkNilLit, nkType, nkCommentStmt: + discard + else: + for child in sons(n): encodeNode(b, child) + b.bld.closeTag() + +proc toTokenBuf*(n: PNode; conf: ConfigRef): BridgeBuf = + ## Encode a whole tree. `n` is not modified and not retained: the buffer holds + ## tokens, and the tables hold the `PSym`/`PType` objects the tree pointed at. + result = initBridgeBuf(conf) + encodeNode(result, n) + +proc rootCursor*(b: var BridgeBuf): Cursor {.inline.} = + ## A read cursor at the encoded root. `beginRead` asserts every tag was + ## closed, so a mis-nested encode is caught here rather than as nonsense + ## further along. + beginRead(b.bld.buf) + +# --------------------------------------------------------------------------- +# Decode +# +# The other direction, for a rewriting pass that has a cursor and needs a tree +# it can mutate. Deliberately NOT written against `bnode`: this module is below +# it (`bnode` reads through a nav, which is exactly the state a decoder should +# not need), and the shape is the encoder's, right here, so the two stay +# legible as a pair. + +proc decodeNode(b: BridgeBuf; c: var Cursor): PNode + +proc decodeTypeSlot(b: BridgeBuf; c: var Cursor): PType = + if nifcore.kind(c) == DotToken: + result = nil + skip c + else: + doAssert nifcore.kind(c) == TagLit and + c.tags.tagName(cursorTagId(c)) == bridgeTypeTagName, + "bridge: type slot expected" + let payload = childCursor(c) + doAssert nifcore.kind(payload) == IntLit, "bridge: (btyp) payload expected" + let idx = int(nifcore.intVal(payload)) + doAssert idx < b.tables.types.len, "bridge: type index out of range" + result = b.tables.types[idx] + skip c + +proc decodeFlags(c: var Cursor): TNodeFlags = + result = nodeFlagsFromCursor(c) + skip c + +proc decodeSym(b: BridgeBuf; c: var Cursor): PNode = + ## Unwinds exactly what `encodeSym` wrote. + var outer = childCursor(c) # inside (nflags + let flags = decodeFlags(outer) + doAssert nifcore.kind(outer) == TagLit and + outer.tags.tagName(cursorTagId(outer)) == hiddenTypeTagName, + "bridge: (ht) expected inside (nflags)" + var ht = childCursor(outer) # inside (ht + let typ = decodeTypeSlot(b, ht) + doAssert nifcore.kind(ht) == TagLit and + ht.tags.tagName(cursorTagId(ht)) == bridgeSymTagName, + "bridge: (bsym) expected inside (ht)" + let payload = childCursor(ht) + doAssert nifcore.kind(payload) == IntLit, "bridge: (bsym) payload expected" + let idx = int(nifcore.intVal(payload)) + doAssert idx < b.tables.syms.len, "bridge: sym index out of range" + result = newSymNode(b.tables.syms[idx], lineInfoFromCursor(program, c)) + result.typField = typ + result.flags = flags + skip c + +proc decodeNode(b: BridgeBuf; c: var Cursor): PNode = + case nifcore.kind(c) + of DotToken: + result = nil + skip c + of TagLit: + let tag = c.tags.tagName(cursorTagId(c)) + if tag == symNodeFlagsTagName: + return decodeSym(b, c) + let kind = parse(TNodeKind, tag) + let info = lineInfoFromCursor(program, c) + var inner = childCursor(c) + let flags = decodeFlags(inner) + let typ = decodeTypeSlot(b, inner) + result = newNodeI(kind, info) + result.flags = flags + result.typField = typ + case kind + of nkCharLit..nkUInt64Lit: + result.intVal = + case nifcore.kind(inner) + of CharLit: BiggestInt(ord(charLit(inner))) + of UIntLit: cast[BiggestInt](nifcore.uintVal(inner)) + else: BiggestInt(nifcore.intVal(inner)) + of nkFloatLit..nkFloat128Lit: + result.floatVal = nifcore.floatVal(inner) + of nkStrLit..nkTripleStrLit: + result.strVal = strVal(inner) + of nkIdent: + result.ident = identFromCursor(program, inner) + else: + while inner.hasMore: + result.sons.add decodeNode(b, inner) + skip c + else: + raiseAssert "bridge: unexpected token " & $nifcore.kind(c) + +proc toPNode*(b: var BridgeBuf): PNode = + ## The tree the buffer encodes, as fresh `PNode`s sharing the ORIGINAL + ## `PSym`s and `PType`s. Round-tripping is therefore identity-preserving for + ## symbols and types and structure-preserving for everything else, which is + ## what a rewriting pass needs: it can rebuild a subtree without the symbols + ## underneath it changing identity. + var c = rootCursor(b) + result = decodeNode(b, c)