From 20b70c8b1edb4a136d33c0f3f9127c6350be784f Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 29 Aug 2026 07:09:23 +0200 Subject: [PATCH] IC: implement the Cursor half of the codegen vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `BNode`'s `Cursor` branch stopped at `{.error.}` stubs. Everything structural now works, so `-d:newIcBackend` no longer fails on the shape of the tree — the remaining stubs (`sym`, `typ`, `info`) are the three that need the decoder's symbol/type/line-info maps, which is the honest next boundary. Two corrections to the seam, both found by looking at what actually reads a `.bif`: * it was pointed at `nifcursors`, the WRITER cursor over `PackedToken`s. `bif.load` produces a `nifcore.TokenBuf` and `ast2nif` decodes it with a `nifcore.Cursor`; that is the type the backend will get. * the cost model said reading child `i` is O(size of children 0.. TNodeKind table once" plan cannot work. It memoizes per pool and drops the memo when the pool changes. Since no call site executes any of this yet, `when isMainModule` walks real `.bif` files and checks the vocabulary against itself and against an uncached tag lookup. Over 70 files (`.s.bif`, `.t.bif`, `.iface.bif`) from the testworkspace corpus: 2.05M nodes, 9.19M assertions, all passing. Both halves of the harness were confirmed live by sabotage — `secondSon` returning child 2 trips it, and so does dropping the tag-pool memo invalidation, which is what proves ids really do differ between files. The default build is untouched: all 216 generated `.c` files still byte- identical to the parent commit. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01FMyRHByv7hhaQJ4Pa1bHbE --- compiler/bnode.nim | 287 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 250 insertions(+), 37 deletions(-) diff --git a/compiler/bnode.nim b/compiler/bnode.nim index b5a9b6ad61..cd7d1b0dec 100644 --- a/compiler/bnode.nim +++ b/compiler/bnode.nim @@ -27,21 +27,33 @@ ## vocabulary on the `Cursor` side. ## ## THE COST MODEL DIFFERS, and that is what the vocabulary is shaped around. A -## `Cursor` is a copyable position in a token buffer, so a child is reached by -## `firstSon` plus one `skip` per preceding sibling — and `skip` steps over a -## whole subtree. Reading child `i` is therefore O(size of children 0..= casePos: break`) rather than counting up to -## the bound. -## * `lastSon(n)` — O(len). Fine once, a trap inside a loop. -## * `len(n)` — O(len) on a `Cursor`, which has to count. Do not put it in a loop -## condition; use `sons`/`sonsFrom`, or `hasSons` for an emptiness test. +## index-yielding `isons` / `isonsButLast` — one pass. ALWAYS use these for a +## loop: `for i in 0..= casePos: break`) rather than counting up +## to the bound. +## * `lastSon(n)` — O(len), because nothing points backwards. Fine once, a trap +## inside a loop; `sonsButLast` is the loop form. +## * `len(n)` — O(len) too: it counts. Do not put it in a loop condition; use +## the iterators, or `hasSons` for an emptiness test. +## +## `kind` is the one accessor whose cost is not structural. A `.bif` carries its +## OWN tag pool, so a tag id means nothing outside the file it came from and +## there can be no process-global id -> `TNodeKind` table; the answer is +## memoized per pool instead, and the memo is dropped when the pool changes. +## `when isMainModule` at the bottom of this file checks that — run it over +## several `.bif` files at once, since one file alone cannot catch a memo that +## fails to notice the switch. ## ## The cgen files hold to one invariant, which is what makes the eventual flip ## mechanical: NO `[]` ON A `PNode` OUTSIDE OF TREE CONSTRUCTION. Every read is @@ -57,35 +69,156 @@ import ast, lineinfos when defined(newIcBackend): - import "../dist/nimony/src/lib" / nifcursors + import "../dist/nimony/src/lib/nifcore" except pool + import ic / enum2nif # Imported only under the define: `cgen` is compiled during the koch # bootstrap, where the nimony libs are unavailable (`ast2nif` is guarded the - # same way). + # same way). `nifcore` — NOT `nifcursors` — is the reader half of NIF: `bif` + # loads a `.bif` into a `nifcore.TokenBuf`, and `ast2nif` decodes it with a + # `nifcore.Cursor`. `nifcursors` is the writer/builder cursor over + # `PackedToken`s and is a different type entirely. type BNode* = Cursor - # Migrated one accessor per step. Until then the `{.error.}` stubs make - # flipping the define report the exact missing piece AT ITS CALL SITE, rather - # than collapsing into a cascade of unrelated type errors. - proc kind*(n: BNode): TNodeKind {.error: - "BNode.kind: not implemented for Cursor yet — map the tag id to TNodeKind. " & - "`ic/enum2nif.parse(TNodeKind, string)` is the reverse of `toNifTag`, but a " & - "per-call string compare is too slow here: build a tag-id -> TNodeKind table once.".} = discard - proc len*(n: BNode): int {.error: "BNode.len: not implemented for Cursor yet (counts children; prefer sons/hasSons)".} = discard - proc safeLen*(n: BNode): int {.error: "BNode.safeLen: not implemented for Cursor yet".} = discard - proc son*(n: BNode; i: int): BNode {.error: "BNode.son: not implemented for Cursor yet (firstSon + i skips)".} = discard - proc firstSon*(n: BNode): BNode {.error: "BNode.firstSon: not implemented for Cursor yet".} = discard - proc secondSon*(n: BNode): BNode {.error: "BNode.secondSon: not implemented for Cursor yet".} = discard - proc lastSon*(n: BNode): BNode {.error: "BNode.lastSon: not implemented for Cursor yet (O(len))".} = discard - proc hasSons*(n: BNode): bool {.error: "BNode.hasSons: not implemented for Cursor yet".} = discard - proc sym*(n: BNode): PSym {.error: "BNode.sym: not implemented for Cursor yet".} = discard - proc typ*(n: BNode): PType {.error: "BNode.typ: not implemented for Cursor yet".} = discard - proc info*(n: BNode): TLineInfo {.error: "BNode.info: not implemented for Cursor yet".} = discard - iterator sons*(n: BNode): BNode {.error: "BNode.sons: not implemented for Cursor yet".} = discard - iterator sonsFrom*(n: BNode; start: int): BNode {.error: "BNode.sonsFrom: not implemented for Cursor yet".} = discard - iterator sonsButLast*(n: BNode; count = 1): BNode {.error: "BNode.sonsButLast: not implemented for Cursor yet (one pass with `count` nodes of lookahead)".} = discard - iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] {.error: "BNode.isons: not implemented for Cursor yet".} = discard - iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] {.error: "BNode.isonsButLast: not implemented for Cursor yet".} = discard + # `nifcore` also has a `kind(c: Cursor): NifKind` — the TOKEN kind (TagLit, + # SymUse, IntLit, ...). It and `kind(n: BNode): TNodeKind` below differ only + # in return type, which Nim cannot overload on, so inside this module the + # nifcore one is always spelled `nifcore.kind`. Modules that import `bnode` + # do not import `nifcore`, so they see only the `TNodeKind` one. + + var kindCachePool: TagPool = nil + var kindCache: seq[int16] = @[] + ## `TagId -> TNodeKind` for ONE tag pool, -1 where not yet resolved. + ## Not a process-global table: a `.bif` carries its OWN tag pool, so ids + ## only mean anything relative to the pool the cursor came from. Codegen + ## works through one module at a time, so a single-entry memo is enough; + ## a pool switch just drops the cache. + + proc kind*(n: BNode): TNodeKind = + ## The `TNodeKind` a `.bif` tag encodes — the inverse of `toNifTag`, which + ## is what wrote it (`ast2nif`: `pool.tags.getOrIncl(toNifTag(n.kind))`). + ## `parse` is a compare against ~180 strings, far too much per node, so the + ## answer is memoized per tag id. + if nifcore.kind(n) != TagLit: return nkEmpty + let pool = n.tags + if pool != kindCachePool: + kindCachePool = pool + kindCache = @[] + let id = int(uint32(cursorTagId(n))) + if id >= kindCache.len: + let oldLen = kindCache.len + kindCache.setLen(id + 1) + for i in oldLen ..< kindCache.len: kindCache[i] = -1'i16 + if kindCache[id] < 0: + kindCache[id] = int16(ord(parse(TNodeKind, pool.tagName(cursorTagId(n))))) + result = TNodeKind(kindCache[id]) + + proc hasSons*(n: BNode): bool {.inline.} = + nifcore.kind(n) == TagLit and n.cursorJump > 0 + + proc firstSon*(n: BNode): BNode {.inline.} = childCursor(n) + + proc son*(n: BNode; i: int): BNode = + ## Child `i`. O(i) — `skip` is a single pointer add, because a `TagLit` + ## token carries the width of its whole subtree. + result = childCursor(n) + for _ in 0 ..< i: skip result + + proc secondSon*(n: BNode): BNode {.inline.} = son(n, 1) + + proc len*(n: BNode): int = + ## Counts the children — O(len). Never put this in a loop condition; the + ## iterators below and `hasSons` exist so it is not needed there. + result = 0 + if nifcore.kind(n) != TagLit: return 0 + var c = childCursor(n) + while c.hasMore: + inc result + skip c + + proc safeLen*(n: BNode): int {.inline.} = len(n) + ## Same as `len`: a non-`TagLit` token has no children and answers 0, so the + ## `PNode` distinction (`len` faults on a literal, `safeLen` does not) has + ## nothing to guard here. + + proc lastSon*(n: BNode): BNode = + ## O(len) — the token stream has no back pointer. Fine once per node, a + ## trap inside a loop; `sonsButLast` is the loop form. + var c = childCursor(n) + while true: + result = c + skip c + if not c.hasMore: break + + iterator sons*(n: BNode): BNode = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + while c.hasMore: + yield c + skip c + + iterator sonsFrom*(n: BNode; start: int): BNode = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + for _ in 0 ..< start: + if not c.hasMore: break + skip c + while c.hasMore: + yield c + skip c + + iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var i = 0 + while i < start and c.hasMore: + skip c + inc i + while c.hasMore: + yield (i, c) + skip c + inc i + + iterator sonsButLast*(n: BNode; count = 1): BNode = + ## One pass with `count` nodes of lookahead — the token stream cannot be + ## walked backwards, so the tail is held back instead of subtracted. + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var pending: seq[BNode] = @[] + while c.hasMore: + pending.add c + skip c + if pending.len > count: + yield pending[0] + pending.delete(0) + + iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var pending: seq[BNode] = @[] + var i = 0 + while c.hasMore: + pending.add c + skip c + if pending.len > count: + yield (i, pending[0]) + inc i + pending.delete(0) + + # Still to migrate. The `{.error.}` stubs make flipping the define report the + # exact missing piece AT ITS CALL SITE, rather than collapsing into a cascade + # of unrelated type errors. + proc sym*(n: BNode): PSym {.error: + "BNode.sym: not implemented for Cursor yet — a SymUse token holds a NAME " & + "(`symName(n)`), so this needs the decoder's name -> PSym map, which lives " & + "in ast2nif's DecodeContext and is not reachable from here yet.".} = discard + proc typ*(n: BNode): PType {.error: + "BNode.typ: not implemented for Cursor yet — types are not inline in the " & + "node stream; they are resolved through the module's type index.".} = discard + proc info*(n: BNode): TLineInfo {.error: + "BNode.info: not implemented for Cursor yet — `rawLineInfo(n)` gives a " & + "NifLineInfo whose FileId must be mapped to a compiler FileIndex.".} = discard + else: type BNode* = PNode @@ -100,3 +233,83 @@ else: ## Emptiness test that does not compute a length — `len` counts on a ## `Cursor`. n.safeLen > 0 + +when isMainModule and defined(newIcBackend): + ## Self-test for the `Cursor` half, which nothing else can reach yet: the + ## backend still runs on `PNode`s, so these accessors have no call sites that + ## a normal build type-checks, let alone executes. Run it against real `.bif` + ## files: + ## + ## nim c -d:newIcBackend compiler/bnode.nim + ## ./compiler/bnode /*.s.bif + ## + ## Pass SEVERAL files — each `.bif` carries its own tag pool, so one file + ## alone cannot catch a `kind` cache that fails to notice the pool changed. + import std / [os, syncio, assertions] + from "../dist/nimony/src/lib" / bif import load, BifModule + + var nodes = 0 + var checks = 0 + + proc walk(n: BNode; base: TokenBuf) = + inc nodes + if nifcore.kind(n) != TagLit: return + template pos(c: BNode): int = cursorToPosition(base, c) + + var listed: seq[int] = @[] + for ch in sons(n): listed.add pos(ch) + doAssert listed.len == len(n), "sons/len disagree" + doAssert (listed.len > 0) == hasSons(n), "hasSons/len disagree" + inc checks, 2 + + if listed.len > 0: + doAssert pos(firstSon(n)) == listed[0], "firstSon" + doAssert pos(lastSon(n)) == listed[^1], "lastSon" + inc checks, 2 + if listed.len > 1: + doAssert pos(secondSon(n)) == listed[1], "secondSon" + inc checks + for i in 0 ..< listed.len: + doAssert pos(son(n, i)) == listed[i], "son " & $i + inc checks, listed.len + + for start in 0 .. min(3, listed.len): + var got: seq[int] = @[] + for ch in sonsFrom(n, start): got.add pos(ch) + doAssert got == listed[start .. ^1], "sonsFrom " & $start + var gotI: seq[int] = @[] + for i, ch in isons(n, start): + doAssert i == start + gotI.len, "isons index" + gotI.add pos(ch) + doAssert gotI == listed[start .. ^1], "isons " & $start + inc checks, 2 + + for count in 1 .. 2: + let want = if listed.len > count: listed[0 ..< listed.len - count] + else: newSeq[int]() + var got: seq[int] = @[] + for ch in sonsButLast(n, count): got.add pos(ch) + doAssert got == want, "sonsButLast " & $count + var gotI: seq[int] = @[] + for i, ch in isonsButLast(n, count): + doAssert i == gotI.len, "isonsButLast index" + gotI.add pos(ch) + doAssert gotI == want, "isonsButLast " & $count + inc checks, 2 + + # `kind` against an uncached lookup — this is what catches a stale cache + # when the tag pool changes from one file to the next. + doAssert kind(n) == parse(TNodeKind, n.tags.tagName(cursorTagId(n))), "kind" + inc checks + + for ch in sons(n): walk(ch, base) + + let files = commandLineParams() + if files.len == 0: + quit "usage: bnode [more.bif ...]" + for f in files: + var m = bif.load(f) + var c = beginRead(m.buf) + walk(c, m.buf) + endRead c + echo "bnode: files=", files.len, " nodes=", nodes, " checks=", checks, " OK"