mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-09-01 19:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

IC: implement the Cursor half of the codegen vocabulary

Browse Source 
`BNode`'s `Cursor` branch stopped at `{.error.}` stubs. Everything structural
now works, so `-d:newIcBackend` no longer fails on the shape of the tree — the
remaining stubs (`sym`, `typ`, `info`) are the three that need the decoder's
symbol/type/line-info maps, which is the honest next boundary.

Two corrections to the seam, both found by looking at what actually reads a
`.bif`:

  * it was pointed at `nifcursors`, the WRITER cursor over `PackedToken`s.
    `bif.load` produces a `nifcore.TokenBuf` and `ast2nif` decodes it with a
    `nifcore.Cursor`; that is the type the backend will get.
  * the cost model said reading child `i` is O(size of children 0..<i). It is
    O(i): a `TagLit` token stores the width of its whole subtree, so
    `nifcore.skip` is one pointer add no matter how big the subtree is. Indexed
    loops are still quadratic and still worth removing, but in the number of
    children, not in tree size.

`kind` is the accessor the stub called pivotal, and the note on it was wrong in
a way that matters: a `.bif` carries its OWN tag pool, so a tag id means
nothing outside its file and the "build a tag-id -> TNodeKind table once" plan
cannot work. It memoizes per pool and drops the memo when the pool changes.

Since no call site executes any of this yet, `when isMainModule` walks real
`.bif` files and checks the vocabulary against itself and against an uncached
tag lookup. Over 70 files (`.s.bif`, `.t.bif`, `.iface.bif`) from the
testworkspace corpus: 2.05M nodes, 9.19M assertions, all passing. Both halves
of the harness were confirmed live by sabotage — `secondSon` returning child 2
trips it, and so does dropping the tag-pool memo invalidation, which is what
proves ids really do differ between files.

The default build is untouched: all 216 generated `.c` files still byte-
identical to the parent commit.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FMyRHByv7hhaQJ4Pa1bHbE
This commit is contained in:
araq
2026-08-29 07:09:23 +02:00
parent 9e076d74c0
commit 20b70c8b1e
1 changed files with 250 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
compiler/bnode.nim
View File

@@ -27,21 +27,33 @@
## vocabulary on the `Cursor` side.
##
## THE COST MODEL DIFFERS, and that is what the vocabulary is shaped around. A
## `Cursor` is a copyable position in a token buffer, so a child is reached by
## `firstSon` plus one `skip` per preceding sibling — and `skip` steps over a
## whole subtree. Reading child `i` is therefore O(size of children 0..<i):
## `Cursor` is a position in a token stream, and a child is reached by stepping
## over each preceding sibling. Stepping is cheap — a `TagLit` token stores the
## width of its whole subtree, so `nifcore.skip` is a single pointer add
## regardless of how big that subtree is — but there is no random access and no
## way to walk backwards:
##
## * `firstSon` / `secondSon` / `son(n, k)` with small constant `k` — cheap, and
## how nearly all structural access in the cgen files now reads.
## * `firstSon` / `secondSon` / `son(n, k)` — O(k) in the NUMBER of preceding
## siblings, not in their size. Cheap for the small constant `k` that nearly
## all structural access in the cgen files uses.
## * `for x in sons(n)` / `sonsFrom(n, k)` / `sonsButLast(n, k)`, and the
## index-yielding `isons` / `isonsButLast` — one linear pass. ALWAYS use these
## for a loop: `for i in 0..<n.len: n[i]` is O(n^2) once `BNode` is a `Cursor`.
## A loop that stops at a computed position walks forward and breaks
## (`for i, it in isons(n): if i >= casePos: break`) rather than counting up to
## the bound.
## * `lastSon(n)` — O(len). Fine once, a trap inside a loop.
## * `len(n)` — O(len) on a `Cursor`, which has to count. Do not put it in a loop
## condition; use `sons`/`sonsFrom`, or `hasSons` for an emptiness test.
## index-yielding `isons` / `isonsButLast` — one pass. ALWAYS use these for a
## loop: `for i in 0..<n.len: n[i]` is O(children^2). A loop that stops at a
## computed position walks forward and breaks
## (`for i, it in isons(n): if i >= casePos: break`) rather than counting up
## to the bound.
## * `lastSon(n)` — O(len), because nothing points backwards. Fine once, a trap
## inside a loop; `sonsButLast` is the loop form.
## * `len(n)` — O(len) too: it counts. Do not put it in a loop condition; use
## the iterators, or `hasSons` for an emptiness test.
##
## `kind` is the one accessor whose cost is not structural. A `.bif` carries its
## OWN tag pool, so a tag id means nothing outside the file it came from and
## there can be no process-global id -> `TNodeKind` table; the answer is
## memoized per pool instead, and the memo is dropped when the pool changes.
## `when isMainModule` at the bottom of this file checks that — run it over
## several `.bif` files at once, since one file alone cannot catch a memo that
## fails to notice the switch.
##
## The cgen files hold to one invariant, which is what makes the eventual flip
## mechanical: NO `[]` ON A `PNode` OUTSIDE OF TREE CONSTRUCTION. Every read is
@@ -57,35 +69,156 @@
import ast, lineinfos
when defined(newIcBackend):
import "../dist/nimony/src/lib" / nifcursors
import "../dist/nimony/src/lib/nifcore" except pool
import ic / enum2nif
# Imported only under the define: `cgen` is compiled during the koch
# bootstrap, where the nimony libs are unavailable (`ast2nif` is guarded the
# same way).
# same way). `nifcore` — NOT `nifcursors` — is the reader half of NIF: `bif`
# loads a `.bif` into a `nifcore.TokenBuf`, and `ast2nif` decodes it with a
# `nifcore.Cursor`. `nifcursors` is the writer/builder cursor over
# `PackedToken`s and is a different type entirely.
type BNode* = Cursor
# Migrated one accessor per step. Until then the `{.error.}` stubs make
# flipping the define report the exact missing piece AT ITS CALL SITE, rather
# than collapsing into a cascade of unrelated type errors.
proc kind*(n: BNode): TNodeKind {.error:
"BNode.kind: not implemented for Cursor yet — map the tag id to TNodeKind. " &
"`ic/enum2nif.parse(TNodeKind, string)` is the reverse of `toNifTag`, but a " &
"per-call string compare is too slow here: build a tag-id -> TNodeKind table once.".} = discard
proc len*(n: BNode): int {.error: "BNode.len: not implemented for Cursor yet (counts children; prefer sons/hasSons)".} = discard
proc safeLen*(n: BNode): int {.error: "BNode.safeLen: not implemented for Cursor yet".} = discard
proc son*(n: BNode; i: int): BNode {.error: "BNode.son: not implemented for Cursor yet (firstSon + i skips)".} = discard
proc firstSon*(n: BNode): BNode {.error: "BNode.firstSon: not implemented for Cursor yet".} = discard
proc secondSon*(n: BNode): BNode {.error: "BNode.secondSon: not implemented for Cursor yet".} = discard
proc lastSon*(n: BNode): BNode {.error: "BNode.lastSon: not implemented for Cursor yet (O(len))".} = discard
proc hasSons*(n: BNode): bool {.error: "BNode.hasSons: not implemented for Cursor yet".} = discard
proc sym*(n: BNode): PSym {.error: "BNode.sym: not implemented for Cursor yet".} = discard
proc typ*(n: BNode): PType {.error: "BNode.typ: not implemented for Cursor yet".} = discard
proc info*(n: BNode): TLineInfo {.error: "BNode.info: not implemented for Cursor yet".} = discard
iterator sons*(n: BNode): BNode {.error: "BNode.sons: not implemented for Cursor yet".} = discard
iterator sonsFrom*(n: BNode; start: int): BNode {.error: "BNode.sonsFrom: not implemented for Cursor yet".} = discard
iterator sonsButLast*(n: BNode; count = 1): BNode {.error: "BNode.sonsButLast: not implemented for Cursor yet (one pass with `count` nodes of lookahead)".} = discard
iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] {.error: "BNode.isons: not implemented for Cursor yet".} = discard
iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] {.error: "BNode.isonsButLast: not implemented for Cursor yet".} = discard
# `nifcore` also has a `kind(c: Cursor): NifKind` — the TOKEN kind (TagLit,
# SymUse, IntLit, ...). It and `kind(n: BNode): TNodeKind` below differ only
# in return type, which Nim cannot overload on, so inside this module the
# nifcore one is always spelled `nifcore.kind`. Modules that import `bnode`
# do not import `nifcore`, so they see only the `TNodeKind` one.
var kindCachePool: TagPool = nil
var kindCache: seq[int16] = @[]
## `TagId -> TNodeKind` for ONE tag pool, -1 where not yet resolved.
## Not a process-global table: a `.bif` carries its OWN tag pool, so ids
## only mean anything relative to the pool the cursor came from. Codegen
## works through one module at a time, so a single-entry memo is enough;
## a pool switch just drops the cache.
proc kind*(n: BNode): TNodeKind =
## The `TNodeKind` a `.bif` tag encodes — the inverse of `toNifTag`, which
## is what wrote it (`ast2nif`: `pool.tags.getOrIncl(toNifTag(n.kind))`).
## `parse` is a compare against ~180 strings, far too much per node, so the
## answer is memoized per tag id.
if nifcore.kind(n) != TagLit: return nkEmpty
let pool = n.tags
if pool != kindCachePool:
kindCachePool = pool
kindCache = @[]
let id = int(uint32(cursorTagId(n)))
if id >= kindCache.len:
let oldLen = kindCache.len
kindCache.setLen(id + 1)
for i in oldLen ..< kindCache.len: kindCache[i] = -1'i16
if kindCache[id] < 0:
kindCache[id] = int16(ord(parse(TNodeKind, pool.tagName(cursorTagId(n)))))
result = TNodeKind(kindCache[id])
proc hasSons*(n: BNode): bool {.inline.} =
nifcore.kind(n) == TagLit and n.cursorJump > 0
proc firstSon*(n: BNode): BNode {.inline.} = childCursor(n)
proc son*(n: BNode; i: int): BNode =
## Child `i`. O(i) — `skip` is a single pointer add, because a `TagLit`
## token carries the width of its whole subtree.
result = childCursor(n)
for _ in 0 ..< i: skip result
proc secondSon*(n: BNode): BNode {.inline.} = son(n, 1)
proc len*(n: BNode): int =
## Counts the children — O(len). Never put this in a loop condition; the
## iterators below and `hasSons` exist so it is not needed there.
result = 0
if nifcore.kind(n) != TagLit: return 0
var c = childCursor(n)
while c.hasMore:
inc result
skip c
proc safeLen*(n: BNode): int {.inline.} = len(n)
## Same as `len`: a non-`TagLit` token has no children and answers 0, so the
## `PNode` distinction (`len` faults on a literal, `safeLen` does not) has
## nothing to guard here.
proc lastSon*(n: BNode): BNode =
## O(len) — the token stream has no back pointer. Fine once per node, a
## trap inside a loop; `sonsButLast` is the loop form.
var c = childCursor(n)
while true:
result = c
skip c
if not c.hasMore: break
iterator sons*(n: BNode): BNode =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
while c.hasMore:
yield c
skip c
iterator sonsFrom*(n: BNode; start: int): BNode =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
for _ in 0 ..< start:
if not c.hasMore: break
skip c
while c.hasMore:
yield c
skip c
iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
var i = 0
while i < start and c.hasMore:
skip c
inc i
while c.hasMore:
yield (i, c)
skip c
inc i
iterator sonsButLast*(n: BNode; count = 1): BNode =
## One pass with `count` nodes of lookahead — the token stream cannot be
## walked backwards, so the tail is held back instead of subtracted.
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
var pending: seq[BNode] = @[]
while c.hasMore:
pending.add c
skip c
if pending.len > count:
yield pending[0]
pending.delete(0)
iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] =
if nifcore.kind(n) == TagLit:
var c = childCursor(n)
var pending: seq[BNode] = @[]
var i = 0
while c.hasMore:
pending.add c
skip c
if pending.len > count:
yield (i, pending[0])
inc i
pending.delete(0)
# Still to migrate. The `{.error.}` stubs make flipping the define report the
# exact missing piece AT ITS CALL SITE, rather than collapsing into a cascade
# of unrelated type errors.
proc sym*(n: BNode): PSym {.error:
"BNode.sym: not implemented for Cursor yet — a SymUse token holds a NAME " &
"(`symName(n)`), so this needs the decoder's name -> PSym map, which lives " &
"in ast2nif's DecodeContext and is not reachable from here yet.".} = discard
proc typ*(n: BNode): PType {.error:
"BNode.typ: not implemented for Cursor yet — types are not inline in the " &
"node stream; they are resolved through the module's type index.".} = discard
proc info*(n: BNode): TLineInfo {.error:
"BNode.info: not implemented for Cursor yet — `rawLineInfo(n)` gives a " &
"NifLineInfo whose FileId must be mapped to a compiler FileIndex.".} = discard
else:
type BNode* = PNode
@@ -100,3 +233,83 @@ else:
## Emptiness test that does not compute a length — `len` counts on a
## `Cursor`.
n.safeLen > 0
when isMainModule and defined(newIcBackend):
## Self-test for the `Cursor` half, which nothing else can reach yet: the
## backend still runs on `PNode`s, so these accessors have no call sites that
## a normal build type-checks, let alone executes. Run it against real `.bif`
## files:
##
## nim c -d:newIcBackend compiler/bnode.nim
## ./compiler/bnode <nimcache>/*.s.bif
##
## Pass SEVERAL files — each `.bif` carries its own tag pool, so one file
## alone cannot catch a `kind` cache that fails to notice the pool changed.
import std / [os, syncio, assertions]
from "../dist/nimony/src/lib" / bif import load, BifModule
var nodes = 0
var checks = 0
proc walk(n: BNode; base: TokenBuf) =
inc nodes
if nifcore.kind(n) != TagLit: return
template pos(c: BNode): int = cursorToPosition(base, c)
var listed: seq[int] = @[]
for ch in sons(n): listed.add pos(ch)
doAssert listed.len == len(n), "sons/len disagree"
doAssert (listed.len > 0) == hasSons(n), "hasSons/len disagree"
inc checks, 2
if listed.len > 0:
doAssert pos(firstSon(n)) == listed[0], "firstSon"
doAssert pos(lastSon(n)) == listed[^1], "lastSon"
inc checks, 2
if listed.len > 1:
doAssert pos(secondSon(n)) == listed[1], "secondSon"
inc checks
for i in 0 ..< listed.len:
doAssert pos(son(n, i)) == listed[i], "son " & $i
inc checks, listed.len
for start in 0 .. min(3, listed.len):
var got: seq[int] = @[]
for ch in sonsFrom(n, start): got.add pos(ch)
doAssert got == listed[start .. ^1], "sonsFrom " & $start
var gotI: seq[int] = @[]
for i, ch in isons(n, start):
doAssert i == start + gotI.len, "isons index"
gotI.add pos(ch)
doAssert gotI == listed[start .. ^1], "isons " & $start
inc checks, 2
for count in 1 .. 2:
let want = if listed.len > count: listed[0 ..< listed.len - count]
else: newSeq[int]()
var got: seq[int] = @[]
for ch in sonsButLast(n, count): got.add pos(ch)
doAssert got == want, "sonsButLast " & $count
var gotI: seq[int] = @[]
for i, ch in isonsButLast(n, count):
doAssert i == gotI.len, "isonsButLast index"
gotI.add pos(ch)
doAssert gotI == want, "isonsButLast " & $count
inc checks, 2
# `kind` against an uncached lookup — this is what catches a stale cache
# when the tag pool changes from one file to the next.
doAssert kind(n) == parse(TNodeKind, n.tags.tagName(cursorTagId(n))), "kind"
inc checks
for ch in sons(n): walk(ch, base)
let files = commandLineParams()
if files.len == 0:
quit "usage: bnode <file.bif> [more.bif ...]"
for f in files:
var m = bif.load(f)
var c = beginRead(m.buf)
walk(c, m.buf)
endRead c
echo "bnode: files=", files.len, " nodes=", nodes, " checks=", checks, " OK"