diff --git a/compiler/bnode.nim b/compiler/bnode.nim index b5a9b6ad61..cd7d1b0dec 100644 --- a/compiler/bnode.nim +++ b/compiler/bnode.nim @@ -27,21 +27,33 @@ ## vocabulary on the `Cursor` side. ## ## THE COST MODEL DIFFERS, and that is what the vocabulary is shaped around. A -## `Cursor` is a copyable position in a token buffer, so a child is reached by -## `firstSon` plus one `skip` per preceding sibling — and `skip` steps over a -## whole subtree. Reading child `i` is therefore O(size of children 0..= casePos: break`) rather than counting up to -## the bound. -## * `lastSon(n)` — O(len). Fine once, a trap inside a loop. -## * `len(n)` — O(len) on a `Cursor`, which has to count. Do not put it in a loop -## condition; use `sons`/`sonsFrom`, or `hasSons` for an emptiness test. +## index-yielding `isons` / `isonsButLast` — one pass. ALWAYS use these for a +## loop: `for i in 0..= casePos: break`) rather than counting up +## to the bound. +## * `lastSon(n)` — O(len), because nothing points backwards. Fine once, a trap +## inside a loop; `sonsButLast` is the loop form. +## * `len(n)` — O(len) too: it counts. Do not put it in a loop condition; use +## the iterators, or `hasSons` for an emptiness test. +## +## `kind` is the one accessor whose cost is not structural. A `.bif` carries its +## OWN tag pool, so a tag id means nothing outside the file it came from and +## there can be no process-global id -> `TNodeKind` table; the answer is +## memoized per pool instead, and the memo is dropped when the pool changes. +## `when isMainModule` at the bottom of this file checks that — run it over +## several `.bif` files at once, since one file alone cannot catch a memo that +## fails to notice the switch. ## ## The cgen files hold to one invariant, which is what makes the eventual flip ## mechanical: NO `[]` ON A `PNode` OUTSIDE OF TREE CONSTRUCTION. Every read is @@ -57,35 +69,156 @@ import ast, lineinfos when defined(newIcBackend): - import "../dist/nimony/src/lib" / nifcursors + import "../dist/nimony/src/lib/nifcore" except pool + import ic / enum2nif # Imported only under the define: `cgen` is compiled during the koch # bootstrap, where the nimony libs are unavailable (`ast2nif` is guarded the - # same way). + # same way). `nifcore` — NOT `nifcursors` — is the reader half of NIF: `bif` + # loads a `.bif` into a `nifcore.TokenBuf`, and `ast2nif` decodes it with a + # `nifcore.Cursor`. `nifcursors` is the writer/builder cursor over + # `PackedToken`s and is a different type entirely. type BNode* = Cursor - # Migrated one accessor per step. Until then the `{.error.}` stubs make - # flipping the define report the exact missing piece AT ITS CALL SITE, rather - # than collapsing into a cascade of unrelated type errors. - proc kind*(n: BNode): TNodeKind {.error: - "BNode.kind: not implemented for Cursor yet — map the tag id to TNodeKind. " & - "`ic/enum2nif.parse(TNodeKind, string)` is the reverse of `toNifTag`, but a " & - "per-call string compare is too slow here: build a tag-id -> TNodeKind table once.".} = discard - proc len*(n: BNode): int {.error: "BNode.len: not implemented for Cursor yet (counts children; prefer sons/hasSons)".} = discard - proc safeLen*(n: BNode): int {.error: "BNode.safeLen: not implemented for Cursor yet".} = discard - proc son*(n: BNode; i: int): BNode {.error: "BNode.son: not implemented for Cursor yet (firstSon + i skips)".} = discard - proc firstSon*(n: BNode): BNode {.error: "BNode.firstSon: not implemented for Cursor yet".} = discard - proc secondSon*(n: BNode): BNode {.error: "BNode.secondSon: not implemented for Cursor yet".} = discard - proc lastSon*(n: BNode): BNode {.error: "BNode.lastSon: not implemented for Cursor yet (O(len))".} = discard - proc hasSons*(n: BNode): bool {.error: "BNode.hasSons: not implemented for Cursor yet".} = discard - proc sym*(n: BNode): PSym {.error: "BNode.sym: not implemented for Cursor yet".} = discard - proc typ*(n: BNode): PType {.error: "BNode.typ: not implemented for Cursor yet".} = discard - proc info*(n: BNode): TLineInfo {.error: "BNode.info: not implemented for Cursor yet".} = discard - iterator sons*(n: BNode): BNode {.error: "BNode.sons: not implemented for Cursor yet".} = discard - iterator sonsFrom*(n: BNode; start: int): BNode {.error: "BNode.sonsFrom: not implemented for Cursor yet".} = discard - iterator sonsButLast*(n: BNode; count = 1): BNode {.error: "BNode.sonsButLast: not implemented for Cursor yet (one pass with `count` nodes of lookahead)".} = discard - iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] {.error: "BNode.isons: not implemented for Cursor yet".} = discard - iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] {.error: "BNode.isonsButLast: not implemented for Cursor yet".} = discard + # `nifcore` also has a `kind(c: Cursor): NifKind` — the TOKEN kind (TagLit, + # SymUse, IntLit, ...). It and `kind(n: BNode): TNodeKind` below differ only + # in return type, which Nim cannot overload on, so inside this module the + # nifcore one is always spelled `nifcore.kind`. Modules that import `bnode` + # do not import `nifcore`, so they see only the `TNodeKind` one. + + var kindCachePool: TagPool = nil + var kindCache: seq[int16] = @[] + ## `TagId -> TNodeKind` for ONE tag pool, -1 where not yet resolved. + ## Not a process-global table: a `.bif` carries its OWN tag pool, so ids + ## only mean anything relative to the pool the cursor came from. Codegen + ## works through one module at a time, so a single-entry memo is enough; + ## a pool switch just drops the cache. + + proc kind*(n: BNode): TNodeKind = + ## The `TNodeKind` a `.bif` tag encodes — the inverse of `toNifTag`, which + ## is what wrote it (`ast2nif`: `pool.tags.getOrIncl(toNifTag(n.kind))`). + ## `parse` is a compare against ~180 strings, far too much per node, so the + ## answer is memoized per tag id. + if nifcore.kind(n) != TagLit: return nkEmpty + let pool = n.tags + if pool != kindCachePool: + kindCachePool = pool + kindCache = @[] + let id = int(uint32(cursorTagId(n))) + if id >= kindCache.len: + let oldLen = kindCache.len + kindCache.setLen(id + 1) + for i in oldLen ..< kindCache.len: kindCache[i] = -1'i16 + if kindCache[id] < 0: + kindCache[id] = int16(ord(parse(TNodeKind, pool.tagName(cursorTagId(n))))) + result = TNodeKind(kindCache[id]) + + proc hasSons*(n: BNode): bool {.inline.} = + nifcore.kind(n) == TagLit and n.cursorJump > 0 + + proc firstSon*(n: BNode): BNode {.inline.} = childCursor(n) + + proc son*(n: BNode; i: int): BNode = + ## Child `i`. O(i) — `skip` is a single pointer add, because a `TagLit` + ## token carries the width of its whole subtree. + result = childCursor(n) + for _ in 0 ..< i: skip result + + proc secondSon*(n: BNode): BNode {.inline.} = son(n, 1) + + proc len*(n: BNode): int = + ## Counts the children — O(len). Never put this in a loop condition; the + ## iterators below and `hasSons` exist so it is not needed there. + result = 0 + if nifcore.kind(n) != TagLit: return 0 + var c = childCursor(n) + while c.hasMore: + inc result + skip c + + proc safeLen*(n: BNode): int {.inline.} = len(n) + ## Same as `len`: a non-`TagLit` token has no children and answers 0, so the + ## `PNode` distinction (`len` faults on a literal, `safeLen` does not) has + ## nothing to guard here. + + proc lastSon*(n: BNode): BNode = + ## O(len) — the token stream has no back pointer. Fine once per node, a + ## trap inside a loop; `sonsButLast` is the loop form. + var c = childCursor(n) + while true: + result = c + skip c + if not c.hasMore: break + + iterator sons*(n: BNode): BNode = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + while c.hasMore: + yield c + skip c + + iterator sonsFrom*(n: BNode; start: int): BNode = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + for _ in 0 ..< start: + if not c.hasMore: break + skip c + while c.hasMore: + yield c + skip c + + iterator isons*(n: BNode; start = 0): tuple[i: int, n: BNode] = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var i = 0 + while i < start and c.hasMore: + skip c + inc i + while c.hasMore: + yield (i, c) + skip c + inc i + + iterator sonsButLast*(n: BNode; count = 1): BNode = + ## One pass with `count` nodes of lookahead — the token stream cannot be + ## walked backwards, so the tail is held back instead of subtracted. + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var pending: seq[BNode] = @[] + while c.hasMore: + pending.add c + skip c + if pending.len > count: + yield pending[0] + pending.delete(0) + + iterator isonsButLast*(n: BNode; count = 1): tuple[i: int, n: BNode] = + if nifcore.kind(n) == TagLit: + var c = childCursor(n) + var pending: seq[BNode] = @[] + var i = 0 + while c.hasMore: + pending.add c + skip c + if pending.len > count: + yield (i, pending[0]) + inc i + pending.delete(0) + + # Still to migrate. The `{.error.}` stubs make flipping the define report the + # exact missing piece AT ITS CALL SITE, rather than collapsing into a cascade + # of unrelated type errors. + proc sym*(n: BNode): PSym {.error: + "BNode.sym: not implemented for Cursor yet — a SymUse token holds a NAME " & + "(`symName(n)`), so this needs the decoder's name -> PSym map, which lives " & + "in ast2nif's DecodeContext and is not reachable from here yet.".} = discard + proc typ*(n: BNode): PType {.error: + "BNode.typ: not implemented for Cursor yet — types are not inline in the " & + "node stream; they are resolved through the module's type index.".} = discard + proc info*(n: BNode): TLineInfo {.error: + "BNode.info: not implemented for Cursor yet — `rawLineInfo(n)` gives a " & + "NifLineInfo whose FileId must be mapped to a compiler FileIndex.".} = discard + else: type BNode* = PNode @@ -100,3 +233,83 @@ else: ## Emptiness test that does not compute a length — `len` counts on a ## `Cursor`. n.safeLen > 0 + +when isMainModule and defined(newIcBackend): + ## Self-test for the `Cursor` half, which nothing else can reach yet: the + ## backend still runs on `PNode`s, so these accessors have no call sites that + ## a normal build type-checks, let alone executes. Run it against real `.bif` + ## files: + ## + ## nim c -d:newIcBackend compiler/bnode.nim + ## ./compiler/bnode /*.s.bif + ## + ## Pass SEVERAL files — each `.bif` carries its own tag pool, so one file + ## alone cannot catch a `kind` cache that fails to notice the pool changed. + import std / [os, syncio, assertions] + from "../dist/nimony/src/lib" / bif import load, BifModule + + var nodes = 0 + var checks = 0 + + proc walk(n: BNode; base: TokenBuf) = + inc nodes + if nifcore.kind(n) != TagLit: return + template pos(c: BNode): int = cursorToPosition(base, c) + + var listed: seq[int] = @[] + for ch in sons(n): listed.add pos(ch) + doAssert listed.len == len(n), "sons/len disagree" + doAssert (listed.len > 0) == hasSons(n), "hasSons/len disagree" + inc checks, 2 + + if listed.len > 0: + doAssert pos(firstSon(n)) == listed[0], "firstSon" + doAssert pos(lastSon(n)) == listed[^1], "lastSon" + inc checks, 2 + if listed.len > 1: + doAssert pos(secondSon(n)) == listed[1], "secondSon" + inc checks + for i in 0 ..< listed.len: + doAssert pos(son(n, i)) == listed[i], "son " & $i + inc checks, listed.len + + for start in 0 .. min(3, listed.len): + var got: seq[int] = @[] + for ch in sonsFrom(n, start): got.add pos(ch) + doAssert got == listed[start .. ^1], "sonsFrom " & $start + var gotI: seq[int] = @[] + for i, ch in isons(n, start): + doAssert i == start + gotI.len, "isons index" + gotI.add pos(ch) + doAssert gotI == listed[start .. ^1], "isons " & $start + inc checks, 2 + + for count in 1 .. 2: + let want = if listed.len > count: listed[0 ..< listed.len - count] + else: newSeq[int]() + var got: seq[int] = @[] + for ch in sonsButLast(n, count): got.add pos(ch) + doAssert got == want, "sonsButLast " & $count + var gotI: seq[int] = @[] + for i, ch in isonsButLast(n, count): + doAssert i == gotI.len, "isonsButLast index" + gotI.add pos(ch) + doAssert gotI == want, "isonsButLast " & $count + inc checks, 2 + + # `kind` against an uncached lookup — this is what catches a stale cache + # when the tag pool changes from one file to the next. + doAssert kind(n) == parse(TNodeKind, n.tags.tagName(cursorTagId(n))), "kind" + inc checks + + for ch in sons(n): walk(ch, base) + + let files = commandLineParams() + if files.len == 0: + quit "usage: bnode [more.bif ...]" + for f in files: + var m = bif.load(f) + var c = beginRead(m.buf) + walk(c, m.buf) + endRead c + echo "bnode: files=", files.len, " nodes=", nodes, " checks=", checks, " OK"