From 98b1e0ab50c03df5a6fc8d57bec322daf0653ac8 Mon Sep 17 00:00:00 2001 From: araq Date: Mon, 31 Aug 2026 09:30:12 +0200 Subject: [PATCH] IC: own nifstreams here, and move the nimony pin to 1721aab3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `nifstreams` is the classic NIF streaming surface — global `pool`, `PackedToken`, `next(s)` — that this compiler's IC modules (ast2nif, deps, modulegraphs, pipelines) are written against. It lived in `dist/nimony/src/lib` even though nimony's own code is under standing orders never to import it, so nothing over there ever ran a line of it. That is how it broke. Nimony's nifcore refactoring unified the token kinds, and the adapter's `next` started returning `tagLitToken(...)` for an opener — kind `TagLit`, not the `ParLe` its own header promises structural scanners. Nothing in nimony noticed. Here, `deps.nim` walks the import graph by testing exactly `t.kind == ParLe`, and the failure has no error path: an opener it does not recognise is just an unknown token, so it skips the subtree and reads on. Every `.deps.nif` still "parsed"; the graph came out missing most of its edges. `nim c --ic:on` then scheduled modules ahead of their imports and died on a `.s.bif` nothing had written yet: Error: nim m requires precompiled NIF for import: .../streams.nim (expected: .../str6a47nh.s.bif) `json`'s rule listed 5 of its 10 dependencies. So the adapter moves into `compiler/`, where its one consumer lives, its test runs, and its contract is ours to keep. `next` now emits a real `ParLe`, with the tag id in the full 28-bit payload rather than `TagLit`'s 9-bit field and a `tagId` shadowing nifpools' to match: `globalTags` holds 355 tags before this compiler registers one of its own dialect, and a 512-tag ceiling is not one this surface can live under. The `when isMainModule` self-test states the promise; reinstating `tagLitToken` fails it on the first assertion. Everything the adapter adapts (nifpools, nifreader, lineinfos) still comes from `dist/nimony` — only the adapter moved. With it here, the pin can move to nimony master: `bif.load` there fills a loaded module's pools with `addOrdered` instead of hashing every entry it just read back in stored-id order. Verified: `koch boot -d:release` reaches "executables are equal", `testament cat ic` is 40/40, and a 67-module `--ic:on` build produces `.c` files byte-identical to the old pin's. Interleaved on that build, old pin 8.40/8.39/8.43 s vs new 8.17/8.14/8.14 s. --- compiler/ast2nif.nim | 3 +- compiler/deps.nim | 3 +- compiler/modulegraphs.nim | 3 +- compiler/nifstreams.nim | 247 ++++++++++++++++++++++++++++++++++++++ compiler/pipelines.nim | 3 +- koch.nim | 5 +- 6 files changed, 258 insertions(+), 6 deletions(-) create mode 100644 compiler/nifstreams.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 75cc121359..1b33da0769 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -20,7 +20,8 @@ import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs -import "../dist/nimony/src/lib" / [bitabs, nifstreams, lineinfos, +import nifstreams +import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifindexes, nifreader] # Step 2b: the READER speaks nifcore; the WRITER keeps nifstreams (global `pool`, # PackedToken/PackedLineInfo). nifstreams does NOT export Cursor/TokenBuf/NifKind, diff --git a/compiler/deps.nim b/compiler/deps.nim index c7d357e82b..d9187ab248 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -15,7 +15,8 @@ from std/sha1 import secureHash, `$` import options, msgs, lineinfos, pathutils, condsyms, modulepaths, extccomp, cnif, platform -import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder] +import nifstreams +import "../dist/nimony/src/lib" / [bitabs, nifreader, nifbuilder] import icmodnames import icnifcore from ic/replayer import BackendActionsExt diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 55a5293674..64b21c60aa 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -17,7 +17,8 @@ import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, when not defined(nimKochBootstrap): import ast2nif - import "../dist/nimony/src/lib" / [nifstreams, bitabs] + import nifstreams + import "../dist/nimony/src/lib" / bitabs import typekeys diff --git a/compiler/nifstreams.nim b/compiler/nifstreams.nim new file mode 100644 index 0000000000..b5ef0f67f4 --- /dev/null +++ b/compiler/nifstreams.nim @@ -0,0 +1,247 @@ +## nifstreams — the classic NIF streaming surface, used ONLY by this compiler's +## IC modules: ast2nif, deps, modulegraphs and pipelines import it and must keep +## compiling unchanged across nimony's own refactorings. +## +## It used to live in `dist/nimony/src/lib`, which is where the rest of the NIF +## stack still is. It does not belong there: nimony's own code imports nifpools +## (via nifprelude) and is under standing orders never to import this file, so +## nothing over there ever exercised it — which is exactly how it came to hand +## out `TagLit` where every caller here tests for `ParLe` (see `next`), silently +## emptying the IC build graph. A compatibility shim with exactly one consumer +## belongs in the consumer's repo, where its tests run and its contract is +## somebody's problem. +## +## Everything it adapts (`nifpools`, `nifreader`, `lineinfos`) still comes from +## `dist/nimony`; only the adapter moved. +## +## Everything here is an honest adapter, not a fake: +## * Floats get a REAL interning pool: `pool.floats.getOrIncl` returns a +## `FloatId` index, `floatToken` packs it into a genuine `FloatLit` NifToken +## (transit-only: it must never enter a TokenBuf, whose float encoding is +## inline multi-token), and `pool.floats[t.floatId]` decodes it — lossless. +## * `Stream`/`next` wrap the textual nifreader; the unified NifKind has real +## `ParLe`/`ParRi`/`EofToken` members, so structural scanners (deps.nim) +## see the exact classic kinds. Ident/StringLit/Symbol payloads are interned +## into the global `pool`, so `pool.strings[t.litId]` works as before. +## Number tokens keep their KIND only (a 4-byte token cannot always carry +## the value); classic scanners never read those payloads. + +import std / tables +import "../dist/nimony/src/lib" / nifpools +# `except`: the frontend went all-NifLineInfo; the classic side keeps speaking +# PackedLineInfo, so nifpools' same-name/same-params variants must not leak +# through (`info(n: NifToken)` differs only in return type, `NoLineInfo` is a +# same-name const of a different type — either would be ambiguous or wrong for +# ast2nif). The classic replacements are defined below / come from lineinfos. +# `tagId` is excluded for a different reason: nifpools decodes the 9-bit field +# of a real `TagLit`, but this surface hands out `ParLe` tokens whose tag id +# fills the whole 28-bit payload (see `next`), so the decode below is the only +# correct one here. +export nifpools except info, NoLineInfo, tagId +import "../dist/nimony/src/lib" / lineinfos +export lineinfos + +from "../dist/nimony/src/lib" / nifreader import Reader, ExpandedToken, decodeStr + +# ── Classic names the Nim compiler side still uses ─────────────────────── + +type + PackedToken* = NifToken ## ast2nif still says PackedToken + +# Raw payload decodes, sound ONLY on this surface. Every token here comes from +# `next` or the classic `symToken`/`strToken`/`identToken` constructors, which +# intern EVERY literal — including names of at most `StrInlineMaxLen` bytes, +# which the nifcore builders would instead store inside the token. On such an +# inline token the payload is packed bytes, not an id, so nifpools (nimony's own +# surface, where buffers come from the builders) deliberately has no equivalent: +# there it must go through a `Cursor`, which handles both encodings. +proc tagId*(n: NifToken): TagId {.inline.} = TagId(uoperand(n)) + ## Classic `ParLe` tokens (see `next`) keep the tag id in the full 28-bit + ## payload rather than in `TagLit`'s 9-bit field: `globalTags` already holds + ## 355 tags before the Nim compiler registers its own dialect, so a 512-tag + ## ceiling is not a ceiling this surface can live under. +proc litId*(n: NifToken): StrId {.inline.} = StrId(uoperand(n) shr 1) +proc symId*(n: NifToken): SymId {.inline.} = SymId(uoperand(n) shr 1) +proc litId*(c: Cursor): StrId {.inline.} = strId(c) +proc firstSon*(n: Cursor): Cursor {.inline.} = childCursor(n) + +var lineMan*: LineInfoManager + ## The classic packed line-info side channel (`pool.man`). Frontend code no + ## longer uses it — it lives here purely for ast2nif's writer, which packs + ## `TLineInfo` into `PackedLineInfo` and unpacks on emit. + +template files*(p: Pool): untyped = p.filenames +template tags*(p: Pool): untyped = globalTags.tags +template man*(p: Pool): untyped = lineMan + +proc info*(n: NifToken): PackedLineInfo {.inline.} = lineinfos.NoLineInfo + ## Classic tokens carried their line info inline; a bare 4-byte nifcore + ## token cannot, so reading it back yields `NoLineInfo` (ast2nif's + ## `emitInfo(t.info)` then emits nothing — matching the writer, which + ## attaches real positions at the builder level instead). + +proc info*(c: Cursor): PackedLineInfo {.inline.} = + ## Classic packed view of a cursor's line info (ast2nif shadows this with + ## its own NifLineInfo template; kept for any other classic reader). + let li = rawLineInfo(c) + if li.file.isValid: pack(lineMan, li.file, li.line, li.col) + else: lineinfos.NoLineInfo + +type + IntId* = distinct int64 ## value carriers (nifcore stores inline) + UIntId* = distinct uint64 + + ## Identity proxies: the id already carries the value, `[]` returns it. + IntegersProxy* = object + UIntegersProxy* = object + +func `==`*(a, b: IntId): bool {.borrow.} +func `==`*(a, b: UIntId): bool {.borrow.} + +template integers*(p: Pool): IntegersProxy = IntegersProxy() +template uintegers*(p: Pool): UIntegersProxy = UIntegersProxy() + +template `[]`*(x: IntegersProxy; id: IntId): int64 = int64(id) +template `[]`*(x: UIntegersProxy; id: UIntId): uint64 = uint64(id) + +# nifcore stores integers inline: the "id" is the value itself. +template getOrIncl*(x: IntegersProxy; v: int64): IntId = IntId(v) +template getOrIncl*(x: UIntegersProxy; v: uint64): UIntId = UIntId(v) + +proc intId*(n: NifToken): IntId {.inline.} = IntId(n.soperand) +proc uintId*(n: NifToken): UIntId {.inline.} = UIntId(uoperand(n)) +proc intId*(c: Cursor): IntId {.inline.} = IntId(intVal(c)) +proc uintId*(c: Cursor): UIntId {.inline.} = UIntId(uintVal(c)) + +proc addIntLit*(dest: var TokenBuf; id: IntId; info: PackedLineInfo) = + addIntLit(dest, int64(id)) + if info.isValid: + let u = unpack(lineMan, info) + appendLineInfo(dest, u.file, u.line, u.col) + +# Classic single-token constructors with a (dropped) line-info argument. +proc strToken*(s: StrId; info: PackedLineInfo): NifToken {.inline.} = strLitToken(s) +proc symToken*(id: SymId; info: PackedLineInfo): NifToken {.inline.} = symToken(id) +proc identToken*(id: StrId; info: PackedLineInfo): NifToken {.inline.} = identToken(id) +proc dotToken*(info: PackedLineInfo): NifToken {.inline.} = dotToken() +proc charToken*(ch: char; info: PackedLineInfo): NifToken {.inline.} = charToken(ch) + +# ── Classic interned float literals (ast2nif) ──────────────────────────── + +type + FloatId* = distinct uint32 ## 1-based index into the global float pool + FloatPool* = object + values: seq[float64] + lookup: Table[uint64, uint32] # bit pattern -> 1-based id + +func `==`*(a, b: FloatId): bool {.borrow.} + +var globalFloats*: FloatPool + +template floats*(p: Pool): var FloatPool = globalFloats + +proc getOrIncl*(fp: var FloatPool; v: float64): FloatId = + let bits = cast[uint64](v) + let existing = fp.lookup.getOrDefault(bits, 0'u32) + if existing != 0'u32: + result = FloatId(existing) + else: + fp.values.add v + let id = uint32(fp.values.len) + fp.lookup[bits] = id + result = FloatId(id) + +proc `[]`*(fp: FloatPool; id: FloatId): float64 {.inline.} = + fp.values[int(uint32(id)) - 1] + +proc floatToken*(id: FloatId; info: PackedLineInfo): NifToken {.inline.} = + ## Transit-only token: carries the pool index so the receiver can decode it + ## via `pool.floats[t.floatId]`. It must never be appended to a TokenBuf + ## (nifcore stores floats inline as a multi-token encoding); the line info + ## is dropped like in the other classic token constructors. + NifToken((uint32(id) shl KindBits) or uint32(FloatLit)) + +proc floatId*(n: NifToken): FloatId {.inline.} = FloatId(uoperand(n)) + +# ── Classic streaming text reader (deps.nim) ───────────────────────────── + +type + Stream* = object + r*: Reader + +proc parLeToken*(t: TagId): NifToken {.inline.} = + ## The classic surface's opening-tag token: kind `ParLe`, tag id in the + ## payload. Transit-only, like `floatToken` — a `ParLe` never appears in a + ## binary token stream, so this must not be appended to a TokenBuf. + NifToken((uint32(t) shl KindBits) or uint32(ParLe)) + +proc open*(filename: string): Stream = + Stream(r: nifreader.open(filename)) + +proc close*(s: var Stream) = + nifreader.close(s.r) + +proc next*(s: var Stream): NifToken = + ## One classic packed token per call. Pool-referencing kinds are interned + ## into the global `pool`/`globalTags`, so `.litId`/`.tagId` accessors and + ## `pool.strings[...]`/`pool.tags[...]` lookups behave exactly as classic + ## nifstreams did. Kinds without a pool payload come back kind-only. + var t = default(ExpandedToken) + nifreader.next(s.r, t) + case t.tk + of ParLe: + # NOT `tagLitToken`: that would set the kind to `TagLit`, and every classic + # structural scanner tests for `ParLe` (deps.nim walks the import graph that + # way). Emitting `TagLit` here made every one of those tests silently fail — + # the scanner saw an unknown token, skipped the subtree, and the Nim + # compiler's IC build graph came out missing most of its edges. + result = parLeToken(registerTag(globalTags, decodeStr(s.r, t))) + of Ident: + result = identToken(pool.strings.getOrIncl(decodeStr(s.r, t))) + of StrLit: + result = strLitToken(pool.strings.getOrIncl(decodeStr(s.r, t))) + of Symbol: + result = symToken(pool.syms.getOrIncl(decodeStr(s.r, t))) + of SymbolDef: + result = symdefToken(pool.syms.getOrIncl(decodeStr(s.r, t))) + else: + # ParRi/EofToken/DotToken/CharLit/numbers: correct kind, no payload. + result = NifToken(uint32(t.tk)) + +when isMainModule: + # `nim c -r compiler/nifstreams.nim`. + # + # The promise this checks: structural scanners see the CLASSIC kinds. Nim's deps.nim walks + # the import graph by testing `t.kind == ParLe` and then reading + # `pool.tags[t.tagId]`. Hand out nifcore's own `TagLit` instead and every one + # of those tests falls through silently — the scanner treats the opener as an + # unknown token, skips the subtree, and Nim's IC build graph comes out missing + # most of its edges while each individual file still "parses" fine. + import std / [os, syncio] + from "../dist/nimony/src/lib" / nifreader import processDirectives + from std / assertions import assert + + let f = getTempDir() / "nifstreams_selftest.nif" + syncio.writeFile f, "(.nif27)\n(stmts (import (infix / std (bracket os osproc))) (x \"s\" y))\n" + + var kinds: seq[NifKind] = @[] + var tagNames: seq[string] = @[] + var lits: seq[string] = @[] + var s = nifstreams.open(f) + discard processDirectives(s.r) + while true: + let t = next(s) + if t.kind == EofToken: break + kinds.add t.kind + case t.kind + of ParLe: tagNames.add pool.tags[t.tagId] + of Ident, StrLit: lits.add pool.strings[t.litId] + else: discard + nifstreams.close(s) + removeFile f + + assert tagNames == @["stmts", "import", "infix", "bracket", "x"], $tagNames + assert lits == @["/", "std", "os", "osproc", "s", "y"], $lits + assert ParRi in kinds, "closers must stay classic too" + assert TagLit notin kinds, "an opener must arrive as ParLe, not TagLit" + echo "success" diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index b28b60644a..f636ada22d 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -6,7 +6,8 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, when not defined(nimKochBootstrap): import vmdef import ast2nif - import "../dist/nimony/src/lib" / [nifstreams, bitabs] + import nifstreams + import "../dist/nimony/src/lib" / bitabs import pipelineutils diff --git a/koch.nim b/koch.nim index d5c5cd0228..b8429de1cf 100644 --- a/koch.nim +++ b/koch.nim @@ -16,11 +16,12 @@ const ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2 SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936" - NimonyStableCommit = "f831b953d7c21d9a4b11d0042039e7f84d7c8dc9" # unversioned \ + NimonyStableCommit = "1721aab3cad18663da92c2b85508b1f2ff73e3df" # unversioned \ # Note that Nimony uses Nim as a git submodule but we don't want to install # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive # is **required** here. - # Commit from 2026-07-10 -- stable .bif file format + # Commit from 2026-08-31 -- nifcore-based lib; `bif.load` fills pools with + # `addOrdered` instead of hashing every entry it just read back in order. # examples of possible values for fusion: #head, #ea82b54, 1.2.3 FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"