diff --git a/compiler/icnifcore.nim b/compiler/icnifcore.nim new file mode 100644 index 0000000000..b0cdbdf745 --- /dev/null +++ b/compiler/icnifcore.nim @@ -0,0 +1,242 @@ +# +# +# The Nim Compiler +# (c) Copyright 2026 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## nifcore-based IC serialization helpers — Stage 1 of porting the IC backend +## from the old `nifstreams`/`nifcursors` NIF stack to `nifcore` (see +## `doc/ic_nifcore_port.md`). +## +## It hosts: +## * the process-wide shared `Pool`/`TagPool` that stands in for the old global +## `nifstreams.pool`, +## * `writeFileStable`, the content-stable file writer mirroring +## `nifcursors.writeFile(..., OnlyIfChanged)`, +## * the first ported writer (`writeSemDeps`), used as the migration spike. +## +## No `nifstreams`/`nifcursors` types cross this module's boundary: callers pass +## plain Nim values (config, ids, string lists), so it can coexist with the +## still-old-API `ast2nif.nim` during the migration. + +import std / [syncio, algorithm] +from std / os import removeFile, moveFile +import options, pathutils, typekeys +import "../dist/nimony/src/lib" / [nifcore, nifcoreparse, nifreader, bif] + +# One shared literals pool + tag pool for the whole process — the nifcore +# analogue of the old global `nifstreams.pool`. A single shared pool keeps +# string/symbol/file ids stable across every TokenBuf the IC backend builds, +# preserving the old global-pool semantics during the migration. (Stage 6 may +# move to fresh per-file pools for bif's fast path; see doc/ic_nifcore_port.md.) +let icPool* = newPool() +let icTags* = newTagPool() + +proc createIcBuf*(cap = 16): TokenBuf {.inline.} = + ## A `TokenBuf` bound to the shared IC pools. + createTokenBuf(cap, icPool, icTags) + +proc tagId*(s: string): TagId {.inline.} = + ## Intern a tag name in the shared tag pool. + icTags.registerTag(s) + +type + IcBuilder* = object + ## A thin nifcore `TokenBuf` builder whose surface is *primitive types only* + ## (strings/ints/floats). It lets the still-old-API `ast2nif.nim` drive a + ## nifcore buffer without any nifcore type crossing the module boundary — + ## the bridge that routes IC output onto the nifcore serializer (Stage 2). + buf*: TokenBuf + +proc newIcBuilder*(cap = 16): IcBuilder = IcBuilder(buf: createIcBuf(cap)) + +proc openTag*(b: var IcBuilder; tag: string) {.inline.} = b.buf.openTag(tagId(tag)) +proc closeTag*(b: var IcBuilder) {.inline.} = b.buf.closeTag() +proc addSymUse*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymUse(s) +proc addSymDef*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymDef(s) +proc addIdent*(b: var IcBuilder; s: string) {.inline.} = b.buf.addIdent(s) +proc addStrLit*(b: var IcBuilder; s: string) {.inline.} = b.buf.addStrLit(s) +proc addIntLit*(b: var IcBuilder; v: int64) {.inline.} = b.buf.addIntLit(v) +proc addUIntLit*(b: var IcBuilder; v: uint64) {.inline.} = b.buf.addUIntLit(v) +proc addFloatLit*(b: var IcBuilder; v: float64) {.inline.} = b.buf.addFloatLit(v) +proc addCharLit*(b: var IcBuilder; c: char) {.inline.} = b.buf.addCharLit(c) +proc addDotToken*(b: var IcBuilder) {.inline.} = b.buf.addDotToken() + +proc lineInfo*(b: var IcBuilder; file: string; line, col: int32; comment = "") = + ## Attach line info (+ optional `#comment#`) to the head just emitted. No-op + ## when `file` is empty (matches the old "emit only when info is valid"). + ## Strings are interned in the shared pools; the file/comment ids reproduce + ## the old `pool.files`/`pool.strings` entries by string value. + if file.len == 0: return + let fid = icPool.filenames.getOrIncl(file) + let cid = if comment.len > 0: icPool.strings.getOrIncl(comment) else: StrId(0) + b.buf.appendLineInfo(fid, line, col, cid) + +proc writeFileStable*(b: var TokenBuf; path: string; onlyIfChanged = false) = + ## Serialize `b` to canonical module NIF text and write it. Mirrors + ## `nifcursors.writeFile`: the module suffix is derived from `path` + ## (`"." & extractModuleSuffix`), and `onlyIfChanged` skips the write when the + ## on-disk bytes already match — the content-stability nifmake's incremental + ## rebuild depends on. + let content = toModuleString(b, "." & extractModuleSuffix(path)) + if onlyIfChanged: + let existing = + try: readFile(path) + except CatchableError: "" + if existing == content: return + writeFile(path, content) + +proc writeStable*(b: var IcBuilder; path: string; onlyIfChanged = false) {.inline.} = + writeFileStable(b.buf, path, onlyIfChanged) + +proc cursorPool*(c: Cursor): Pool {.inline.} = nifcore.pool(c) + ## The literals pool the cursor's buffer was built against. `ast2nif.nim` + ## imports `nifcore` with `except pool` (to keep nifstreams' global `pool` + ## var the writer uses), so the reader reaches a cursor's pool through here — + ## needed once `bif`-loaded buffers carry their OWN fresh pool rather than the + ## shared `icPool`. + +proc freshModuleCopy(b: var IcBuilder): TokenBuf = + ## Re-home `b.buf` into a PRIVATE, module-local pool via `addSubtree` (which + ## re-interns only the literals/tags this buffer actually uses). `b.buf` is bound + ## to the process-wide shared `icPool`/`icTags`; storing it directly would embed + ## the WHOLE shared pool (correct but huge — see `bif.storeToFile`). The copy's + ## fresh-pool reload reproduces ids verbatim (the bif fresh-pool INVARIANT). + result = createTokenBuf(b.buf.len, newPool(), newTagPool()) + var c = b.buf.beginRead() + while c.hasMore: + addSubtree(result, c) + skip c + +proc storeBif*(b: var IcBuilder; path: string; dottedSuffix: string) = + ## Persist the buffer as a compact, self-contained binary NIF (`.bif`). + var fresh = freshModuleCopy(b) + bif.store(fresh, path, dottedSuffix) + +proc storeBifStable*(b: var IcBuilder; path: string; dottedSuffix: string) = + ## Content-stable `bif` write — the binary analogue of `writeFileStable`'s + ## `onlyIfChanged`: only replace `path` when the encoded bytes differ, so an + ## unchanged sidecar keeps its mtime and nifmake prunes the dependent rebuild + ## cascade. Used for the iface/impl cookies + dep sidecars whose byte-stability + ## gates incremental builds. (bif encoding is deterministic for a given buffer + ## under fresh pools, so equal content ⇒ equal bytes.) + var fresh = freshModuleCopy(b) + let tmp = path & ".tmp" + bif.store(fresh, tmp, dottedSuffix) + let newBytes = readFile(tmp) + let oldBytes = + try: readFile(path) + except CatchableError: "" + if newBytes == oldBytes: + removeFile(tmp) + else: + moveFile(tmp, path) + +# --- subtree splicing (shared pool, so a raw subtree copy is exact) ---------- + +proc addAll*(dest: var IcBuilder; src: var IcBuilder) = + ## Append every top-level subtree of `src` into `dest` — the nifcore analogue + ## of the old `dest.add wholeBuffer` splice. + var c = src.buf.beginRead() + while c.hasMore: + addSubtree(dest.buf, c) + skip c + +proc addStmtsBody*(dest: var IcBuilder; src: var IcBuilder) = + ## Append the BODY of a `(stmts . . )` builder into `dest`, dropping the + ## wrapper tag and its two leading dot slots (flags/type) — the nifcore + ## analogue of the old `for i in 3 ..< content.len-1: dest.add content[i]`. + var c = src.buf.beginRead() # at (stmts + c.into: + skip c # flags dot + skip c # type dot + while c.hasMore: + addSubtree(dest.buf, c) + skip c + +# --- cookie input: a line-info-free logical token list of the module --------- +# The cookie hashers (ast2nif) need a flat, ParRi-bearing, index-addressable +# view of the serialized module. nifcore has no ParRi kind and variable-width +# tokens, so we flatten the buffer here (in the clean nifcore world) into a +# neutral `CookieTok` list — no nifcore type crosses into ast2nif. + +type + CookieKind* = enum + ckParLe, ckParRi, ckSym, ckSymDef, ckIdent, ckStr, ckInt, ckUInt, ckFloat, ckChar, ckDot + CookieTok* = object + kind*: CookieKind + tag*: string # ckParLe + name*: string # ckSym / ckSymDef + sym*: uint32 # ckSym / ckSymDef id (identity key) + str*: string # ckIdent / ckStr + ival*: int64 + uval*: uint64 + fval*: float64 + cval*: uint32 + +proc flattenGo(c: var Cursor; b: TokenBuf; acc: var seq[CookieTok]) = + while c.hasMore: + case c.kind + of TagLit: + acc.add CookieTok(kind: ckParLe, tag: b.tags.tagName(c.cursorTagId)) + c.into: + flattenGo(c, b, acc) + acc.add CookieTok(kind: ckParRi) + of Symbol: + acc.add CookieTok(kind: ckSym, name: symName(c, b.pool), sym: uint32(symId(c, b.pool))) + skip c + of SymbolDef: + acc.add CookieTok(kind: ckSymDef, name: symName(c, b.pool), sym: uint32(symId(c, b.pool))) + skip c + of Ident: + acc.add CookieTok(kind: ckIdent, str: strVal(c, b.pool)); skip c + of StrLit: + acc.add CookieTok(kind: ckStr, str: strVal(c, b.pool)); skip c + of IntLit: + acc.add CookieTok(kind: ckInt, ival: intVal(c)); skip c + of UIntLit: + acc.add CookieTok(kind: ckUInt, uval: uintVal(c)); skip c + of FloatLit: + acc.add CookieTok(kind: ckFloat, fval: floatVal(c)); skip c + of CharLit: + acc.add CookieTok(kind: ckChar, cval: uint32(ord(charLit(c)))); skip c + of DotToken: + acc.add CookieTok(kind: ckDot); skip c + else: + skip c # LineInfoLit / ExtendedSuffix ride on heads, never standalone + +proc flattenForCookie*(b: var IcBuilder): seq[CookieTok] = + ## Flatten the nifcore module buffer to the cookie hashers' flat token list. + result = newSeqOfCap[CookieTok](b.buf.len) + var cur = b.buf.beginRead() + flattenGo(cur, b.buf, result) + +proc collectBifStrLits*(path: string): seq[string] = + ## Read a small `(tag "s" "s" …)` bif sidecar (`semdeps`/`edges`) and return every + ## string literal it holds, in order — the binary analogue of the old nifstreams + ## scan that collected `StrLit`s. Keeps nifcore types out of `deps.nim`, which + ## only needs the recorded string list. + result = @[] + var m = bif.load(path) + var c = m.buf.beginRead() + while c.hasMore: + if c.kind == StrLit: result.add strVal(c) + inc c + +proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) = + ## Stage 1 spike: the nifcore port of `ast2nif.writeSemDeps`. Serializes the + ## module's resolved direct imports as `(semdeps "path" ...)`. Byte-identical + ## to the old writer (verified), so `nim ic` build graphs are unaffected. + let selfSuffix = modname(thisModule, config) + var paths = importPaths + sort paths + var dest = newIcBuilder(4 + 2*paths.len) + dest.openTag "semdeps" + for p in paths: + dest.addStrLit p + dest.closeTag() + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".s.deps.bif").string + storeBifStable(dest, path, "." & extractModuleSuffix(path))