From adda34bcb84fec12e92be74181256a9413e741b0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 20 Jul 2026 14:01:30 +0300 Subject: [PATCH] add --genBif for semantic BIF output on non-IC builds (#26001) ## Summary Adds `--genBif:on|off`, allowing regular compiler builds to generate per-module semantic BIF artifacts in `nimcache`. This reuses the semantic artifact format produced by incremental compilation without enabling IC or changing the normal code-generation and linking pipeline. In comparison to `nim check --compress ...` this new flag `nim c --genBif:on --compileOnly yourlib.nim` is considerably more useful for tooling. That produced full semantic proc declarations, Nim visibility, signatures, overload disambiguators, and pragmas. For a proc that was actually code-generated, it also recorded the exact backend name, for example. ## Motivation External tools such as language servers, debuggers, and binding generators can benefit from resolved symbol and type information produced during an ordinary build. Previously, these semantic BIF artifacts were tied to the incremental compiler workflow. ## Details With the option enabled: ```sh nim c --genBif:on project.nim ``` the compiler writes semantic `.s.bif` files and their supporting sidecars for each semantically checked module while continuing with the requested backend normally. The option: - Works with non-IC builds. - Does not enable incremental compilation. - Does not change generated program behavior. - Does not enable or introduce native ABI exports. - Does not generate `.abi.nif` manifests. - Is ignored for NimScript compilation. The `genBif` name follows existing artifact-generation options such as `genScript`, `genMapping`, and `genCDeps`. ## Testing Added a focused C backend test that runs a regular build with `--genBif:on` and verifies that semantic `.s.bif` artifacts are generated. A release-mode temporary compiler build and the focused Testament test both pass. --- compiler/ast2nif.nim | 27 +++++++++++++-------------- compiler/commands.nim | 2 ++ compiler/options.nim | 1 + compiler/pipelines.nim | 5 +++-- compiler/semdata.nim | 13 ++++++++----- compiler/semstmts.nim | 3 ++- doc/advopt.txt | 1 + doc/ic.md | 10 ++++++++++ tests/codegen/tgenbif.nim | 14 ++++++++++++++ 9 files changed, 54 insertions(+), 22 deletions(-) create mode 100644 tests/codegen/tgenbif.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index d8015285fc..fe3ab0fb8a 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -222,9 +222,8 @@ type decodedFileIndices: HashSet[FileIndex] locals: HashSet[ItemId] # track proc-local symbols inProc: int - writtenTypes: seq[PType] # types sealed during this emit; under ideActive - writtenSyms: seq[PSym] # they are reset to Complete afterwards so nimsuggest - # can keep mutating its still-live query targets + writtenTypes: seq[PType] # types sealed during a non-owning emit + writtenSyms: seq[PSym] # reset afterwards so their owner can keep using them writtenPackages: HashSet[string] depSuffixes: HashSet[string] # module suffixes already emitted as `(import ...)` deps emittedBackendTypes: HashSet[(int32, int32)] # backend-local types already def'd this @@ -473,6 +472,9 @@ proc writeNode(w: var Writer; dest: var IcBuilder; n: PNode; forAst = false) proc writeType(w: var Writer; dest: var IcBuilder; typ: PType) proc writeSym(w: var Writer; dest: var IcBuilder; sym: PSym) +func restoresWrittenState(config: ConfigRef): bool {.inline.} = + config.ideActive or optGenBif in config.globalOptions + proc writeLoc(w: var Writer; dest: var IcBuilder; loc: TLoc) = dest.addIdent toNifTag(loc.k) dest.addIdent toNifTag(loc.storage) @@ -569,7 +571,7 @@ proc writeType(w: var Writer; dest: var IcBuilder; typ: PType) = # module (or nowhere), leaving dangling references (e.g. `symbol has no # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed - if w.infos.config.ideActive: w.writtenTypes.add typ + if restoresWrittenState(w.infos.config): w.writtenTypes.add typ writeTypeDef(w, dest, typ) else: dest.addSymUse pool.syms.getOrIncl(nifTypeName(w, typ)), NoLineInfo @@ -734,7 +736,7 @@ proc writeSym(w: var Writer; dest: var IcBuilder; sym: PSym) = dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo elif shouldWriteSymDef(w, sym): sym.state = Sealed - if w.infos.config.ideActive: w.writtenSyms.add sym + if restoresWrittenState(w.infos.config): w.writtenSyms.add sym writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, @@ -769,7 +771,7 @@ proc writeSymNode(w: var Writer; dest: var IcBuilder; n: PNode; sym: PSym) = else: shouldWriteSymDef(w, sym) if wantDef: if not sym.itemId.isBackendMinted and not isField: sym.state = Sealed - if w.infos.config.ideActive: w.writtenSyms.add sym + if restoresWrittenState(w.infos.config): w.writtenSyms.add sym if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): writeType(w, dest, nodeTyp) @@ -1788,17 +1790,15 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; let s = op.sym if s.state != Sealed: s.state = Sealed - if config.ideActive: w.writtenSyms.add s + if restoresWrittenState(config): w.writtenSyms.add s writeSymDef w, dest, s dest.addParRi() - # nimsuggest reuses these symbols/types as live, mutable query targets (sem - # re-runs, usage tracking, flag updates). Sealing is only needed for intra-emit - # dedup; once the NIF is built, un-seal so suggest can keep mutating them - # (matches `loadedState` loading Complete under ideActive). The `Sealed` guard - # stays in force for a real `nim m`/`nim nifc` build. - if config.ideActive: + # Nimsuggest and normal code generation reuse these symbols/types as live, + # mutable targets. Sealing is only needed for intra-emit dedup; once the NIF + # is built, un-seal them. The guard stays in force for a real `nim m` build. + if restoresWrittenState(config): for s in w.writtenSyms: if s.state == Sealed: s.state = Complete for t in w.writtenTypes: @@ -3540,4 +3540,3 @@ when isMainModule: echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") echo objb.name, " ", objb.module, " ", objb.count - diff --git a/compiler/commands.nim b/compiler/commands.nim index ccf57142da..123e404141 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -826,6 +826,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; localError(conf, info, "expected nim|cpp but found " & arg) of "compress": conf.globalOptions.incl optCompress + of "genbif": + processOnOffSwitchG(conf, {optGenBif}, arg, pass, info) of "g": # alias for --debugger:native conf.globalOptions.incl optCDebug conf.options.incl optLineDir diff --git a/compiler/options.nim b/compiler/options.nim index 10ff8b9d88..ad3163e842 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -140,6 +140,7 @@ type # please make sure we have under 32 options optDocRaw # for documentation: Don't render markdown for JSON output optItaniumMangle # mangling follows the Itanium spec optCompress # turn on AST compression by converting it to NIF + optGenBif # generate semantic BIF alongside ordinary code generation optWithinConfigSystem # we still compile within the configuration system TGlobalOptions* = set[TGlobalOption] diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 894e7cc49d..29959fb76f 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -167,7 +167,8 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator s = stream graph.interactive = stream.kind == llsStdIn var topLevelStmts = - if optCompress in graph.config.globalOptions or graph.config.cmd == cmdM: + if {optCompress, optGenBif} * graph.config.globalOptions != {} or + graph.config.cmd == cmdM: newNodeI(nkStmtList, module.info) else: nil @@ -255,7 +256,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator graph.config.cmd == cmdM and graph.config.errorCounter == 0 and graph.config.m.fileInfos[module.position].dirtyFile.isEmpty else: - (optCompress in graph.config.globalOptions) or + ({optCompress, optGenBif} * graph.config.globalOptions != {}) or (graph.config.cmd == cmdM and (sfMainModule in module.flags or (graph.config.icGroup.len > 0 and diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 21421fbdb8..5117c3db0c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -382,8 +382,9 @@ proc addImportFileDep*(c: PContext; f: FileIndex) = if f notin deps[]: deps[].add f proc addPragmaComputation*(c: PContext; n: PNode) = - # Also store for NIF-based IC (cmdM mode or optCompress) - if optCompress in c.config.globalOptions or c.config.cmd == cmdM: + # Also store whenever the semchecked module is serialized to NIF/BIF. + if {optCompress, optGenBif} * c.config.globalOptions != {} or + c.config.cmd == cmdM: addNifReplayAction(c.graph, c.module.position.int32, n) proc inclSym(sq: var seq[PSym], s: PSym): bool = @@ -670,10 +671,12 @@ proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) = ## delegated to the "NIF" file mechanism. ## ## We only bother when a NIF file is actually going to be written (IC / `nim m`, - ## `--compress`, or a running suggestion engine); a plain `nim c` throws the - ## record away, so recording it would be pure overhead. + ## `--compress`, semantic BIF output, or a running suggestion engine); a plain + ## `nim c` throws the record away, so recording it would be pure overhead. if info.fileIndex == InvalidFileIdx: return - if c.config.cmd == cmdM or optCompress in c.config.globalOptions or c.config.ideActive: + if c.config.cmd == cmdM or + {optCompress, optGenBif} * c.config.globalOptions != {} or + c.config.ideActive: c.graph.nifExpansions.mgetOrPut(c.module.position.int32, @[]).add (expandedSym, info) const diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ab3a30a29e..4b9dd3e7a4 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2892,7 +2892,8 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult, resolvedIncStmt proc evalInclude(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) var resolvedIncStmt: PNode = nil - if optCompress in c.config.globalOptions or c.config.cmd == cmdM: + if {optCompress, optGenBif} * c.config.globalOptions != {} or + c.config.cmd == cmdM: # New resolve the include filenames to string literals that contain absolute paths, # nicer for IC: resolvedIncStmt = newNodeI(nkIncludeStmt, n.info) diff --git a/doc/advopt.txt b/doc/advopt.txt index 06eb0f78f5..b04bcbb436 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -122,6 +122,7 @@ Advanced options: --lineDir:on|off generation of #line directive on|off --embedsrc:on|off embeds the original source code as comments in the generated output + --genBif:on|off generate per-module semantic BIF metadata in nimcache --tlsEmulation:on|off turn thread local storage emulation on|off --implicitStatic:on|off turn implicit compile time evaluation on|off --trmacros:on|off turn term rewriting macros on|off diff --git a/doc/ic.md b/doc/ic.md index a2e6cd0592..e2846f3235 100644 --- a/doc/ic.md +++ b/doc/ic.md @@ -39,6 +39,16 @@ debugging a build). Artifacts (the NIF zoo) ======================= +Semantic BIF from regular builds +-------------------------------- + +``--genBif:on`` makes a regular compiler invocation write each semantically +checked module as ``.s.bif`` under the build's nimcache directory. This +reuses the semantic artifact format used by IC without enabling incremental +compilation or changing how the program is generated and linked. Tools such as +language servers, debuggers, and binding generators can request these artifacts +when they need resolved symbols and types from an ordinary build. + Per module ```` (a content hash of the path; see *NIF symbols* below), under the nimcache directory: diff --git a/tests/codegen/tgenbif.nim b/tests/codegen/tgenbif.nim new file mode 100644 index 0000000000..f23c304dbf --- /dev/null +++ b/tests/codegen/tgenbif.nim @@ -0,0 +1,14 @@ +discard """ + output: "ok" + targets: "c" + matrix: "--genBif:on" +""" + +import std/[compilesettings, os] + +let cache = querySetting(nimcacheDir) +var hasSemanticBif = false +for path in walkFiles(cache / "*.s.bif"): + hasSemanticBif = true +doAssert hasSemanticBif +echo "ok"