From 4434e2d6bd70846aafdb777465f9119be4aa3eb7 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 13 Jun 2026 21:47:55 +0200 Subject: [PATCH] IC: per-module backend merge stage (Phase 2b, B2) --icBackendStage:merge reads every module's .c.nif (the cg stages' emit-everywhere output), computes the global live set and, for each externally-linked definition that several cg processes emitted, the single artifact allowed to embed its body; it writes ic.backend.merge.nif for the emit stage to consume. This is the cross-process replacement for the whole-program backend's in-process icSharedDefOwner/DCE coordination. Mechanism: - cgen marks every unique program-wide definition (callConv != ccInline and not a dispatcher) with a new 'u' cdef flag. Its complement -- inline procs (static per-TU) and method dispatchers (main-only) -- is emitted into every using TU and must never be deduplicated, so it carries no flag. The flag is inert for the whole-program path (renderMarkedC/computeLiveFromCArtifacts ignore it). - cnif.computeMergeDecision does one mark&sweep pass over all artifacts (same liveness as computeLiveFromCArtifacts) plus owner assignment: the owner of a 'u' definition is the lexicographically smallest artifact that emits it -- a pure function of the claimant set, stable across rebuilds. writeMergeDecision/readMergeDecision serialize the result as (merge (live ...) (owners (own Symbol StrLit)*)). - generateMergeStage is a pure artifact operation (no module graph loaded): glob the nimcache's .c.nif, compute, write the decision. Validated on a diamond (lib.shared called from sibling modules a and b, both with top-level demands): cg emits shared into a, b and lib; merge assigns owner = lib (smallest claimant) so a/b will prototype it, while the inline nimFrame stays out of the owners map (kept everywhere). Whole-program backend path unchanged (dispatch guarded on icBackendStage); koch ic thallo green. Co-Authored-By: Claude Opus 4.8 --- compiler/cgen.nim | 11 +++ compiler/cnif.nim | 187 ++++++++++++++++++++++++++++++++++++++++ compiler/nifbackend.nim | 27 ++++++ 3 files changed, 225 insertions(+) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 9579515c04..94edbd9375 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1560,6 +1560,17 @@ proc genProcLvl3*(m: BModule, prc: PSym) = if sfExportc in prc.flags or sfConstructor in prc.flags: defFlags.add 'x' if sfCompilerProc in prc.flags: defFlags.add 'c' if prc.kind == skMethod or sfDispatcher in prc.flags: defFlags.add 'm' + if (prc.typ == nil or prc.typ.callConv != ccInline) and + sfDispatcher notin prc.flags: + # A unique program-wide definition: external linkage, so exactly one + # translation unit may embed its body and everyone else declares it. + # In the whole-program backend `icSharedDefOwner` (first claimant wins) + # enforces this in process; in the per-module backend each module's `cg` + # process emits the body (emit-everywhere), so this flag tells the merge + # stage which definitions to assign a single owner and prototype in the + # rest. The complement — inline procs and method dispatchers — is emitted + # into every using TU (`static`/main-only) and must never be deduplicated. + defFlags.add 'u' if not hasCnifMarks(prc.loc.snippet): # The C name was not minted through `fillBackendName` (e.g. set by an # `extern`/`rtl` pragma at sem time), so its uses are invisible to the diff --git a/compiler/cnif.nim b/compiler/cnif.nim index 7868cd86c4..c0318c2067 100644 --- a/compiler/cnif.nim +++ b/compiler/cnif.nim @@ -459,3 +459,190 @@ proc computeLiveFromCArtifacts*(files: openArray[string]): CnifLiveness = result.defs = defs.len for d in defs: if d in result.live: inc result.liveDefs + +# ---- The merge stage: liveness + owner assignment ------------------------- + +type + MergeDecision* = object + ## What the per-module backend's `merge` stage computes from every + ## module's `.c.nif` and what its `emit` stage consumes to render the + ## final `.c` of one module. + live*: HashSet[string] ## globally reachable C names (dead cdefs + ## are dropped from every module) + owners*: Table[string, string] ## for each `'u'`-flagged (unique, + ## externally-linked) definition, the single + ## artifact base name allowed to embed its + ## body; every other module prototypes it + broken*: bool ## an artifact was missing or unparsable — + ## the caller should fall back / regenerate + defs*, liveDefs*: int + +proc computeMergeDecision*(files: openArray[string]): MergeDecision = + ## One pass over every `.c.nif`: the same mark&sweep as + ## `computeLiveFromCArtifacts` plus, per definition, owner assignment. + ## + ## Each `cg` process emits the body of every definition it demands + ## (emit-everywhere), so the same externally-linked definition appears in + ## several artifacts. A `'u'` flag on the `(cdef ...)` marks those that need + ## exactly one owner (the whole-program backend's `icSharedDefOwner` + ## invariant, here recomputed across processes); the owner is the + ## lexicographically smallest artifact that emits it — a pure function of the + ## claimant set, hence stable across rebuilds. Definitions without `'u'` + ## (inline procs, dispatchers) are `static`/main-only and emitted into every + ## using TU, so they get no owner entry and are never deduplicated. + result = MergeDecision(live: initHashSet[string](), + owners: initTable[string, string]()) + var pool = newPool() + var tags = newTagPool() + let stmtsTag = tags.registerTag("stmts") + let cdefTag = tags.registerTag("cdef") + let cdataTag = tags.registerTag("cdata") + let crefTag = tags.registerTag("cref") + let cdepsTag = tags.registerTag("cdeps") + let metaTag = tags.registerTag("meta") + var uses = initTable[string, HashSet[string]]() + var roots = initHashSet[string]() + var defs = initHashSet[string]() + for f in files: + if not fileExists(f): + result.broken = true + return + let owner = extractFilename(f) + var buf = parseFromFile(f, 1000, pool, tags) + var c = beginRead(buf) + if c.kind != TagLit or c.cursorTagId != stmtsTag: + result.broken = true + endRead(c) + return + c.loopInto: + case c.kind + of Symbol, Ident: + roots.incl symOrIdentName(c) + inc c + of TagLit: + if c.cursorTagId == metaTag or c.cursorTagId == cdataTag or + c.cursorTagId == crefTag or c.cursorTagId == cdepsTag: + skip c + elif c.cursorTagId == cdefTag: + var ownerName = "" + var flagsSeen = false + var isUnique = false + c.loopInto: + case c.kind + of SymbolDef: + ownerName = symName(c) + defs.incl ownerName + flagsSeen = false + inc c + of Symbol, Ident: + let name = symOrIdentName(c) + if not flagsSeen: + flagsSeen = true + for ch in name: + if ch in {'x', 'c', 'm'}: roots.incl ownerName + elif ch == 'u': isUnique = true + else: + uses.mgetOrPut(ownerName, initHashSet[string]()).incl name + inc c + of DotToken: + flagsSeen = true # empty flags field + inc c + else: + skip c + if isUnique and ownerName.len > 0: + # smallest claimant wins; ties impossible (one entry per name) + let prev = result.owners.getOrDefault(ownerName, "") + if prev.len == 0 or owner < prev: + result.owners[ownerName] = owner + else: + c.loopInto: + if c.kind in {Symbol, Ident}: + roots.incl symOrIdentName(c) + inc c + else: + skip c + else: + skip c + endRead(c) + var work = newSeqOfCap[string](roots.len) + for r in roots: work.add r + while work.len > 0: + let s = work.pop() + if not result.live.containsOrIncl(s): + if uses.hasKey(s): + for dep in uses[s]: + if dep notin result.live: + work.add dep + result.defs = defs.len + for d in defs: + if d in result.live: inc result.liveDefs + +const MergeDecisionFile* = "ic.backend.merge.nif" + ## Fixed name of the merge stage's output in the nimcache, read by `emit`. + +proc writeMergeDecision*(outfile: string; d: MergeDecision) = + ## Serializes the merge decision: `(merge (live Symbol*) (owners (own + ## Symbol StrLit)*))`. C names are mangled (no dots) so they serialize as + ## symbols; owner artifact base names go in string literals. + var live: seq[string] = @[] + for n in d.live: live.add n + sort live + var keys: seq[string] = @[] + for k in d.owners.keys: keys.add k + sort keys + var b = nifbuilder.open(outfile) + b.withTree "merge": + b.withTree "live": + for n in live: b.addSymbol n, "" + b.withTree "owners": + for k in keys: + b.withTree "own": + b.addSymbol k, "" + b.addStrLit d.owners[k] + b.close() + +proc readMergeDecision*(f: string): MergeDecision = + ## Reads back a `writeMergeDecision` file; `broken=true` if absent/unparsable. + result = MergeDecision(live: initHashSet[string](), + owners: initTable[string, string]()) + if not fileExists(f): + result.broken = true + return + var pool = newPool() + var tags = newTagPool() + let mergeTag = tags.registerTag("merge") + let liveTag = tags.registerTag("live") + let ownersTag = tags.registerTag("owners") + let ownTag = tags.registerTag("own") + var buf = parseFromFile(f, 1000, pool, tags) + var c = beginRead(buf) + if c.kind != TagLit or c.cursorTagId != mergeTag: + result.broken = true + endRead(c) + return + c.loopInto: + if c.kind == TagLit and c.cursorTagId == liveTag: + c.loopInto: + if c.kind in {Symbol, Ident}: + result.live.incl symOrIdentName(c) + inc c + else: + skip c + elif c.kind == TagLit and c.cursorTagId == ownersTag: + c.loopInto: + if c.kind == TagLit and c.cursorTagId == ownTag: + var key = "" + c.loopInto: + if c.kind in {Symbol, Ident}: + key = symOrIdentName(c) + inc c + elif c.kind == StrLit: + if key.len > 0: result.owners[key] = strVal(c) + inc c + else: + skip c + else: + skip c + else: + skip c + endRead(c) diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index aedd930f5f..ab59a2e6d4 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -569,12 +569,39 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) = # so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here. cgenWriteModules(g.backend, g.config) +proc generateMergeStage(g: ModuleGraph) = + ## Per-module backend merge (`--icBackendStage:merge`): a pure artifact + ## operation, no module graph loaded. Reads every `.c.nif` the `cg` stages + ## wrote, computes the global live set and — for each `'u'`-flagged unique + ## definition that several `cg` processes emitted (emit-everywhere) — the one + ## artifact allowed to embed its body, and writes the decision the `emit` + ## stages consume. This replaces the whole-program backend's in-process + ## `icSharedDefOwner`/DCE coordination with a cross-process rule. + let nimcache = getNimcacheDir(g.config).string + var files: seq[string] = @[] + for artifact in walkFiles(nimcache / "*.c.nif"): + files.add artifact + sort files + let decision = computeMergeDecision(files) + if decision.broken: + rawMessage(g.config, errGenerated, + "per-module backend merge: a .c.nif artifact is missing or unparsable") + return + writeMergeDecision(nimcache / MergeDecisionFile, decision) + if isDefined(g.config, "icDceCheck"): + stderr.writeLine "[icMerge] artifacts: " & $files.len & + " live: " & $decision.live.len & " defs: " & $decision.defs & + " liveDefs: " & $decision.liveDefs & " owned: " & $decision.owners.len + proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. ## Traverses the module dependency graph and generates C code. if g.config.icBackendStage == "cg": generateCgStage(g, mainFileIdx) return + elif g.config.icBackendStage == "merge": + generateMergeStage(g) + return elif g.config.icBackendStage.len > 0: rawMessage(g.config, errGenerated, "per-module backend stage not implemented yet: " & g.config.icBackendStage)