IC: render every module's .c in one emit process

`emit` is the one backend stage with nothing to get wrong. It loads no
module graph — it derives its output path rather than resolving it — and
`renderCFromArtifact` filters text against the merge decision without
touching an AST. It is a pure function of one `.c.nif` and one global
artifact, so grouping the calls cannot change what comes out.

Measured rather than assumed: 67 separate emit processes and one process
with all 67 members produce byte-identical `.c`, in 0.502 s against
0.041 s. So emit takes a single nifmake rule covering every live module.

Where that shows up is narrower than the 12x suggests. Cold serial 8.21 s
-> 7.82 s; cold parallel 2.81 s -> 2.83 s, i.e. nothing, because 0.46 s
spread over 16 cores is already invisible. An edit to `system.nim` is
2.15 s either way: emit was never what made that slow. The honest case is
CPU rather than wall clock, plus one fewer thing to pay for on a machine
that is not idle — the fire-all is structural (every `emit` re-fires
whenever `merge` rewrites the decision, by design) and now costs one
process start instead of 67.

Also fixes a bug this exposed: `isMain` was a per-BATCH flag. That is the
right question for `lower`/`cg`, where main loads the whole program and
is never batched with anything, but emit batches freely — and main in a
mixed batch had its `.c` path derived from its module SUFFIX instead of
its source file, so it was never written at all.

`-d:icBatchSize:N` still splits emit the old way, for comparing against
the fan-out. ic 40/40, `koch boot -d:release` equal executables, and the
67 generated `.c` are byte-identical to the previous commit's.
This commit is contained in:
araq
2026-08-31 13:59:46 +02:00
parent 5d77a71043
commit 8299401888
2 changed files with 37 additions and 16 deletions

View File

@@ -1337,6 +1337,22 @@ proc backendBatchSize(conf: ConfigRef; liveCount: int): int =
result = (liveCount + jobs - 1) div jobs
result = max(1, result)
proc emitBatches(c: DepContext; live: seq[bool];
shared: seq[seq[int]]): seq[seq[int]] =
## emit's partition. Unlike `lower`/`cg` it takes the MAIN module too and, by
## default, puts every live node in one batch: emit owns no decisions, so
## there is nothing for a grouping to get wrong (see the rule that uses this).
## An explicit `-d:icBatchSize` reuses the shared partition instead, plus main,
## so the fan-out remains available to compare against.
if isDefined(c.config, "icBatchSize"):
result = shared
if live.len > 0 and live[0]: result.add @[0]
else:
var all: seq[int] = @[]
for i in 0 ..< c.nodes.len:
if live[i]: all.add i
result = if all.len > 0: @[all] else: @[]
proc backendBatches(c: DepContext; live: seq[bool]): seq[seq[int]] =
## Partition the live non-main nodes into batches of node indices. The main
## module is never in one: it loads the whole program, so batching it with
@@ -1584,7 +1600,18 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.endTree()
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
for batch in batches:
#
# ONE rule for everything, main included. emit is a pure function of a
# `.c.nif` and the merge decision — `renderCFromArtifact` filters text and
# touches no AST, and the stage loads no module graph at all — so batching it
# cannot change what it produces, and measurement agrees: 67 processes and one
# process give byte-identical `.c`, in 0.502 s versus 0.041 s. What that buys
# is not the cold build (where 0.5 s serial is ~0.05 s across cores) but the
# fire-all: every `emit` re-fires whenever `merge` rewrites the decision, which
# is every edit that reaches the backend. That now costs one process start.
#
# `-d:icBatchSize:N` still splits it, for A/B-ing against the fan-out.
for batch in emitBatches(c, live, batches):
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
@@ -1605,19 +1632,6 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
# and produced identical bytes looks exactly like a rule that never ran.
outputStr cFiles[idx] & ".stamp"
b.endTree()
block:
let i = 0
if live[i]:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:emit"
b.addStrLit "--icBackendModules:" & c.nodes[i].files[0].modname
inputStr cnifFiles[i]
inputStr mergeFile
outputStr cFiles[i]
outputStr cFiles[i] & ".stamp"
b.endTree()
# link: compile + link every emitted `.c` in one process.
b.addTree "do"

View File

@@ -813,9 +813,16 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
return
let members = if batch.isMain: @[mainSuffix] else: batch.members
let members = if batch.members.len == 0: @[mainSuffix] else: batch.members
for member in members:
emitOneModule(g, mainFileIdx, member, batch.isMain, decision)
# Per MEMBER, not per batch. `backendBatch.isMain` answers "is this
# invocation the main-module invocation", which is the right question for
# `lower`/`cg` (main loads the whole program, so it is never batched with
# anything). emit has no such constraint and batches freely, so main can sit
# in a batch with others — and then the batch-wide flag sent main's `.c` to
# the path derived from its SUFFIX rather than from its source file, and its
# `.c` was never written.
emitOneModule(g, mainFileIdx, member, member == mainSuffix, decision)
proc emitOneModule(g: ModuleGraph; mainFileIdx: FileIndex; member: string;
isMain: bool; decision: MergeDecision) =