mirror of
https://github.com/nim-lang/Nim.git
synced 2026-09-08 14:37:29 +00:00
IC: render every module's .c in one emit process
`emit` is the one backend stage with nothing to get wrong. It loads no module graph — it derives its output path rather than resolving it — and `renderCFromArtifact` filters text against the merge decision without touching an AST. It is a pure function of one `.c.nif` and one global artifact, so grouping the calls cannot change what comes out. Measured rather than assumed: 67 separate emit processes and one process with all 67 members produce byte-identical `.c`, in 0.502 s against 0.041 s. So emit takes a single nifmake rule covering every live module. Where that shows up is narrower than the 12x suggests. Cold serial 8.21 s -> 7.82 s; cold parallel 2.81 s -> 2.83 s, i.e. nothing, because 0.46 s spread over 16 cores is already invisible. An edit to `system.nim` is 2.15 s either way: emit was never what made that slow. The honest case is CPU rather than wall clock, plus one fewer thing to pay for on a machine that is not idle — the fire-all is structural (every `emit` re-fires whenever `merge` rewrites the decision, by design) and now costs one process start instead of 67. Also fixes a bug this exposed: `isMain` was a per-BATCH flag. That is the right question for `lower`/`cg`, where main loads the whole program and is never batched with anything, but emit batches freely — and main in a mixed batch had its `.c` path derived from its module SUFFIX instead of its source file, so it was never written at all. `-d:icBatchSize:N` still splits emit the old way, for comparing against the fan-out. ic 40/40, `koch boot -d:release` equal executables, and the 67 generated `.c` are byte-identical to the previous commit's.
This commit is contained in:
@@ -1337,6 +1337,22 @@ proc backendBatchSize(conf: ConfigRef; liveCount: int): int =
|
||||
result = (liveCount + jobs - 1) div jobs
|
||||
result = max(1, result)
|
||||
|
||||
proc emitBatches(c: DepContext; live: seq[bool];
|
||||
shared: seq[seq[int]]): seq[seq[int]] =
|
||||
## emit's partition. Unlike `lower`/`cg` it takes the MAIN module too and, by
|
||||
## default, puts every live node in one batch: emit owns no decisions, so
|
||||
## there is nothing for a grouping to get wrong (see the rule that uses this).
|
||||
## An explicit `-d:icBatchSize` reuses the shared partition instead, plus main,
|
||||
## so the fan-out remains available to compare against.
|
||||
if isDefined(c.config, "icBatchSize"):
|
||||
result = shared
|
||||
if live.len > 0 and live[0]: result.add @[0]
|
||||
else:
|
||||
var all: seq[int] = @[]
|
||||
for i in 0 ..< c.nodes.len:
|
||||
if live[i]: all.add i
|
||||
result = if all.len > 0: @[all] else: @[]
|
||||
|
||||
proc backendBatches(c: DepContext; live: seq[bool]): seq[seq[int]] =
|
||||
## Partition the live non-main nodes into batches of node indices. The main
|
||||
## module is never in one: it loads the whole program, so batching it with
|
||||
@@ -1584,7 +1600,18 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
|
||||
b.endTree()
|
||||
|
||||
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
|
||||
for batch in batches:
|
||||
#
|
||||
# ONE rule for everything, main included. emit is a pure function of a
|
||||
# `.c.nif` and the merge decision — `renderCFromArtifact` filters text and
|
||||
# touches no AST, and the stage loads no module graph at all — so batching it
|
||||
# cannot change what it produces, and measurement agrees: 67 processes and one
|
||||
# process give byte-identical `.c`, in 0.502 s versus 0.041 s. What that buys
|
||||
# is not the cold build (where 0.5 s serial is ~0.05 s across cores) but the
|
||||
# fire-all: every `emit` re-fires whenever `merge` rewrites the decision, which
|
||||
# is every edit that reaches the backend. That now costs one process start.
|
||||
#
|
||||
# `-d:icBatchSize:N` still splits it, for A/B-ing against the fan-out.
|
||||
for batch in emitBatches(c, live, batches):
|
||||
b.addTree "do"
|
||||
b.addIdent "nim_nifc"
|
||||
b.withTree "args":
|
||||
@@ -1605,19 +1632,6 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
|
||||
# and produced identical bytes looks exactly like a rule that never ran.
|
||||
outputStr cFiles[idx] & ".stamp"
|
||||
b.endTree()
|
||||
block:
|
||||
let i = 0
|
||||
if live[i]:
|
||||
b.addTree "do"
|
||||
b.addIdent "nim_nifc"
|
||||
b.withTree "args":
|
||||
b.addStrLit "--icBackendStage:emit"
|
||||
b.addStrLit "--icBackendModules:" & c.nodes[i].files[0].modname
|
||||
inputStr cnifFiles[i]
|
||||
inputStr mergeFile
|
||||
outputStr cFiles[i]
|
||||
outputStr cFiles[i] & ".stamp"
|
||||
b.endTree()
|
||||
|
||||
# link: compile + link every emitted `.c` in one process.
|
||||
b.addTree "do"
|
||||
|
||||
@@ -813,9 +813,16 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
|
||||
rawMessage(g.config, errGenerated,
|
||||
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
|
||||
return
|
||||
let members = if batch.isMain: @[mainSuffix] else: batch.members
|
||||
let members = if batch.members.len == 0: @[mainSuffix] else: batch.members
|
||||
for member in members:
|
||||
emitOneModule(g, mainFileIdx, member, batch.isMain, decision)
|
||||
# Per MEMBER, not per batch. `backendBatch.isMain` answers "is this
|
||||
# invocation the main-module invocation", which is the right question for
|
||||
# `lower`/`cg` (main loads the whole program, so it is never batched with
|
||||
# anything). emit has no such constraint and batches freely, so main can sit
|
||||
# in a batch with others — and then the batch-wide flag sent main's `.c` to
|
||||
# the path derived from its SUFFIX rather than from its source file, and its
|
||||
# `.c` was never written.
|
||||
emitOneModule(g, mainFileIdx, member, member == mainSuffix, decision)
|
||||
|
||||
proc emitOneModule(g: ModuleGraph; mainFileIdx: FileIndex; member: string;
|
||||
isMain: bool; decision: MergeDecision) =
|
||||
|
||||
Reference in New Issue
Block a user