diff --git a/compiler/deps.nim b/compiler/deps.nim index d52d0df269..afc1c4a591 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -1337,6 +1337,22 @@ proc backendBatchSize(conf: ConfigRef; liveCount: int): int = result = (liveCount + jobs - 1) div jobs result = max(1, result) +proc emitBatches(c: DepContext; live: seq[bool]; + shared: seq[seq[int]]): seq[seq[int]] = + ## emit's partition. Unlike `lower`/`cg` it takes the MAIN module too and, by + ## default, puts every live node in one batch: emit owns no decisions, so + ## there is nothing for a grouping to get wrong (see the rule that uses this). + ## An explicit `-d:icBatchSize` reuses the shared partition instead, plus main, + ## so the fan-out remains available to compare against. + if isDefined(c.config, "icBatchSize"): + result = shared + if live.len > 0 and live[0]: result.add @[0] + else: + var all: seq[int] = @[] + for i in 0 ..< c.nodes.len: + if live[i]: all.add i + result = if all.len > 0: @[all] else: @[] + proc backendBatches(c: DepContext; live: seq[bool]): seq[seq[int]] = ## Partition the live non-main nodes into batches of node indices. The main ## module is never in one: it loads the whole program, so batching it with @@ -1584,7 +1600,18 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string b.endTree() # emit: render each module's `.c` from its `.c.nif` + the merge decision. - for batch in batches: + # + # ONE rule for everything, main included. emit is a pure function of a + # `.c.nif` and the merge decision — `renderCFromArtifact` filters text and + # touches no AST, and the stage loads no module graph at all — so batching it + # cannot change what it produces, and measurement agrees: 67 processes and one + # process give byte-identical `.c`, in 0.502 s versus 0.041 s. What that buys + # is not the cold build (where 0.5 s serial is ~0.05 s across cores) but the + # fire-all: every `emit` re-fires whenever `merge` rewrites the decision, which + # is every edit that reaches the backend. That now costs one process start. + # + # `-d:icBatchSize:N` still splits it, for A/B-ing against the fan-out. + for batch in emitBatches(c, live, batches): b.addTree "do" b.addIdent "nim_nifc" b.withTree "args": @@ -1605,19 +1632,6 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string # and produced identical bytes looks exactly like a rule that never ran. outputStr cFiles[idx] & ".stamp" b.endTree() - block: - let i = 0 - if live[i]: - b.addTree "do" - b.addIdent "nim_nifc" - b.withTree "args": - b.addStrLit "--icBackendStage:emit" - b.addStrLit "--icBackendModules:" & c.nodes[i].files[0].modname - inputStr cnifFiles[i] - inputStr mergeFile - outputStr cFiles[i] - outputStr cFiles[i] & ".stamp" - b.endTree() # link: compile + link every emitted `.c` in one process. b.addTree "do" diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 611d3d0b44..55246c9846 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -813,9 +813,16 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) = rawMessage(g.config, errGenerated, "per-module emit: missing or unparsable merge decision " & MergeDecisionFile) return - let members = if batch.isMain: @[mainSuffix] else: batch.members + let members = if batch.members.len == 0: @[mainSuffix] else: batch.members for member in members: - emitOneModule(g, mainFileIdx, member, batch.isMain, decision) + # Per MEMBER, not per batch. `backendBatch.isMain` answers "is this + # invocation the main-module invocation", which is the right question for + # `lower`/`cg` (main loads the whole program, so it is never batched with + # anything). emit has no such constraint and batches freely, so main can sit + # in a batch with others — and then the batch-wide flag sent main's `.c` to + # the path derived from its SUFFIX rather than from its source file, and its + # `.c` was never written. + emitOneModule(g, mainFileIdx, member, member == mainSuffix, decision) proc emitOneModule(g: ModuleGraph; mainFileIdx: FileIndex; member: string; isMain: bool; decision: MergeDecision) =