mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-09-02 03:43:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

ic fixes3 (#26157)

Browse Source
This commit is contained in:
Andreas Rumpf
2026-09-01 16:47:03 +02:00
committed by GitHub
parent 859b0ba270
commit 8f72860d7d
36 changed files with 2942 additions and 1143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
compiler/ast.nim
View File

@@ -359,6 +359,18 @@ proc `flags=`*(t: PType, val: TTypeFlags) {.inline.} =
t.flagsImpl = val
proc sons*(t: PType): var TTypeSeq {.inline.} =
## The RAW child seq. Despite the name this is NOT the counterpart of the
## `sons` ITERATOR over a `PNode`, and it is not the way to walk a type's
## children — use `kids` / `ikids` / `paramTypes` / `signature`, or the named
## accessors (`returnType`, `baseClass`, `elementType`, `indexType`,
## `genericHead`, ...), which say WHICH child they mean.
##
## The difference is not cosmetic. A `tyProc` keeps its parameter types in
## `n`, not here — `setSons` asserts `sonsImpl.len <= 1` for one — so `[]`,
## `len` and every iterator built on them route parameters through
## `n[i].sym.typ`, while this seq holds only the return type. `for x in
## t.sons` therefore compiles, looks like the `PNode` idiom, and silently
## visits a different set of types.
if t.state == Partial: loadType(t)
result = t.sonsImpl
@@ -765,10 +777,28 @@ when false:
echo k
echo v
when defined(icSymCount):
import std / [syncio, exitprocs, tables as symCountTables]
var symMints*: symCountTables.CountTable[string]
var symMintTotal*: int
var symCountHooked = false
proc newSym*(symKind: TSymKind, name: PIdent, idgen: IdGenerator; owner: PSym,
info: TLineInfo; options: TOptions = {}): PSym =
# generates a symbol and initializes the hash field too
assert not name.isNil
when defined(icSymCount):
# Counting symbol MINTS, not their names in the output: a gensym's number is
# its item id, so one extra symbol anywhere shifts every later name. A count
# is therefore far more sensitive than diffing generated C, and it localises
# the extra mint by kind instead of by whatever file happened to show it.
inc symMintTotal
symMints.inc $symKind
if not symCountHooked:
symCountHooked = true
addExitProc proc () =
stderr.writeLine "SYMMINT total=" & $symMintTotal
for k, v in symMints: stderr.writeLine "SYMMINT " & k & "=" & $v
let id = nextSymId idgen
result = PSym(name: name, kindImpl: symKind, flagsImpl: {}, infoImpl: info, itemId: id,
optionsImpl: options, ownerFieldImpl: owner, offsetImpl: defaultOffset,
@@ -1634,7 +1664,7 @@ proc isImportedException*(t: PType; conf: ConfigRef): bool =
result = base.sym != nil and {sfCompileToCpp, sfImportc} * base.sym.flags != {}
proc isInfixAs*(n: PNode): bool =
return n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.id == ord(wAs)
return n.kind == nkInfix and n.firstSon.kind == nkIdent and n.firstSon.ident.id == ord(wAs)
proc skipColon*(n: PNode): PNode =
result = n
@@ -1714,32 +1744,83 @@ proc addParam*(procType: PType; param: PSym) =
const magicsThatCanRaise = {
mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mEcho}
# `canRaise` reaches the effect list through `effectsOf` / `raisesNothing`
# rather than by subscripting `fn.typ.n`, so the layout is written down in one
# place. Under `--ic:on` that list came back from a `.bif`, and whether it came
# back intact is checked separately: `-d:icCanRaiseLog` logs every verdict, and
# the same program built with and without `--ic:on` must produce the same ones.
when defined(icCanRaiseLog):
var canRaiseBranch* = 0
## Which branch decided the last answer: 1 = the symbol's magic/flags,
## 2 = `mEcho`, 3 = the EFFECT LIST reached through `effectsOf`, 4 = the
## conservative predicate, 5 = short-circuited in `canRaiseDisp` before
## either predicate ran, 0 = fell through. Only branch 3 reads anything
## that had to survive a `.bif` round trip, so a differential in which no
## callee reaches it would prove nothing about the writer — which is the
## whole point of running the differential. See `-d:icCanRaiseLog`.
template markCanRaiseBranch*(n: int) =
when defined(icCanRaiseLog): canRaiseBranch = n
proc canRaiseConservative*(fn: PNode): bool =
if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
result = false
else:
result = true
markCanRaiseBranch 4
result = not (fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise)
proc effectsOf*(t: PType): PNode {.inline.} =
## The `nkEffectList` a proc type carries as child 0 of its formal-params
## node, with the parameters following from index 1 (`newProcType` builds it
## that way; `cgen` reads the params back with `sonsFrom(prc.typ.n, 1)`).
##
## Named rather than subscripted so that the layout is written down in ONE
## place. `.n` here is a TYPE's node, never a routine body, so it is always
## fully materialised and `firstSon` is safe — the `nfLazyBody` hazard that
## makes raw child access dangerous elsewhere (see `astdef.sons`) cannot reach
## it. A proc type always has this child; `t.n` with no children is not a
## shape the writer or sem produces, and this deliberately does not paper over
## one appearing.
result = if t.n == nil: nil else: t.n.firstSon
proc raisesNothing*(effects: PNode): bool =
## Whether an effect list says DEFINITIVELY that nothing is raised: it is long
## enough to have a raises slot at all, the slot is present, and it is empty.
##
## Every other shape — a list too short to carry the slot, an absent slot, a
## non-empty one — means the effects are unspecified or non-empty, and a
## caller must assume a raise. Stating it as the NEGATIVE is the point: the
## safe default has to be "can raise", so the one narrow case that licenses
## dropping an exception check is the one spelled out here, and a shape nobody
## anticipated falls on the conservative side by construction rather than by
## luck.
result = effects != nil and effects.len >= effectListLen and
effects[exceptionEffects] != nil and
effects[exceptionEffects].safeLen == 0
proc canRaise*(fn: PNode): bool =
if fn.kind == nkSym and (fn.sym.magic notin magicsThatCanRaise or
{sfImportc, sfInfixCall} * fn.sym.flags == {sfImportc} or
sfGeneratedOp in fn.sym.flags):
markCanRaiseBranch 1
result = false
elif fn.kind == nkSym and fn.sym.magic == mEcho:
markCanRaiseBranch 2
result = true
elif fn.typ != nil and fn.typ.kind == tyProc and fn.typ.n != nil:
# TODO check for n having sons? or just return false for now if not
if fn.typ.n[0].kind == nkSym:
markCanRaiseBranch 3
let effects = effectsOf(fn.typ)
if effects.kind == nkSym:
# The historical shape: slot 0 used to be an `nkType` before the effects
# moved in (see `newProcType`). Nothing to read, so nothing licenses a
# raise.
result = false
else:
# A proc-typed value with no explicit raises slot still has
# unspecified effects, which sempass2 treats conservatively.
# Codegen needs to do the same in order to keep goto-exception
# checks after indirect/closure calls.
result = ((fn.typ.n[0].len < effectListLen) or
fn.typ.n[0][exceptionEffects] == nil or
fn.typ.n[0][exceptionEffects].safeLen > 0)
result = not raisesNothing(effects)
else:
markCanRaiseBranch 0
result = false
proc toHumanStrImpl[T](kind: T, num: static int): string =
@@ -1756,7 +1837,7 @@ proc toHumanStr*(kind: TTypeKind): string =
result = toHumanStrImpl(kind, 2)
proc skipHiddenAddr*(n: PNode): PNode {.inline.} =
(if n.kind == nkHiddenAddr: n[0] else: n)
(if n.kind == nkHiddenAddr: n.firstSon else: n)
proc isNewStyleConcept*(n: PNode): bool {.inline.} =
assert n.kind == nkTypeClassTy

491
compiler/ast2nif.nim
View File

@@ -20,7 +20,8 @@ import "../dist/checksums/src/checksums" / sha1
import astdef, idents, msgs, options
import lineinfos as astli
import pathutils #, modulegraphs
import "../dist/nimony/src/lib" / [bitabs, nifstreams, lineinfos,
import nifstreams
import "../dist/nimony/src/lib" / [bitabs, lineinfos,
nifindexes, nifreader]
# Step 2b: the READER speaks nifcore; the WRITER keeps nifstreams (global `pool`,
# PackedToken/PackedLineInfo). nifstreams does NOT export Cursor/TokenBuf/NifKind,
@@ -28,12 +29,13 @@ import "../dist/nimony/src/lib" / [bitabs, nifstreams, lineinfos,
# `pool(c: Cursor)` accessor would shadow nifstreams' global `pool` var the writer
# uses; the reader reaches pools via `symName(c)`/`strVal(c)` etc.
import "../dist/nimony/src/lib/nifcore" except pool
from "../dist/nimony/src/lib" / bif import load, BifModule
from "../dist/nimony/src/lib" / bif import load, BifModule, IndexVis, ivHidden
import icmodnames
import "../dist/nimony/src/models" / nifindex_tags
import typekeys
import icnifcore
import ic / [enum2nif]
import icprof
const SysModuleSuffix* = "@sys"
const BackendLocalMarker* = "@bk"
@@ -130,6 +132,16 @@ type
revTab: Table[FileId, FileIndex] # reverse mapping for oldLineInfo
man: LineInfoManager
config: ConfigRef
# The READ direction's cache, which `revTab` cannot serve: `revTab` is keyed
# by a `FileId` in the WRITER's global `pool.files`, while a decoded token's
# `FileId` indexes the buffer's OWN filename pool. So the cache has to be
# keyed by (pool, FileId), and it is a `seq` because `FileId`s are small and
# dense within one pool. `readPool` holds a REFERENCE rather than a raw
# pointer on purpose: it keeps the pool alive, so a freed pool cannot be
# replaced by a new one at the same address and silently answer from the
# wrong file table.
readPool: Pool
readTab: seq[FileIndex]
proc newLineInfoWriter(config: ConfigRef): LineInfoWriter =
# `fileK` starts invalid so the one-entry cache never collides with a real
@@ -178,12 +190,26 @@ proc oldLineInfo(w: var LineInfoWriter; info: NifLineInfo; p: Pool): TLineInfo =
## it to a `TLineInfo`. `info.file` indexes the loaded buffer's OWN filename
## pool `p` (= `cursorPool(n)`), which is the shared `icPool` for a text-parsed
## module but a fresh per-file pool for a `bif`-loaded one.
##
## Memoized per pool. Resolving a name costs a string copy out of the pool
## plus a hash of a full path, and the generator asks for a node's line info
## on essentially every statement it emits — 259k times on a 68-module build,
## which was 1.36s of the 1.88s the cursor-driven generator spent.
if info.file == NoFile:
result = unknownLineInfo
else:
let filePath = p.filenames[info.file]
let fileIdx = msgs.fileInfoIdx(w.config, AbsoluteFile filePath)
result = TLineInfo(line: info.line.uint16, col: info.col.int16, fileIndex: fileIdx)
if p != w.readPool:
w.readPool = p
w.readTab = @[]
let id = int(uint32(info.file))
if id >= w.readTab.len:
let oldLen = w.readTab.len
w.readTab.setLen(id + 1)
for i in oldLen ..< w.readTab.len: w.readTab[i] = astli.InvalidFileIdx
if w.readTab[id] == astli.InvalidFileIdx:
w.readTab[id] = msgs.fileInfoIdx(w.config, AbsoluteFile p.filenames[info.file])
result = TLineInfo(line: info.line.uint16, col: info.col.int16,
fileIndex: w.readTab[id])
# ------------- Writer ---------------------------------------------------------------
@@ -205,11 +231,23 @@ will tell us the precise offsets anyway.
]#
const
hiddenTypeTagName = "ht"
symDefTagName = "sd"
typeDefTagName = "td"
hiddenTypeTagName* = "ht"
symDefTagName* = "sd"
typeDefTagName* = "td"
bindingIdTagName = "bid"
bridgeSymTagName* = "bsym"
## `(bsym <intlit>)` — a symbol reference in the IN-PROCESS bridge format
## (`nodebridge.nim`), where the payload is an INDEX into the bridge's own
## `seq[PSym]` rather than a NIF name. Never written to a file: a `.bif` has
## to name symbols because the reader is a different process, but a bridged
## buffer is read by the process that built it, so it can hand back the very
## same `PSym` object. That is what makes the bridge lossless, and
## incidentally what makes `sym` idempotent for FIELDS on a bridged buffer —
## the file path cannot be, because `loadFieldStub` mints per use.
bridgeTypeTagName* = "btyp"
## `(btyp <intlit>)` — the same for a node's type slot.
var
sdefTag = registerTag(symDefTagName)
tdefTag = registerTag(typeDefTagName)
@@ -239,6 +277,9 @@ type
inTypeReclist: int # >0 while writing a type's OWN reclist: fields must be SELF-CONTAINED
# defs (the type can be seek-loaded in isolation), not entry-deduped uses
emittedCanonTypes: Table[string, int32] # canonical type name -> itemId.item of the def
extraExports: HashSet[ItemId] # symbols made importable by an explicit `export s`
# rather than by a `*` on the declaration; see
# `modulegraphs.reexportedLocalSyms`
proc isLocalSym(sym: PSym): bool {.inline.} =
@@ -314,22 +355,15 @@ proc toNifSymName(w: var Writer; sym: PSym): string =
# during a VM transform): re-home to the current module with the `@bk`
# marker so each referencing module self-contains it. See transformBody.
#
# Use `itemId.item` (the writer's dedup identity, see `emittedBackendSyms`)
# as the numeric name component, NOT `disamb`: closure `:env` syms in one
# module are minted from TWO id spaces — the backend lower stage's
# `tb.idgen` and sem's `vmTransfIdgen` (transf.transformBody) — whose
# `disambTable`s each start `:env` at the same low count, so a macro-lowered
# `:env` (e.g. `implementSendProcBody`) and a backend-lowered one
# (`peerTrimmerHeartbeat`) collide on `:env.2.<mod>@bk`. Two distinct syms
# then share a NIF name; the loader's name-keyed index/`c.syms` return the
# first for both, so one proc's `:env` gets the OTHER proc's env type
# (mismatched-pointer C, "has no member colonup_" at link). `itemId.item` is
# unique per `@bk` sym (both are emitted as defs, see writeSym), mirroring
# how `@bk` TYPES already key off `itemId.item` (nifTypeName). The loader
# copies this back into `disamb` (sn.count), so `globalName` round-trips.
# The numeric name component comes from `astdef.backendMintedDisamb` — the
# ONE definition of which integer identifies a backend-minted symbol, shared
# with the two C-name manglers (`mangleProcNameExt`, `ccgutils.makeUnique`)
# so the NIF name and the C name cannot disagree. `@bk` TYPES key off
# `itemId.item` the same way (see `nifTypeName`). The loader copies this back
# into `disamb` (sn.count), so `globalName` round-trips.
result = sym.name.s
result.add '.'
result.addInt sym.itemId.item
result.addInt backendMintedDisamb(sym)
result.add '.'
result.add modname(w.currentModule, w.infos.config)
result.add BackendLocalMarker
@@ -401,7 +435,7 @@ proc stripFieldMarker(rawName: string): string {.inline.} =
else:
rawName[0 ..< rawName.len - FieldMarker.len]
proc isFieldNifName(name: string): bool {.inline.} =
proc isFieldNifName*(name: string): bool {.inline.} =
## True for an object field's local NIF name `<ident>`f.<disamb>` (see
## `FieldMarker`): no module suffix, marker on the ident.
let sn = parseSymName(name)
@@ -1062,8 +1096,13 @@ proc writeSymDef(w: var Writer; dest: var IcBuilder; sym: PSym) =
# ("undeclared field 'Number'").
let isPureEnumField = sym.kindImpl == skEnumField and sym.typImpl != nil and
sym.typImpl.symImpl != nil and sfPure in sym.typImpl.symImpl.flagsImpl
# `sfExported` is the declaration's `*`. An explicit `export s` makes a symbol
# importable WITHOUT it (semExport -> reexportSym -> the interface table only),
# so ask the interface as well or those symbols ship as non-importable and the
# importer reports "undeclared identifier".
if sym.kindImpl != skField and not isPureEnumField and
{sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}:
({sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported} or
sym.itemId in w.extraExports):
dest.addIdent "x"
else:
dest.addDotToken
@@ -1370,7 +1409,7 @@ var modFlagsTag = registerTag("modflags")
# instead of a plain construction, and no read was ever recognised as a move.
# Only wrap when there is something to say, so the common sym use stays a bare
# token.
const symNodeFlagsTagName = "nflags"
const symNodeFlagsTagName* = "nflags"
var symNodeFlagsTag = registerTag(symNodeFlagsTagName)
const PersistedSymNodeFlags = PersistentNodeFlags - {nfLazyType, nfHasComment}
@@ -2144,8 +2183,10 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
resolvedImportDeps: seq[FileIndex] = @[];
firstUnusedId: int32 = 0;
expansions: seq[(PSym, TLineInfo)] = @[];
moduleFlags: int32 = 0) =
moduleFlags: int32 = 0;
extraExports: seq[ItemId] = @[]) =
var w = Writer(infos: newLineInfoWriter(config), currentModule: thisModule)
for id in extraExports: w.extraExports.incl id
w.deps = newIcBuilder(64)
var content = newIcBuilder(300)
@@ -2552,6 +2593,18 @@ proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifInd
type
LoadFlag* = enum
LoadFullAst, AlwaysLoadInterface
SkipInterfaceTables
## Do not eagerly build the module's interface string tables. Set by
## `modulegraphs.loadTransitiveHooks`, which loads a module only to
## register its hooks / macro-cache replay / generic-instance offers and
## throws the tables away — the module is a dep-of-a-dep, not an import, so
## none of its symbols are visible to the module being semchecked.
##
## The eager pass calls `loadSymFromIndexEntry` for EVERY index entry, and
## its only other effect is pre-populating the name-keyed `c.syms` cache —
## which `resolveSym` fills lazily on a miss anyway, straight from the same
## index. So for these loads it is pure work: on a 219-module program a
## one-line edit paid it 209 times over.
proc isGlobalIndexSym(s, dottedSuffix: string): bool =
## Mirror of `nifbuilder.addSymbolDefRetIsGlobal` / `bif.isGlobalSymbol`: a sym
@@ -2566,14 +2619,9 @@ proc isGlobalIndexSym(s, dottedSuffix: string): bool =
if s[i] == '.': inc dots
dots >= 2
proc buildPosIndex(buf: var TokenBuf; suffix: string): Table[string, NifIndexEntry] =
## Step 2a token-position index: scan the eagerly-parsed module `buf` for the
## global `SymbolDef`s it OWNS and record each at the token position of its
## enclosing tag (`(sd`/`(td`), with visibility from the marker that follows
## the def. Replaces `readEmbeddedIndex` (whose byte offsets are meaningless
## once the file is parsed); mirrors `bif.buildIndex` and the text writer's
## `(.index …)`. Foreign symbols appear only as `Symbol` uses (never
## `SymbolDef`s) so they are naturally excluded.
proc rescanPosIndex(buf: var TokenBuf; suffix: string): Table[string, NifIndexEntry] =
## VERIFICATION ONLY (`-d:icIndexCheck`): the old full-token-stream rescan,
## kept so `indexFromBif` can be graded against it over a whole real build.
result = initTable[string, NifIndexEntry]()
let dotted = "." & suffix
if buf.len == 0: return
@@ -2583,17 +2631,53 @@ proc buildPosIndex(buf: var TokenBuf; suffix: string): Table[string, NifIndexEnt
case c.kind
of TagLit:
mostRecentTagPos = cursorToPosition(buf, c)
inc c # descend into the body (visit every token)
inc c
of SymbolDef:
let nm = symName(c)
let tagPos = mostRecentTagPos
inc c # advance to the marker / next sibling
inc c
if isGlobalIndexSym(nm, dotted):
let vis = if c.hasMore and c.kind == DotToken: Hidden else: Exported
result[nm] = NifIndexEntry(offset: tagPos, info: NoLineInfo, vis: vis)
else:
inc c
proc indexFromBif(m: BifModule): Table[string, NifIndexEntry] =
## The module's name -> token-position index, taken from the index the `.bif`
## ALREADY CARRIES rather than recomputed.
##
## `bif.store` builds that index in one forward traversal at write time
## (`bif.buildIndex`) and writes it into the file; `bif.load` reads it back as
## `BifModule.index`, with `pos` already a TOKEN index of the declaration's
## enclosing tag — the very thing this used to rescan the whole token stream
## to recompute, once per module per backend process. That rescan was 909ms of
## a 10.1s cold `--ic:on` build (`-d:icBNodeProf`, `tPosIndex`).
##
## The two agree by construction, and it is worth saying exactly why, because
## "the file has an index" would not be enough on its own: the writer filters
## with `bif.isGlobalSymbol(name, dottedSuffix)` and every `storeBif` call site
## passes `"." & extractModuleSuffix(path)`, which is the same `dottedSuffix`
## the reader would have formed — so the two filters select the same symbols,
## and the `vis` rule (a `DotToken` marker after the def means hidden) is the
## same test on the same token.
result = initTable[string, NifIndexEntry](m.index.len)
for e in m.index:
result[poolSym(m.buf.pool, e.sym)] =
NifIndexEntry(offset: int(e.pos), info: NoLineInfo,
vis: (if e.vis == ivHidden: Hidden else: Exported))
proc indexFromBif(m: var BifModule; suffix: string): Table[string, NifIndexEntry] =
result = indexFromBif(m)
when defined(icIndexCheck):
let want = rescanPosIndex(m.buf, suffix)
doAssert result.len == want.len,
"index size differs for " & suffix & ": carried " & $result.len &
" rescanned " & $want.len
for k, v in want:
let got = result.getOrDefault(k)
doAssert got.offset == v.offset and got.vis == v.vis,
"index entry differs for " & k & " in " & suffix
proc readUnusedId(buf: var TokenBuf): int32 =
## Find the module's `(unusedid <int>)` directive — emitted as the FIRST child
## of the top-level `(stmts ...)` by writeNifModule/writeLoweredModule — and
@@ -2632,7 +2716,7 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}):
# This mirrors `toNifFilename` (kept in sync). `bif.load` mints FRESH per-file
# pools, so the buffer's literals/tags resolve through its own
# `cursorPool(n)`/`n.tags` (the reader is pool-agnostic); the token-position
# index is rebuilt name-based via `buildPosIndex`.
# index is taken from the one the file carries (`indexFromBif`).
let conf = c.infos.config
let useLowered = conf.cmd == cmdNifC and
(conf.icBackendStage == "cg" or conf.icBackendStage == "emit")
@@ -2644,8 +2728,12 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}):
raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile &
". This can happen when loading a module from NIF that references another module " &
"whose NIF file hasn't been written yet."
icProfStart(tBifLoad)
var m = bif.load(modFile)
let index = buildPosIndex(m.buf, suffix)
icProfStop(tBifLoad)
icProfStart(tPosIndex)
let index = indexFromBif(m, suffix)
icProfStop(tPosIndex)
# Seed the backend id counters ABOVE every id the file already uses, so a
# freshly-minted backend sym/type (closure env, RTTI hook, temp) can never
# share a `toId` with a loaded one. See `readUnusedId` / `(unusedid)`.
@@ -2670,7 +2758,7 @@ proc ensureSemBuf(c: var DecodeContext; module: FileIndex) =
let semFile = (getNimcacheDir(c.infos.config) / RelativeFile(m.suffix & ".s.bif")).string
if not fileExists(semFile): return
var sm = bif.load(semFile)
m.semIndex = buildPosIndex(sm.buf, m.suffix)
m.semIndex = indexFromBif(sm, m.suffix)
m.semBuf = ensureMove sm.buf
proc hasTypeOffset(c: var DecodeContext; module: FileIndex; nifName: string): bool =
@@ -3258,6 +3346,22 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
s = c.loadSymStub(n, thisModule, localSyms)
result = newSymNode(s, info)
result.typField = typ
# `(ht . <sym>)` — an EXPLICITLY nil node type — is left exactly as the
# writer meant it: NIL. The wrapper is only emitted when the node's own
# type differed from its symbol's (`writeSymNode`), so a nil here says
# the node genuinely had no type while the symbol had one, and that is
# load-bearing: a type symbol used as a VALUE (`newException(KeyError,
# ...)`) is exactly that shape, and handing it `sym.typ` makes sem read
# the typedesc as an expression of the type it denotes ("only a 'ref
# object' can be raised").
#
# There IS a load-order dependence here — `newSymNode` above marks the
# node lazy when the symbol was still an unloaded stub, so `ast.typ`
# answers `sym.typ` for that population and `nil` for the rest — and it
# is NOT fixed by pinning the flag either way: setting it breaks sem as
# above, and clearing it would strip the fallback from the stub
# population that `nifcBackendActive` exists to serve. Left alone
# deliberately.
elif tagIs(n, symDefTagName):
let info = c.infos.oldLineInfo(n.info, cursorPool(n))
let name = n.firstSon
@@ -3501,23 +3605,60 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex;
# (moduleId can add to c.mods which would invalidate Table iterators)
var indexTab = move c.mods[module].index
# Add all symbols to interf (exported interface) and interfHidden
# Only the EXPORTED half; `buildHiddenInterface` below does the rest, on
# demand. Exported symbols go into both tables, which costs little and leaves
# `interfHidden` a coherent view of a module with no hidden symbols rather
# than an empty one.
prof pIfaceModules
for nifName, entry in indexTab:
if entry.vis == Exported:
prof pIfaceExported
let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule)
if sym != nil:
strTableAdd(interf, sym)
strTableAdd(interfHidden, sym)
elif not nifName.startsWith("`t"):
# do not load types, they are not part of an interface but an implementation detail!
#echo "LOADING SYM ", nifName, " ", entry.offset
let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule)
if sym != nil:
strTableAdd(interfHidden, sym)
# Move index table back
c.mods[module].index = move indexTab
proc buildHiddenInterface*(c: var DecodeContext; suffix: string;
interfHidden: var TStrTable): bool {.discardable.} =
## The hidden-only half of a loaded module's interface, materialised on
## demand. Deferred because almost nothing reads it: `interfHidden` is reached
## exclusively through `modulegraphs.interfSelect`, which picks it only when
## `optImportHidden` is in the module's options, and that flag is set in
## exactly one place — an `import x {.all.}`. Building it eagerly was 1.05s of
## a cold Atlas build: 1.70M hidden stubs against 0.29M exported ones, made by
## every `nim m` for every module it imports and read by none of them.
##
## Takes the module SUFFIX, not a FileIndex, and that is the whole trick. A
## module has TWO FileIndexes: `registerNifSuffix` keys
## `filenameToIndexTbl` by the suffix string and mints a `fikNifModule` entry,
## while the graph indexes `g.ifaces` by the module's `fikSource` file. `c.mods`
## is keyed by the former. Asking it with the latter misses every single time,
## silently, and an `import x {.all.}` then reports "undeclared identifier"
## for a symbol that is right there.
##
## Returns false when the artifact is not on disk yet — an import the build
## has not produced. The caller must leave the request PENDING then: writing
## it off on that first miss costs the module its hidden symbols for the rest
## of the process.
let conf = c.infos.config
if not fileExists((getNimcacheDir(conf) / RelativeFile(suffix & ".s.bif")).string):
return false
let module = moduleId(c, suffix, {})
if not c.mods.hasKey(module): return false
var indexTab = move c.mods[module].index
for nifName, entry in indexTab:
if entry.vis != Exported and not nifName.startsWith("`t"):
prof pIfaceHidden
# do not load types, they are not part of an interface but an implementation detail!
let sym = loadSymFromIndexEntry(c, module, nifName, entry, suffix)
if sym != nil:
strTableAdd(interfHidden, sym)
c.mods[module].index = move indexTab
result = true
proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] =
## Stubs for every non-type symbol serialized in `module`'s NIF index. The
## per-module backend uses this to emit the routines a module OWNS: procs are
@@ -3772,12 +3913,71 @@ proc nifModuleHasIncludes*(config: ConfigRef; fileIdx: FileIndex): bool =
done = true
skip c
proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) =
proc peekSymKind(c: var DecodeContext; module: FileIndex;
entry: NifIndexEntry): TSymKind =
## The kind a symbol's `(sd …)` header records, WITHOUT decoding the symbol.
##
## The layout is `(sd <SymbolDef name> <marker: `x` | `.`> <kind> …)`, which is
## exactly what `loadSymFromCursor` walks — that proc is the definition this
## mirrors, so the two must be changed together. Anything unexpected answers
## `skUnknown` and the caller falls back to a real load rather than guessing.
var n = cursorFromIndexEntry(c, module, entry)
if n.kind != TagLit or not tagIs(n, symDefTagName): return skUnknown
var k = childCursor(n)
if not k.hasMore or k.kind != SymbolDef: return skUnknown
skip k # the name
if not k.hasMore: return skUnknown
skip k # the `x` / `.` export marker
if not k.hasMore or k.kind != TagLit: return skUnknown
result = parse(TSymKind, cursorTag(k))
proc symKindFast(c: var DecodeContext; sym: PSym; symAsStr: string): TSymKind =
## `sym`'s kind, taken from its def header while it is still `Partial` rather
## than by forcing the full decode. An already-loaded symbol answers from the
## field, and anything the peek cannot read falls back to loading.
##
## `-d:icPeekKindCheck` grades the peek against the load it replaces, on every
## call: the loaded kind is authoritative, so a disagreement is the peek's bug.
## The oracle has to be run for the answer to mean anything — and broken on
## purpose once, to confirm it fires.
if sym.state != Partial:
prof pPeekLoaded
return sym.kindImpl
let e = c.syms.getOrDefault(symAsStr)
if e[1].offset == 0:
prof pPeekFallback
loadSym(c, sym)
return sym.kindImpl
result = peekSymKind(c, sym.itemId.module.FileIndex, e[1])
if result == skUnknown:
# The peek could not read the header. Correct, but it is also how a walk
# that has drifted out of step with `loadSymFromCursor` would present, so
# the rate is counted rather than shrugged at: `-d:icBNodeProf` reports
# `PeekFallback` beside `PeekKind`, and it should stay at zero.
prof pPeekFallback
loadSym(c, sym)
return sym.kindImpl
prof pPeekKind
when defined(icPeekKindCheck):
let peeked = result
loadSym(c, sym)
doAssert peeked == sym.kindImpl,
"peekSymKind disagrees for " & symAsStr & ": peeked " & $peeked &
" but the load says " & $sym.kindImpl
proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; symAsStr: string;
interf: var TStrTable) =
## When a non-pure enum type is (re-)exported, its fields must also become
## visible (unqualified) to importers. In a from-source build this happens via
## `rawImportSymbol`'s enum handling when the type is imported; the lazy IC
## importer never runs that, so we materialise the fields into the interface
## here, when the export list is processed.
##
## Only a TYPE can contribute fields, and almost none of an export list is
## types — so the kind is read off the def header first (`symKindFast`) rather
## than by forcing every exported symbol through a full decode to find out.
## That decode was 290ms of an 8.6s build over 34815 symbols.
if symKindFast(c, sym, symAsStr) != skType: return
loadSym(c, sym)
if sym.kindImpl != skType or sfPure in sym.flagsImpl: return
let et = sym.typImpl
@@ -3791,6 +3991,79 @@ proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTa
if f != nil and f.kind == nkSym and f.sym != nil:
strTableAdd(interf, f.sym)
type
TopTag = enum
## Which top-level directive a tag names. `processTopLevel` used to decide
## this with an `elif` chain of ~20 `tagIs` calls, i.e. up to twenty tag-NAME
## string comparisons per node, and the common cases (a real statement, or
## `implementation`) sit at the END of the chain so the average node walked
## all of it — 1.46M nodes on a 68-module build. Resolved once per tag id
## instead, and the chain becomes a `case`.
ttOther, ttReplay, ttUnusedId, ttModFlags,
ttRepConverter, ttRepDestroy, ttRepWasMoved, ttRepCopy, ttRepSink, ttRepDup,
ttRepTrace, ttRepDeepCopy, ttRepEnumToStr, ttRepMethod, ttRepPureEnum,
ttRepCppMember, ttExport, ttInclude, ttImport, ttReexpMod, ttOffer, ttTOffer,
ttModuleSrc, ttExpansion, ttSig, ttImplementation,
ttLetSection, ttVarSection, ttPragma
const
letSectionTag = toNifTag(nkLetSection)
varSectionTag = toNifTag(nkVarSection)
pragmaTag = toNifTag(nkPragma)
proc classifyTopTag(name: string): TopTag =
case name
of "replay": ttReplay
of "unusedid": ttUnusedId
of "modflags": ttModFlags
of "repconverter": ttRepConverter
of "repdestroy": ttRepDestroy
of "repwasmoved": ttRepWasMoved
of "repcopy": ttRepCopy
of "repsink": ttRepSink
of "repdup": ttRepDup
of "reptrace": ttRepTrace
of "repdeepcopy": ttRepDeepCopy
of "repenumtostr": ttRepEnumToStr
of "repmethod": ttRepMethod
of "reppureenum": ttRepPureEnum
of "repcppmember": ttRepCppMember
of "export": ttExport
of "include": ttInclude
of "import": ttImport
of "reexpmod": ttReexpMod
of "offer": ttOffer
of "toffer": ttTOffer
of "modulesrc": ttModuleSrc
of "expansion": ttExpansion
of "sig": ttSig
of "implementation": ttImplementation
else:
if name == letSectionTag: ttLetSection
elif name == varSectionTag: ttVarSection
elif name == pragmaTag: ttPragma
else: ttOther
var topTagPool: TagPool = nil
var topTagCache: seq[int8] = @[]
## `TagId -> TopTag`, -1 unresolved, for ONE tag pool. `topTagPool` holds the
## pool by REFERENCE so it stays alive and a freed pool cannot be replaced at
## the same address — the same argument `indexFromBif`'s memo rests on.
proc topTagAt(cur: Cursor): TopTag =
let pool {.cursor.} = cur.tags
if pool != topTagPool:
topTagPool = pool
topTagCache = @[]
let id = int(uint32(cursorTagId(cur)))
if id >= topTagCache.len:
let oldLen = topTagCache.len
topTagCache.setLen(id + 1)
for i in oldLen ..< topTagCache.len: topTagCache[i] = -1'i8
if topTagCache[id] < 0:
topTagCache[id] = int8(ord(classifyTopTag(pool.tagName(cursorTagId(cur)))))
result = TopTag(topTagCache[id])
proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag];
interf: var TStrTable; suffix: string; module: int): PrecompiledModule =
## Step 2 phase 2: walk the module body directly over the resident `buf` cursor
@@ -3808,59 +4081,101 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
# `topLevel`. They sit in the module header before `(implementation)`.
var cont = true
while cont and cur.hasMore:
prof pTopNodes
if cur.kind != TagLit:
cont = false
else:
if tagIs(cur, "replay"):
case topTagAt(cur)
of ttReplay:
# Always load replay actions (macro cache operations)
icProfStart(tTopReplay)
cur.into:
while cur.hasMore:
let replayNode = loadNode(c, cur, suffix, localSyms)
if replayNode != nil:
result.topLevel.sons.add replayNode
elif tagIs(cur, "unusedid"):
icProfStop(tTopReplay)
of ttUnusedId:
# backend id seed — consumed eagerly by `moduleId`/`readUnusedId`; just
# skip past it here so the rest of the header still loads.
skip cur
elif tagIs(cur, "modflags"):
of ttModFlags:
cur.into:
if cur.hasMore and cur.kind == IntLit:
result.moduleFlags = int32 intVal(cur)
skip cur
while cur.hasMore: skip cur
elif tagIs(cur, "repconverter"): loadLogOp(c, result.logOps, cur, ConverterEntry, attachedTrace, module)
elif tagIs(cur, "repdestroy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDestructor, module)
elif tagIs(cur, "repwasmoved"): loadLogOp(c, result.logOps, cur, HookEntry, attachedWasMoved, module)
elif tagIs(cur, "repcopy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedAsgn, module)
elif tagIs(cur, "repsink"): loadLogOp(c, result.logOps, cur, HookEntry, attachedSink, module)
elif tagIs(cur, "repdup"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDup, module)
elif tagIs(cur, "reptrace"): loadLogOp(c, result.logOps, cur, HookEntry, attachedTrace, module)
elif tagIs(cur, "repdeepcopy"): loadLogOp(c, result.logOps, cur, HookEntry, attachedDeepCopy, module)
elif tagIs(cur, "repenumtostr"): loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module)
elif tagIs(cur, "repmethod"): loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module)
elif tagIs(cur, "reppureenum"): loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module)
elif tagIs(cur, "repcppmember"): loadLogOp(c, result.logOps, cur, CppMemberEntry, attachedTrace, module)
elif tagIs(cur, "export"):
of ttRepConverter:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, ConverterEntry, attachedTrace, module)
of ttRepDestroy:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedDestructor, module)
of ttRepWasMoved:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedWasMoved, module)
of ttRepCopy:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedAsgn, module)
of ttRepSink:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedSink, module)
of ttRepDup:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedDup, module)
of ttRepTrace:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedTrace, module)
of ttRepDeepCopy:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, HookEntry, attachedDeepCopy, module)
of ttRepEnumToStr:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, EnumToStrEntry, attachedTrace, module)
of ttRepMethod:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, MethodEntry, attachedTrace, module)
of ttRepPureEnum:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, PureEnumEntry, attachedTrace, module)
of ttRepCppMember:
timed tTopLogOps:
loadLogOp(c, result.logOps, cur, CppMemberEntry, attachedTrace, module)
of ttExport:
if SkipInterfaceTables in flags:
# Same reason the interface tables are skipped: `interf` is a scratch
# table this caller throws away, so every `resolveSym` here (one per
# exported symbol, plus `addReexportedEnumFields`) only warms the
# name-keyed `c.syms` cache that `resolveSym` refills lazily on a miss.
skip cur
continue
icProfStart(tExportBranch)
cur.into:
while cur.hasMore and cur.kind == DotToken: skip cur # flags / type
while cur.hasMore:
if cur.kind == Symbol:
prof pExportSyms
let symAsStr = symName(cur)
# Skip symbols re-exported by this dependency but owned by the module
# being compiled fresh (they would collide with the fresh originals).
if c.mainModuleSuffix.len == 0 or
parseSymName(symAsStr).module != c.mainModuleSuffix:
icProfStart(tResolveSym)
let sym = resolveSym(c, symAsStr, false)
icProfStop(tResolveSym)
if sym != nil:
strTableAdd(interf, sym)
addReexportedEnumFields(c, sym, interf)
icProfStart(tEnumFields)
addReexportedEnumFields(c, sym, symAsStr, interf)
icProfStop(tEnumFields)
skip cur
else:
raiseAssert "expected Symbol or ParRi but got " & $cur.kind &
" in export list of module " & suffix
elif tagIs(cur, "include"): loadInclude(c, cur, result.includes)
elif tagIs(cur, "import"): loadImport(c, cur, result.deps)
elif tagIs(cur, "reexpmod"):
icProfStop(tExportBranch)
of ttInclude: loadInclude(c, cur, result.includes)
of ttImport: loadImport(c, cur, result.deps)
of ttReexpMod:
# a re-exported MODULE: (reexpmod "name" "suffix"); the module sym is a
# qualifier in this module's interface — materialized by modulegraphs.
var mname, msuffix = ""
@@ -3869,7 +4184,7 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
if cur.hasMore and cur.kind == StrLit: (msuffix = strVal(cur); skip cur)
if mname.len > 0 and msuffix.len > 0:
result.reexportedModules.add (mname, msuffix)
elif tagIs(cur, "offer"):
of ttOffer:
# (offer <genericSym> <instSym> <genericParamsCount> <type>...) — resolve
# to PSyms/PTypes; modulegraphs registers them into `procInstCache`.
# Best-effort: a type that fails to resolve drops the whole offer.
@@ -3878,6 +4193,7 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
var cts: seq[PType] = @[]
var idx = 0
var ok = true
icProfStart(tTopOffers)
cur.into:
while cur.hasMore:
if cur.kind == Symbol:
@@ -3895,12 +4211,14 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
else: skip cur
if ok and genSym != nil and instSym != nil:
result.genericOffers.add (genSym, instSym, cts, paramsCount)
elif tagIs(cur, "toffer"):
icProfStop(tTopOffers)
of ttTOffer:
# (toffer "<genericBodySym>" "<instType>") — intern the two full names,
# resolve, FULLY load the instance (so `searchInstTypes` can match its
# params). Best-effort: a failure to resolve drops the offer.
var genName, instName = ""
var idx = 0
icProfStart(tTopOffers)
cur.into:
while cur.hasMore:
if cur.kind == StrLit:
@@ -3915,33 +4233,43 @@ proc processTopLevel(c: var DecodeContext; cur: var Cursor; flags: set[LoadFlag]
if genSym != nil and inst != nil:
loadType(c, inst)
result.typeOffers.add (genSym, inst)
elif tagIs(cur, "modulesrc"):
icProfStop(tTopOffers)
of ttModuleSrc:
prof pTopToolingSkip
# self-identification record for the standalone include-graph scanner;
# not needed by the loader, just skip past it.
skip cur
elif tagIs(cur, "expansion"):
of ttExpansion:
prof pTopToolingSkip
# template/macro expansion usage record for tooling (`idetools` scans it
# as a `Symbol` use); the loader itself needs nothing from it.
skip cur
elif tagIs(cur, "sig"):
of ttSig:
prof pTopToolingSkip
# signature-symbol occurrence record for tooling (`idetools` scans it as a
# `Symbol` use); the loader itself needs nothing from it.
skip cur
elif tagIs(cur, "implementation"):
of ttImplementation:
cont = false
elif LoadFullAst in flags or tagIs(cur, toNifTag(nkLetSection)) or
tagIs(cur, toNifTag(nkVarSection)) or tagIs(cur, toNifTag(nkPragma)):
of ttLetSection, ttVarSection, ttPragma:
# Parse the full statement. let/var sections are loaded unconditionally
# (see above) so `{.compileTime.}` globals reach the eager initializer.
# Top-level pragmas are loaded too: a module-level `{.emit.}` (and the
# `{.push/pop.}` around it) must reach the `cg` stage's genPragma/genEmit,
# else e.g. a `#include` is dropped and the generated C won't compile.
# writeToplevelNode routes these into this header section.
icProfStart(tTopStmts)
let stmtNode = loadNode(c, cur, suffix, localSyms)
if stmtNode != nil:
result.topLevel.sons.add stmtNode
else:
cont = false
icProfStop(tTopStmts)
of ttOther:
if LoadFullAst in flags:
let stmtNode = loadNode(c, cur, suffix, localSyms)
if stmtNode != nil:
result.topLevel.sons.add stmtNode
else:
cont = false
proc registerModuleSelfSym*(c: var DecodeContext; suffix: string; m: PSym) =
## Bind the module's NIF name to the ONE module symbol the graph registered.
@@ -3963,7 +4291,9 @@ proc registerModuleSelfSym*(c: var DecodeContext; suffix: string; m: PSym) =
proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable;
flags: set[LoadFlag] = {}): PrecompiledModule =
# Ensure module index is loaded - moduleId returns the FileIndex for this suffix
icProfStart(tModuleId)
let module = moduleId(c, string(suffix), flags)
icProfStop(tModuleId)
# Load the module AST (or just replay actions if loadFullAst is false).
# processTopLevel also collects export instructions. Step 2 phase 2: read the
@@ -3973,14 +4303,19 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi
if cur.kind == TagLit and tagIs(cur, toNifTag(nkStmtList)):
inc cur # enter (stmts (past the tag head, onto the flags dot)
skip cur # flags dot (processTopLevel skips the type dot itself)
icProfStart(tTopLevel)
result = processTopLevel(c, cur, flags, interf, string(suffix), module.int)
icProfStop(tTopLevel)
else:
result = PrecompiledModule(topLevel: newNode(nkStmtList))
# Populate interface tables from the NIF index structure
# Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym
# Use exports collected by processTopLevel
populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix))
if SkipInterfaceTables notin flags:
icProfStart(tInterfTables)
populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix))
icProfStop(tInterfTables)
proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable;
flags: set[LoadFlag] = {}): PrecompiledModule =

4
compiler/astalgo.nim
View File

@@ -741,6 +741,6 @@ proc listSymbolNames*(symbols: openArray[PSym]): string =
result.add sym.name.s
proc isDiscriminantField*(n: PNode): bool =
if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
if n.kind == nkCheckedFieldExpr: sfDiscriminant in n.firstSon.secondSon.sym.flags
elif n.kind == nkDotExpr: sfDiscriminant in n.secondSon.sym.flags
else: false

97
compiler/astdef.nim
View File

@@ -957,13 +957,54 @@ iterator items*(n: PNode): PNode =
iterator sons*(n: PNode): PNode =
## Iterates over the children of `n`. Preferred over `for i in 0..<n.len: n[i]`
## as it does not rely on random indexed access (see doc/ic_backend_nif_native.md).
## as it does not rely on random indexed access, and over `for x in n.sons`,
## which reads the raw FIELD and so skips the `len` hook that materialises a
## deferred `nfLazyBody` body — over such a body that loop silently visits
## nothing.
for i in 0..<n.safeLen: yield n[i]
iterator isons*(n: PNode): tuple[i: int, n: PNode] =
## Like `sons` but also yields the child index. Replaces
## `for i in 0..<n.len: ... n[i] ...` when `i` itself is still needed.
for i in 0..<n.safeLen: yield (i, n[i])
iterator isons*(n: PNode; start = 0): tuple[i: int, n: PNode] =
## Like `sons` but also yields the child index, and optionally skips the first
## `start` children. Replaces `for i in start..<n.len: ... n[i] ...` when `i`
## itself is still needed — for a parameter position, a `needTmp[i-1]` lookup,
## a parallel index into the routine's `PType`, and so on. `start` is almost
## always 1, to step over a call's callee or a case statement's selector.
##
## Use `sonsFrom` instead when the index is only ever used to subscript `n`.
for i in start..<n.safeLen: yield (i, n[i])
iterator sonsFrom*(n: PNode; start: int): PNode =
## `sons` skipping the first `start` children. Replaces
## `for i in start..<n.len: ... n[i] ...`, which is by far the commonest
## indexed shape in the code generator — `start` is almost always 1, to step
## over a case/try statement's selector or a call's callee.
for i in start..<n.safeLen: yield n[i]
iterator sonsButLast*(n: PNode; count = 1): PNode =
## `sons` without the last `count` children. Replaces `for i in 0..<n.len-1:
## ... n[i] ...`, which is what an `nkOfBranch`/`nkExceptBranch` walk looks
## like: the last child is the branch BODY, the ones before it are the labels
## it matches. `count = 2` is the `nkVarTuple`/`nkIdentDefs` shape, whose last
## two children are the type and the value. A `Cursor` can serve this with a
## single pass and `count` nodes of lookahead; the indexed form has to re-walk
## the children for every label.
##
## Use `isonsButLast` instead when the index is still needed.
for i in 0 ..< n.safeLen - count: yield n[i]
iterator isonsButLast*(n: PNode; count = 1): tuple[i: int, n: PNode] =
## Like `sonsButLast` but also yields the child index — for a tuple field
## position, a parallel index into the tuple's `PType`, and so on.
for i in 0 ..< n.safeLen - count: yield (i, n[i])
template son*(n: PNode; i: int): PNode =
## Named indexed access to child `i`, for the small constant positions that
## `firstSon`/`secondSon`/`lastSon` do not cover.
n[i]
template hasSons*(n: PNode): bool =
## Emptiness test; goes through `safeLen` so a deferred body is materialised.
n.safeLen > 0
when defined(useNodeIds):
const nodeIdToDebug* = -1 # 2322968
@@ -1046,6 +1087,52 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode =
# handling for IC, they end up in IC indexes etc. Thus we "log" them in the module graph
# and to pass them around to the NIF writer. This is not very elegant but it works.
const
InstanceDisambBit* = 0x4000_0000'i32
## Set in the `disamb` of routine instances whose value is content-derived
## (see `modulegraphs.setInstanceDisamb`); keeps them disjoint from the
## small counter range ordinary symbols draw from, so the NIF name
## `name.disamb.module` stays collision-free within a module.
HookDisambBit* = 0x2000_0000'i32
## Set in the `disamb` of synthesized type-bound operators and `$enum`
## procs whose value is content-derived (see `modulegraphs.setHookDisamb`);
## disjoint from both the small counter range and `InstanceDisambBit`.
##
## Both live here rather than in `modulegraphs` because `ast2nif` — which
## cannot import that module — names symbols by them.
proc backendMintedDisamb*(s: PSym): int32 {.inline.} =
## The integer that identifies a BACKEND-MINTED symbol (`isBackendMinted`) in
## every name derived from it: its NIF name (`ast2nif.toNifSymName`) and its C
## name (`mangleutils.mangleProcNameExt`, `ccgutils.makeUnique`).
##
## Two cases, and the whole point of having ONE function is that all three
## sites take the same one:
##
## * A lifted HOOK's `disamb` is CONTENT-derived (`modulegraphs.setHookDisamb`),
## so it is identical in every process. Such a hook really does cross process
## boundaries — `lower` mints the env hooks of nested routines while `cg`
## mints those of the module's top level, and both land in the same
## translation unit — and its C name is also baked into emit-everywhere RTTI
## tables. `itemId.item` would differ per process, so two unrelated hooks
## collided on one `_c<item>` and the merge stage kept a single body for both
## (C accepted the mistyped call, C++ rejected it).
## * Otherwise `itemId.item` — the writer's dedup identity, unique per `@bk`
## sym. `disamb` cannot serve here: a module's `:env` syms are minted from TWO
## id spaces (the backend `lower` stage's idgen and sem's `vmTransfIdgen`)
## whose `disambTable`s each start `:env` at the same low count, so a
## macro-lowered and a backend-lowered `:env` collide on `:env.2.<mod>@bk`.
##
## The loader copies the name's numeric component back into `disamb`, so after a
## round trip `disamb` equals this value and `ast2nif.globalName` — which always
## reads `disamb` — agrees with the name the writer produced.
##
## This rule used to be written out at each of the three sites. They drifted:
## `toNifSymName` lacked the hook exception, so a content-derived value was
## overwritten by the loader and two backend hooks merged into one C function.
if (s.disamb and HookDisambBit) != 0'i32: s.disamb
else: s.itemId.item
type
LogEntryKind* = enum
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry,

142
compiler/ccgcalls.nim
View File

@@ -11,6 +11,12 @@
proc canRaiseDisp(p: BProc; n: PNode): bool =
# we assume things like sysFatal cannot raise themselves
# 5 = "decided here, neither predicate ran". Without resetting, the marker
# keeps whatever the PREVIOUS call left in it and the early return below
# attributes this answer to a branch that did not execute — which is how the
# first run of this differential came to claim effect-list coverage it did
# not have. Both short-circuits below leave it at 5.
markCanRaiseBranch 5
if n.kind == nkSym and n.sym.kind == skMethod:
# A base method may be overridden by a branch with a wider exception set.
# Its inferred effects describe only the base body, not every vtable target.
@@ -25,6 +31,13 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
else:
# we have to be *very* conservative:
result = canRaiseConservative(n)
when defined(icCanRaiseLog):
# `canRaise` reads the raises spec off `fn.typ.n`, and under `--ic:on` that
# node came back from a `.bif`. The only oracle for whether it came back
# INTACT is the same program built without IC. Log the verdict per callee;
# the two builds must produce the same one.
if n.kind == nkSym:
logCanRaise(n.sym, result)
proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
@@ -46,15 +59,14 @@ proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
nkCheckedFieldExpr:
n = n.firstSon
of nkHiddenStdConv, nkHiddenSubConv, nkConv:
n = n[1]
n = n.secondSon
else:
# cannot analyse the location; assume the worst
return true
result = false
if le != nil:
for i in 1..<ri.len:
let r = ri[i]
for r in sonsFrom(ri, 1):
if isPartOf(le, r, {pfStructural}) != arNo: return true
# we use the weaker 'canRaise' here in order to prevent too many
# annoying warnings, see #14514
@@ -63,8 +75,7 @@ proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
message(p.config, le.info, warnObservableStores, $le)
# bug #19613 prevent dangerous aliasing too:
if dest != nil and dest != le:
for i in 1..<ri.len:
let r = ri[i]
for r in sonsFrom(ri, 1):
if isPartOf(dest, r, {pfStructural}) != arNo: return true
proc hasNoInit(call: PNode): bool {.inline.} =
@@ -99,7 +110,7 @@ proc cleanupTemp(p: BProc; returnType: PType, tmp: TLoc): bool =
else:
result = false
proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
proc fixupCall(p: BProc, le: PNode, ri: PNode, d: var TLoc,
result: var Builder, call: var CallBuilder) =
let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri.firstSon)
genLineDir(p, ri)
@@ -190,7 +201,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} =
of {nkAddr, nkHiddenAddr, nkHiddenDeref}:
x = x.firstSon
of nkHiddenStdConv:
x = x[1]
x = x.secondSon
else:
break
if x.kind == nkSym and x.sym.kind == skParam:
@@ -199,9 +210,9 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} =
result = true
proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) =
var a = initLocExpr(p, q[1])
var b = initLocExpr(p, q[2])
var c = initLocExpr(p, q[3])
var a = initLocExpr(p, q.secondSon)
var b = initLocExpr(p, son(q, 2))
var c = initLocExpr(p, son(q, 3))
# bug #23321: In the function mapType, ptrs (tyPtr, tyVar, tyLent, tyRef)
# are mapped into ctPtrToArray, the dereference of which is skipped
# in the `genDeref`. We need to skip these ptrs here
@@ -227,7 +238,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
let lit = cIntLiteral(first)
result = (cCast(ptrType(dest), cOp(Add, NimInt, ra, cOp(Sub, NimInt, rb, lit))), lengthExpr)
of tyOpenArray, tyVarargs:
let data = if reifiedOpenArray(q[1]): dotField(ra, "Field0") else: ra
let data = if reifiedOpenArray(q.secondSon): dotField(ra, "Field0") else: ra
result = (cCast(ptrType(dest), cOp(Add, NimInt, data, rb)), lengthExpr)
of tyUncheckedArray, tyCstring:
result = (cCast(ptrType(dest), cOp(Add, NimInt, ra, rb)), lengthExpr)
@@ -264,23 +275,23 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Builder) =
var q = skipConv(n)
var skipped = false
while q.kind == nkStmtListExpr and q.len > 0:
while q.kind == nkStmtListExpr and q.hasSons:
skipped = true
q = q.lastSon
if getMagic(q) == mSlice:
# magic: pass slice to openArray:
if skipped:
q = skipConv(n)
while q.kind == nkStmtListExpr and q.len > 0:
for i in 0..<q.len-1:
genStmts(p, q[i])
while q.kind == nkStmtListExpr and q.hasSons:
for it in sonsButLast(q):
genStmts(p, it)
q = q.lastSon
let (x, y) = genOpenArraySlice(p, q, formalType, n.typ.elementType)
result.add(x)
result.addArgumentSeparator()
result.add(y)
else:
var a = initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n)
var a = initLocExpr(p, if n.kind == nkHiddenStdConv: n.secondSon else: n)
case skipTypes(a.t, abstractVar+{tyStatic}).kind
of tyOpenArray, tyVarargs:
let ra = rdLoc(a)
@@ -445,7 +456,7 @@ proc skipTrivialIndirections(n: PNode): PNode =
of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
result = result.firstSon
of nkHiddenStdConv, nkHiddenSubConv:
result = result[1]
result = result.secondSon
else: break
proc getPotentialReads(n: PNode; result: var seq[PNode]) =
@@ -453,44 +464,47 @@ proc getPotentialReads(n: PNode; result: var seq[PNode]) =
of nkLiterals, nkIdent, nkFormalParams: discard
of nkSym: result.add n
else:
for s in n:
for s in sons(n):
getPotentialReads(s, result)
proc genParams(p: BProc, ri: PNode, typ: PType; result: var Builder, argBuilder: var CallBuilder) =
# We must generate temporaries in cases like #14396
# to keep the strict Left-To-Right evaluation
var needTmp = newSeq[bool](ri.len - 1)
# The arguments are walked BACKWARDS below; collect them once and index that.
var args: seq[PNode] = @[]
for it in sonsFrom(ri, 1): args.add it
var needTmp = newSeq[bool](args.len)
var potentialWrites: seq[PNode] = @[]
for i in countdown(ri.len - 1, 1):
if ri[i].skipTrivialIndirections.kind == nkSym:
needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
for i in countdown(args.high, 0):
if args[i].skipTrivialIndirections.kind == nkSym:
needTmp[i] = potentialAlias(args[i], potentialWrites)
else:
#if not ri[i].typ.isCompileTimeOnly:
#if not args[i].typ.isCompileTimeOnly:
var potentialReads: seq[PNode] = @[]
getPotentialReads(ri[i], potentialReads)
getPotentialReads(args[i], potentialReads)
for n in potentialReads:
if not needTmp[i - 1]:
needTmp[i - 1] = potentialAlias(n, potentialWrites)
getPotentialWrites(ri[i], false, potentialWrites)
if not needTmp[i]:
needTmp[i] = potentialAlias(n, potentialWrites)
getPotentialWrites(args[i], false, potentialWrites)
when false:
# this optimization is wrong, see bug #23748
if ri[i].kind in {nkHiddenAddr, nkAddr}:
if args[i].kind in {nkHiddenAddr, nkAddr}:
# Optimization: don't use a temp, if we would only take the address anyway
needTmp[i - 1] = false
needTmp[i] = false
for i in 1..<ri.len:
for i, it in isons(ri, 1):
if i < typ.n.len:
assert(typ.n[i].kind == nkSym)
let paramType = typ.n[i]
assert(son(typ.n, i).kind == nkSym)
let paramType = son(typ.n, i)
if not paramType.typ.isCompileTimeOnly:
var arg = newBuilder("")
genArg(p, ri[i], paramType.sym, ri, arg, needTmp[i-1])
genArg(p, it, paramType.sym, ri, arg, needTmp[i-1])
if arg.buf.len != 0:
result.addArgument(argBuilder):
result.add(extract(arg))
else:
var arg = newBuilder("")
genArgNoParam(p, ri[i], arg, needTmp[i-1])
genArgNoParam(p, it, arg, needTmp[i-1])
if arg.buf.len != 0:
result.addArgument(argBuilder):
result.add(extract(arg))
@@ -500,7 +514,7 @@ proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
(sym.typ.callConv == ccInline or sym.owner.id == module.id):
res = res & "_actual".rope
proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
proc genPrefixCall(p: BProc, le: PNode, ri: PNode, d: var TLoc) =
# this is a hotspot in the compiler
var op = initLocExpr(p, ri.firstSon)
# getUniqueType() is too expensive here:
@@ -516,7 +530,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
genParams(p, ri, typ, res, call)
fixupCall(p, le, ri, d, res, call)
proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
proc genClosureCall(p: BProc, le: PNode, ri: PNode, d: var TLoc) =
template callProc(rp, params, pTyp: Snippet): Snippet =
let e = dotField(rp, "ClE_0")
@@ -551,6 +565,12 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
var argBuilder = default(CallBuilder) # not initCallBuilder, we just want the params
genParams(p, ri, typ, params, argBuilder)
# `rawProc` is bound BEFORE the `{.dirty.}` template that uses it. Inside a
# generic proc a dirty template's identifiers resolve at instantiation, and a
# local declared after the template loses to the module-level `rawProc` proc
# — which type-checks as a completely different thing.
let rawProc = getClosureType(p.module, typ, clHalf)
template genCallPattern {.dirty.} =
let rp = rdLoc(op)
let pars = extract(params)
@@ -559,8 +579,6 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
p.s(cpsStmts).add(callIter(rp, pars))
else:
p.s(cpsStmts).add(callProc(rp, pars, rawProc))
let rawProc = getClosureType(p.module, typ, clHalf)
let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri.firstSon)
if typ.returnType != nil:
if isInvalidReturnType(p.config, typ):
@@ -617,22 +635,22 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Builder;
if i < typ.n.len:
# 'var T' is 'T&' in C++. This means we ignore the request of
# any nkHiddenAddr when it's a 'var T'.
let paramType = typ.n[i]
let paramType = son(typ.n, i)
assert(paramType.kind == nkSym)
if paramType.typ.isCompileTimeOnly:
discard
elif paramType.typ.kind in {tyVar} and ri[i].kind == nkHiddenAddr:
elif paramType.typ.kind in {tyVar} and son(ri, i).kind == nkHiddenAddr:
result.addArgument(argBuilder):
genArgNoParam(p, ri[i].firstSon, result)
genArgNoParam(p, son(ri, i).firstSon, result)
else:
result.addArgument(argBuilder):
genArgNoParam(p, ri[i], result) #, typ.n[i].sym)
genArgNoParam(p, son(ri, i), result) #, son(typ.n, i).sym)
else:
if tfVarargs notin typ.flags:
localError(p.config, ri.info, "wrong argument count")
else:
result.addArgument(argBuilder):
genArgNoParam(p, ri[i], result)
genArgNoParam(p, son(ri, i), result)
discard """
Dot call syntax in C++
@@ -694,10 +712,10 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Builder) =
# However manual wrappers may also use 'ptr T'. In any case we support both
# for convenience.
internalAssert p.config, i < typ.n.len
assert(typ.n[i].kind == nkSym)
assert(son(typ.n, i).kind == nkSym)
# if the parameter is lying (tyVar) and thus we required an additional deref,
# skip the deref:
var ri = ri[i]
var ri = son(ri, i)
while ri.kind == nkObjDownConv: ri = ri.firstSon
let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
if t.kind in {tyVar}:
@@ -721,7 +739,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Builder) =
else:
ri = skipAddrDeref(ri)
if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri.firstSon
genArgNoParam(p, ri, result) #, typ.n[i].sym)
genArgNoParam(p, ri, result) #, son(typ.n, i).sym)
result.add(".")
proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Builder) =
@@ -731,12 +749,12 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Bu
case pat[i]
of '@':
var callBuilder = default(CallBuilder) # not init call builder
for k in j..<ri.len:
for k, _ in isons(ri, j):
genOtherArg(p, ri, k, typ, result, callBuilder)
inc i
of '#':
if i+1 < pat.len and pat[i+1] in {'+', '@'}:
let ri = ri[j]
let ri = son(ri, j)
if ri.kind in nkCallKinds:
let typ = skipTypes(ri.firstSon.typ, abstractInst)
if pat[i+1] == '+': genArgNoParam(p, ri.firstSon, result)
@@ -744,7 +762,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Bu
if 1 < ri.len:
var callBuilder: CallBuilder = default(CallBuilder)
genOtherArg(p, ri, 1, typ, result, callBuilder)
for k in j+1..<ri.len:
for k, _ in isons(ri, j+1):
var callBuilder: CallBuilder = default(CallBuilder)
genOtherArg(p, ri, k, typ, result, callBuilder)
result.add(")")
@@ -755,7 +773,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Bu
genThisArg(p, ri, j, typ, result)
inc i
elif i+1 < pat.len and pat[i+1] == '[':
var arg = ri[j].skipAddrDeref
var arg = son(ri, j).skipAddrDeref
while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg.firstSon
genArgNoParam(p, arg, result)
#result.add debugTree(arg, 0, 10)
@@ -778,7 +796,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Bu
if i - 1 >= start:
result.add(substr(pat, start, i - 1))
proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
proc genInfixCall(p: BProc, le: PNode, ri: PNode, d: var TLoc) =
var op = initLocExpr(p, ri.firstSon)
# getUniqueType() is too expensive here:
var typ = skipTypes(ri.firstSon.typ, abstractInst)
@@ -815,7 +833,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
pl.add(op.snippet)
var res = newBuilder("")
var call = initCallBuilder(res, extract(pl))
for i in 2..<ri.len:
for i, _ in isons(ri, 2):
genOtherArg(p, ri, i, typ, res, call)
fixupCall(p, le, ri, d, res, call)
@@ -836,25 +854,25 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
pl.add(op.snippet)
if ri.len > 1:
pl.add(": ")
genArg(p, ri[1], typ.n[1].sym, ri, pl)
genArg(p, ri.secondSon, typ.n.secondSon.sym, ri, pl)
start = 2
else:
if ri.len > 1:
genArg(p, ri[1], typ.n[1].sym, ri, pl)
genArg(p, ri.secondSon, typ.n.secondSon.sym, ri, pl)
pl.add(" ")
pl.add(op.snippet)
if ri.len > 2:
pl.add(": ")
genArg(p, ri[2], typ.n[2].sym, ri, pl)
for i in start..<ri.len:
genArg(p, son(ri, 2), son(typ.n, 2).sym, ri, pl)
for i, it in isons(ri, start):
if i >= typ.n.len:
internalError(p.config, ri.info, "varargs for objective C method?")
assert(typ.n[i].kind == nkSym)
var param = typ.n[i].sym
assert(son(typ.n, i).kind == nkSym)
var param = son(typ.n, i).sym
pl.add(" ")
pl.add(param.name.s)
pl.add(": ")
genArg(p, ri[i], param, ri, pl)
genArg(p, it, param, ri, pl)
if typ.returnType != nil:
if isInvalidReturnType(p.config, typ):
if ri.len > 1: pl.add(" ")
@@ -907,10 +925,10 @@ proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
We want to return early but the 'finally' section is traversed before
the 'let args = ...' statement. We exploit this to generate better
code for 'return'. ]#
result = e.len == 2 and e.firstSon.kind == nkSym and
e.firstSon.sym.name.s == "=destroy" and notYetAlive(e[1].skipAddr)
result = e.safeLen == 2 and e.firstSon.kind == nkSym and
e.firstSon.sym.name.s == "=destroy" and notYetAlive(e.secondSon.skipAddr)
proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
proc genAsgnCall(p: BProc, le: PNode, ri: PNode, d: var TLoc) =
if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
return
when defined(icDbgHash):

588
compiler/ccgexprs.nim
View File

File diff suppressed because it is too large Load Diff

11
compiler/ccgreset.nim
View File

@@ -19,18 +19,17 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
if n == nil: return
case n.kind
of nkRecList:
for i in 0..<n.len:
specializeResetN(p, accessor, n[i], typ)
for it in sons(n):
specializeResetN(p, accessor, it, typ)
of nkRecCase:
if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
let disc = n[0].sym
if (n.firstSon.kind != nkSym): internalError(p.config, n.info, "specializeResetN")
let disc = n.firstSon.sym
if disc.loc.snippet == "": fillObjectFields(p.module, typ)
if disc.loc.t == nil:
internalError(p.config, n.info, "specializeResetN()")
let discField = dotField(accessor, disc.loc.snippet)
p.s(cpsStmts).addSwitchStmt(discField):
for i in 1..<n.len:
let branch = n[i]
for branch in sonsFrom(n, 1):
assert branch.kind in {nkOfBranch, nkElse}
var caseBuilder: SwitchCaseBuilder
p.s(cpsStmts).addSwitchCase(caseBuilder):

351
compiler/ccgstmts.nim
View File

@@ -98,8 +98,8 @@ proc genVarTuple(p: BProc, n: PNode) =
if n.kind != nkVarTuple: internalError(p.config, n.info, "genVarTuple")
# if we have a something that's been captured, use the lowering instead:
for i in 0..<n.len-2:
if n[i].kind != nkSym:
for it in sonsButLast(n, 2):
if it.kind != nkSym:
genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.module.idgen, p.prc))
return
@@ -120,10 +120,9 @@ proc genVarTuple(p: BProc, n: PNode) =
initElifBranch(p.s(cpsStmts), hcrIf, hcrCond)
genLineDir(p, n)
var tup = initLocExpr(p, n[^1])
var tup = initLocExpr(p, n.lastSon)
var t = tup.t.skipTypes(abstractInst)
for i in 0..<n.len-2:
let vn = n[i]
for i, vn in isonsButLast(n, 2):
let v = vn.sym
if sfCompileTime in v.flags: continue
backendEnsureMutable v
@@ -133,7 +132,7 @@ proc genVarTuple(p: BProc, n: PNode) =
registerTraverseProc(p, v)
else:
assignLocalVar(p, vn)
initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[^1]))
initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n.lastSon))
var field = initLoc(locExpr, vn, tup.storage)
let rtup = rdLoc(tup)
let fieldName =
@@ -173,7 +172,9 @@ proc genVarTuple(p: BProc, n: PNode) =
cCast(ptrType(CPointer), cAddr(curr.loc.snippet))))
proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
proc loadInto(p: BProc, le: PNode, ri: PNode, a: var TLoc) {.inline.} =
## `le` is the DESTINATION and stays a `PNode` — it only ever reaches
## `genAsgnCall`, which keeps it a `PNode` for the alias analysis.
if ri.kind in nkCallKinds and (ri.firstSon.kind != nkSym or
ri.firstSon.sym.magic == mNone):
genAsgnCall(p, le, ri, a)
@@ -278,9 +279,9 @@ proc genGotoState(p: BProc, n: PNode) =
howManyExcepts = p.inExceptBlockLen)
p.s(cpsStmts).addGoto("BeforeRet_")
var statesCounter = lastOrd(p.config, n.firstSon.typ)
if n.len >= 2 and n[1].kind == nkIntLit:
statesCounter = getInt(n[1])
let prefix = if n.len == 3 and n[2].kind == nkStrLit: n[2].strVal.rope
if n.len >= 2 and n.secondSon.kind == nkIntLit:
statesCounter = getInt(n.secondSon)
let prefix = if n.len == 3 and son(n, 2).kind == nkStrLit: son(n, 2).strVal.rope
else: rope"STATE"
for i in 0i64..toInt64(statesCounter):
p.s(cpsStmts).addSingleSwitchCase(cIntValue(i)):
@@ -291,7 +292,7 @@ proc genBreakState(p: BProc, n: PNode, d: var TLoc) =
d = initLoc(locExpr, n, OnUnknown)
if n.firstSon.kind == nkClosure:
a = initLocExpr(p, n.firstSon[1])
a = initLocExpr(p, n.firstSon.secondSon)
let ra = a.rdLoc
d.snippet = cOp(LessThan,
subscript(
@@ -329,18 +330,18 @@ proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): Snippet =
var argBuilder = default(CallBuilder) # not init, only building params
let typ = skipTypes(call.firstSon.typ, abstractInst)
assert(typ.kind == tyProc)
for i in 1..<call.len:
for i, child in isons(call, 1):
#if it's a type we can just generate here another initializer as we are in an initializer context
if call[i].kind == nkCall and call[i].firstSon.kind == nkSym and call[i].firstSon.sym.kind == skType:
if child.kind == nkCall and child.firstSon.kind == nkSym and child.firstSon.sym.kind == skType:
res.addArgument(argBuilder):
res.add genCppInitializer(p.module, p, call[i].firstSon.sym.typ, didGenTemp)
res.add genCppInitializer(p.module, p, child.firstSon.sym.typ, didGenTemp)
else:
#We need to test for temp in globals, see: #23657
let param =
if typ[i].kind in {tyVar} and call[i].kind == nkHiddenAddr:
call[i].firstSon
if typ[i].kind in {tyVar} and child.kind == nkHiddenAddr:
child.firstSon
else:
call[i]
child
if not param.typ.isCompileTimeOnly and (param.kind != nkBracketExpr or param.typ.kind in
{tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray,
tyVarargs, tySequence, tyString, tyCstring, tyTuple}):
@@ -349,7 +350,7 @@ proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): Snippet =
genOtherArg(p, call, i, typ, res, argBuilder)
result = extract(res)
proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
proc genSingleVar(p: BProc, v: PSym; vn: PNode; value: PNode) =
if sfGoto in v.flags:
# translate 'var state {.goto.} = X' into 'goto LX':
genGotoVar(p, value)
@@ -478,19 +479,19 @@ proc genSingleVar(p: BProc, a: PNode) =
discard
else:
return
genSingleVar(p, v, a.firstSon, a[2])
genSingleVar(p, v, a.firstSon, son(a, 2))
proc genClosureVar(p: BProc, a: PNode) =
var immediateAsgn = a[2].kind != nkEmpty
var immediateAsgn = son(a, 2).kind != nkEmpty
var v: TLoc = initLocExpr(p, a.firstSon)
genLineDir(p, a)
if immediateAsgn:
loadInto(p, a.firstSon, a[2], v)
elif sfNoInit notin a.firstSon[1].sym.flags:
loadInto(p, a.firstSon, son(a, 2), v)
elif sfNoInit notin a.firstSon.secondSon.sym.flags:
constructLoc(p, v)
proc genVarStmt(p: BProc, n: PNode) =
for it in n:
for it in sons(n):
case it.kind
of nkCommentStmt: discard
of nkIdentDefs:
@@ -523,7 +524,7 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
d = getTemp(p, n.typ)
genLineDir(p, n)
let lend = getLabel(p)
for it in n.sons:
for it in sons(n):
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
if it.len == 2:
@@ -538,9 +539,9 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
if p.module.compileToCpp:
# avoid "jump to label crosses initialization" error:
p.s(cpsStmts).addScope():
expr(p, it[1], d)
expr(p, it.secondSon, d)
else:
expr(p, it[1], d)
expr(p, it.secondSon, d)
endSimpleBlock(p, scope)
if n.len > 1:
p.s(cpsStmts).addGoto(lend)
@@ -574,15 +575,15 @@ proc genReturnStmt(p: BProc, t: PNode) =
p.s(cpsStmts).addGoto("BeforeRet_")
proc genGotoForCase(p: BProc; caseStmt: PNode) =
for i in 1..<caseStmt.len:
for child in sonsFrom(caseStmt, 1):
var scope: ScopeBuilder
startSimpleBlock(p, scope)
let it = caseStmt[i]
for j in 0..<it.len-1:
if it[j].kind == nkRange:
let it = child
for label in sonsButLast(it):
if label.kind == nkRange:
localError(p.config, it.info, "range notation not available for computed goto")
return
let val = getOrdValue(it[j])
let val = getOrdValue(label)
p.s(cpsStmts).addLabel("NIMSTATE_" & $val)
genStmts(p, it.lastSon)
endSimpleBlock(p, scope)
@@ -590,11 +591,10 @@ proc genGotoForCase(p: BProc; caseStmt: PNode) =
iterator fieldValuePairs(n: PNode): tuple[memberSym, valueSym: PNode] =
assert(n.kind in {nkLetSection, nkVarSection})
for identDefs in n:
for identDefs in sons(n):
if identDefs.kind == nkIdentDefs:
let valueSym = identDefs[^1]
for i in 0..<identDefs.len-2:
let memberSym = identDefs[i]
let valueSym = identDefs.lastSon
for memberSym in sonsButLast(identDefs, 2):
yield((memberSym: memberSym, valueSym: valueSym))
proc genComputedGoto(p: BProc; n: PNode) =
@@ -602,6 +602,8 @@ proc genComputedGoto(p: BProc; n: PNode) =
# flatten the loop body because otherwise let and var sections
# wrapped inside stmt lists by inject destructors won't be recognised
# REBUILDS the statement list, so from here this proc works on
# a fresh `PNode` tree — there is nothing in the buffer corresponding to it.
let n = n.flattenStmts()
var casePos = -1
var arraySize: int = 0
@@ -637,56 +639,57 @@ proc genComputedGoto(p: BProc; n: PNode) =
p.s(cpsStmts).addField(labelsInit, ""):
p.s(cpsStmts).add(cLabelAddr("TMP" & $(id+i) & "_"))
for j in 0..<casePos:
genStmts(p, n[j])
for j, it in isons(n):
if j >= casePos: break
genStmts(p, it)
let caseStmt = n[casePos]
let caseStmt = son(n, casePos)
var a: TLoc = initLocExpr(p, caseStmt.firstSon)
let ra = a.rdLoc
# first goto:
p.s(cpsStmts).addComputedGoto(subscript(tmp, ra))
for i in 1..<caseStmt.len:
for child in sonsFrom(caseStmt, 1):
var scope: ScopeBuilder
startSimpleBlock(p, scope)
let it = caseStmt[i]
for j in 0..<it.len-1:
if it[j].kind == nkRange:
let it = child
for label in sonsButLast(it):
if label.kind == nkRange:
localError(p.config, it.info, "range notation not available for computed goto")
return
let val = getOrdValue(it[j])
let val = getOrdValue(label)
let lit = cIntLiteral(toInt64(val)+id+1)
p.s(cpsStmts).addLabel("TMP" & lit & "_")
genStmts(p, it.lastSon)
for j in casePos+1..<n.len:
genStmts(p, n[j])
for after in sonsFrom(n, casePos+1):
genStmts(p, after)
for j in 0..<casePos:
for j, before in isons(n):
if j >= casePos: break
# prevent new local declarations
# compile declarations as assignments
let it = n[j]
if it.kind in {nkLetSection, nkVarSection}:
let asgn = copyNode(it)
if before.kind in {nkLetSection, nkVarSection}:
let asgn = copyNode(before)
asgn.transitionSonsKind(nkAsgn)
asgn.sons.setLen 2
for sym, value in it.fieldValuePairs:
for sym, value in before.fieldValuePairs:
if value.kind != nkEmpty:
asgn[0] = sym
asgn[1] = value
asgn.secondSon = value
genStmts(p, asgn)
else:
genStmts(p, it)
genStmts(p, before)
var a: TLoc = initLocExpr(p, caseStmt.firstSon)
let ra = a.rdLoc
p.s(cpsStmts).addComputedGoto(subscript(tmp, ra))
endSimpleBlock(p, scope)
for j in casePos+1..<n.len:
genStmts(p, n[j])
for it in sonsFrom(n, casePos+1):
genStmts(p, it)
proc genWhileStmt(p: BProc, t: PNode) =
@@ -699,12 +702,12 @@ proc genWhileStmt(p: BProc, t: PNode) =
genLineDir(p, t)
preserveBreakIdx:
var loopBody = t[1]
var loopBody = t.secondSon
if loopBody.stmtsContainPragma(wComputedGoto) and
hasComputedGoto in CC[p.config.cCompiler].props:
# for closure support weird loop bodies are generated:
if loopBody.len == 2 and loopBody.firstSon.kind == nkEmpty:
loopBody = loopBody[1]
loopBody = loopBody.secondSon
genComputedGoto(p, loopBody)
else:
var stmt: WhileBuilder
@@ -746,7 +749,7 @@ proc genBlock(p: BProc, n: PNode, d: var TLoc) =
sym.locImpl.k = locOther
sym.positionImpl = p.breakIdx+1
# ^ IC: review this
expr(p, n[1], d)
expr(p, n.secondSon, d)
endSimpleBlock(p, scope)
proc genParForStmt(p: BProc, t: PNode) =
@@ -759,21 +762,21 @@ proc genParForStmt(p: BProc, t: PNode) =
assignLocalVar(p, t.firstSon)
#initLoc(forLoopVar.loc, locLocalVar, forLoopVar.typ, onStack)
#discard mangleName(forLoopVar)
let call = t[1]
let call = t.secondSon
assert(call.len == 4 or call.len == 5)
var rangeA = initLocExpr(p, call[1])
var rangeB = initLocExpr(p, call[2])
var rangeA = initLocExpr(p, call.secondSon)
var rangeB = initLocExpr(p, son(call, 2))
var stepNode: PNode = nil
# $n at the beginning because of #9710
if call.len == 4: # procName(a, b, annotation)
if call.safeLen == 4: # procName(a, b, annotation)
if call.firstSon.sym.name.s == "||": # `||`(a, b, annotation)
p.s(cpsStmts).addCPragma("omp " & call[3].getStr)
p.s(cpsStmts).addCPragma("omp " & son(call, 3).getStr)
else:
p.s(cpsStmts).addCPragma(call[3].getStr)
p.s(cpsStmts).addCPragma(son(call, 3).getStr)
else: # `||`(a, b, step, annotation)
stepNode = call[3]
p.s(cpsStmts).addCPragma("omp " & call[4].getStr)
stepNode = son(call, 3)
p.s(cpsStmts).addCPragma("omp " & son(call, 4).getStr)
p.breakIdx = startBlockWith(p):
if stepNode == nil:
@@ -782,7 +785,7 @@ proc genParForStmt(p: BProc, t: PNode) =
var step: TLoc = initLocExpr(p, stepNode)
initForStep(p.s(cpsStmts), forLoopVar.loc.rdLoc, rangeA.rdLoc, rangeB.rdLoc, step.rdLoc, true)
p.blocks[p.breakIdx].isLoop = true
genStmts(p, t[2])
genStmts(p, son(t, 2))
endBlockWith(p):
finishFor(p.s(cpsStmts))
@@ -909,17 +912,17 @@ proc genRaiseStmt(p: BProc, t: PNode) =
template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
rangeFormat, eqFormat: untyped) =
var x, y: TLoc
for i in 0..<b.len - 1:
for it in sonsButLast(b):
let rlabel {.inject.} = labl
if b[i].kind == nkRange:
x = initLocExpr(p, b[i].firstSon)
y = initLocExpr(p, b[i][1])
if it.kind == nkRange:
x = initLocExpr(p, it.firstSon)
y = initLocExpr(p, it.secondSon)
let ra {.inject.} = rdCharLoc(e)
let rb {.inject.} = rdCharLoc(x)
let rc {.inject.} = rdCharLoc(y)
rangeFormat
else:
x = initLocExpr(p, b[i])
x = initLocExpr(p, it)
let ra {.inject.} = rdCharLoc(e)
let rb {.inject.} = rdCharLoc(x)
eqFormat
@@ -927,15 +930,16 @@ template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
proc genCaseSecondPass(p: BProc, t: PNode, d: var TLoc,
labId, until: int): TLabel =
var lend = getLabel(p)
for i in 1..until:
for i, branch in isons(t, 1):
if i > until: break
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
p.s(cpsStmts).addLabel("LA" & $(labId + i) & "_")
if t[i].kind == nkOfBranch:
exprBlock(p, t[i][^1], d)
if branch.kind == nkOfBranch:
exprBlock(p, branch.lastSon, d)
p.s(cpsStmts).addGoto(lend)
else:
exprBlock(p, t[i].firstSon, d)
exprBlock(p, branch.firstSon, d)
result = lend
template genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
@@ -944,11 +948,12 @@ template genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
# generate a C-if statement for a Nim case statement
var res: TLabel
var labId = p.labels
for i in 1..until:
for i, branch in isons(t, 1):
if i > until: break
inc(p.labels)
let lab = "LA" & $p.labels & "_"
if t[i].kind == nkOfBranch: # else statement
genCaseGenericBranch(p, t[i], a, lab, rangeFormat, eqFormat)
if branch.kind == nkOfBranch: # else statement
genCaseGenericBranch(p, branch, a, lab, rangeFormat, eqFormat)
else:
p.s(cpsStmts).addGoto(lab)
if until < t.len-1:
@@ -964,20 +969,20 @@ template genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
template genCaseGeneric(p: BProc, t: PNode, d: var TLoc,
rangeFormat, eqFormat: untyped) =
var a: TLoc = initLocExpr(p, t.firstSon)
var lend = genIfForCaseUntil(p, t, d, t.len-1, a, rangeFormat, eqFormat)
var lend = genIfForCaseUntil(p, t, d, t.safeLen-1, a, rangeFormat, eqFormat)
fixLabel(p, lend)
proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
stringKind: TTypeKind,
branches: var openArray[Builder]) =
var x: TLoc
for i in 0..<b.len - 1:
assert(b[i].kind != nkRange)
x = initLocExpr(p, b[i])
for it in sonsButLast(b):
assert(it.kind != nkRange)
x = initLocExpr(p, it)
var j: int = 0
case b[i].kind
case it.kind
of nkStrLit..nkTripleStrLit:
j = int(hashString(p.config, b[i].strVal) and high(branches))
j = int(hashString(p.config, it.strVal) and high(branches))
of nkNilLit: j = 0
else:
assert false, "invalid string case branch node kind"
@@ -992,18 +997,18 @@ proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
# count how many constant strings there are in the case:
var strings = 0
for i in 1..<t.len:
if t[i].kind == nkOfBranch: inc(strings, t[i].len - 1)
for it in sonsFrom(t, 1):
if it.kind == nkOfBranch: inc(strings, it.len - 1)
if strings > stringCaseThreshold:
var bitMask = math.nextPowerOfTwo(strings) - 1
var branches: seq[Builder]
newSeq(branches, bitMask + 1)
var a: TLoc = initLocExpr(p, t.firstSon) # first pass: generate ifs+goto:
var labId = p.labels
for i in 1..<t.len:
for it in sonsFrom(t, 1):
inc(p.labels)
if t[i].kind == nkOfBranch:
genCaseStringBranch(p, t[i], a, "LA" & rope(p.labels) & "_",
if it.kind == nkOfBranch:
genCaseStringBranch(p, it, a, "LA" & rope(p.labels) & "_",
stringKind, branches)
else:
# else statement: nothing to do yet
@@ -1022,7 +1027,7 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
p.s(cpsStmts).add(extract(branches[j]))
p.s(cpsStmts).addBreak()
# else statement:
if t[^1].kind != nkOfBranch:
if t.lastSon.kind != nkOfBranch:
p.s(cpsStmts).addGoto("LA" & rope(p.labels) & "_")
# third pass: generate statements
var lend = genCaseSecondPass(p, t, d, labId, t.len-1)
@@ -1040,16 +1045,15 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
proc branchHasTooBigRange(b: PNode): bool =
result = false
for it in b:
for it in sons(b):
# last son is block
if (it.kind == nkRange) and
it[1].intVal - it.firstSon.intVal > RangeExpandLimit:
it.secondSon.intVal - it.firstSon.intVal > RangeExpandLimit:
return true
proc ifSwitchSplitPoint(p: BProc, n: PNode): int =
result = 0
for i in 1..<n.len:
var branch = n[i]
for i, branch in isons(n, 1):
var stmtBlock = lastSon(branch)
if stmtBlock.stmtsContainPragma(wLinearScanEnd):
result = i
@@ -1058,24 +1062,24 @@ proc ifSwitchSplitPoint(p: BProc, n: PNode): int =
result = i
proc genCaseRange(p: BProc, branch: PNode, info: var SwitchCaseBuilder) =
for j in 0..<branch.len-1:
if branch[j].kind == nkRange:
for it in sonsButLast(branch):
if it.kind == nkRange:
if hasSwitchRange in CC[p.config.cCompiler].props:
var litA = newBuilder("")
var litB = newBuilder("")
genLiteral(p, branch[j].firstSon, litA)
genLiteral(p, branch[j][1], litB)
genLiteral(p, it.firstSon, litA)
genLiteral(p, it.secondSon, litB)
p.s(cpsStmts).addCaseRange(info, extract(litA), extract(litB))
else:
var v = copyNode(branch[j].firstSon)
while v.intVal <= branch[j][1].intVal:
var v = copyNode(it.firstSon)
while v.intVal <= it.secondSon.intVal:
var litA = newBuilder("")
genLiteral(p, v, litA)
p.s(cpsStmts).addCase(info, extract(litA))
inc(v.intVal)
else:
var litA = newBuilder("")
genLiteral(p, branch[j], litA)
genLiteral(p, it, litA)
p.s(cpsStmts).addCase(info, extract(litA))
proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
@@ -1101,10 +1105,9 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
let rca = rdCharLoc(a)
p.s(cpsStmts).addSwitchStmt(rca):
var hasDefault = false
for i in splitPoint+1..<n.len:
for branch in sonsFrom(n, splitPoint+1):
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
var branch = n[i]
var caseBuilder: SwitchCaseBuilder
p.s(cpsStmts).addSwitchCase(caseBuilder):
if branch.kind == nkOfBranch:
@@ -1197,7 +1200,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
#init on locals, fixes #23306
lineCg(p, cpsLocals, "std::exception_ptr T$1_;$n", [etmp])
let fin = if t[^1].kind == nkFinally: t[^1] else: nil
let fin = if t.lastSon.kind == nkFinally: t.lastSon else: nil
p.nestedTryStmts.add((fin, false, t.kind == nkHiddenTryStmt, 0.Natural))
if t.kind == nkHiddenTryStmt:
@@ -1222,10 +1225,11 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
var ifStmt = default(IfBuilder)
var hasIf = false
var hasElse = false
while (i < t.len) and (t[i].kind == nkExceptBranch):
while i < t.len and son(t, i).kind == nkExceptBranch:
let exceptBranch = son(t, i)
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
if t[i].len == 1:
if exceptBranch.len == 1:
hasImportedCppExceptions = true
hasElse = true
# general except section:
@@ -1237,7 +1241,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
scope = initScope(p.s(cpsStmts))
# we handled the error:
linefmt(p, cpsStmts, "T$1_ = nullptr;$n", [etmp])
expr(p, t[i].firstSon, d)
expr(p, exceptBranch.firstSon, d)
linefmt(p, cpsStmts, "#popCurrentException();$n", [])
endBlockWith(p):
if hasIf:
@@ -1247,11 +1251,11 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
else:
var orExpr = newRopeAppender()
var exvar = PNode(nil)
for j in 0..<t[i].len - 1:
var typeNode = t[i][j]
if t[i][j].isInfixAs():
typeNode = t[i][j][1]
exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
for label in sonsButLast(exceptBranch):
var typeNode = label
if label.isInfixAs():
typeNode = label.secondSon
exvar = son(label, 2) # ex1 in `except ExceptType as ex1:`
assert(typeNode.kind == nkType)
if isImportedException(typeNode.typ, p.config):
hasImportedCppExceptions = true
@@ -1279,7 +1283,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
rdLoc(exvar.sym.loc), rope(etmp+1)])
# we handled the error:
linefmt(p, cpsStmts, "T$1_ = nullptr;$n", [etmp])
expr(p, t[i][^1], d)
expr(p, exceptBranch.lastSon, d)
linefmt(p, cpsStmts, "#popCurrentException();$n", [])
endBlockWith(p):
finishBranch(p.s(cpsStmts), ifStmt)
@@ -1300,46 +1304,46 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
var catchAllPresent = false
incl p.flags, noSafePoints # mark as not needing 'popCurrentException'
if hasImportedCppExceptions:
for i in 1..<t.len:
if t[i].kind != nkExceptBranch: break
for it in sonsFrom(t, 1):
if it.kind != nkExceptBranch: break
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
if t[i].len == 1:
if it.len == 1:
# general except section:
startBlockWith(p):
p.s(cpsStmts).add("catch (...) {\n")
genExceptBranchBody(t[i].firstSon)
genExceptBranchBody(it.firstSon)
endBlockWith(p):
p.s(cpsStmts).add("}\n")
catchAllPresent = true
else:
for j in 0..<t[i].len-1:
var typeNode = t[i][j]
if t[i][j].isInfixAs():
typeNode = t[i][j][1]
for label in sonsButLast(it):
var typeNode = label
if label.isInfixAs():
typeNode = label.secondSon
if isImportedException(typeNode.typ, p.config):
let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
let exvar = son(label, 2) # ex1 in `except ExceptType as ex1:`
fillLocalName(p, exvar.sym)
backendEnsureMutable exvar.sym
fillLoc(exvar.sym.locImpl, locTemp, exvar, OnStack)
startBlockWith(p):
lineCg(p, cpsStmts, "catch ($1& $2) {$n", [getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc)])
genExceptBranchBody(t[i][^1]) # exception handler body will duplicated for every type
genExceptBranchBody(it.lastSon) # exception handler body will duplicated for every type
endBlockWith(p):
p.s(cpsStmts).add("}\n")
elif isImportedException(typeNode.typ, p.config):
startBlockWith(p):
lineCg(p, cpsStmts, "catch ($1&) {$n", [getTypeDesc(p.module, t[i][j].typ)])
genExceptBranchBody(t[i][^1]) # exception handler body will duplicated for every type
lineCg(p, cpsStmts, "catch ($1&) {$n", [getTypeDesc(p.module, label.typ)])
genExceptBranchBody(it.lastSon) # exception handler body will duplicated for every type
endBlockWith(p):
p.s(cpsStmts).add("}\n")
excl p.flags, noSafePoints
discard pop(p.nestedTryStmts)
# general finally block:
if t.len > 0 and t[^1].kind == nkFinally:
if t.hasSons and t.lastSon.kind == nkFinally:
if not catchAllPresent:
startBlockWith(p):
p.s(cpsStmts).add("catch (...) {\n")
@@ -1350,7 +1354,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
var scope: ScopeBuilder
startSimpleBlock(p, scope)
genStmts(p, t[^1].firstSon)
genStmts(p, t.lastSon.firstSon)
linefmt(p, cpsStmts, "if (T$1_) std::rethrow_exception(T$1_);$n", [etmp])
endSimpleBlock(p, scope)
@@ -1360,23 +1364,23 @@ proc bodyCanRaise(p: BProc; n: PNode): bool =
result = canRaiseDisp(p, n.firstSon)
if not result:
# also check the arguments:
for i in 1 ..< n.len:
if bodyCanRaise(p, n[i]): return true
for it in sonsFrom(n, 1):
if bodyCanRaise(p, it): return true
of nkRaiseStmt:
result = true
of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
result = false
else:
for i in 0 ..< safeLen(n):
if bodyCanRaise(p, n[i]): return true
result = false
for it in sons(n):
if bodyCanRaise(p, it): return true
proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
let fin = if t[^1].kind == nkFinally: t[^1] else: nil
let fin = if t.lastSon.kind == nkFinally: t.lastSon else: nil
inc p.labels
let lab = p.labels
let hasExcept = t[1].kind == nkExceptBranch
let hasExcept = t.secondSon.kind == nkExceptBranch
if hasExcept: inc p.withinTryWithExcept
p.nestedTryStmts.add((fin, false, t.kind == nkHiddenTryStmt, Natural lab))
@@ -1390,7 +1394,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
var ifStmt = default(IfBuilder)
var scope = default(ScopeBuilder)
var isIf = false
if 1 < t.len and t[1].kind == nkExceptBranch:
if 1 < t.len and t.secondSon.kind == nkExceptBranch:
startBlockWith(p):
isIf = true
ifStmt = initIfStmt(p.s(cpsStmts))
@@ -1405,7 +1409,8 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
var innerIfStmt = default(IfBuilder)
var innerScope = default(ScopeBuilder)
var innerIsIf = false
while (i < t.len) and (t[i].kind == nkExceptBranch):
while i < t.len and son(t, i).kind == nkExceptBranch:
let exceptBranch = son(t, i)
inc p.labels
let nextExcept = p.labels
@@ -1414,7 +1419,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
var isScope = false
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
if t[i].len == 1:
if exceptBranch.len == 1:
# general except section:
startBlockWith(p):
if innerIsIf:
@@ -1424,14 +1429,14 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
innerScope = initScope(p.s(cpsStmts))
# we handled the exception, remember this:
p.s(cpsStmts).addAssignment(cDeref("nimErr_"), NimFalse)
expr(p, t[i].firstSon, d)
expr(p, exceptBranch.firstSon, d)
else:
if not innerIsIf:
innerIsIf = true
innerIfStmt = initIfStmt(p.s(cpsStmts))
var orExpr: Snippet = ""
for j in 0..<t[i].len - 1:
assert(t[i][j].kind == nkType)
for label in sonsButLast(exceptBranch):
assert(label.kind == nkType)
var excVal = cCall(cgsymValue(p.module, "nimBorrowCurrentException"))
let member =
if p.module.compileToCpp:
@@ -1440,13 +1445,13 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
dotField(derefField(excVal, "Sup"), "m_type")
var branch: Snippet = ""
if optTinyRtti in p.config.globalOptions:
let checkFor = $getObjDepth(t[i][j].typ)
let checkFor = $getObjDepth(label.typ)
branch = cCall(cgsymValue(p.module, "isObjDisplayCheck"),
member,
checkFor,
$genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config))))
$genDisplayElem(MD5Digest(hashType(label.typ, p.config))))
else:
let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
let checkFor = genTypeInfoV1(p.module, label.typ, label.info)
branch = cCall(cgsymValue(p.module, "isObj"),
member,
checkFor)
@@ -1459,7 +1464,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
initElifBranch(p.s(cpsStmts), innerIfStmt, orExpr)
# we handled the exception, remember this:
p.s(cpsStmts).addAssignment(cDeref("nimErr_"), NimFalse)
expr(p, t[i][^1], d)
expr(p, exceptBranch.lastSon, d)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "popCurrentException"))
p.s(cpsStmts).addLabel("LA" & $nextExcept & "_")
@@ -1480,19 +1485,20 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
else:
finishScope(p.s(cpsStmts), scope)
if i < t.len and t[i].kind == nkFinally:
if i < t.len and son(t, i).kind == nkFinally:
let finallyBranch = son(t, i)
var finallyScope: ScopeBuilder
startSimpleBlock(p, finallyScope)
if not bodyCanRaise(p, t[i].firstSon):
if not bodyCanRaise(p, finallyBranch.firstSon):
# this is an important optimization; most destroy blocks are detected not to raise an
# exception and so we help the C optimizer by not mutating nimErr_ pointlessly:
genStmts(p, t[i].firstSon)
genStmts(p, finallyBranch.firstSon)
else:
# pretend we did handle the error for the safe execution of the 'finally' section:
p.procSec(cpsLocals).addVar(kind = Local, name = "oldNimErrFin" & $lab & "_", typ = NimBool)
p.s(cpsStmts).addAssignment("oldNimErrFin" & $lab & "_", cDeref("nimErr_"))
p.s(cpsStmts).addAssignment(cDeref("nimErr_"), NimFalse)
genStmts(p, t[i].firstSon)
genStmts(p, finallyBranch.firstSon)
# this is correct for all these cases:
# 1. finally is run during ordinary control flow
# 2. finally is run after 'except' block handling: these however set the
@@ -1583,7 +1589,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
nonQuirkyIf = initIfStmt(p.s(cpsStmts))
initElifBranch(p.s(cpsStmts), nonQuirkyIf, removeSinglePar(
cOp(Equal, dotField(safePoint, "status"), cIntValue(0))))
let fin = if t[^1].kind == nkFinally: t[^1] else: nil
let fin = if t.lastSon.kind == nkFinally: t.lastSon else: nil
p.nestedTryStmts.add((fin, quirkyExceptions, t.kind == nkHiddenTryStmt, 0.Natural))
expr(p, t.firstSon, d)
var quirkyIf = default(IfBuilder)
@@ -1596,7 +1602,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
initElseBranch(p.s(cpsStmts), nonQuirkyIf)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "popSafePoint"))
genRestoreFrameAfterException(p)
elif 1 < t.len and t[1].kind == nkExceptBranch:
elif 1 < t.len and t.secondSon.kind == nkExceptBranch:
startBlockWith(p):
quirkyIf = initIfStmt(p.s(cpsStmts))
initElifBranch(p.s(cpsStmts), quirkyIf,
@@ -1609,10 +1615,11 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
var i = 1
var exceptIf = default(IfBuilder)
var exceptIfInited = false
while (i < t.len) and (t[i].kind == nkExceptBranch):
while i < t.len and son(t, i).kind == nkExceptBranch:
let exceptBranch = son(t, i)
# bug #4230: avoid false sharing between branches:
if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
if t[i].len == 1:
if exceptBranch.len == 1:
# general except section:
var scope = default(ScopeBuilder)
startBlockWith(p):
@@ -1622,7 +1629,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
scope = initScope(p.s(cpsStmts))
if not quirkyExceptions:
p.s(cpsStmts).addFieldAssignment(safePoint, "status", cIntValue(0))
expr(p, t[i].firstSon, d)
expr(p, exceptBranch.firstSon, d)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "popCurrentException"))
endBlockWith(p):
if exceptIfInited:
@@ -1631,8 +1638,8 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
finishScope(p.s(cpsStmts), scope)
else:
var orExpr: Snippet = ""
for j in 0..<t[i].len - 1:
assert(t[i][j].kind == nkType)
for label in sonsButLast(exceptBranch):
assert(label.kind == nkType)
var excVal = cCall(cgsymValue(p.module, "nimBorrowCurrentException"))
let member =
if p.module.compileToCpp:
@@ -1641,13 +1648,13 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
dotField(derefField(excVal, "Sup"), "m_type")
var branch: Snippet = ""
if optTinyRtti in p.config.globalOptions:
let checkFor = $getObjDepth(t[i][j].typ)
let checkFor = $getObjDepth(label.typ)
branch = cCall(cgsymValue(p.module, "isObjDisplayCheck"),
member,
checkFor,
$genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config))))
$genDisplayElem(MD5Digest(hashType(label.typ, p.config))))
else:
let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
let checkFor = genTypeInfoV1(p.module, label.typ, label.info)
branch = cCall(cgsymValue(p.module, "isObj"),
member,
checkFor)
@@ -1663,7 +1670,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
initElifBranch(p.s(cpsStmts), exceptIf, orExpr)
if not quirkyExceptions:
p.s(cpsStmts).addFieldAssignment(safePoint, "status", cIntValue(0))
expr(p, t[i][^1], d)
expr(p, exceptBranch.lastSon, d)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "popCurrentException"))
endBlockWith(p):
finishBranch(p.s(cpsStmts), exceptIf)
@@ -1681,11 +1688,12 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
else:
finishBranch(p.s(cpsStmts), quirkyIf)
finishIfStmt(p.s(cpsStmts), quirkyIf)
if i < t.len and t[i].kind == nkFinally:
if i < t.len and son(t, i).kind == nkFinally:
let finallyBranch = son(t, i)
p.finallySafePoints.add(safePoint)
var finallyScope: ScopeBuilder
startSimpleBlock(p, finallyScope)
genStmts(p, t[i].firstSon)
genStmts(p, finallyBranch.firstSon)
# pretend we handled the exception in a 'finally' so that we don't
# re-raise the unhandled one but instead keep the old one (it was
# not popped either):
@@ -1710,8 +1718,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
if isAsmStmt: 1 # first son is pragmas
else: 0
for i in offset..<t.len:
let it = t[i]
for it in sonsFrom(t, offset):
case it.kind
of nkStrLit..nkTripleStrLit:
res.add(it.strVal)
@@ -1758,9 +1765,9 @@ proc genAsmStmt(p: BProc, t: PNode) =
var asmSyntax = ""
if (let p = t.firstSon; p.kind == nkPragma):
for i in p:
for i in sons(p):
if whichPragma(i) == wAsmSyntax:
asmSyntax = i[1].strVal
asmSyntax = i.secondSon.strVal
if asmSyntax != "" and
not (
@@ -1791,10 +1798,10 @@ proc determineSection(n: PNode): TCFileSection =
proc genEmit(p: BProc, t: PNode) =
var s = newRopeAppender()
genAsmOrEmitStmt(p, t[1], false, s)
genAsmOrEmitStmt(p, t.secondSon, false, s)
if p.prc == nil:
# top level emit pragma?
let section = determineSection(t[1])
let section = determineSection(t.secondSon)
genCLineDir(p.module.s[section], t.info, p.config)
p.module.s[section].add(s)
else:
@@ -1837,9 +1844,9 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
if dotExpr.kind == nkCheckedFieldExpr: dotExpr = dotExpr.firstSon
var a = initLocExpr(p, e.firstSon)
var tmp: TLoc = getTemp(p, a.t)
expr(p, e[1], tmp)
expr(p, e.secondSon, tmp)
if p.inUncheckedAssignSection == 0:
let field = dotExpr[1].sym
let field = dotExpr.secondSon.sym
genDiscriminantCheck(p, a, tmp, dotExpr.firstSon.typ, field)
message(p.config, e.info, warnCaseTransition)
genAssignment(p, a, tmp, {})
@@ -1847,7 +1854,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
if e.firstSon.kind == nkSym and sfGoto in e.firstSon.sym.flags:
genLineDir(p, e)
genGotoVar(p, e[1])
genGotoVar(p, e.secondSon)
elif optFieldCheck in p.options and isDiscriminantField(e.firstSon):
genLineDir(p, e)
asgnFieldDiscriminant(p, e)
@@ -1856,13 +1863,13 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
# nimsso: s[i] = c → nimStrPutV3(&s, i, c) (handles COW internally)
genLineDir(p, e)
var base = initLocExpr(p, e.firstSon.firstSon)
var idx = initLocExpr(p, e.firstSon[1])
var rhs = initLocExpr(p, e[1])
var idx = initLocExpr(p, e.firstSon.secondSon)
var rhs = initLocExpr(p, e.secondSon)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimStrPutV3"),
byRefLoc(p, base), rdLoc(idx), rdCharLoc(rhs))
else:
let le = e.firstSon
let ri = e[1]
let ri = e.secondSon
var a: TLoc = initLoc(locNone, le, OnUnknown)
discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs), dkVar)
a.flags.incl(lfEnforceDeref)

11
compiler/ccgtrav.nim
View File

@@ -31,19 +31,18 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
if n == nil: return
case n.kind
of nkRecList:
for i in 0..<n.len:
genTraverseProc(c, accessor, n[i], typ)
for it in sons(n):
genTraverseProc(c, accessor, it, typ)
of nkRecCase:
if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
if (n.firstSon.kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
var p = c.p
let disc = n[0].sym
let disc = n.firstSon.sym
if disc.loc.snippet == "": fillObjectFields(c.p.module, typ)
if disc.loc.t == nil:
internalError(c.p.config, n.info, "genTraverseProc()")
let discField = dotField(accessor, disc.loc.snippet)
p.s(cpsStmts).addSwitchStmt(discField):
for i in 1..<n.len:
let branch = n[i]
for branch in sonsFrom(n, 1):
assert branch.kind in {nkOfBranch, nkElse}
var caseBuilder: SwitchCaseBuilder
p.s(cpsStmts).addSwitchCase(caseBuilder):

73
compiler/ccgtypes.nim
View File

@@ -59,10 +59,10 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string =
result = "_Z" # Common prefix in Itanium ABI
var params = ""
var staticLists = ""
if s.typ.len > 1: #we dont care about the return param
for i in 1..<s.typ.len:
if s.typ[i].isNil: continue
params.add encodeType(m, s.typ[i], staticLists)
if s.typ.paramsLen > 0: # we dont care about the return param
for _, pt in paramTypes(s.typ):
if pt.isNil: continue
params.add encodeType(m, pt, staticLists)
result.add encodeSym(m, s, makeUnique, staticLists)
result.add params
@@ -311,7 +311,7 @@ proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
var rettype = typ
var isAllowedCall = true
if isProc:
rettype = rettype[0]
rettype = rettype.returnType
isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall}
if rettype == nil or (isAllowedCall and
getSize(conf, rettype) > conf.target.floatSize*3):
@@ -480,7 +480,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TypeDescKind
of tySequence:
let sig = hashType(t, m.config)
if optSeqDestructors in m.config.globalOptions:
if skipTypes(etB[0], typedescInst).kind == tyEmpty:
if skipTypes(etB.elementType, typedescInst).kind == tyEmpty:
internalError(m.config, "cannot map the empty seq type to a C type")
result = cacheGetType(m.forwTypeCache, sig)
@@ -524,7 +524,7 @@ proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) =
if result == "":
discard getTypeDescAux(m, t, check, dkVar)
else:
let dataTyp = getTypeDescAux(m, t.skipTypes(abstractInst)[0], check, dkVar)
let dataTyp = getTypeDescAux(m, t.skipTypes(abstractInst).elementType, check, dkVar)
m.s[cfsTypes].addSimpleStruct(m, name = result & "_Content", baseType = ""):
m.s[cfsTypes].addField(name = "cap", typ = NimInt)
m.s[cfsTypes].addField(name = "data",
@@ -598,10 +598,10 @@ proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params
rettype = runtimeFormat(rettype.replace("'0", "$1"), [getTypeDescAux(m, t.returnType, check, dkResult)])
var types, names, args: seq[string] = @[]
if not isCtor:
var this = t.n[1].sym
var this = t.n.secondSon.sym
backendEnsureMutable this
fillParamName(m, this)
fillLoc(this.locImpl, locParam, t.n[1],
fillLoc(this.locImpl, locParam, t.n.secondSon,
this.paramStorageLoc)
if this.typ.kind == tyPtr:
this.locImpl.snippet = "this"
@@ -611,9 +611,9 @@ proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params
types.add getTypeDescWeak(m, this.typ, check, dkParam)
let firstParam = if isCtor: 1 else: 2
for i in firstParam..<t.n.len:
if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genMemberProcParams")
var param = t.n[i].sym
for it in sonsFrom(t.n, firstParam):
if it.kind != nkSym: internalError(m.config, t.n.info, "genMemberProcParams")
var param = it.sym
var descKind = dkParam
if optByRef in param.options:
if param.typ.kind == tyGenericInst:
@@ -623,7 +623,7 @@ proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params
var typ, name: string
backendEnsureMutable param
fillParamName(m, param)
fillLoc(param.locImpl, locParam, t.n[i],
fillLoc(param.locImpl, locParam, it,
param.paramStorageLoc)
if ccgIntroducedPtr(m.config, param, t.returnType) and descKind == dkParam:
typ = getTypeDescWeak(m, param.typ, check, descKind) & "*"
@@ -668,9 +668,9 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
rettype = getTypeDescWeak(m, t.returnType, check, dkResult)
var paramBuilder: ProcParamBuilder
params.addProcParams(paramBuilder):
for i in 1..<t.n.len:
if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
var param = t.n[i].sym
for child in sonsFrom(t.n, 1):
if child.kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
var param = child.sym
# The hidden closure environment param (`:envP`) is not a real C parameter:
# the environment is passed via the trailing `ClE_0` (added below) and
# `closureSetup` materialises `:envP` as a local cast of it. In a from-source
@@ -692,7 +692,7 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
if isCompileTimeOnly(param.typ): continue
backendEnsureMutable param
fillParamName(m, param)
fillLoc(param.locImpl, locParam, t.n[i],
fillLoc(param.locImpl, locParam, child,
param.paramStorageLoc)
if isClosureEnv: continue # name/loc filled, but not part of the C signature
var typ: Rope
@@ -715,7 +715,7 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
# need to pass hidden parameter:
params.addParam(paramBuilder, name = param.locImpl.snippet & "Len_" & $j, typ = NimInt)
inc(j)
arr = arr[0].skipTypes({tySink})
arr = arr.elementType.skipTypes({tySink})
if t.returnType != nil and isInvalidReturnType(m.config, t):
var arr = t.returnType
var typ: Snippet
@@ -767,7 +767,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
check: var IntSet; result: var Builder; unionPrefix = "") =
case n.kind
of nkRecList:
for ni in n.sons:
for ni in sons(n):
genRecordFieldsAux(m, ni, rectype, check, result, unionPrefix)
of nkRecCase:
if n.firstSon.kind != nkSym: internalError(m.config, n.info, "genRecordFieldsAux")
@@ -775,10 +775,10 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
# prefix mangled name with "_U" to avoid clashes with other field names,
# since identifiers are not allowed to start with '_'
var unionBody = newBuilder("")
for i in 1..<n.len:
case n[i].kind
for i, it in isons(n, 1):
case it.kind
of nkOfBranch, nkElse:
let k = lastSon(n[i])
let k = lastSon(it)
if k.kind != nkSym:
let structName = "_" & mangleRecFieldName(m, n.firstSon.sym) & "_" & $i
var a = newBuilder("")
@@ -915,7 +915,7 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType =
result = typ[idx]
for i in 1..stars:
if result != nil and result.kidsLen > 0:
result = if result.kind == tyGenericInst: result[FirstGenericParamAt]
result = if result.kind == tyGenericInst: result.firstGenericParam
else: result.elemType
proc getOpenArrayDesc(m: BModule; t: PType, check: var IntSet; kind: TypeDescKind): Rope =
@@ -1075,7 +1075,7 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes
let owner = hashOwner(t.sym)
if not gDebugInfo.hasEnum(t.sym.name.s, t.sym.info.line, owner):
var vals: seq[(string, int)] = @[]
for son in t.n.sons:
for son in sons(t.n):
assert(son.kind == nkSym)
let field = son.sym
vals.add((field.name.s, field.position.int))
@@ -1267,7 +1267,7 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool
var check = initIntSet()
fillBackendName(m, prc)
backendEnsureMutable prc
fillLoc(prc.locImpl, locProc, prc.ast[namePos], OnUnknown)
fillLoc(prc.locImpl, locProc, son(prc.ast, namePos), OnUnknown)
var memberOp = "#." #only virtual
var typ: PType
if isCtor:
@@ -1321,7 +1321,7 @@ proc genProcHeader(m: BModule; prc: PSym; result: var Builder; visibility: var D
var check = initIntSet()
fillBackendName(m, prc)
backendEnsureMutable prc
fillLoc(prc.locImpl, locProc, prc.ast[namePos], OnUnknown)
fillLoc(prc.locImpl, locProc, son(prc.ast, namePos), OnUnknown)
var rettype: Snippet = ""
var desc = newBuilder("")
genProcParams(m, prc.typ, rettype, desc, check, true, false)
@@ -1462,7 +1462,7 @@ proc discriminatorTableName(m: BModule; objtype: PType, d: PSym): Rope =
# bugfix: we need to search the type that contains the discriminator:
var objtype = objtype.skipTypes(abstractPtrs)
while lookupInRecord(objtype.n, d.name) == nil:
objtype = objtype[0].skipTypes(abstractPtrs)
objtype = objtype.baseClass.skipTypes(abstractPtrs)
if objtype.sym == nil:
internalError(m.config, d.info, "anonymous obj with discriminator")
result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)]
@@ -1552,23 +1552,22 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope;
else:
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
for i in 1..<n.len:
var b = n[i] # branch
for b in sonsFrom(n, 1):
var tmp2 = getNimNode(m)
genObjectFields(m, typ, origType, lastSon(b), tmp2, info)
case b.kind
of nkOfBranch:
if b.len < 2:
internalError(m.config, b.info, "genObjectFields; nkOfBranch broken")
for j in 0..<b.len - 1:
if b[j].kind == nkRange:
var x = toInt(getOrdValue(b[j].firstSon))
var y = toInt(getOrdValue(b[j][1]))
for label in sonsButLast(b):
if label.kind == nkRange:
var x = toInt(getOrdValue(label.firstSon))
var y = toInt(getOrdValue(label.secondSon))
while x <= y:
m.s[cfsTypeInit3].addSubscriptAssignment(tmp, cIntValue(x), cAddr(tmp2))
inc(x)
else:
m.s[cfsTypeInit3].addSubscriptAssignment(tmp, cIntValue(getOrdValue(b[j])), cAddr(tmp2))
m.s[cfsTypeInit3].addSubscriptAssignment(tmp, cIntValue(getOrdValue(label)), cAddr(tmp2))
of nkElse:
m.s[cfsTypeInit3].addSubscriptAssignment(tmp, cIntValue(L), cAddr(tmp2))
else: internalError(m.config, n.info, "genObjectFields(nkRecCase)")
@@ -1862,7 +1861,7 @@ proc getObjDepth(t: PType): int16 =
result = -1
while x != nil:
x = skipTypes(x, skipPtrs)
x = x[0]
x = x.baseClass
inc(result)
proc genDisplayElem(d: MD5Digest): uint32 =
@@ -1878,7 +1877,7 @@ proc genDisplay(result: var Builder, m: BModule; t: PType, depth: int) =
while x != nil:
x = skipTypes(x, skipPtrs)
seqs[i] = cIntValue(genDisplayElem(MD5Digest(hashType(x, m.config))))
x = x[0]
x = x.baseClass
inc i
var arr: StructInitializer
@@ -2290,7 +2289,7 @@ proc genTypeInfo*(config: ConfigRef, m: BModule; t: PType; info: TLineInfo): Rop
proc retrieveSym(n: PNode): PSym =
case n.kind
of nkPostfix: result = retrieveSym(n[1])
of nkPostfix: result = retrieveSym(n.secondSon)
of nkPragmaExpr, nkTypeDef: result = retrieveSym(n.firstSon)
of nkSym: result = n.sym
else: result = nil

39
compiler/ccgutils.nim
View File

@@ -22,13 +22,13 @@ proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
case n.kind
of nkStmtList:
result = nil
for i in 0..<n.len:
result = getPragmaStmt(n[i], w)
for it in sons(n):
result = getPragmaStmt(it, w)
if result != nil: break
of nkPragma:
result = nil
for i in 0..<n.len:
if whichPragma(n[i]) == w: return n[i]
for it in sons(n):
if whichPragma(it) == w: return it
else:
result = nil
@@ -92,7 +92,7 @@ proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
result = true
elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
result = true # requested anyway
elif (tfFinal in pt.flags) and (pt[0] == nil):
elif (tfFinal in pt.flags) and (pt.baseClass == nil):
result = false # no need, because no subtyping possible
else:
result = true # ordinary objects are always passed by reference,
@@ -113,20 +113,12 @@ proc encodeName*(name: string): string =
proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
result = if name == "": s.name.s else: name
# keep backend-minted ids out of the `_u` namespace; their item counter
# restarts at 0 and would collide with loaded symbols' ids
# restarts at 0 and would collide with loaded symbols' ids. Which integer
# identifies such a symbol is decided ONCE, in `astdef.backendMintedDisamb`,
# shared with `mangleProcNameExt` and `ast2nif.toNifSymName`.
if s.itemId.isBackendMinted:
result.add "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# A backend-minted sym whose `disamb` is content-derived (setHookDisamb gave
# it HookDisambBit) — e.g. the `rttiDestroy` wrapper. Its `itemId.item` is a
# PER-PROCESS backend counter, so using it makes the C name diverge across
# the emit-everywhere processes: the type's RTTI table (emit-everywhere,
# merge-deduped) ends up referencing one process's `_c<item>` while the
# wrapper is defined with another's -> undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes, so use it.
result.add $s.disamb
else:
result.add $s.itemId.item
result.add $backendMintedDisamb(s)
else:
result.add "_u"
# Mirror `mangleProcNameExt`: use the per-(module,name) `disamb`, NOT
@@ -156,7 +148,7 @@ proc encodeType*(m: BModule; t: PType; staticLists: var string): string =
of tyObject, tyEnum, tyDistinct, tyUserTypeClass, tyGenericParam:
result = encodeSym(m, t.sym)
of tyGenericInst, tyUserTypeClassInst, tyGenericBody:
result = encodeName(t[0].sym.name.s)
result = encodeName(t.genericHead.sym.name.s)
result.add "I"
for i in 1..<t.len - 1:
result.add encodeType(m, t[i], staticLists)
@@ -168,8 +160,7 @@ proc encodeType*(m: BModule; t: PType; staticLists: var string): string =
of tySequence: encodeName("seq")
else: encodeName(kindName)
result.add "I"
for i in 0..<t.len:
let s = t[i]
for s in kids(t):
if s.isNil: continue
result.add encodeType(m, s, staticLists)
result.add "E"
@@ -180,12 +171,12 @@ proc encodeType*(m: BModule; t: PType; staticLists: var string): string =
raiseAssert "unreachable"
of tyRange:
var val = "range_"
if t.n[0].typ.kind in {tyFloat..tyFloat128}:
val.addFloat t.n[0].floatVal
if t.n.firstSon.typ.kind in {tyFloat..tyFloat128}:
val.addFloat t.n.firstSon.floatVal
val.add "_"
val.addFloat t.n[1].floatVal
val.addFloat t.n.secondSon.floatVal
else:
val.add $t.n[0].intVal & "_" & $t.n[1].intVal
val.add $t.n.firstSon.intVal & "_" & $t.n.secondSon.intVal
result = encodeName(val)
of tyString..tyUInt64, tyPointer, tyBool, tyChar, tyVoid, tyAnything, tyNil, tyEmpty:
result = encodeName(kindName)

455
compiler/cgen.nim
View File

@@ -16,7 +16,7 @@ import
rodutils, renderer, cgendata, aliases,
lowerings, lineinfos, pathutils, transf,
injectdestructors, astmsgs, modulepaths, pushpoppragmas,
mangleutils, cbuilderbase, modulegraphs
mangleutils, cbuilderbase, modulegraphs, icprof
from expanddefaults import caseObjDefaultBranch
from ast2nif import globalName, toNifFilename, icNifTypeName
@@ -67,28 +67,51 @@ proc addForwardedProc(m: BModule, prc: PSym) =
proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef; idgen: IdGenerator): BModule
proc getCFile*(m: BModule): AbsoluteFile
proc ownerModule(m: BModule; s: PSym): BModule =
## The BModule of `s`'s own module, created on demand. A NIF backend loads
## modules lazily, so the owner may have no BModule yet even though the symbol
## resolved.
var ms = getModule(s)
registerModule m.g.graph, ms
if ms.position >= m.g.mods.len:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
else:
result = m.g.mods[ms.position]
if result == nil:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
proc findPendingModule(m: BModule, s: PSym): BModule =
# TODO fixme
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Per-module backend codegen: only module M (`m`) is emitted in this
# process, so every demanded definition — whether a normal proc owned by
# another (here unwritten) module or a minted instance/hook — is emitted
# into M's TU. Definitions owned elsewhere are emitted again by their own
# module's cg process; the merge stage keeps one per C name and turns the
# rest into prototypes (which already live in the unmarked protos section).
# Per-module backend codegen. `m.g.icEmitted` is the set of modules THIS
# process writes a TU for, so it — not the identity of whichever TU happened
# to demand `s` — decides where the definition goes:
#
# * owner in `icEmitted`: this process is writing that module's TU, so the
# definition belongs in it and nowhere else. That is the ordinary
# whole-program routing below, and honouring it is what lets one process
# emit SEVERAL modules without their definitions collapsing into the first
# TU to ask for them. With the set at its current size of one, the owner
# IS `m` and this returns exactly what the old unconditional `return m`
# did — the point of the branch is that it stops being a special case.
#
# * owner elsewhere: the module is not written in this process, so the
# definition has nowhere else to go and is emitted here as well
# (emit-everywhere). The process that owns it emits it too; `merge` keeps
# one per C name and turns the rest into prototypes, which already live in
# the unmarked protos section.
#
# `getModule` walks the owner chain and yields nil if it never reaches a
# module (backend-minted symbols can be parented outside one), which is a
# definition with no owning TU: emit it here.
let ms = getModule(s)
if ms != nil and ms.kind == skModule and m.g.icEmitted.contains(ms.position):
return ownerModule(m, s)
return m
if m.config.symbolFiles == v2Sf or optCompress in m.config.globalOptions:
let ms = s.itemId.module #getModule(s)
result = m.g.mods[ms]
elif m.config.cmd in {cmdNifC, cmdM}:
var ms = getModule(s)
registerModule m.g.graph, ms
if ms.position >= m.g.mods.len:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
else:
result = m.g.mods[ms.position]
if result == nil:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
result = ownerModule(m, s)
else:
var ms = getModule(s)
result = m.g.mods[ms.position]
@@ -113,57 +136,151 @@ proc icNifName(m: BModule; t: PType): string =
result = ""
proc signatureHasMetaType*(t: PType; depth: int = 0): bool =
## Whether a routine signature mentions a compile-time/meta element type
## (`typed`/`untyped` — e.g. `echo`'s `varargs[typed]` — typedesc, static,
## generic param). Such routines are expanded at their call sites and never
## emitted standalone, so the per-module owned-routine seeding must skip them
## (`getTypeDescAux(tyTyped)` otherwise). `tfHasMeta` alone misses the varargs
## element case, hence the explicit scan.
result = false
if t == nil or depth > 8: return false
if t.kind == tyGenericBody:
# The uninstantiated template carried as a `tyGenericInst`'s first child
# always mentions its `tyGenericParam` placeholders, but the instance
# itself is fully concrete (e.g. `var CountTable[SigHash]`). Descending
# here would wrongly flag every routine with a generic-instance parameter
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind == tyStatic:
# A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in
# `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise
# fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal
# runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter
# (no bound value, `t.n == nil`) is meta. Without this, every routine whose
# signature touches a `static`-parameterized generic instance (the bulk of
# the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up
# an undefined reference at link (mirrors the tyGenericBody case above).
return t.n == nil
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc ownsRuntimeRoutine*(s: PSym; modPos: int): bool =
## A concrete, non-generic, runtime routine with a real body, OWNED by the
## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
## routine called only from other modules is still emitted by somebody) and
## the `lower` stage's owned-routine enumeration, so both stages see exactly
## the same set. The exclusions:
## - nested/closure procs (owner is a proc, not a module): emitted via their
## enclosing routine's lambda-lifting, never standalone;
## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge;
## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures:
## not real codegen targets.
## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into
## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never
## per module. A dispatcher is a `copySym` clone of the method that shares the
## method's body sub-tree (incl. its closure iterator); transforming it here
## would lambda-lift that SHARED iterator a SECOND time under a different owner
## identity, baking a conflicting `up` field → "up references do not agree"
## (the divergence is impossible in non-IC, where the dispatcher body is empty
## at lift time). So a dispatcher is never an owned runtime routine.
## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
## iterator, which is expanded at each call site) and must be emitted by its
## owner — else a cross-module `for` over it links to nothing.
##
## Generic INSTANCES (`sfFromGeneric`) are NEVER an owned runtime routine — not
## in `cg` and not in the `lower` stage. They are demanded by the backend's
## emit-everywhere path and deduped by `merge` (content C name); the frontend
## materialises them through the `(offer)` mechanism. The `lower` stage must
## not transform an instance: a not-fully-concrete instance (a closure factory
## over a `static` param, or a `$`/`=` op instance whose body resolves only at
## its further-specialised use sites) still carries unresolved overload choices
## and crashes `transformBody` (empty-`namePos` lambda, nil-typed const-fold).
s.itemId.module == modPos and
(s.kind in {skProc, skFunc, skConverter, skMethod} or
(s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
sfFromGeneric notin s.flags and
sfDispatcher notin s.flags and
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
son(s.ast, genericParamsPos).kind == nkEmpty
# NOTE: an `nkEmpty` body is NOT a disqualifier. A concrete, owned, non-
# forward/-importc/-magic routine whose body folds to nothing is still a real
# definition the owner must emit (`void f(void){}`), exactly as whole-program
# cgen does — else a cross-module caller links to nothing. This bites e.g.
# Nimbus' `extras.incInternalErrors`, a plain `proc` whose sole statement is a
# metrics-counter `.inc()` that the `metrics` library expands to a no-op when
# the importing tool (ncli) builds with `-u:metrics`; the body is then a bare
# `nkEmpty`, but `state_transition_epoch` still calls it. Forward declarations
# (the other empty-body case) carry `sfForward` and are excluded above.
proc bodyIsSeededByItsOwner(prc: PSym): bool =
## Whether SOME module's `cg` is guaranteed to emit `prc`'s body on its own,
## without this TU asking for it. There are exactly two seeders in the
## per-module backend, and this enumerates them:
##
## * `nifbackend.generateCodeForModule` walks its module's index and
## `requestProcDef`s every `ownsRuntimeRoutine` — the SAME predicate the
## `lower` stage uses to decide what it transforms into that module's
## `.t.bif`. So asking it about `prc`'s OWN defining module answers
## "will that module's cg seed this?".
## * `nifbackend.emitMethodDispatchers` synthesizes every method dispatcher
## into the MAIN TU. A dispatcher is a `copySym` clone that no module's
## index enumerates, so the first rule cannot see it.
##
## Anything else — a generic instance, a synthesized hook, a nested routine
## (emitted as part of its enclosing routine's lambda-lifted body), an inline
## iterator (expanded at each call site) — is seeded by nobody. Those are
## emitted by EVERY demander and `merge` keeps one per content-addressed C
## name. That is the single default, and it is the safe direction: emitting a
## body twice costs a merge dedup, while emitting it nowhere is a link error.
##
## A BACKEND-MINTED routine (a hook or nested proc that lambda-lifting /
## `injectDestructorCalls` created during `lower`) exists in no module's semmed
## NIF: it is written into the `.t.bif` of every module that references it,
## re-homed there with `@bk`. Its `itemId.module` therefore names whichever
## `.t.bif` it was read from rather than a module that seeds it, so it must not
## be routed through the ownership question at all.
if isBackendMinted(prc.itemId): return false
result = sfDispatcher in prc.flags or
ownsRuntimeRoutine(prc, prc.itemId.module)
proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
## Per-module backend codegen is concerned with ONE module: it emits the
## bodies of the routines that module OWNS (its own top-level defs) and only
## *prototypes* a routine owned by another module — that routine's body is
## emitted by its own module's `cg` process, and the merge stage's DCE prunes
## whatever ends up globally dead. The funnel where the main module re-emitted
## its entire transitive closure (≈1.8 GB, a 56 MB `.c.nif`) is exactly this
## rule being absent.
## Whether the translation unit `m` emits `prc`'s BODY, as opposed to only a
## prototype for a body some other `cg` process emits. The funnel where the
## main module re-emitted its entire transitive closure (~1.8 GB, a 56 MB
## `.c.nif`) is exactly this rule being absent.
##
## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
## owning-module top-level — they are minted on demand — so each demander emits
## them and the merge stage deduplicates by their content-addressed C name.
## `m` is the TU the body would go INTO — `findPendingModule`'s answer — not
## the one that demanded it. The two were the same module for as long as a `cg`
## process wrote exactly one TU, and asking with the demander was harmless.
## With a batch they differ, and asking with the demander is the bug: a
## definition routed to its owner inside the batch was marked declared there
## and then emitted by nobody, since the demander is not the owner and the
## owner never gets asked again (18 undefined symbols at link, batch size 4).
##
## A NESTED routine is not emitted on its own: it is lambda-lifted and emitted
## as part of its ENCLOSING routine's body, into the same TU. So the decision
## must follow the OUTERMOST enclosing routine (the one directly under the
## module — `skipGenericOwner` stops at a generic *instance*, not its
## originating generic), never the nested symbol's own identity. Otherwise a
## nested proc whose enclosing is a generic instance (content-addressed,
## emitted by every demander) — e.g. nim-serialization's per-field `readField`
## inside the `makeFieldReadersTable[R,W]` instance, whose address fills the
## returned table — is gated out (its own `itemId.module` is the minting module
## and its disamb is a plain counter), so the enclosing's lift degrades it to a
## prototype and its body lands in no TU → undefined at link.
## The decision is a lookup against `bodyIsSeededByItsOwner`, i.e. against the
## very predicates that drive the seeding, rather than a re-derivation from
## symbol ancestry. Re-derivation is what made this function a five-clause
## tower and the source of a run of "emitted by nobody" / "two hooks on one C
## name" bugs: the walk answered a question about who WILL emit by inspecting
## who DECLARED, and the two drifted apart for every symbol the backend mints.
if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
return true
# The symbol may ITSELF be content-addressed (a synthesized hook or a generic
# instance carries `Hook/InstanceDisambBit` on its OWN `disamb`): then it has no
# single owning module and every demander emits it (merge dedups by C name),
# regardless of what it is nested under. This must be checked on `prc` directly,
# not on `top`: a `=destroy`/`=sink` lifted while compiling some enclosing proc
# (e.g. system's `isZeroMemory` destroying a `ptr array`) has that PROC as its
# `skipGenericOwner`, so `top` walks up to a plain routine whose own disamb has
# no bit — gating the hook to that routine's owner module, which mints it
# on demand and emits it nowhere → undefined at link.
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32:
return true
var top = prc
while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
top = top.skipGenericOwner
result = top.itemId.module == m.module.position or
(top.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 or
# An INLINE iterator has no standalone body — it is expanded at each
# call site — so it is materialized in every module that iterates over
# it, never in its owner. A proc nested in one (e.g. std/uri's
# `parseData` inside `iterator decodeQuery`) is lambda-lifted into each
# of those consumer TUs and must be emitted there (its stable
# owner-suffixed name + `'u'` flag let the merge stage keep one); gating
# it to the iterator's owner module leaves it in no TU → undefined.
(top.kind == skIterator and top.typ != nil and
top.typ.callConv != ccClosure)
if not bodyIsSeededByItsOwner(prc):
# Seeded by nobody: every demander emits it, merge keeps one.
result = true
elif sfDispatcher in prc.flags:
result = sfMainModule in m.module.flags
else:
result = prc.itemId.module == m.module.position
proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
result = TLoc(k: k, storage: s, lode: lode,
@@ -418,7 +535,7 @@ proc genCLineDir(r: var Builder, p: BProc, info: TLineInfo; conf: ConfigRef) =
if freshLineInfo(p, info):
genCLineDir(r, info.fileIndex, info.safeLineNm, p, info, lastFileIndex)
proc genLineDir(p: BProc, t: PNode) =
proc genLineDir(p: BProc; t: PNode) =
if p == p.module.preInitProc: return
let line = t.info.safeLineNm
@@ -538,8 +655,8 @@ type
needAssignCall
TAssignmentFlags = set[TAssignmentFlag]
proc genObjConstr(p: BProc, e: PNode, d: var TLoc)
proc rawConstExpr(p: BProc, n: PNode; d: var TLoc)
proc genObjConstr(p: BProc; e: PNode, d: var TLoc)
proc rawConstExpr(p: BProc; n: PNode; d: var TLoc)
proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags)
type
@@ -558,9 +675,9 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
if mode == constructRefObj: r = cDeref(r)
var s = skipTypes(t, abstractInst)
if not p.module.compileToCpp:
while s.kind == tyObject and s[0] != nil:
while s.kind == tyObject and s.baseClass != nil:
r = dotField(r, "Sup")
s = skipTypes(s[0], skipPtrs)
s = skipTypes(s.baseClass, skipPtrs)
if optTinyRtti in p.config.globalOptions:
p.s(section).addFieldAssignment(r, "m_type", genTypeInfoV2(p.module, t, a.lode.info))
else:
@@ -593,9 +710,9 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
if mode == constructRefObj: r = cDeref(r)
var s = skipTypes(t, abstractInst)
if not p.module.compileToCpp:
while s.kind == tyObject and s[0] != nil and s.sym.magic != mException:
while s.kind == tyObject and s.baseClass != nil and s.sym.magic != mException:
r = dotField(r, "Sup")
s = skipTypes(s[0], skipPtrs)
s = skipTypes(s.baseClass, skipPtrs)
p.s(section).addFieldAssignment(r, "name", makeCString(t.skipTypes(abstractInst).sym.name.s))
proc genRefAssign(p: BProc, dest, src: TLoc)
@@ -767,6 +884,16 @@ proc localVarDecl(res: var Builder, p: BProc; n: PNode,
backendEnsureMutable s
fillLoc(s.locImpl, locLocalVar, n, OnStack)
if s.kind == skLet: incl(s, lfNoDeepCopy)
else:
# Already named by an EARLIER emission of this same routine — an inline proc
# regenerated per user, or (under a batched `cg`) a definition emitted into
# two of this process's TUs. `fillLocalName` caches the C name on the PSym
# but takes the uniquifying counter from the BProc, and this BProc is a new
# one whose `sigConflicts` never saw that name. Claim it, or the next local
# of the same base name minted HERE starts from `_1` again and redeclares
# it: gcc "redeclaration of 'i_1' with no linkage", 64 of Atlas's 204 `.c`
# at batch size 4.
p.sigConflicts.inc(s.name.s.mangle)
genCLineDir(res, p, n.info, p.config)
@@ -776,7 +903,7 @@ proc localVarDecl(res: var Builder, p: BProc; n: PNode,
initializer = initializer,
initializerKind = initializerKind)
proc assignLocalVar(p: BProc, n: PNode) =
proc assignLocalVar(p: BProc; n: PNode) =
#assert(s.loc.k == locNone) # not yet assigned
# this need not be fulfilled for inline procs; they are regenerated
# for each module that uses them!
@@ -800,7 +927,7 @@ proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool =
# and s.owner.kind == skModule # owner isn't always a module (global pragma on local var)
# and s.loc.k == locGlobalVar # loc isn't always initialized when this proc is used
proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
proc genGlobalVarDecl(res: var Builder, p: BProc; n: PNode; td: Snippet;
initializer: Snippet = "",
initializerKind: VarInitializerKind = Assignment,
allowConst = true) =
@@ -841,7 +968,7 @@ proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
initializer = initializer,
initializerKind = initializerKind)
proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
proc assignGlobalVar(p: BProc; n: PNode; value: Rope) =
let s = n.sym
if s.loc.k == locNone:
fillBackendName(p.module, s)
@@ -905,7 +1032,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
backendEnsureMutable s
resetLoc(p, s.locImpl)
proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode; didGenTemp: var bool) =
proc callGlobalVarCppCtor(p: BProc; v: PSym; vn: PNode; value: PNode; didGenTemp: var bool) =
let s = vn.sym
fillBackendName(p.module, s)
backendEnsureMutable s
@@ -944,16 +1071,16 @@ proc genStmts(p: BProc, t: PNode)
proc expr(p: BProc, n: PNode, d: var TLoc)
proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc)
proc genLiteral(p: BProc, n: PNode; result: var Builder)
proc genLiteral(p: BProc; n: PNode; result: var Builder)
proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Builder; argBuilder: var CallBuilder)
proc raiseExit(p: BProc)
proc raiseExitCleanup(p: BProc, destroy: string)
proc initLocExpr(p: BProc, e: PNode, flags: TLocFlags = {}): TLoc =
proc initLocExpr(p: BProc; e: PNode, flags: TLocFlags = {}): TLoc =
result = initLoc(locNone, e, OnUnknown, flags)
expr(p, e, result)
proc initLocExprSingleUse(p: BProc, e: PNode): TLoc =
proc initLocExprSingleUse(p: BProc; e: PNode): TLoc =
result = initLoc(locNone, e, OnUnknown)
if e.kind in nkCallKinds and (e.firstSon.kind != nkSym or e.firstSon.sym.magic == mNone):
# We cannot check for tfNoSideEffect here because of mutable parameters.
@@ -966,6 +1093,22 @@ proc initLocExprSingleUse(p: BProc, e: PNode): TLoc =
result.flags.incl lfSingleUse
expr(p, e, result)
when defined(icCanRaiseLog):
import std / syncio
proc logCanRaise(s: PSym; verdict: bool) =
## One line per verdict, keyed by name + disamb + OWNING MODULE, and carrying
## the magic that usually decides the answer.
##
## The module is not decoration: `disamb` is a per-module counter, so `len.0`
## names a different routine in every module that has one, and a key without
## the module reports a collision as a disagreement. NOT the itemId — that is
## a per-build counter and would make every line differ for no reason.
let m = getModule(s)
stderr.writeLine "CANRAISE " & s.name.s & "." & $s.disamb & "." &
(if m == nil: "?" else: m.name.s) & "|" & $verdict & "|" & $s.magic &
"|b" & $canRaiseBranch
include ccgcalls, "ccgstmts.nim"
proc initFrame(p: BProc, procname, filename: Rope): Rope =
@@ -1107,8 +1250,11 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
var a: TLoc = initLocExpr(m.initProc, n.firstSon)
let callee = rdLoc(a)
var params: seq[Snippet] = @[]
for i in 1..<n.len-1:
a = initLocExpr(m.initProc, n[i])
var remaining = n.len - 2 # children 1 ..< len-1
for it in sonsFrom(n, 1):
if remaining <= 0: break
dec remaining
a = initLocExpr(m.initProc, it)
params.add(rdLoc(a))
params.add(makeCString($extname))
template load(builder: var Builder) =
@@ -1117,7 +1263,7 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
cCast(getTypeDesc(m, sym.typ, dkVar),
cCall(callee, params)))
var last = lastSon(n)
if last.kind == nkHiddenStdConv: last = last[1]
if last.kind == nkHiddenStdConv: last = last.secondSon
internalAssert(m.config, last.kind == nkStrLit)
let idx = last.strVal
if idx.len == 0:
@@ -1222,14 +1368,14 @@ proc closeNamespaceNim(result: var Builder) =
proc closureSetup(p: BProc, prc: PSym) =
if tfCapturesEnv notin prc.typ.flags: return
# prc.ast[paramsPos].last contains the type we're after — BUT a closure loaded
# The `paramsPos` child of `prc.ast` has the type we're after — BUT a closure loaded
# from a `.t.bif` (a lambda-lifted nested proc / generic instance the `lower`
# stage transformed) can arrive with an EMPTY AST param node: the lifted hidden
# `:env` param lives in `typ.n`, the authoritative signature (`genProc` already
# reads `typ.n`, not the AST). The two param nodes diverge across the NIF
# boundary; fall back to `typ.n` so the env param resolves instead of indexing
# an empty container.
var params = prc.ast[paramsPos]
var params = son(prc.ast, paramsPos)
if params.safeLen == 0 and prc.typ.n != nil and prc.typ.n.kind == nkFormalParams:
params = prc.typ.n
var ls = lastSon(params)
@@ -1260,14 +1406,14 @@ proc containsResult(n: PNode): bool =
of succ(nkEmpty)..pred(nkSym), succ(nkSym)..nkNilLit, harmless:
discard
of nkReturnStmt:
for ni in n.sons:
for ni in sons(n):
if containsResult(ni): return true
result = n.len > 0 and n.firstSon.kind == nkEmpty
result = n.hasSons and n.firstSon.kind == nkEmpty
of nkSym:
if n.sym.kind == skResult:
result = true
else:
for ni in n.sons:
for ni in sons(n):
if containsResult(ni): return true
proc easyResultAsgn(n: PNode): PNode =
@@ -1278,11 +1424,11 @@ proc easyResultAsgn(n: PNode): PNode =
while i < n.len and n[i].kind in harmless: inc i
if i < n.len: result = easyResultAsgn(n[i])
of nkAsgn, nkFastAsgn, nkSinkAsgn:
if n.firstSon.kind == nkSym and n.firstSon.sym.kind == skResult and not containsResult(n[1]):
if n.firstSon.kind == nkSym and n.firstSon.sym.kind == skResult and not containsResult(n.secondSon):
incl n.flags, nfPreventCg
return n[1]
return n.secondSon
of nkReturnStmt:
if n.len > 0:
if n.hasSons:
result = easyResultAsgn(n.firstSon)
if result != nil: incl n.flags, nfPreventCg
else: discard
@@ -1316,13 +1462,13 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
result = Unknown
case n.kind
of nkStmtList, nkStmtListExpr:
for it in n:
for it in sons(n):
result = allPathsAsgnResult(p, it)
if result != Unknown: return result
of nkAsgn, nkFastAsgn, nkSinkAsgn:
if n.firstSon.kind == nkSym and n.firstSon.sym.kind == skResult:
if not containsResult(n[1]):
if allPathsAsgnResult(p, n[1]) == InitRequired:
if not containsResult(n.secondSon):
if allPathsAsgnResult(p, n.secondSon) == InitRequired:
result = InitRequired
else:
result = InitSkippable
@@ -1330,9 +1476,9 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
elif containsResult(n):
result = InitRequired
else:
result = allPathsAsgnResult(p, n[1])
result = allPathsAsgnResult(p, n.secondSon)
of nkReturnStmt:
if n.len > 0:
if n.hasSons:
if n.firstSon.kind == nkEmpty and result != InitSkippable:
# This is a bare `return` statement, if `result` was not initialized
# anywhere else (or if we're not sure about this) let's require it to be
@@ -1343,7 +1489,7 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
of nkIfStmt, nkIfExpr:
var exhaustive = false
result = InitSkippable
for it in n:
for it in sons(n):
# Every condition must not use 'result':
if it.len == 2 and containsResult(it.firstSon):
return InitRequired
@@ -1357,8 +1503,7 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
result = InitSkippable
var exhaustive = skipTypes(n.firstSon.typ,
abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString, tyCstring}
for i in 1..<n.len:
let it = n[i]
for it in sonsFrom(n, 1):
allPathsInBranch(it.lastSon)
if it.kind == nkElse: exhaustive = true
if not exhaustive: result = Unknown
@@ -1367,7 +1512,7 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
# condition and that would be fine. Everything else isn't:
result = allPathsAsgnResult(p, n.firstSon)
if result == Unknown:
result = allPathsAsgnResult(p, n[1])
result = allPathsAsgnResult(p, n.secondSon)
# we cannot assume that the 'while' loop is really executed at least once:
if result == InitSkippable: result = Unknown
of harmless:
@@ -1390,11 +1535,11 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
# is 'finally: result = x'
result = InitSkippable
allPathsInBranch(n.firstSon)
for i in 1..<n.len:
if n[i].kind == nkFinally:
result = allPathsAsgnResult(p, n[i].lastSon)
for it in sonsFrom(n, 1):
if it.kind == nkFinally:
result = allPathsAsgnResult(p, it.lastSon)
else:
allPathsInBranch(n[i].lastSon)
allPathsInBranch(it.lastSon)
of nkCallKinds:
if canRaiseDisp(p, n.firstSon) or
(n.firstSon.kind == nkSym and sfNoReturn in n.firstSon.sym.flags):
@@ -1406,8 +1551,8 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
# arithmetic operations may raise exceptions
result = InitRequired
else:
for i in 0..<n.safeLen:
allPathsInBranch(n[i])
for it in sons(n):
allPathsInBranch(it)
of nkRaiseStmt:
result = InitRequired
of nkChckRangeF, nkChckRange64, nkChckRange:
@@ -1415,8 +1560,8 @@ proc allPathsAsgnResult(p: BProc; n: PNode): InitResultEnum =
# bug #22852
result = InitRequired
else:
for i in 0..<n.safeLen:
allPathsInBranch(n[i])
for it in sons(n):
allPathsInBranch(it)
proc getProcTypeCast(m: BModule, prc: PSym): Rope =
result = getTypeDesc(m, prc.loc.t)
@@ -1490,9 +1635,11 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
# it there would WRONGLY skip destructor injection and miscompile (orc
# decref-on-freed). The `.t.bif`-loaded-body concept exists only under cmdNifC.
let wasLoaded = m.config.cmd == cmdNifC and prc.transformedBody != nil
icProfStart(tTransform)
var procBody = transformBody(m.g.graph, m.idgen, prc, {})
if sfInjectDestructors in prc.flags and not wasLoaded:
procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
icProfStop(tTransform)
let tmpInfo = prc.info
discard freshLineInfo(p, prc.info)
@@ -1500,7 +1647,7 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
if sfPure notin prc.flags and prc.typ.returnType != nil:
if resultPos >= prc.ast.len:
internalError(m.config, prc.info, "proc has no result symbol")
let resNode = prc.ast[resultPos]
let resNode = son(prc.ast, resultPos)
let res = resNode.sym # get result symbol
if not isInvalidReturnType(m.config, prc.typ) and sfConstructor notin prc.flags:
if sfNoInit in prc.flags: incl(res, sfNoInit)
@@ -1512,8 +1659,9 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
# declare the result symbol:
assignLocalVar(p, resNode)
assert(res.loc.snippet != "")
let paths = allPathsAsgnResult(p, procBody)
if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc} and
allPathsAsgnResult(p, procBody) == InitSkippable:
paths == InitSkippable:
# In an ideal world the codegen could rely on injectdestructors doing its job properly
# and then the analysis step would not be required.
discard "result init optimized out"
@@ -1548,8 +1696,8 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
backendEnsureMutable res
res.locImpl.storage = OnUnknown
for i in 1..<prc.typ.n.len:
let param = prc.typ.n[i].sym
for paramNode in sonsFrom(prc.typ.n, 1):
let param = paramNode.sym
if param.typ.isCompileTimeOnly: continue
if prc.typ.callConv == ccClosure and param.name.s == ":envP":
# The hidden closure-env param is materialised by `closureSetup`, never a
@@ -1564,7 +1712,9 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
continue
assignParam(p, param, prc.typ.returnType)
closureSetup(p, prc)
icProfStart(tGenBody)
genProcBody(p, procBody)
icProfStop(tGenBody)
# IC: spurious write, seems fine for now:
prc.infoImpl = tmpInfo
@@ -1685,13 +1835,23 @@ proc genProcPrototype(m: BModule, sym: PSym) =
genMemberProcHeader(m, sym, scratch, false, true)
return
if lfDynamicLib in sym.loc.flags:
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Under IC per-module cg every demander emits the dynlib proc's DEFINITION
# locally (findPendingModule returns `m`, so symInDynamicLib follows this
# call and the merge stage keeps one def per C name). Emitting the
# cross-module `extern` proto here would register `sym.id` in
# `m.declaredThings` and thereby make that `symInDynamicLib` skip, leaving
# the `Dl_*` symbol declared-but-never-defined -> undefined at link.
# Does THIS TU emit the dynlib proc's definition? Under IC cg it does
# whenever `findPendingModule` routes the symbol here — which it does unless
# the owner is another member of this process's batch. Mirrored rather than
# called, because `findPendingModule` creates a `BModule` on demand and a
# prototype has no business doing that.
let owner = getModule(sym)
let emittedByABatchSibling =
owner != nil and owner.kind == skModule and
owner.position != m.module.position and
m.g.icEmitted.contains(owner.position)
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg" and
not emittedByABatchSibling:
# This TU emits the DEFINITION itself: `symInDynamicLib` follows this call
# and the merge stage keeps one def per C name. Emitting the cross-module
# `extern` proto here would register `sym.id` in `m.declaredThings` and
# thereby make that `symInDynamicLib` skip, leaving the `Dl_*` symbol
# declared-but-never-defined -> undefined at link.
discard "definition emitted by symInDynamicLib"
elif sym.itemId.module != m.module.position and
not containsOrIncl(m.declaredThings, sym.id):
@@ -1740,17 +1900,17 @@ include inliner
proc genProcLvl2(m: BModule, prc: PSym) =
if lfImportCompilerProc in prc.loc.flags:
fillProcLoc(m, prc.ast[namePos])
fillProcLoc(m, son(prc.ast, namePos))
useHeader(m, prc)
# dependency to a compilerproc:
cgsym(m, prc.name.s)
return
if lfNoDecl in prc.loc.flags:
fillProcLoc(m, prc.ast[namePos])
fillProcLoc(m, son(prc.ast, namePos))
genProcPrototype(m, prc)
elif lfDynamicLib in prc.loc.flags:
var q = findPendingModule(m, prc)
fillProcLoc(q, prc.ast[namePos])
fillProcLoc(q, son(prc.ast, namePos))
genProcPrototype(m, prc)
if q != nil and not containsOrIncl(q.declaredThings, prc.id):
symInDynamicLib(q, prc)
@@ -1777,13 +1937,13 @@ proc genProcLvl2(m: BModule, prc: PSym) =
# not on the first module that uses it
if m.module.itemId.module != prc.itemId.module and optCompress in m.config.globalOptions:
let prcCopy = prc # copyInlineProc(prc, m.idgen)
fillProcLoc(m, prcCopy.ast[namePos])
fillProcLoc(m, son(prcCopy.ast, namePos))
genProcPrototype(m, prcCopy)
genProcLvl3(m, prcCopy)
else:
let m2 = if m.config.symbolFiles != disabledSf: m
else: findPendingModule(m, prc)
fillProcLoc(m2, prc.ast[namePos])
fillProcLoc(m2, son(prc.ast, namePos))
#elif {sfExportc, sfImportc} * prc.flags == {}:
# # reset name to restore consistency in case of hashing collisions:
# #echo "resetting ", prc.id, " by ", m.module.name.s
@@ -1793,7 +1953,7 @@ proc genProcLvl2(m: BModule, prc: PSym) =
genProcLvl3(m, prc)
elif sfImportc notin prc.flags:
var q = findPendingModule(m, prc)
fillProcLoc(q, prc.ast[namePos])
fillProcLoc(q, son(prc.ast, namePos))
# generate a getProc call to initialize the pointer for this
# externally-to-the-current-module defined proc, also important
# to do the declaredProtos check before the call to genProcPrototype
@@ -1811,10 +1971,11 @@ proc genProcLvl2(m: BModule, prc: PSym) =
# which will actually become a function pointer
if isReloadable(m, prc):
genProcPrototype(q, prc)
if emitsBodyInThisModule(m, prc):
# Ask about `q`, the TU the body goes into. Outside a batch `q` IS `m`.
if emitsBodyInThisModule(q, prc):
genProcLvl3(q, prc)
else:
fillProcLoc(m, prc.ast[namePos])
fillProcLoc(m, son(prc.ast, namePos))
useHeader(m, prc)
if sfInfixCall notin prc.flags: genProcPrototype(m, prc)
@@ -1836,7 +1997,7 @@ proc genProc(m: BModule, prc: PSym) =
if sfBorrow in prc.flags or not isActivated(prc): return
if sfForward in prc.flags:
addForwardedProc(m, prc)
fillProcLoc(m, prc.ast[namePos])
fillProcLoc(m, son(prc.ast, namePos))
else:
genProcLvl2(m, prc)
if {sfExportc, sfCompilerProc} * prc.flags == {sfExportc} and
@@ -1910,12 +2071,27 @@ proc headerTop(): Rope =
proc getCopyright(conf: ConfigRef; cfile: Cfile): Rope =
result = headerTop()
if optCompileOnly notin conf.globalOptions:
result.add ("/* Compiled for: $1, $2, $3 */$N" &
"/* Command for C compiler:$n $4 */$N") %
result.add ("/* Compiled for: $1, $2, $3 */$N") %
[rope(platform.OS[conf.target.targetOS].name),
rope(platform.CPU[conf.target.targetCPU].name),
rope(extccomp.CC[conf.cCompiler].name),
rope(getCompileCFileCmd(conf, cfile))]
rope(extccomp.CC[conf.cCompiler].name)]
# The per-module IC backend cannot write this line truthfully. A global
# `{.passC.}` (system's `-pthread`, say) reaches `conf.compileOptions` only
# in a process that compiled the module declaring it, and a `cg` process
# sees one module's import closure — so the command it would print is a
# partial snapshot, and WHICH part depends on how modules were grouped into
# processes. Measured on a 67-module program: 2 of 67 `.c` carried
# `-pthread` at batch size 1, 4 at size 4, 5 at size 8, against 16 of 16 for
# a whole-program `nim c`. The real command is assembled by the `link`
# stage, which applies every module's recorded directives first
# (`replayer.applyBackendActions`) — so the object files were always
# correct; only this comment was wrong, and non-deterministically so.
if conf.cmd == cmdNifC and conf.icBackendStage.len > 0:
result.add "/* Command for C compiler: assembled by the link stage\L" &
" from every module's recorded C directives. */\L"
else:
result.add ("/* Command for C compiler:$n $1 */$N") %
[rope(getCompileCFileCmd(conf, cfile))]
proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope =
var res = newBuilder(getCopyright(conf, cfile))
@@ -2376,7 +2552,7 @@ proc genDatInitCode(m: BModule) =
proc hcrGetProcLoadCode(builder: var Builder, m: BModule, sym, prefix, handle, getProcFunc: string) =
let prc = magicsys.getCompilerProc(m.g.graph, sym)
assert prc != nil
fillProcLoc(m, prc.ast[namePos])
fillProcLoc(m, son(prc.ast, namePos))
var tmp = mangleDynLibProc(prc)
backendEnsureMutable prc
@@ -2743,9 +2919,9 @@ when false:
readMergeInfo(getCFile(m), m)
result = m
proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool, init: var IfBuilder) =
proc addHcrInitGuards(p: BProc; n: PNode, inInitGuard: var bool, init: var IfBuilder) =
if n.kind == nkStmtList:
for child in n:
for child in sons(n):
addHcrInitGuards(p, child, inInitGuard, init)
else:
let stmtShouldExecute = n.kind in {nkVarSection, nkLetSection} or
@@ -2842,6 +3018,23 @@ proc genModuleCode(m: BModule; cf: var Cfile): string =
proc registerModuleCode(m: BModule; cf: var Cfile; code: string) =
## Second half of `writeModule`: writes the .c file if it changed and
## registers it for compilation.
##
## NOT under the per-module backend's `cg` stage. There the `.c` belongs to
## `emit`, which renders it from the `.c.nif` using the GLOBAL merge decision;
## `cg` can only filter by the liveness its own process can see, so writing
## here puts a second, differently-filtered `.c` at the very path `emit`
## declares as its nifmake output. Two stages then claim one output, and the
## `.c` ends up newer than `emit`'s own `.c.nif` input — so any build in which
## `emit` is not forced to run anyway keeps `cg`'s unfiltered text and hands it
## to the linker ("multiple definition of eqdup__…").
##
## Today nothing surfaces this: `merge` rewrites the decision file on every
## run and every `emit` lists it as an input, so all of them re-fire and
## overwrite the stray file. That makes the fire-all load-bearing rather than
## the "insurance" it is documented as, and it silently blocks making the
## decision content-stable. `cg`'s product is the `.c.nif`; the compile
## registration is likewise the `link` stage's job.
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg": return
if code != "" or m.config.symbolFiles != disabledSf:
when hasTinyCBackend:
if m.config.cmd == cmdTcc:

10
compiler/cgendata.nim
View File

@@ -142,6 +142,13 @@ type
# not a list of IDs nor can it be made to be one.
mangledPrcs*: HashSet[string]
icEmitted*: IntSet
## Under `--icBackendStage:cg`: the positions of the modules THIS process
## writes a translation unit for. `cgen.findPendingModule` consults it to
## decide where a demanded definition goes — see the comment there. Empty
## outside that stage, which is why every other backend keeps the ordinary
## whole-program routing.
TCGen = object of PPassContext # represents a C source file
s*: TCFileSections # sections of the C file
flags*: set[CodegenFlag]
@@ -238,7 +245,8 @@ proc newProc*(prc: PSym, module: BModule): BProc =
proc newModuleList*(g: ModuleGraph): BModuleList =
BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](),
config: g.config, graph: g, nimtvDeclared: initIntSet())
config: g.config, graph: g, nimtvDeclared: initIntSet(),
icEmitted: initIntSet())
iterator cgenModules*(g: BModuleList): BModule =
for m in g.modulesClosed:

12
compiler/commands.nim
View File

@@ -989,12 +989,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendStage = arg
of "icbackendmodule":
# `nim nifc` only: the NIF module suffix the cg/emit stage operates on (see
# options.icBackendModule).
of "icbackendmodule", "icbackendmodules":
# `nim nifc` only: the NIF module suffixes the lower/cg/emit stage operates
# on, comma-separated — the invocation's batch (see
# options.icBackendModules). The singular spelling is the same switch: a
# one-module batch is what the per-module fan-out passes.
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendModule = arg
conf.icBackendModules = @[]
for suffix in arg.split(','):
if suffix.len > 0: conf.icBackendModules.add suffix
of "import":
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:

234
compiler/deps.nim
View File

@@ -15,9 +15,11 @@ from std/sha1 import secureHash, `$`
import options, msgs, lineinfos, pathutils, condsyms,
modulepaths, extccomp, cnif, platform
import "../dist/nimony/src/lib" / [nifstreams, bitabs, nifreader, nifbuilder]
import nifstreams
import "../dist/nimony/src/lib" / [bitabs, nifreader, nifbuilder]
import icmodnames
import icnifcore
from ic/replayer import BackendActionsExt
type
FilePair = object
@@ -63,6 +65,13 @@ proc depsFile(c: DepContext; f: FilePair): string =
proc parsedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".p.nif"
proc parsedDepsFile(c: DepContext; f: FilePair): string =
## The deps sidecar `nifler parse --deps <src> <out>.p.nif` actually writes: it
## appends `.deps.nif` to the OUTPUT path, giving `<mod>.p.deps.nif`. Not to be
## confused with `depsFile` (`<mod>.deps.nif`), which the driver's own
## `nifler deps` pre-scan writes.
parsedFile(c, f).changeFileExt("") & ".deps.nif"
proc semmedFile(c: DepContext; f: FilePair): string =
getNimcacheDir(c.config).string / f.modname & ".s.bif"
@@ -513,6 +522,18 @@ proc parseImportPath(s: var Stream; t: var PackedToken): seq[string] =
for r in parseImportPath(s, t):
result.add op & r
if t.kind == ParRi: t = next(s) # skip closing ')'
elif tag == "pragmax":
# `import x {.all.}` serialises as `(pragmax x (pragmas all))`. Without
# this it fell into the unknown-subtree skip below and the import was
# DROPPED from the static graph: the build only learned about it from the
# `.s.deps` sidecar a round later, after a round that failed with
# "requires precompiled NIF for import". Correct, but a wasted round and
# an alarming error line for an ordinary import.
t = next(s) # skip 'pragmax' tag
result = parseImportPath(s, t) # the path is the first child
while t.kind != ParRi and t.kind != EofToken:
discard parseImportPath(s, t) # the pragma list; consumed, not a path
if t.kind == ParRi: t = next(s) # skip closing ')'
elif tag == "bracket":
t = next(s) # skip 'bracket' tag
while t.kind != ParRi and t.kind != EofToken:
@@ -803,18 +824,35 @@ proc pruneDeadSpeculative(c: var DepContext) =
for d in c.nodes[v].deps:
if not dead[d] and not alive[d]: stack.add d
# Drop the scan artifacts of a module that just left the graph, so an
# edit-accumulated cache does not differ from a clean one for no reason
# (`tests/ic/tdead_when_import` pins that). Re-running nifler if it ever comes
# back costs a single parse.
#
# But a FILE can belong to several nodes, and only the NODE is dead.
# `lib/system/inclrtl.nim` is `include`d by dozens of live stdlib modules and
# also sits in the file set of a dead-speculative one; a clean build therefore
# has its `.p.nif`, and deleting it here does not tidy the cache, it corrupts
# it. The consequences compound: the missing output re-fires that file's
# `nifler` rule, which rewrites the parsed file with a fresh mtime, which
# re-fires every `nim_m` rule listing it as an input — 16 full module re-sems
# (system, os, times, strutils, macros, unicode, ...) on every warm build, for
# ever, because the scanner is stateless and rediscovers the dead node each
# run. Measured on a 219-module program: an 11 s NO-OP build. So delete only
# what no live node claims.
var liveFiles = initHashSet[string]()
for i in 0 ..< n:
if alive[i]:
for f in c.nodes[i].files: liveFiles.incl f.nimFile
var cascaded = 0
for i in 0 ..< n:
if not alive[i]:
# Drop the scan artifacts of a module that just left the graph. `nifler`
# ran on it during `traverseDeps` (that is how we learned it cannot
# build), and leaving its `.p.nif`/`.deps.nif` behind makes an
# edit-accumulated cache differ from a clean one for no reason. Re-running
# nifler if it ever comes back costs a single parse.
for f in c.nodes[i].files:
if f.nimFile in liveFiles: continue
removeFile(c.parsedFile(f))
removeFile(c.depsFile(f))
removeFile(c.parsedFile(f).changeFileExt("") & ".deps.nif")
removeFile(c.parsedDepsFile(f))
if c.nodes[i].missingImport.len > 0:
rawMessage(c.config, hintSuccess,
"ic: skipping " & c.nodes[i].files[0].nimFile &
@@ -1102,8 +1140,13 @@ proc generateFrontendBuildFile(c: DepContext; forwardedArgs: seq[string]): strin
b.addTree "output"
b.addStrLit parsed
b.endTree()
# The deps sidecar this command really produces is `<mod>.p.deps.nif`,
# not `<mod>.deps.nif` (which only the driver's `nifler deps` pre-scan
# writes). Declaring the latter made the rule permanently stale — a
# missing output is nifmake's strongest rebuild trigger — for every
# module the pre-scan does not also cover.
b.addTree "output"
b.addStrLit c.depsFile(pair)
b.addStrLit c.parsedDepsFile(pair)
b.endTree()
b.endTree()
@@ -1277,6 +1320,76 @@ proc computeLiveBackendNodes(c: DepContext): seq[bool] =
let idx = c.processedModules.getOrDefault(c.toPair(p).modname, -1)
if idx >= 0: stack.add idx
proc intDefine(conf: ConfigRef; name: string; fallback: int): int =
## `-d:<name>:N` as an int, or `fallback` when unset or unparsable.
result = fallback
if isDefined(conf, name):
try: result = parseInt(conf.symbols[name])
except ValueError: result = fallback
proc backendBatchSize(conf: ConfigRef; liveCount: int): int =
## How many modules share one backend process. 1 is the historical per-module
## fan-out; larger batches amortise the process floor and the dependency
## closure load (measured on a 67-module program: 7.6 ms of process startup
## and ~10 ms of closure loading per child, against 3.5 ms of actual codegen).
##
## `-d:icBatchSize:N` pins it. The default is 1 — the plumbing is in place but
## the policy is not yet validated. `-d:icBatchSize:0` means "one batch per
## job", which is the shape a tuned default will take: enough batches to keep
## every core busy and no more, since a batch beyond that only buys
## amortisation at the price of parallelism.
if not isDefined(conf, "icBatchSize"): return 1
result = intDefine(conf, "icBatchSize", 1)
if result == 0:
let jobs =
if isDefined(conf, "icNoParallel"): 1
elif isDefined(conf, "icJobs"): max(1, intDefine(conf, "icJobs", 1))
elif conf.numberOfProcessors > 0: conf.numberOfProcessors
else: 1
result = (liveCount + jobs - 1) div jobs
result = max(1, result)
proc emitBatches(c: DepContext; live: seq[bool];
shared: seq[seq[int]]): seq[seq[int]] =
## emit's partition. Unlike `lower`/`cg` it takes the MAIN module too and, by
## default, puts every live node in one batch: emit owns no decisions, so
## there is nothing for a grouping to get wrong (see the rule that uses this).
## An explicit `-d:icBatchSize` reuses the shared partition instead, plus main,
## so the fan-out remains available to compare against.
if isDefined(c.config, "icBatchSize"):
result = shared
if live.len > 0 and live[0]: result.add @[0]
else:
var all: seq[int] = @[]
for i in 0 ..< c.nodes.len:
if live[i]: all.add i
result = if all.len > 0: @[all] else: @[]
proc backendBatches(c: DepContext; live: seq[bool]): seq[seq[int]] =
## Partition the live non-main nodes into batches of node indices. The main
## module is never in one: it loads the whole program, so batching it with
## anything defeats the memory bound the per-module split exists to give.
##
## Contiguous runs of `c.nodes`, which is import-traversal order, so a batch's
## members tend to share dependencies and its union closure stays close to one
## member's. A smarter partition (by closure overlap, or by the dirty set on an
## incremental build) belongs here and nowhere else — every stage already takes
## whatever grouping this returns.
var liveIdx: seq[int] = @[]
for i in 0 ..< c.nodes.len:
if live[i] and c.nodes[i].id != 0: liveIdx.add i
let size = backendBatchSize(c.config, liveIdx.len)
result = @[]
var i = 0
while i < liveIdx.len:
var batch: seq[int] = @[]
var j = i
while j < liveIdx.len and batch.len < size:
batch.add liveIdx[j]
inc j
result.add batch
i = j
proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string =
## Per-module backend build file. One `nim_nifc` command template (the actual
## stage/module switches ride in each rule's `(args …)`), then the stages of
@@ -1332,6 +1445,8 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
if fileExists(cnifFiles[i]) or fileExists(cFiles[i]): prunedStale = true
removeFile(cnifFiles[i])
removeFile(cFiles[i])
removeFile(cFiles[i] & ".stamp")
removeFile(cFiles[i] & BackendActionsExt)
# The merge decision is a pure function of the set of `.c.nif`s present; if we
# just removed an over-approximated module's artifacts, a decision computed
# while they were present is stale — it can name a now-absent module as a
@@ -1393,17 +1508,39 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
# frontend writes `.s.nif`s content-stably, so an interface change to a
# dependency re-sems (and re-emits the `.s.nif` of) every transitive importer;
# a module whose own `.s.nif` is unchanged genuinely needs no re-lowering.
for i, node in c.nodes:
if not live[i]: continue
let batches = backendBatches(c, live)
template suffixList(batch: seq[int]): string =
var acc = ""
for k, idx in batch:
if k > 0: acc.add ","
acc.add c.nodes[idx].files[0].modname
acc
for batch in batches:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:lower"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr c.semmedFile(node.files[0])
b.addStrLit "--icBackendModules:" & suffixList(batch)
for idx in batch:
inputStr c.semmedFile(c.nodes[idx].files[0])
inputStr argsFile
outputStr tFiles[i]
for idx in batch:
outputStr tFiles[idx]
b.endTree()
# The main module is its own rule in every stage: it loads the whole program.
block:
let i = 0
if live[i]:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:lower"
b.addStrLit "--icBackendModules:" & c.nodes[i].files[0].modname
inputStr c.semmedFile(c.nodes[i].files[0])
inputStr argsFile
outputStr tFiles[i]
b.endTree()
# cg: one rule per module. Input is this module's OWN `.t.nif`. cg DOES read
# its dependencies' `.t.nif`s at runtime (loadDepClosure), but ordering is
@@ -1415,21 +1552,38 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
# emit-everywhere'd but does not own is dropped by `emit` regardless, so a
# stale copy here is harmless. The main module additionally depends on every
# other `.c.nif` (it reads their init/datInit metas to wire up NimMain).
for i, node in c.nodes:
if not live[i]: continue
for batch in batches:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:cg"
b.addStrLit "--icBackendModule:" & node.files[0].modname
inputStr tFiles[i]
b.addStrLit "--icBackendModules:" & suffixList(batch)
for idx in batch:
inputStr tFiles[idx]
inputStr argsFile
if node.id == 0:
for idx in batch:
outputStr cnifFiles[idx]
# The module's C compile/link directives (`{.passL.}` etc.), recorded so
# the `link` stage recovers them without loading the module graph. See
# `replayer.writeBackendActions`.
outputStr cFiles[idx] & BackendActionsExt
b.endTree()
block:
let i = 0
if live[i]:
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:cg"
b.addStrLit "--icBackendModules:" & c.nodes[i].files[0].modname
inputStr tFiles[i]
inputStr argsFile
for j in 0 ..< c.nodes.len:
if c.nodes[j].id != 0 and live[j]:
inputStr cnifFiles[j]
outputStr cnifFiles[i]
b.endTree()
outputStr cnifFiles[i]
outputStr cFiles[i] & BackendActionsExt
b.endTree()
# merge: read the live modules' `.c.nif`, write the ownership/liveness
# decision. The list is handed over as a FILE (`LiveModulesFile`) because the
@@ -1458,21 +1612,37 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
b.endTree()
# emit: render each module's `.c` from its `.c.nif` + the merge decision.
for i, node in c.nodes:
if not live[i]: continue
#
# ONE rule for everything, main included. emit is a pure function of a
# `.c.nif` and the merge decision — `renderCFromArtifact` filters text and
# touches no AST, and the stage loads no module graph at all — so batching it
# cannot change what it produces, and measurement agrees: 67 processes and one
# process give byte-identical `.c`, in 0.502 s versus 0.041 s. What that buys
# is not the cold build (where 0.5 s serial is ~0.05 s across cores) but the
# fire-all: every `emit` re-fires whenever `merge` rewrites the decision, which
# is every edit that reaches the backend. That now costs one process start.
#
# `-d:icBatchSize:N` still splits it, for A/B-ing against the fan-out.
for batch in emitBatches(c, live, batches):
b.addTree "do"
b.addIdent "nim_nifc"
b.withTree "args":
b.addStrLit "--icBackendStage:emit"
b.addStrLit "--icBackendModule:" & node.files[0].modname
# Inputs: this module's OWN `.c.nif` and the global merge decision. emit also
# loads `.t.nif`s at runtime (getCFile/type resolution), but those are depth 1
# and emit is past the merge barrier, so they always exist — no need to list
# them. (emit still re-fires for every module whenever `merge` rewrites the
# decision file; making that incremental is a separate concern.)
inputStr cnifFiles[i]
b.addStrLit "--icBackendModules:" & suffixList(batch)
# Inputs: each member's OWN `.c.nif` and the global merge decision. emit
# reads nothing else — it derives its output paths rather than loading a
# module graph. (It still re-fires for every module whenever `merge` rewrites
# the decision file; making that incremental is a separate concern — though
# batching is what makes the re-fire cheap.)
for idx in batch:
inputStr cnifFiles[idx]
inputStr mergeFile
outputStr cFiles[i]
for idx in batch:
outputStr cFiles[idx]
# The freshness proof for this rule; see nifbackend.generateEmitStage. The
# `.c` alone cannot serve: it is written OnlyIfChanged, so a rule that ran
# and produced identical bytes looks exactly like a rule that never ran.
outputStr cFiles[idx] & ".stamp"
b.endTree()
# link: compile + link every emitted `.c` in one process.
@@ -1486,7 +1656,9 @@ proc generateBackendBuildFile(c: DepContext; forwardedArgs: seq[string]): string
# path splits back into outDir+outFile in the child).
b.addStrLit "--out:" & exeFile
for i in 0 ..< c.nodes.len:
if live[i]: inputStr cFiles[i]
if live[i]:
inputStr cFiles[i]
inputStr cFiles[i] & BackendActionsExt
inputStr argsFile
outputStr exeFile
b.endTree()

68
compiler/ic/replayer.nim
View File

@@ -14,11 +14,77 @@
import ".." / [ast, modulegraphs, trees, extccomp, btrees,
msgs, lineinfos, pathutils, options, cgmeth]
import std/tables
import std/[tables, os, strutils, syncio]
when defined(nimPreviewSlimSystem):
import std/assertions
const BackendActionsExt* = ".cflags"
## Sidecar written by a module's `cg` stage next to its `.c`, carrying the C
## compile/link directives that module's `{.passL.}`/`{.compile.}`/… pragmas
## recorded. See `writeBackendActions`.
proc writeBackendActions*(g: ModuleGraph; module: PSym; list: PNode;
outfile: string) =
## Serialize the backend-relevant replay actions of ONE module to `outfile`,
## one tab-separated action per line.
##
## The `link` stage used to recover these by loading the whole import closure
## as `PrecompiledModule`s and re-running `replayBackendActions` over each —
## a 3.7s whole-program graph load, per link, purely to recover a handful of
## strings and the modules' `.c` paths. The producing `cg` process already has
## them in hand, so it writes them down instead and `link` reads them back
## (`applyBackendActions`). Written unconditionally, even when empty: it is a
## declared nifmake output of the `cg` rule, and a missing output re-fires the
## rule for ever.
##
## `localpassc` needs the module's own source path, which only the writer can
## resolve, so it is baked in here as a third field.
var content = ""
if list != nil:
for n in list:
if n.kind == nkReplayAction and n.len >= 2 and
n[0].kind == nkStrLit and n[1].kind == nkStrLit:
case n[0].strVal
of "compile":
if n.len == 4 and n[2].kind == nkStrLit and n[3].kind == nkStrLit:
content.add "compile\t" & n[1].strVal & "\t" & n[2].strVal & "\t" &
n[3].strVal & "\n"
of "link", "passl", "passc", "cppdefine":
content.add n[0].strVal & "\t" & n[1].strVal & "\n"
of "localpassc":
content.add "localpassc\t" & n[1].strVal & "\t" &
toFullPathConsiderDirty(g.config, module.info.fileIndex).string & "\n"
else: discard
writeFile(outfile, content)
proc applyBackendActions*(g: ModuleGraph; infile: string) =
## Apply one module's recorded C directives (see `writeBackendActions`). The
## `link` stage's replacement for loading that module and replaying its AST.
if not fileExists(infile): return
for line in lines(infile):
if line.len == 0: continue
let f = line.split('\t')
case f[0]
of "compile":
if f.len == 4:
let cname = AbsoluteFile f[1]
var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
obj: AbsoluteFile f[2],
flags: {CfileFlag.External}, customArgs: f[3])
extccomp.addExternalFileToCompile(g.config, cf)
of "link":
if f.len == 2: extccomp.addExternalFileToLink(g.config, AbsoluteFile f[1])
of "passl":
if f.len == 2: extccomp.addLinkOption(g.config, f[1])
of "passc":
if f.len == 2: extccomp.addCompileOption(g.config, f[1])
of "localpassc":
if f.len == 3: extccomp.addLocalCompileOption(g.config, f[1], AbsoluteFile f[2])
of "cppdefine":
if f.len == 2: options.cppDefine(g.config, f[1])
else: discard
proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/
## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a

111
compiler/icprof.nim Normal file
View File

@@ -0,0 +1,111 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Opt-in instrumentation for the IC backend, enabled with `-d:icBNodeProf`.
## Off, every template below is `discard` and nothing is linked in.
##
## It lives in its own module with NO compiler imports so that any stage can
## use it without creating a cycle — `ast2nif` for the loader, `nifbackend` for
## the stage phases, `cgen` for what happens per routine.
##
## Each backend process appends ONE line to `$NIM_IC_BNODE_PROF` at exit (or to
## stderr when that is unset), because a `--ic:on` build fans out a process per
## module per stage and interleaved writes would tear. Use `-d:icNoParallel`
## when the numbers need to be attributable to a particular module.
##
## Counts are for volume, timings for cost, and the two answer different
## questions: a call count alone once pointed at the wrong accessor (700k calls
## worth 8ms) while the real cost was 259k `info` resolutions worth 1.36s.
when defined(icBNodeProf):
import std / [envvars, exitprocs, syncio, monotimes]
from std / times import inNanoseconds
type
ProfSlot* = enum
pTyp, pIfaceExported, pIfaceHidden, pIfaceModules,
pTopNodes, pExportSyms, pPeekKind, pPeekFallback, pPeekLoaded,
pTopToolingSkip
TimeSlot* = enum
tLoadClosure, tModuleId, tBifLoad, tPosIndex, tTopLevel, tInterfTables,
tTransform, tGenBody, tExportBranch, tResolveSym, tEnumFields,
# Coarse phases, added to find where a backend process spends the time
# that none of the slots above account for. `tStage` is the whole stage
# body, so `Process - tStage` is everything before it: exec, the Nim
# runtime, config replay, `registerNifSuffix`/graph setup.
tStage,
tLowerOwned, tLowerHooks, tLowerWrite,
tCgGen, tCgInit, tCgFinish, tCgWrite,
tMergeStage, tEmitRender, tLinkStage,
# `nim m` (the frontend): the sem pass as a whole, and writing the module's
# `.s.bif`. `Stage - WriteNif - <the loading slots>` is then sem proper.
tWriteNif,
# `processTopLevel`'s branches: which part of a module HEADER costs what.
tTopReplay, tTopLogOps, tTopOffers, tTopStmts
let procStart = getMonoTime()
## Set when this module initialises, i.e. essentially at process start, so
## the dump can report total process wall time and the startup share can be
## derived as `Process - Stage`.
var profStageName* = "frontend"
## Which invocation this is: the backend stage name, or "frontend" for a
## `nim m` process, which arms the profiler through ast2nif but never enters
## a backend stage. Without it the `Process - Stage` startup figure is
## meaningless — 204 frontend processes' whole runtime lands in it.
var profCounts: array[ProfSlot, int]
var profNanos: array[TimeSlot, int64]
var profStart: array[TimeSlot, MonoTime]
var profArmed = false
proc profDump() =
var line = "BNODEPROF stage=" & profStageName
for s in ProfSlot: line.add " " & ($s)[1..^1] & "=" & $profCounts[s]
for s in TimeSlot: line.add " " & ($s)[1..^1] & "ms=" & $(profNanos[s] div 1_000_000)
line.add " Processms=" & $((getMonoTime() - procStart).inNanoseconds div 1_000_000)
let f = getEnv("NIM_IC_BNODE_PROF")
if f.len > 0:
let h = open(f, fmAppend)
h.writeLine line
h.close()
else:
stderr.writeLine line
template armProf() =
if not profArmed:
profArmed = true
addExitProc profDump
template prof*(s: ProfSlot; n = 1) =
armProf()
inc profCounts[s], n
template icProfStart*(s: TimeSlot) =
armProf()
profStart[s] = getMonoTime()
template icProfStop*(s: TimeSlot) =
profNanos[s] += (getMonoTime() - profStart[s]).inNanoseconds
template timed*(s: TimeSlot; body: untyped) =
## Leaf timing. NOT re-entrant, and the phase slots are not disjoint —
## `tTransform` contains body materialization. Read them as nested, not
## additive.
##
## Arms the dump like `prof`/`icProfStart` do. It did not, and so a process
## whose ONLY instrumentation is a `timed` never reported at all: the
## `merge`, `emit` and `link` stages were silently absent from every profile.
armProf()
let t0 = getMonoTime()
body
profNanos[s] += (getMonoTime() - t0).inNanoseconds
else:
template prof*(s: untyped; n = 1) = discard
template icProfStart*(s: untyped) = discard
template icProfStop*(s: untyped) = discard
template timed*(s: untyped; body: untyped) = body

15
compiler/lambdalifting.nim
View File

@@ -675,6 +675,17 @@ proc rawClosureCreation(owner: PSym;
if up != nil and upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) == up.typ.skipTypes({tyOwned, tyRef, tyPtr}):
result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
up, env.info))
# That assignment stores a real `ref`, so `injectDestructorCalls` has to
# find the up-field type's ops — otherwise it stays a raw pointer store,
# the enclosing env's refcount is one too low, and at teardown the two
# envs' mutually recursive `=destroy`s each believe they hold the last
# reference and recurse until the stack is gone. Whole-program cgen never
# noticed: some LATER lifting pass creates this very ref type's ops, and it
# runs before any routine's destructor injection. The per-module backend
# injects a routine right after lifting it (the `lower` stage), long before
# the module's top level is transformed at all (that is `cg`).
if up.typ != nil and up.typ.kind == tyRef and up.typ.elementType != nil:
createTypeBoundOpsLL(d.graph, up.typ, env.info, d.idgen, owner)
#elif oldenv != nil and oldenv.typ == upField.typ:
# result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
# oldenv, env.info))
@@ -732,6 +743,10 @@ proc closureCreationForIter(owner: PSym, iter: PNode;
if u != nil and u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == expectedUpTyp:
result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
u, iter.info))
# See the identical call in `rawClosureCreation`: the up-field's ops must
# exist by the time this assignment is destructor-injected.
if u.typ != nil and u.typ.kind == tyRef and u.typ.elementType != nil:
createTypeBoundOpsLL(d.graph, u.typ, iter.info, d.idgen, owner)
else:
localError(d.graph.config, iter.info, "internal error: cannot create up reference for iter")
result.add makeClosure(d.graph, d.idgen, iter.sym, vnode, iter.info)

5
compiler/main.nim
View File

@@ -29,6 +29,7 @@ when defined(nimPreviewSlimSystem):
import ../dist/checksums/src/checksums/sha1
import pipelines
import icprof
from icconfig import produceIcConfig, ensureIcConfig
when not defined(nimKochBootstrap):
@@ -445,7 +446,9 @@ proc mainCommand*(graph: ModuleGraph) =
# per-module compilation model cannot provide (yet); methods dispatch
# through the classic if-chain dispatchers instead
excl conf.features, Feature.vtables
commandCheck(graph)
# `tStage` for a `nim m` process, so `Process - Stage` is its real startup
# (exec, runtime init, config replay) rather than its whole runtime.
timed tStage: commandCheck(graph)
of cmdNifC:
setUseIc(true)
excl conf.features, Feature.vtables

20
compiler/mangleutils.nim
View File

@@ -61,22 +61,12 @@ proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string =
# starts with an EMPTY per-name disamb table, so its `disamb` restarts at 0
# and collides with same-named sem-time symbols loaded from NIFs (two
# `=destroy` hooks both mangling to `_u2` → "conflicting types for ..." in
# the generated C). Most such symbols never cross a process boundary (nifc
# lifts, emits and compiles them in one run), so the per-module-unique
# item id is a safe and deterministic discriminator; the `_c` marker keeps
# the namespace disjoint from `_u<disamb>`.
# the generated C). The `_c` marker keeps the namespace disjoint from
# `_u<disamb>`; `backendMintedDisamb` (astdef) is the ONE definition of which
# integer identifies such a symbol, shared with `ccgutils.makeUnique` and
# `ast2nif.toNifSymName` so the C name and the NIF name cannot drift apart.
result = "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# EXCEPTION: a backend-minted sym whose `disamb` is content-derived
# (setHookDisamb gave it HookDisambBit) — e.g. the `rttiDestroy` wrapper —
# DOES cross process boundaries: its C name is baked into the type's RTTI
# table, which is emit-everywhere and merge-deduped, so one process's
# `_c<item>` (a per-process backend counter) ends up referenced while the
# wrapper is defined with another's → undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes; use it.
result.addInt s.disamb
else:
result.addInt s.itemId.item
result.addInt backendMintedDisamb(s)
else:
result = "_u"
# Use `disamb` rather than `itemId.item`: under incremental compilation a

70
compiler/modulegraphs.nim
View File

@@ -17,7 +17,8 @@ import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils,
when not defined(nimKochBootstrap):
import ast2nif
import "../dist/nimony/src/lib" / [nifstreams, bitabs]
import nifstreams
import "../dist/nimony/src/lib" / bitabs
import typekeys
@@ -35,6 +36,10 @@ type
pureEnums*: seq[PSym]
interf: TStrTable
interfHidden: TStrTable
hiddenPending: bool ## `interfHidden` holds only the exported half so far;
## `ensureHiddenIface` materialises the hidden-only
## symbols on first use. See
## `ast2nif.buildHiddenInterface`.
uniqueName*: Rope
Operators* = object
@@ -256,6 +261,25 @@ proc toBase64a(s: cstring, len: int): string =
result.add cb64[a shr 2]
result.add cb64[(a and 3) shl 4]
proc ensureHiddenIface(g: ModuleGraph; pos: int) =
## Materialise a loaded module's hidden-only interface the first time anything
## asks for it. Every READ of `interfHidden` goes through `interfSelect`, so
## guarding those sites is complete.
if g.ifaces[pos].hiddenPending:
when not defined(nimKochBootstrap):
# By SUFFIX: `c.mods` and `g.ifaces` use different FileIndexes for the
# same module (see `buildHiddenInterface`). Into a LOCAL table, because
# loading symbols can grow `g.ifaces` and a `var` alias into it would then
# point at the freed buffer. Cleared only on success, so an import whose
# `.s.bif` does not exist yet is retried rather than written off.
var tab = g.ifaces[pos].interfHidden
if buildHiddenInterface(ast.program,
cachedModuleSuffix(g.config, FileIndex pos), tab):
g.ifaces[pos].interfHidden = tab
g.ifaces[pos].hiddenPending = false
else:
g.ifaces[pos].hiddenPending = false
template interfSelect(iface: Iface, importHidden: bool): TStrTable =
var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower!
if importHidden: ret = iface.interfHidden.addr
@@ -291,6 +315,7 @@ proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent):
assert m.kind == skModule
mi.modIndex = m.position
mi.importHidden = optImportHidden in m.options
if mi.importHidden: ensureHiddenIface(g, mi.modIndex)
result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name)
proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
@@ -298,6 +323,7 @@ proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
let importHidden = optImportHidden in m.options
if importHidden: ensureHiddenIface(g, m.position)
for s in g.ifaces[m.position].interfSelect(importHidden).data:
if s != nil:
yield s
@@ -314,12 +340,31 @@ proc reexportedModuleSyms*(g: ModuleGraph; m: PSym): seq[(string, string)] =
not seen.containsOrIncl(s.position):
result.add (s.name.s, cachedModuleSuffix(g.config, FileIndex s.position))
proc reexportedLocalSyms*(g: ModuleGraph; m: PSym): seq[ItemId] =
## Symbols DEFINED in `m` that reached `m`'s interface through an explicit
## `export s` rather than through a `*` marker on their declaration.
##
## `semExport` re-exports by `reexportSym`, which adds to the interface table
## and does NOT set `sfExported` — so a symbol can be importable while its
## declaration says otherwise. The NIF writer decides importability from
## `sfExported` alone and therefore missed exactly these. `std/random` does it
## (`proc initRand(): Rand` private, then `since (1, 5, 1): export initRand`),
## which is why `--ic:on` could not compile anything that reached
## `std/tempfiles` — `initRand()` was undeclared in the importer.
result = @[]
for s in g.ifaces[m.position].interf.data:
if s != nil and s.kind != skModule and sfExported notin s.flags and
s.itemId.module == m.position:
result.add s.itemId
proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
let importHidden = optImportHidden in m.options
if importHidden: ensureHiddenIface(g, m.position)
result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
proc someSymAmb*(g: ModuleGraph; m: PSym; name: PIdent; amb: var bool): PSym =
let importHidden = optImportHidden in m.options
if importHidden: ensureHiddenIface(g, m.position)
var ti: TIdentIter = default(TIdentIter)
result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name)
if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil:
@@ -574,12 +619,6 @@ proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
let ownerModule = inst.itemId.module.int
g.opsLog.add LogEntry(kind: GenericInstEntry, module: ownerModule, sym: inst)
const
InstanceDisambBit* = 0x4000_0000'i32
## Set in the `disamb` of routine instances whose value is content-derived
## (see `setInstanceDisamb`); keeps them disjoint from the small counter
## range ordinary symbols draw from, so the NIF name `name.disamb.module`
## stays collision-free within a module.
proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
concreteTypes: openArray[PType]) =
@@ -618,12 +657,6 @@ proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
break
inst.disamb = h
const
HookDisambBit* = 0x2000_0000'i32
## Set in the `disamb` of synthesized type-bound operators and `$enum`
## procs whose value is content-derived (see `setHookDisamb`); disjoint
## from both the small counter range and the `InstanceDisambBit` range.
proc setHookDisamb*(g: ModuleGraph; hook: PSym; opName: string; typ: PType) =
## Under IC, replace a synthesized hook's counter-based `disamb` with a
## content-derived one: a hash of the operation name plus the `typeKey` of
@@ -1047,7 +1080,13 @@ when not defined(nimKochBootstrap):
var isKnownFile = false
let fileIdx = g.config.registerNifSuffix(string suffix, isKnownFile)
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
# `SkipInterfaceTables`: `interf`/`interfHidden` here are scratch tables
# shared by every iteration and never read — this module is a
# dep-of-a-dep, so none of its symbols are visible to the module being
# semchecked. Building them called `loadSymFromIndexEntry` for every
# index entry of every closure member.
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden,
{SkipInterfaceTables})
registerLoadedHooks(g, precomp.logOps)
# Record this transitively-loaded module so the sem driver applies its
# VM-level load effects (macro-cache replay + `{.compileTime.}` global init)
@@ -1110,6 +1149,7 @@ when not defined(nimKochBootstrap):
strTableAdd(interf, inner)
g.ifaces[fIdx.int].interf = interf
g.ifaces[fIdx.int].interfHidden = interfHidden
g.ifaces[fIdx.int].hiddenPending = true
proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
flags: set[LoadFlag] = {}): PrecompiledModule =
@@ -1155,6 +1195,8 @@ when not defined(nimKochBootstrap):
result = loadNifModule(ast.program, fileIdx,
g.ifaces[fileIdx.int].interf,
g.ifaces[fileIdx.int].interfHidden, flags)
# The hidden-only half was not built; `ensureHiddenIface` will, if asked.
g.ifaces[fileIdx.int].hiddenPending = true
result.module = m
# Restore the module symbol's persisted flags (see ast2nif `(modflags)`);
# `cgen.genTopLevelStmt` gates the destructor pass on `sfInjectDestructors`.

497
compiler/nifbackend.nim
View File

@@ -28,6 +28,7 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen,
from cgmeth import generateIfMethodDispatchers
from transf import transformBody
from injectdestructors import injectDestructorCalls
import icprof
import ic / replayer
proc systemNifSuffix(conf: ConfigRef): string =
@@ -134,91 +135,6 @@ proc emitMethodDispatchers(g: ModuleGraph) =
if not containsOrIncl(mainMod.declaredThings, disp.id):
genProcLvl3(mainMod, disp)
proc signatureHasMetaType(t: PType; depth: int = 0): bool =
## Whether a routine signature mentions a compile-time/meta element type
## (`typed`/`untyped` — e.g. `echo`'s `varargs[typed]` — typedesc, static,
## generic param). Such routines are expanded at their call sites and never
## emitted standalone, so the per-module owned-routine seeding must skip them
## (`getTypeDescAux(tyTyped)` otherwise). `tfHasMeta` alone misses the varargs
## element case, hence the explicit scan.
result = false
if t == nil or depth > 8: return false
if t.kind == tyGenericBody:
# The uninstantiated template carried as a `tyGenericInst`'s first child
# always mentions its `tyGenericParam` placeholders, but the instance
# itself is fully concrete (e.g. `var CountTable[SigHash]`). Descending
# here would wrongly flag every routine with a generic-instance parameter
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind == tyStatic:
# A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in
# `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise
# fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal
# runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter
# (no bound value, `t.n == nil`) is meta. Without this, every routine whose
# signature touches a `static`-parameterized generic instance (the bulk of
# the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up
# an undefined reference at link (mirrors the tyGenericBody case above).
return t.n == nil
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
## A concrete, non-generic, runtime routine with a real body, OWNED by the
## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
## routine called only from other modules is still emitted by somebody) and
## the `lower` stage's owned-routine enumeration, so both stages see exactly
## the same set. The exclusions:
## - nested/closure procs (owner is a proc, not a module): emitted via their
## enclosing routine's lambda-lifting, never standalone;
## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge;
## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures:
## not real codegen targets.
## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into
## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never
## per module. A dispatcher is a `copySym` clone of the method that shares the
## method's body sub-tree (incl. its closure iterator); transforming it here
## would lambda-lift that SHARED iterator a SECOND time under a different owner
## identity, baking a conflicting `up` field → "up references do not agree"
## (the divergence is impossible in non-IC, where the dispatcher body is empty
## at lift time). So a dispatcher is never an owned runtime routine.
## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
## iterator, which is expanded at each call site) and must be emitted by its
## owner — else a cross-module `for` over it links to nothing.
##
## Generic INSTANCES (`sfFromGeneric`) are NEVER an owned runtime routine — not
## in `cg` and not in the `lower` stage. They are demanded by the backend's
## emit-everywhere path and deduped by `merge` (content C name); the frontend
## materialises them through the `(offer)` mechanism. The `lower` stage must
## not transform an instance: a not-fully-concrete instance (a closure factory
## over a `static` param, or a `$`/`=` op instance whose body resolves only at
## its further-specialised use sites) still carries unresolved overload choices
## and crashes `transformBody` (empty-`namePos` lambda, nil-typed const-fold).
s.itemId.module == modPos and
(s.kind in {skProc, skFunc, skConverter, skMethod} or
(s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
sfFromGeneric notin s.flags and
sfDispatcher notin s.flags and
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty
# NOTE: an `nkEmpty` body is NOT a disqualifier. A concrete, owned, non-
# forward/-importc/-magic routine whose body folds to nothing is still a real
# definition the owner must emit (`void f(void){}`), exactly as whole-program
# cgen does — else a cross-module caller links to nothing. This bites e.g.
# Nimbus' `extras.incInternalErrors`, a plain `proc` whose sole statement is a
# metrics-counter `.inc()` that the `metrics` library expands to a no-op when
# the importing tool (ncli) builds with `-u:metrics`; the body is then a bare
# `nkEmpty`, but `state_transition_epoch` still calls it. Forward declarations
# (the other empty-body case) carry `sfForward` and are excluded above.
proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
## Generate C code for a single module.
let moduleId = precomp.module.position
@@ -309,24 +225,30 @@ proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, nifFiles)
proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
proc loadDepClosure(g: ModuleGraph; targetSuffixes: seq[string]):
tuple[modules: seq[PrecompiledModule], precompSys: PrecompiledModule,
target: PrecompiledModule] =
## Per-module `cg`/`emit` for a NON-main target: load system + the target
## module + the target's transitive import closure ONLY — not the whole
## program. This is the "process the one file it is passed" model (à la
## Nimony's `hexer c file.nif`): the foreign symbols the target's codegen
## demands are loaded lazily by `ast2nif.moduleId`, which opens any referenced
## module's NIF index on first touch, so a body in a not-loaded module still
## resolves. The closure is loaded as full `BModule`s only so that the
## incidental `g.mods[pos]` accesses during codegen resolve; system's own
## internal closure (allocators, locks, …) is included because a target's
## emit-everywhere codegen can demand those without importing them directly.
targets: seq[PrecompiledModule]] =
## Per-module `lower`/`cg`/`emit` for a NON-main batch: load system + every
## module in the batch + their transitive import closure ONLY — not the whole
## program. This is the "process the files it is passed" model (à la Nimony's
## `hexer c file.nif`): the foreign symbols a target's codegen demands are
## loaded lazily by `ast2nif.moduleId`, which opens any referenced module's NIF
## index on first touch, so a body in a not-loaded module still resolves. The
## closure is loaded as full `BModule`s only so that the incidental
## `g.mods[pos]` accesses during codegen resolve; system's own internal closure
## (allocators, locks, …) is included because a target's emit-everywhere
## codegen can demand those without importing them directly.
##
## The whole program is no longer loaded in this process, which is what bounds
## per-process memory under nifmake's parallel fan-out (the main module's `cg`,
## which still loads everything for NimMain's init list and the method
## dispatchers, runs essentially alone since every other `.c.nif` precedes it).
##
## The batch is loaded as ONE closure: `resetForBackend`, the system load and
## the closure walk happen once no matter how many targets share the process,
## and a module in two targets' closures is loaded once. That amortization is
## the reason batches exist — a per-module process spends far more time here
## than it spends generating code.
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile)
@@ -338,18 +260,30 @@ proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
var visited = initHashSet[string]()
visited.incl systemNifSuffix(g.config)
# Only the target is codegen'd, so only it needs its full AST; the closure is
# loaded interface-only (demanded bodies come lazily from the kept-open
# streams), which is what keeps a per-module process light under parallel fan-out.
var isKnown = false
let targetIdx = registerNifSuffix(g.config, targetSuffix, isKnown)
let target = moduleFromNifFile(g, targetIdx, {LoadFullAst})
visited.incl targetSuffix
# Only the batch is codegen'd, so only it needs full ASTs; the surrounding
# closure is loaded interface-only (demanded bodies come lazily from the
# kept-open streams), which is what keeps the process light under fan-out.
var targets: seq[PrecompiledModule] = @[]
var stack: seq[ModuleSuffix] = @[]
if target.module != nil:
modules.add target
for dep in target.deps: stack.add dep
# Separate from `visited`, which exists to keep the closure walk off modules
# already loaded. System is in `visited` from the start yet can perfectly well
# BE a batch member — it is a live node with its own `.t.bif` and `.c.nif` —
# and then it needs the full-AST load like any other member, on top of the
# interface-only load above. Reusing `visited` to deduplicate members skipped
# it and produced a batch with nothing in it.
var claimed = initHashSet[string]()
for targetSuffix in targetSuffixes:
if claimed.containsOrIncl(targetSuffix): continue
var isKnown = false
let targetIdx = registerNifSuffix(g.config, targetSuffix, isKnown)
let target = moduleFromNifFile(g, targetIdx, {LoadFullAst})
targets.add target
# A member that is also another member's dependency must keep its full AST,
# so claim it before the closure walk can load it interface-only.
visited.incl targetSuffix
if target.module != nil:
modules.add target
for dep in target.deps: stack.add dep
if precompSys.module != nil:
for dep in precompSys.deps: stack.add dep
while stack.len > 0:
@@ -366,7 +300,7 @@ proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
discard setupNifBackendModule(g, m.module)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, target)
result = (modules, precompSys, targets)
proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
precompSys: PrecompiledModule; suffix: string): PrecompiledModule =
@@ -380,6 +314,18 @@ proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix:
return precompSys
proc backendBatch(conf: ConfigRef; mainSuffix: string):
tuple[members: seq[string], isMain: bool] =
## The module suffixes this invocation processes, and whether it is the
## main-module invocation. Main is never batched with anything else: it loads
## the WHOLE program (NimMain's init list and the method dispatchers are
## whole-program facts), so putting another module in with it would defeat the
## bound on per-process memory that the per-module split exists to provide.
let members = conf.icBackendModules
result = (members: members,
isMain: members.len == 0 or
(members.len == 1 and members[0] == mainSuffix))
proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
owner: PSym; seen: var IntSet) =
## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting
@@ -444,8 +390,12 @@ proc reownFromTwin(n: PNode; twin, s: PSym) =
for i in 0 ..< n.safeLen:
reownFromTwin(n[i], twin, s)
proc lowerOneModule(g: ModuleGraph; target: PrecompiledModule;
seenNested: var IntSet)
proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:<suffix>`):
## Backend lowering for this invocation's batch
## (`--icBackendStage:lower --icBackendModules:<a,b,c>`):
## enumerate the routines this module OWNS and write them to `<module>.t.nif`.
## Eventually this transforms each owned routine once, in the owner's id space,
## so `cg` reads the result instead of re-deriving it (re-derivation per
@@ -458,30 +408,46 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## stage does.
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
let batch = backendBatch(g.config, mainSuffix)
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var targets: seq[PrecompiledModule]
if batch.isMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
targets = @[findTargetModule(g, modules, precompSys, mainSuffix)]
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
(modules, precompSys, targets) = block:
icProfStart(tLoadClosure)
let r = loadDepClosure(g, batch.members)
icProfStop(tLoadClosure)
r
# ONE PSym graph for the whole batch, so the guard against transforming a
# nested routine twice has to span it: two members reaching the same nested
# closure would otherwise inject its destructors twice into the same `PSym`.
# (In the one-module-per-process fan-out the two members are two processes
# with two copies, and each injects once.)
var seenNested = initIntSet()
for target in targets:
lowerOneModule(g, target, seenNested)
proc lowerOneModule(g: ModuleGraph; target: PrecompiledModule;
seenNested: var IntSet) =
## Lower the routines `target` OWNS and write its `.t.bif`. One batch member.
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: module not found for suffix: " & g.config.icBackendModule)
"per-module lowering: module not found for suffix")
return
let modPos = target.module.position
let tb = BModuleList(g.backend).mods[modPos]
if tb == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: no backend module for suffix: " & g.config.icBackendModule)
"per-module lowering: no backend module for suffix: " &
cachedModuleSuffix(g.config, FileIndex modPos))
return
# Transform every owned routine ONCE in this single process's id space and
# re-serialize the ENTIRE module as a proper indexed NIF (`writeLoweredModule`)
@@ -497,11 +463,14 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops
# (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly
# the ops THIS stage created (not those loaded from `.s.nif`).
# Per MEMBER, not per batch: each member's `.t.bif` must carry exactly the ops
# ITS lowering lifted, the way its own process would have written them.
let opsLogStart = g.opsLog.len
# Shared across the owned loop so a nested routine reachable from more than one
# owner is transformed + destructor-injected EXACTLY once (double injection
# would emit two `=destroy`/`=copy` runs).
var seenNested = initIntSet()
# `seenNested` comes from the caller and spans the whole batch — see the
# comment at its declaration. Within one module it already served to transform
# + destructor-inject a nested routine reachable from more than one owner
# EXACTLY once (double injection would emit two `=destroy`/`=copy` runs).
icProfStart(tLowerOwned)
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
# REUSE path (`icReuseSemLowering` ON): a routine already transformed during
@@ -545,6 +514,8 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves
# the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift
# further hooks (a field's `=destroy`).
icProfStop(tLowerOwned)
icProfStart(tLowerHooks)
var hooks: seq[LogEntry] = @[]
var i = opsLogStart
while i < g.opsLog.len:
@@ -564,9 +535,11 @@ proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# Re-serialize the whole module to its suffix-based `.t.nif` (the path
# `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals
# routines itself.
icProfStop(tLowerHooks)
let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.bif").string
writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
timed tLowerWrite:
writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " &
$hooks.len & " hooks"
@@ -587,13 +560,19 @@ proc visitDep(suffix: string;
let bm = bl.mods[pm.module.position]
if bm != nil: ordered.add bm
proc cgGenerateModule(g: ModuleGraph; target: PrecompiledModule)
proc cgFinishModule(g: ModuleGraph; target: PrecompiledModule;
modules: seq[PrecompiledModule];
precompSys: PrecompiledModule)
proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:<suffix>`):
## generate C for the single module named by `icBackendModule` and write only
## its `.c.nif` artifact (no merge, no `.c` render, no cc/link — those are
## separate nifmake rules).
## Backend codegen for this invocation's batch
## (`--icBackendStage:cg --icBackendModules:<a,b,c>`): generate C for each
## member and write its `.c.nif` artifact (no merge, no `.c` render, no
## cc/link — those are separate nifmake rules).
##
## `findPendingModule` routes every demand into the target (emit-everywhere).
## `findPendingModule` routes a demand to its owner when the owner is in the
## batch and into the demanding TU otherwise (emit-everywhere).
##
## A NON-main target loads only its own import closure (`loadDepClosure`); the
## whole program is no longer pulled into every parallel `cg` process. The main
@@ -603,12 +582,11 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# gate `newSymNode`'s lazy-type marking to this stage only (see astdef)
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
let batch = backendBatch(g.config, mainSuffix)
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var targets: seq[PrecompiledModule]
if batch.isMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
@@ -619,16 +597,64 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# MERGE stage recomputes the one program-wide live set across all `.c.nif`s.
# Running a whole-program liveness pass over all ~260 NIFs in the main `cg`
# would cost ~900 MB for a result the merge stage throws away.
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
targets = @[findTargetModule(g, modules, precompSys, mainSuffix)]
else:
# No whole-program load, hence no whole-program DCE: the target emits its
# No whole-program load, hence no whole-program DCE: each member emits its
# full demanded closure and the merge stage drops what is globally dead.
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module codegen: module not found for suffix: " & g.config.icBackendModule)
return
(modules, precompSys, targets) = block:
icProfStart(tLoadClosure)
let r = loadDepClosure(g, batch.members)
icProfStop(tLoadClosure)
r
for i, target in targets:
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module codegen: module not found for suffix: " &
(if i < batch.members.len: batch.members[i] else: mainSuffix))
return
let bl = BModuleList(g.backend)
# Declare which modules this process writes a TU for, BEFORE any code is
# generated: `findPendingModule` consults the set on the very first demand, so
# a member added later would have its definitions routed into whichever TU
# asked first — which is precisely what the set exists to prevent.
for target in targets:
bl.icEmitted.incl target.module.position
# Generate EVERY member before finishing ANY of them. `finishModule` closes a
# TU (`finalCodegenActions` puts it in `modulesClosed`), and a later member's
# codegen routes definitions it does not own INTO an earlier member's TU — see
# `findPendingModule`. Finishing as we went closed those TUs first, and the
# definitions that arrived afterwards were silently dropped: 18 undefined
# symbols at link, all of them `_u`-flagged uniques whose owner happened to
# sort earlier in its batch.
timed tCgGen:
for target in targets:
cgGenerateModule(g, target)
timed tCgFinish:
for target in targets:
cgFinishModule(g, target, modules, precompSys)
# Writes each batch member's `.c.nif` (every other loaded module's TU is empty,
# so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here.
timed tCgWrite:
cgenWriteModules(g.backend, g.config)
# Always leave a `.c.nif` for every member, even one whose module has no code
# (a leaf library whose procs all emit into their users): the nifmake graph
# declares a `.c.nif` output per member, so a missing one would re-fire the
# rule forever. An empty artifact renders to an empty `.c`.
for target in targets:
let tb = bl.mods[target.module.position]
if tb != nil:
let artifact = getCFile(tb).string & ".nif"
if not fileExists(artifact):
writeCnifArtifact("", artifact,
semmedNif = toNifFilename(g.config, FileIndex target.module.position),
moduleBase = $getSomeNameForModule(tb))
proc cgGenerateModule(g: ModuleGraph; target: PrecompiledModule) =
## Generate ONE batch member's code. Does NOT finish its TU — see the caller.
# The `lower` stage already wrote each module's transformed bodies + lifted
# hooks into its `.t.nif`, which the loaders above read directly (toNifFilename
# resolves the `.t.nif`); transformed bodies arrive via loadSymFromCursor and
@@ -639,12 +665,22 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# This module's top-level `var`s with a `=destroy` registered their teardown
# in `graph.globalDestructors` during `genTopLevelStmt` above. Main's `cg` is
# a different process and never sees them, so emit them as this TU's own
# exported proc and announce the name in the meta head.
# exported proc and announce the name in the meta head. Stays HERE, in the
# generate pass: it consumes the destructors this module just registered.
let tbm = bl.mods[target.module.position]
if tbm != nil:
tbm.icGlobalDtorName = genIcModuleDestroyGlobals(g, tbm)
proc cgFinishModule(g: ModuleGraph; target: PrecompiledModule;
modules: seq[PrecompiledModule];
precompSys: PrecompiledModule) =
## Close ONE batch member's translation unit, once every member of the batch
## has generated. The artifact write is not here: `cgenWriteModules` is a
## single whole-list operation the caller runs after the whole batch.
let bl = BModuleList(g.backend)
# The main module also owns the whole-program method dispatchers + NimMain.
if sfMainModule in target.module.flags:
icProfStart(tCgInit)
emitMethodDispatchers(g)
# NimMain (generated when the main module is finished) must call every other
# module's init/datInit. Those translation units are produced by their own
@@ -710,24 +746,15 @@ proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# `globalDestructors` list backwards. Main's own destructors come first and
# are added by `finalCodegenActions` itself.
reverse g.icModuleDtors
icProfStop(tCgInit)
let tb = bl.mods[target.module.position]
if tb != nil:
finishModule(g, tb)
# Writes only the target's `.c.nif` (every other loaded module's TU is empty,
# so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here.
cgenWriteModules(g.backend, g.config)
# Always leave a `.c.nif` for the target, even when the module has no code
# (a leaf library whose procs all emit into their users): the per-module
# nifmake graph declares one `.c.nif` output per `cg` rule, so a missing one
# would re-fire the rule forever. An empty artifact renders to an empty `.c`.
if tb != nil:
let artifact = getCFile(tb).string & ".nif"
if not fileExists(artifact):
writeCnifArtifact("", artifact,
semmedNif = toNifFilename(g.config, FileIndex target.module.position),
moduleBase = $getSomeNameForModule(tb))
# Record this module's C compile/link directives next to its `.c` so the
# `link` stage can recover them without loading the module graph. See
# `replayer.writeBackendActions`.
writeBackendActions(g, target.module, target.topLevel,
getCFile(tb).string & BackendActionsExt)
proc generateMergeStage(g: ModuleGraph) =
## Per-module backend merge (`--icBackendStage:merge`): a pure artifact
@@ -764,16 +791,19 @@ proc generateMergeStage(g: ModuleGraph) =
" live: " & $decision.live.len & " defs: " & $decision.defs &
" liveDefs: " & $decision.liveDefs & " owned: " & $decision.owners.len
proc emitOneModule(g: ModuleGraph; mainFileIdx: FileIndex; member: string;
isMain: bool; decision: MergeDecision)
proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend emit (`--icBackendStage:emit --icBackendModule:<suffix>`):
## Backend emit for this invocation's batch
## (`--icBackendStage:emit --icBackendModules:<a,b,c>`):
## render the target module's final `.c` from its `.c.nif` and the merge
## decision. Loads the target the same way `cg` does so `getCFile` returns the
## identical path `cg` wrote to (the main module's source-vs-suffix aliasing in
## particular); no codegen runs. A non-main target loads only its own closure
## (`loadDepClosure`) so emit, like `cg`, stays bounded under parallel fan-out.
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
let batch = backendBatch(g.config, mainSuffix)
# emit renders a module's final `.c` PURELY from its own `.c.nif` and the merge
# decision (see `renderCFromArtifact` — text filtering, no AST is touched). It
# used to load the target's whole transitive import closure as BModules solely
@@ -786,20 +816,37 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# path directly instead — the SAME pure computation `deps.nim.backendCFile`
# uses to DECLARE this stage's output (`getCFile` == that formula) — so an emit
# process loads nothing and the fire-all costs process-startup, not a graph load.
# The decision is read ONCE for the batch: it is a whole-program artifact, and
# re-reading it per member was a per-process cost the batch exists to remove.
let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
return
let members = if batch.members.len == 0: @[mainSuffix] else: batch.members
for member in members:
# Per MEMBER, not per batch. `backendBatch.isMain` answers "is this
# invocation the main-module invocation", which is the right question for
# `lower`/`cg` (main loads the whole program, so it is never batched with
# anything). emit has no such constraint and batches freely, so main can sit
# in a batch with others — and then the batch-wide flag sent main's `.c` to
# the path derived from its SUFFIX rather than from its source file, and its
# `.c` was never written.
emitOneModule(g, mainFileIdx, member, member == mainSuffix, decision)
proc emitOneModule(g: ModuleGraph; mainFileIdx: FileIndex; member: string;
isMain: bool; decision: MergeDecision) =
## Render ONE batch member's final `.c` from its `.c.nif` and the batch's
## merge decision.
let cfilename =
if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx)
else: AbsoluteFile g.config.icBackendModule
if isMain: AbsoluteFile toFullPath(g.config, mainFileIdx)
else: AbsoluteFile member
let cfile = changeFileExt(completeCfilePath(g.config,
mangleModuleName(g.config, cfilename).AbsoluteFile), icCFileExt(g.config)).string
let artifact = cfile & ".nif"
if not fileExists(artifact):
rawMessage(g.config, errGenerated,
"per-module emit: missing .c.nif artifact for suffix: " & g.config.icBackendModule)
return
let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
"per-module emit: missing .c.nif artifact for suffix: " & member)
return
var dropped = 0
let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped)
@@ -814,6 +861,15 @@ proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
# up-to-date check, not a shared prerequisite in nifmake's mtime ordering.
if not fileExists(cfile) or readFile(cfile) != code:
writeFile(cfile, code)
# ... but nifmake needs SOME output whose mtime proves "this rule ran since its
# inputs last moved". With the `.c` as the only output, the content-stable write
# above is indistinguishable from not having run: `merge` rewrites the decision
# file unconditionally, so every `emit` whose `.c` came out byte-identical stays
# older than a declared input and re-fires on every warm build from then on
# (measured: all 218 emit rules of a 219-module program, on a NO-OP build).
# The stamp is written unconditionally and is the rule's freshness proof; the
# `.c` keeps its content-stable mtime so `callCCompiler` still reuses the `.o`.
writeFile(cfile & ".stamp", $code.len & " " & $dropped & "\n")
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
$dropped & " bodies (" & $code.len & " bytes)"
@@ -822,53 +878,62 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend link (`--icBackendStage:link`): the `emit` stages have
## written every module's `.c`; register them and run the C compiler + linker
## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and
## skips up-to-date objects itself). No codegen runs — the graph is loaded only
## so `getCFile` yields each module's emitted `.c` path.
let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
# The per-module `cg` processes each collect their module's C compile/link
# directives (`{.passL: "-lm".}` etc.) via `replayBackendActions`, but those
# live in the cg process and never reach this separate link process. Re-collect
# every loaded module's directives here so the final `callCCompiler` sees them
# (without this, math's `-lm` is lost → undefined `floor`/`pow`/… at link).
for m in modules:
replayBackendActions(g, m.module, m.topLevel)
if precompSys.module != nil:
replayBackendActions(g, precompSys.module, precompSys.topLevel)
let bl = BModuleList(g.backend)
## skips up-to-date objects itself). No codegen runs and NO MODULE GRAPH IS
## LOADED.
##
## It used to load the whole import closure (`loadBackendModules`) for two
## things only: each module's `.c` path via `getCFile`, and its recorded C
## directives via `replayBackendActions`. That was 3.7s of the ~11s serial
## backend critical path on a 219-module program — a whole-program
## deserialization to recover a list of paths and a handful of strings. Both
## are now read from artifacts the earlier stages already produce:
## * the driver's `LiveModulesFile` manifest lists every live module's
## `.c.nif`, and the `.c` sits beside it (`emit`'s output);
## * each module's `cg` wrote its directives to a `.cflags` sidecar.
let nimcache = getNimcacheDir(g.config).string
var cfiles: seq[string] = @[]
let manifest = nimcache / LiveModulesFile
if fileExists(manifest):
for line in lines(manifest):
let p = line.strip()
if p.len > 0 and p.endsWith(".nif"): cfiles.add p[0 ..< p.len - ".nif".len]
else:
# A cache written by an older compiler has no manifest; fall back to the
# `.c` files sitting next to the artifacts.
for artifact in walkFiles(nimcache / ("*" & icCFileExt(g.config) & ".nif")):
cfiles.add artifact[0 ..< artifact.len - ".nif".len]
sort cfiles
var addedCFiles = initHashSet[string]()
for m in bl.mods:
if m != nil:
let cfile = getCFile(m)
# Only modules that are their own cg/emit target produced a `.c`; the rest
# (extra members of system's closure that no build rule targets) had their
# code emit-everywhere'd into the targets, so they have no file to compile.
if not fileExists(cfile.string): continue
addedCFiles.incl extractFilename(cfile.string)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
# `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its
# SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so
# `callCCompiler` skips its compile but still links the existing object. This
# is what makes a localized edit recompile only the handful of `.c`s the
# `emit` stage actually rewrote, instead of every object every time — the
# final piece of per-module backend incrementality after the merge barrier.
addExternalFileToCompile(g.config, cf)
for cpath in cfiles:
# Only modules that are their own cg/emit target produced a `.c`; the rest
# had their code emit-everywhere'd into the targets, so there is nothing to
# compile for them.
if not fileExists(cpath): continue
addedCFiles.incl extractFilename(cpath)
# The directives this module recorded (`{.passL: "-lm".}` etc.); without
# them math's `-lm` is lost -> undefined `floor`/`pow`/… at link.
applyBackendActions(g, cpath & BackendActionsExt)
let cfile = AbsoluteFile cpath
var cf = Cfile(nimname: splitFile(cfile).name, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
# `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its
# SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so
# `callCCompiler` skips its compile but still links the existing object. This
# is what makes a localized edit recompile only the handful of `.c`s the
# `emit` stage actually rewrote, instead of every object every time.
addExternalFileToCompile(g.config, cf)
# deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build
# node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)`
# import) that the NIF-`deps` walk above never reaches because the condition is
# off. Such a node still emitted a `.c`, and it can OWN a live generic instance
# that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by
# `strutils.escape`) — so its body must be at link or that reference is
# node (e.g. `net`'s `when defineSsl: import openssl`) that the manifest above
# may not cover. Such a node still emitted a `.c`, and it can OWN a live generic
# instance that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused
# by `strutils.escape`) — so its body must be at link or that reference is
# undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol;
# a node that owns nothing live (a Windows-only winsock node on Linux) is
# correctly skipped.
block:
let nimcache = getNimcacheDir(g.config).string
let decision = readMergeDecision(nimcache / MergeDecisionFile)
if not decision.broken:
var liveOwners = initHashSet[string]()
@@ -880,6 +945,7 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
if addedCFiles.containsOrIncl(cbase): continue
let cfile = AbsoluteFile(nimcache / cbase)
if not fileExists(cfile.string): continue
applyBackendActions(g, cfile.string & BackendActionsExt)
var cf = Cfile(nimname: cbase, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
@@ -890,20 +956,27 @@ proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
when defined(icBNodeProf): profStageName = g.config.icBackendStage
if g.config.icBackendStage == "lower":
generateLowerStage(g, mainFileIdx)
timed tStage: generateLowerStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "cg":
generateCgStage(g, mainFileIdx)
timed tStage: generateCgStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "merge":
generateMergeStage(g)
timed tStage:
timed tMergeStage:
generateMergeStage(g)
return
elif g.config.icBackendStage == "emit":
generateEmitStage(g, mainFileIdx)
timed tStage:
timed tEmitRender:
generateEmitStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "link":
generateLinkStage(g, mainFileIdx)
timed tStage:
timed tLinkStage:
generateLinkStage(g, mainFileIdx)
return
else:
rawMessage(g.config, errGenerated,

247
compiler/nifstreams.nim Normal file
View File

@@ -0,0 +1,247 @@
## nifstreams — the classic NIF streaming surface, used ONLY by this compiler's
## IC modules: ast2nif, deps, modulegraphs and pipelines import it and must keep
## compiling unchanged across nimony's own refactorings.
##
## It used to live in `dist/nimony/src/lib`, which is where the rest of the NIF
## stack still is. It does not belong there: nimony's own code imports nifpools
## (via nifprelude) and is under standing orders never to import this file, so
## nothing over there ever exercised it — which is exactly how it came to hand
## out `TagLit` where every caller here tests for `ParLe` (see `next`), silently
## emptying the IC build graph. A compatibility shim with exactly one consumer
## belongs in the consumer's repo, where its tests run and its contract is
## somebody's problem.
##
## Everything it adapts (`nifpools`, `nifreader`, `lineinfos`) still comes from
## `dist/nimony`; only the adapter moved.
##
## Everything here is an honest adapter, not a fake:
## * Floats get a REAL interning pool: `pool.floats.getOrIncl` returns a
## `FloatId` index, `floatToken` packs it into a genuine `FloatLit` NifToken
## (transit-only: it must never enter a TokenBuf, whose float encoding is
## inline multi-token), and `pool.floats[t.floatId]` decodes it — lossless.
## * `Stream`/`next` wrap the textual nifreader; the unified NifKind has real
## `ParLe`/`ParRi`/`EofToken` members, so structural scanners (deps.nim)
## see the exact classic kinds. Ident/StringLit/Symbol payloads are interned
## into the global `pool`, so `pool.strings[t.litId]` works as before.
## Number tokens keep their KIND only (a 4-byte token cannot always carry
## the value); classic scanners never read those payloads.
import std / tables
import "../dist/nimony/src/lib" / nifpools
# `except`: the frontend went all-NifLineInfo; the classic side keeps speaking
# PackedLineInfo, so nifpools' same-name/same-params variants must not leak
# through (`info(n: NifToken)` differs only in return type, `NoLineInfo` is a
# same-name const of a different type — either would be ambiguous or wrong for
# ast2nif). The classic replacements are defined below / come from lineinfos.
# `tagId` is excluded for a different reason: nifpools decodes the 9-bit field
# of a real `TagLit`, but this surface hands out `ParLe` tokens whose tag id
# fills the whole 28-bit payload (see `next`), so the decode below is the only
# correct one here.
export nifpools except info, NoLineInfo, tagId
import "../dist/nimony/src/lib" / lineinfos
export lineinfos
from "../dist/nimony/src/lib" / nifreader import Reader, ExpandedToken, decodeStr
# ── Classic names the Nim compiler side still uses ───────────────────────
type
PackedToken* = NifToken ## ast2nif still says PackedToken
# Raw payload decodes, sound ONLY on this surface. Every token here comes from
# `next` or the classic `symToken`/`strToken`/`identToken` constructors, which
# intern EVERY literal — including names of at most `StrInlineMaxLen` bytes,
# which the nifcore builders would instead store inside the token. On such an
# inline token the payload is packed bytes, not an id, so nifpools (nimony's own
# surface, where buffers come from the builders) deliberately has no equivalent:
# there it must go through a `Cursor`, which handles both encodings.
proc tagId*(n: NifToken): TagId {.inline.} = TagId(uoperand(n))
## Classic `ParLe` tokens (see `next`) keep the tag id in the full 28-bit
## payload rather than in `TagLit`'s 9-bit field: `globalTags` already holds
## 355 tags before the Nim compiler registers its own dialect, so a 512-tag
## ceiling is not a ceiling this surface can live under.
proc litId*(n: NifToken): StrId {.inline.} = StrId(uoperand(n) shr 1)
proc symId*(n: NifToken): SymId {.inline.} = SymId(uoperand(n) shr 1)
proc litId*(c: Cursor): StrId {.inline.} = strId(c)
proc firstSon*(n: Cursor): Cursor {.inline.} = childCursor(n)
var lineMan*: LineInfoManager
## The classic packed line-info side channel (`pool.man`). Frontend code no
## longer uses it — it lives here purely for ast2nif's writer, which packs
## `TLineInfo` into `PackedLineInfo` and unpacks on emit.
template files*(p: Pool): untyped = p.filenames
template tags*(p: Pool): untyped = globalTags.tags
template man*(p: Pool): untyped = lineMan
proc info*(n: NifToken): PackedLineInfo {.inline.} = lineinfos.NoLineInfo
## Classic tokens carried their line info inline; a bare 4-byte nifcore
## token cannot, so reading it back yields `NoLineInfo` (ast2nif's
## `emitInfo(t.info)` then emits nothing — matching the writer, which
## attaches real positions at the builder level instead).
proc info*(c: Cursor): PackedLineInfo {.inline.} =
## Classic packed view of a cursor's line info (ast2nif shadows this with
## its own NifLineInfo template; kept for any other classic reader).
let li = rawLineInfo(c)
if li.file.isValid: pack(lineMan, li.file, li.line, li.col)
else: lineinfos.NoLineInfo
type
IntId* = distinct int64 ## value carriers (nifcore stores inline)
UIntId* = distinct uint64
## Identity proxies: the id already carries the value, `[]` returns it.
IntegersProxy* = object
UIntegersProxy* = object
func `==`*(a, b: IntId): bool {.borrow.}
func `==`*(a, b: UIntId): bool {.borrow.}
template integers*(p: Pool): IntegersProxy = IntegersProxy()
template uintegers*(p: Pool): UIntegersProxy = UIntegersProxy()
template `[]`*(x: IntegersProxy; id: IntId): int64 = int64(id)
template `[]`*(x: UIntegersProxy; id: UIntId): uint64 = uint64(id)
# nifcore stores integers inline: the "id" is the value itself.
template getOrIncl*(x: IntegersProxy; v: int64): IntId = IntId(v)
template getOrIncl*(x: UIntegersProxy; v: uint64): UIntId = UIntId(v)
proc intId*(n: NifToken): IntId {.inline.} = IntId(n.soperand)
proc uintId*(n: NifToken): UIntId {.inline.} = UIntId(uoperand(n))
proc intId*(c: Cursor): IntId {.inline.} = IntId(intVal(c))
proc uintId*(c: Cursor): UIntId {.inline.} = UIntId(uintVal(c))
proc addIntLit*(dest: var TokenBuf; id: IntId; info: PackedLineInfo) =
addIntLit(dest, int64(id))
if info.isValid:
let u = unpack(lineMan, info)
appendLineInfo(dest, u.file, u.line, u.col)
# Classic single-token constructors with a (dropped) line-info argument.
proc strToken*(s: StrId; info: PackedLineInfo): NifToken {.inline.} = strLitToken(s)
proc symToken*(id: SymId; info: PackedLineInfo): NifToken {.inline.} = symToken(id)
proc identToken*(id: StrId; info: PackedLineInfo): NifToken {.inline.} = identToken(id)
proc dotToken*(info: PackedLineInfo): NifToken {.inline.} = dotToken()
proc charToken*(ch: char; info: PackedLineInfo): NifToken {.inline.} = charToken(ch)
# ── Classic interned float literals (ast2nif) ────────────────────────────
type
FloatId* = distinct uint32 ## 1-based index into the global float pool
FloatPool* = object
values: seq[float64]
lookup: Table[uint64, uint32] # bit pattern -> 1-based id
func `==`*(a, b: FloatId): bool {.borrow.}
var globalFloats*: FloatPool
template floats*(p: Pool): var FloatPool = globalFloats
proc getOrIncl*(fp: var FloatPool; v: float64): FloatId =
let bits = cast[uint64](v)
let existing = fp.lookup.getOrDefault(bits, 0'u32)
if existing != 0'u32:
result = FloatId(existing)
else:
fp.values.add v
let id = uint32(fp.values.len)
fp.lookup[bits] = id
result = FloatId(id)
proc `[]`*(fp: FloatPool; id: FloatId): float64 {.inline.} =
fp.values[int(uint32(id)) - 1]
proc floatToken*(id: FloatId; info: PackedLineInfo): NifToken {.inline.} =
## Transit-only token: carries the pool index so the receiver can decode it
## via `pool.floats[t.floatId]`. It must never be appended to a TokenBuf
## (nifcore stores floats inline as a multi-token encoding); the line info
## is dropped like in the other classic token constructors.
NifToken((uint32(id) shl KindBits) or uint32(FloatLit))
proc floatId*(n: NifToken): FloatId {.inline.} = FloatId(uoperand(n))
# ── Classic streaming text reader (deps.nim) ─────────────────────────────
type
Stream* = object
r*: Reader
proc parLeToken*(t: TagId): NifToken {.inline.} =
## The classic surface's opening-tag token: kind `ParLe`, tag id in the
## payload. Transit-only, like `floatToken` — a `ParLe` never appears in a
## binary token stream, so this must not be appended to a TokenBuf.
NifToken((uint32(t) shl KindBits) or uint32(ParLe))
proc open*(filename: string): Stream =
Stream(r: nifreader.open(filename))
proc close*(s: var Stream) =
nifreader.close(s.r)
proc next*(s: var Stream): NifToken =
## One classic packed token per call. Pool-referencing kinds are interned
## into the global `pool`/`globalTags`, so `.litId`/`.tagId` accessors and
## `pool.strings[...]`/`pool.tags[...]` lookups behave exactly as classic
## nifstreams did. Kinds without a pool payload come back kind-only.
var t = default(ExpandedToken)
nifreader.next(s.r, t)
case t.tk
of ParLe:
# NOT `tagLitToken`: that would set the kind to `TagLit`, and every classic
# structural scanner tests for `ParLe` (deps.nim walks the import graph that
# way). Emitting `TagLit` here made every one of those tests silently fail —
# the scanner saw an unknown token, skipped the subtree, and the Nim
# compiler's IC build graph came out missing most of its edges.
result = parLeToken(registerTag(globalTags, decodeStr(s.r, t)))
of Ident:
result = identToken(pool.strings.getOrIncl(decodeStr(s.r, t)))
of StrLit:
result = strLitToken(pool.strings.getOrIncl(decodeStr(s.r, t)))
of Symbol:
result = symToken(pool.syms.getOrIncl(decodeStr(s.r, t)))
of SymbolDef:
result = symdefToken(pool.syms.getOrIncl(decodeStr(s.r, t)))
else:
# ParRi/EofToken/DotToken/CharLit/numbers: correct kind, no payload.
result = NifToken(uint32(t.tk))
when isMainModule:
# `nim c -r compiler/nifstreams.nim`.
#
# The promise this checks: structural scanners see the CLASSIC kinds. Nim's deps.nim walks
# the import graph by testing `t.kind == ParLe` and then reading
# `pool.tags[t.tagId]`. Hand out nifcore's own `TagLit` instead and every one
# of those tests falls through silently — the scanner treats the opener as an
# unknown token, skips the subtree, and Nim's IC build graph comes out missing
# most of its edges while each individual file still "parses" fine.
import std / [os, syncio]
from "../dist/nimony/src/lib" / nifreader import processDirectives
from std / assertions import assert
let f = getTempDir() / "nifstreams_selftest.nif"
syncio.writeFile f, "(.nif27)\n(stmts (import (infix / std (bracket os osproc))) (x \"s\" y))\n"
var kinds: seq[NifKind] = @[]
var tagNames: seq[string] = @[]
var lits: seq[string] = @[]
var s = nifstreams.open(f)
discard processDirectives(s.r)
while true:
let t = next(s)
if t.kind == EofToken: break
kinds.add t.kind
case t.kind
of ParLe: tagNames.add pool.tags[t.tagId]
of Ident, StrLit: lits.add pool.strings[t.litId]
else: discard
nifstreams.close(s)
removeFile f
assert tagNames == @["stmts", "import", "infix", "bracket", "x"], $tagNames
assert lits == @["/", "std", "os", "osproc", "s", "y"], $lits
assert ParRi in kinds, "closers must stay classic too"
assert TagLit notin kinds, "an opener must arrive as ParLe, not TagLit"
echo "success"

16
compiler/options.nim
View File

@@ -29,7 +29,7 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "37"
icFormatVersion* = "38"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
@@ -468,10 +468,16 @@ type
# codegen+DCE+cc+link in one process). The stages
# are wired as nifmake rules by `deps.nim`'s backend
# build file. See `compiler/nifbackend.nim`.
icBackendModule*: string # under `nim nifc` with icBackendStage in {cg,emit}:
# the NIF module suffix this invocation codegens or
# emits. The other modules are loaded only so types
# resolve; their definitions are referenced extern.
icBackendModules*: seq[string]
# under `nim nifc` with icBackendStage in
# {lower,cg,emit}: the NIF module suffixes this
# invocation processes — its BATCH. One entry is
# the per-module fan-out; several share one process
# and therefore ONE dependency-closure load between
# them, which is the whole point (see
# `nifbackend.loadDepClosure`). Every other module
# is loaded only so types resolve; its definitions
# are referenced extern. Empty = the main module.
spellSuggestMax*: int # max number of spelling suggestions for typos
cppDefines*: HashSet[string] # (*)

14
compiler/pipelines.nim
View File

@@ -6,9 +6,11 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs,
when not defined(nimKochBootstrap):
import vmdef
import ast2nif
import "../dist/nimony/src/lib" / [nifstreams, bitabs]
import nifstreams
import "../dist/nimony/src/lib" / bitabs
import pipelineutils
import icprof
import ../dist/checksums/src/checksums/sha1
@@ -335,10 +337,12 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
# `injectDestructorCalls` and top-level locals were never destroyed.
let moduleFlags =
if sfInjectDestructors in module.flags: ModFlagInjectDestructors else: 0'i32
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId,
expansions, moduleFlags)
timed tWriteNif:
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId,
expansions, moduleFlags,
reexportedLocalSyms(graph, module))
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]

26
compiler/trees.nim
View File

@@ -87,8 +87,9 @@ proc getMagic*(op: PNode): TMagic =
if op == nil: return mNone
case op.kind
of nkCallKinds:
case op[0].kind
of nkSym: result = op[0].sym.magic
let callee = op.firstSon
case callee.kind
of nkSym: result = callee.sym.magic
else: result = mNone
else: result = mNone
@@ -107,10 +108,11 @@ proc isDeepConstExpr*(n: PNode; preventInheritance = false): bool =
of nkCharLit..nkNilLit:
result = true
of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv:
result = isDeepConstExpr(n[1], preventInheritance)
result = isDeepConstExpr(n.secondSon, preventInheritance)
of nkCurly, nkBracket, nkPar, nkTupleConstr, nkObjConstr, nkClosure, nkRange:
for i in ord(n.kind == nkObjConstr)..<n.len:
if not isDeepConstExpr(n[i], preventInheritance): return false
# `nkObjConstr` carries its TYPE as child 0 and its fields from 1.
for it in sonsFrom(n, ord(n.kind == nkObjConstr)):
if not isDeepConstExpr(it, preventInheritance): return false
if n.typ.isNil: result = true
else:
let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
@@ -140,16 +142,16 @@ proc isRange*(n: PNode): bool {.inline.} =
result = false
proc whichPragma*(n: PNode): TSpecialWord =
let key = if n.kind in nkPragmaCallKinds and n.len > 0: n[0] else: n
let key = if n.kind in nkPragmaCallKinds and n.hasSons: n.firstSon else: n
case key.kind
of nkIdent: result = whichKeyword(key.ident)
of nkSym: result = whichKeyword(key.sym.name)
of nkCast: return wCast
of nkClosedSymChoice, nkOpenSymChoice, nkOpenSym:
return whichPragma(key[0])
return whichPragma(key.firstSon)
of nkBracketExpr:
if n.kind notin nkPragmaCallKinds: return wInvalid
result = whichPragma(key[0])
result = whichPragma(key.firstSon)
if result notin {wHint, wHintAsError, wWarning, wWarningAsError}:
# note bracket pragmas, see processNote
result = wInvalid
@@ -217,11 +219,11 @@ proc getRoot*(n: PNode): PSym =
result = nil
of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr, nkHiddenAddr, nkAddr:
result = getRoot(n[0])
result = getRoot(n.firstSon)
of nkHiddenStdConv, nkHiddenSubConv, nkConv:
result = getRoot(n[1])
result = getRoot(n.secondSon)
of nkCallKinds:
if getMagic(n) == mSlice: result = getRoot(n[1])
if getMagic(n) == mSlice: result = getRoot(n.secondSon)
else: result = nil
else: result = nil
@@ -253,7 +255,7 @@ proc isRunnableExamples*(n: PNode): bool =
n.kind == nkIdent and n.ident.id == ord(wRunnableExamples)
proc skipAddr*(n: PNode): PNode {.inline.} =
result = if n.kind in {nkAddr, nkHiddenAddr}: n[0] else: n
result = if n.kind in {nkAddr, nkHiddenAddr}: n.firstSon else: n
proc getPotentialWrites*(n: PNode; mutate: bool; result: var seq[PNode]) =
case n.kind:

10
compiler/types.nim
View File

@@ -119,7 +119,7 @@ proc getOrdValueAux*(n: PNode, err: var bool): Int128 =
of nkNilLit:
int128.Zero
of nkHiddenStdConv:
getOrdValueAux(n[1], err)
getOrdValueAux(n.secondSon, err)
else:
err = true
int128.Zero
@@ -1398,11 +1398,11 @@ proc skipConv*(n: PNode): PNode =
of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64:
# only skip the conversion if it doesn't lose too important information
# (see bug #1334)
if n[0].typ.classify == n.typ.classify:
result = n[0]
if n.firstSon.typ.classify == n.typ.classify:
result = n.firstSon
of nkHiddenStdConv, nkHiddenSubConv, nkConv:
if n[1].typ.classify == n.typ.classify:
result = n[1]
if n.secondSon.typ.classify == n.typ.classify:
result = n.secondSon
else: discard
proc skipHidden*(n: PNode): PNode =

53
doc/ic.md
View File

@@ -380,6 +380,59 @@ widely-imported module is not, and the cost is almost entirely frontend re-sem.
(see the comment at `generateEmitStage`): partial `emit` leaves inconsistent
ownership across the `.c` set. This path was tried and reverted; do not retry.
Where a cold build's time is (measured)
---------------------------------------
Numbers from `-d:icBNodeProf` (`compiler/icprof.nim`; each process appends a
line to `$NIM_IC_BNODE_PROF` tagged `stage=<name>`), on Atlas, 204 modules,
cold, 2026-08-31. They are recorded here because two obvious optimisations
were tried against them and did not pay.
Per stage, summed process wall, parallel build of 9.66s elapsed:
| stage | procs | wall |
| ----- | ----- | ---- |
| frontend (`nim m`) | 181 | 10.60s |
| lower | 14 | 4.49s |
| cg | 14 | 4.50s |
| merge | 1 | 0.20s |
| emit | 14 | 0.42s |
| link (the whole C compile + link) | 1 | 1.65s |
A `nim m` process splits as: startup 2%, loading imported `.s.bif` 46%,
writing its own `.s.bif` 18%, sem + parse 34% — two thirds of the frontend is
artifact I/O. The loading is not concentrated anywhere (`BifLoad` 695ms,
`PosIndex` 519ms, `ModuleId` 841ms, `TopLevel` 1459ms = offers 569 + export
branch 312 + log ops 137 + the bare cursor walk ~371); it is 180 processes each
re-parsing ~20 modules' interfaces out of 44.7MB of `.s.bif`, i.e. the
amortisation problem that batching solved for the backend
(`loadDepClosure` 10.2s -> 1.3s) and the frontend has not solved.
- **Hidden interface stubs** were 1.05s of that loading (1.70M stubs against
0.29M exported ones) and are now built on demand
(`modulegraphs.ensureHiddenIface`). A module has TWO FileIndexes — the NIF
suffix's `fikNifModule` entry keys `DecodeContext.mods`, the source file's
keys `g.ifaces` — so the lazy builder takes a suffix.
- **The tooling-only header records** (`sig`, `expansion`, `modulesrc`) are
80% of every module header the loader walks (3.36M of 4.19M nodes) and
skipping them entirely was measured at 53ms: `skip` on a `TagLit` is a
jump, ~16ns a node. Not worth a format change.
- **The C compiler** is the largest CPU item (12.2s against a whole-program
build's 10.2s) and the smallest wall lever: it fans out across cores, and the
excess over a whole-program build is ~0.4s of wall. 3.8MB of the 5.4MB of
extra C is per-TU prototypes and typedefs, intrinsic to 204 translation units
instead of 139; 53 of the 204 object files define nothing and compiling all
of them costs 0.23s of user time. Fewer, larger TUs is the only real fix and
trades directly against what IC exists for.
- **Reading routine bodies off a `.bif` cursor instead of a `PNode`** was
built and measured (branch `araq-ic-fixes2`, removed again in
`araq-ic-fixes3`): it reached parity with the tree, not a win, and could
only ever have saved `transformBody` + the body hand-off — under 1% of the
build. The lasting result of that work is the loader's `oldLineInfo`
memoization, which halved a cold `--ic:on` build, and the cgen files'
iterator/named-accessor vocabulary (`sons`/`sonsFrom`/`sonsButLast`,
`firstSon`/`secondSon`/`son`, `baseClass`/`returnType`/`elementType`).
Code, logic & debugging
========================

34
koch.nim
View File

@@ -16,11 +16,12 @@ const
ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2
SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936"
NimonyStableCommit = "f831b953d7c21d9a4b11d0042039e7f84d7c8dc9" # unversioned \
NimonyStableCommit = "1721aab3cad18663da92c2b85508b1f2ff73e3df" # unversioned \
# Note that Nimony uses Nim as a git submodule but we don't want to install
# Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
# is **required** here.
# Commit from 2026-07-10 -- stable .bif file format
# Commit from 2026-08-31 -- nifcore-based lib; `bif.load` fills pools with
# `addOrdered` instead of hashing every entry it just read back in order.
# examples of possible values for fusion: #head, #ea82b54, 1.2.3
FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"
@@ -196,10 +197,31 @@ proc bundleChecksums(latest: bool) =
# to `koch boot`, but `nimCompileFold` spawns a fresh `nim c` that would
# otherwise inherit the ambient configuration.
const nifOptions = "-d:release --noNimblePath --skipUserCfg --skipParentCfg"
if not fileExists("bin/nifler".exe):
nimCompileFold("Compile nifler", "dist/nimony/src/nifler/nifler.nim", options = nifOptions)
if not fileExists("bin/nifmake".exe):
nimCompileFold("Compile nifmake", "dist/nimony/src/nifmake/nifmake.nim", options = nifOptions)
# Rebuilding these only when the binary is ABSENT silently keeps the tools of
# the PREVIOUS pin: bump `NimonyStableCommit` in a checkout that already has
# `bin/nifler`, and the compiler links the new `dist/nimony/src/lib` while
# `nifler`/`nifmake` still speak the old one. A fresh CI checkout has no
# `bin/`, so it builds them and looks green — only the working tree that
# already has them breaks, which is the worst way round to find out. So stamp
# each tool with the nimony commit it came from and rebuild on a mismatch.
# If the commit cannot be determined (a bundled `dist` with no `.git`), fall
# back to the old build-if-absent rule rather than rebuilding every time.
let nimonyHead = block:
let (outp, status) = osproc.execCmdEx(
"git -C " & quoteShell(distDir / "nimony") & " rev-parse HEAD")
if status == 0: outp.strip else: ""
proc bundleNifTool(name, src: string) =
let stamp = "bin" / ("." & name & ".nimony-commit")
let builtFrom = if fileExists(stamp): readFile(stamp).strip else: ""
if not fileExists(("bin" / name).exe) or
(nimonyHead.len > 0 and builtFrom != nimonyHead):
nimCompileFold("Compile " & name, src, options = nifOptions)
if nimonyHead.len > 0: writeFile(stamp, nimonyHead)
bundleNifTool("nifler", "dist/nimony/src/nifler/nifler.nim")
bundleNifTool("nifmake", "dist/nimony/src/nifmake/nifmake.nim")
proc bundleNimsuggest(args: string) =
bundleChecksums(false)

4
tests/ic/mexportprivate.nim Normal file
View File

@@ -0,0 +1,4 @@
proc pub*(x: int): int = x + 1
proc hidden(): int = 42 # no `*` ...
export hidden # ... but explicitly re-exported

4
tests/ic/mimporthidden.nim Normal file
View File

@@ -0,0 +1,4 @@
proc pub*(x: int): int = x + 1
proc secret(): int = 7 # no `*`
proc hiddenToo(x: int): int = x

48
tests/ic/readme.md Normal file
View File

@@ -0,0 +1,48 @@
# Running `tests/ic`
./bin/testament --nim:<your compiler> cat ic
## The metamorphic tests are expensive, and look hung when they are not
16 of the tests carry `#? metamorphic`. Each has 3–4 `#!STEP` directives, and
every step compiles the program **twice** — once under `nim ic`, once with
`nim c` as the reference oracle. That is 100+ full compilations for the
category. Under `--ic:on` each compilation additionally fans out one backend
process per module per stage, and each of those is a compiler holding its own
module graph (~800MB peak).
**A `nim ic` parent sitting at 0% CPU is normal.** It is waiting on its
children. It is not a deadlock, and neither is a metamorphic test that occupies
the runner for many minutes. Before concluding anything is stuck, check that the
test NAME changes over a few minutes — that is the difference between slow and
hung, and it is easy to get wrong.
On a memory-constrained machine the fan-out will swap. The symptoms are exactly
the ones that read as a deadlock: several processes at 0% CPU, no output, a
different test "stuck" on every run, and the same compilation finishing in
seconds when run on its own. Check `vm_stat` (page-ins per second) and
`sysctl vm.swapusage` before looking for a bug. This was diagnosed as a
testament/`nim ic` interaction more than once before anyone measured.
Cap the fan-out to fit the machine — precedence documented at `deps.nim`'s
`let parallel`:
--parallelBuild:N # standard flag, given meaning under IC
-d:icJobs:N # same cap, legacy define
-d:icNoParallel # serial, and non-interleaved child output
Serial output matters for a second reason: the parallel backend processes share
one stderr, so any per-process diagnostic printing (`-d:icCanRaiseLog`)
interleaves and produces torn lines. Either use
`-d:icNoParallel` or parse defensively and count what you dropped.
## Running a single test
`testament r tests/ic/<file>.nim` works for the ordinary tests. It does NOT work
for the metamorphic ones — the multi-step files carry several `discard """`
spec blocks and the single-test path rejects them with "duplicate `specStart`".
Those only run through `cat ic`.
Files matching `tests/ic/*_temp.nim` are ignored by git (see `.gitignore`) and
are scratch, not tests: several import helper modules that do not exist and fail
for that reason alone.

101
tests/ic/tclosure_hooks.nim Normal file
View File

@@ -0,0 +1,101 @@
discard """
description: '''IC vs `nim c`: closure environments, their hooks and their owners'''
"""
#? metamorphic
# A closure's environment type — and the `=destroy`/`=copy` the compiler lifts
# for it — is minted by the BACKEND, during the `lower` stage, and exists in no
# module's semmed NIF. The per-module backend has to decide which translation
# unit emits such a routine, and the owner walk it uses lands on the module of
# the ORIGINAL generic: for a generic closure iterator defined in one module and
# instantiated in another, that is a module which never sees the instance, so the
# env's `=destroy` was emitted by nobody (`undefined reference to
# eqdestroy__c485__…`). Every referencing TU emits it now.
#
# The steps then move the captured state around, because the env's LAYOUT is what
# decides whether those hooks are trivial: a body-only edit that adds a capture
# changes the env type of a routine whose importers do not re-sem.
#!FILE clleaf.nim
type Ev* = proc (s: string): string {.closure.}
proc leafMaker*(tag: string): Ev =
var n = 0
proc outer(s: string): string =
proc inner(t: string): string =
inc n
tag & ":" & t & ":" & $n
inner(s)
result = outer
iterator leafIter*[T](xs: seq[T]): T {.closure.} =
for x in xs: yield x
#!FILE clmid.nim
import clleaf
proc midMaker*(tag: string): Ev =
let base = leafMaker(tag & "/mid")
var calls = 0
result = proc (s: string): string =
inc calls
base(s) & "#" & $calls
proc midIter*(): seq[string] =
# instantiates `leafIter[string]` HERE, not where it is defined
result = @[]
for x in leafIter(@["p", "q"]): result.add x
#!FILE main.nim
import clleaf, clmid
let t = midMaker("top")
echo t("Alpha")
echo t("Beta")
echo midIter()
# an instance only the main module has
var fs: seq[float] = @[]
for x in leafIter(@[1.5, 2.5]): fs.add x
echo fs
#!STEP
# body-only edit that GROWS the environment: a second captured local
#!FILE clleaf.nim
type Ev* = proc (s: string): string {.closure.}
proc leafMaker*(tag: string): Ev =
var n = 0
var seen: seq[string] = @[]
proc outer(s: string): string =
proc inner(t: string): string =
inc n
seen.add t
tag & ":" & t & ":" & $n & ":" & $seen.len
inner(s)
result = outer
iterator leafIter*[T](xs: seq[T]): T {.closure.} =
var i = 0
for x in xs:
inc i
yield x
#!STEP
# and shrink it again
#!FILE clleaf.nim
type Ev* = proc (s: string): string {.closure.}
proc leafMaker*(tag: string): Ev =
var n = 0
proc outer(s: string): string =
proc inner(t: string): string =
inc n
tag & ":" & t & ":" & $n
inner(s)
result = outer
iterator leafIter*[T](xs: seq[T]): T {.closure.} =
for x in xs: yield x
#!STEP

90
tests/ic/tclosure_nested_iter.nim Normal file
View File

@@ -0,0 +1,90 @@
discard """
description: '''IC vs `nim c`: a closure iterator nested in a closure iterator'''
"""
#? metamorphic
# `env.:up = enclosingEnv` links a nested routine's environment to its parent,
# and the two environments then reference each other. That assignment has to go
# through `=copy` (with the cyclic increment) or the parent's refcount is one too
# low, and at teardown both `=destroy`s believe they hold the last reference and
# recurse until the stack is gone — a SIGSEGV, after the program's own output has
# already been printed. (`tests/iter/tnestedclosures.nim`, "Test 3".)
#
# Whether it becomes a `=copy` depends on the up-field type's hooks existing when
# the routine is destructor-injected. Whole-program cgen got that for free: a
# LATER lifting pass creates them, and it runs before any routine's injection.
# The per-module backend injects a routine right after lifting it (the `lower`
# stage), long before the module's top level is transformed at all (that is
# `cg`) — so the hooks are created at the assignment site now.
#!FILE main.nim
iterator foo(): int {.closure.} =
let x = 34
proc bar() = echo "bar sees ", x
iterator bar2(): int {.closure.} =
bar()
yield x
for y in bar2():
yield y
for v in foo(): echo v
# a closure iterator nested in a closure iterator, inside a proc
proc factory() =
iterator outerIt(): int {.closure.} =
iterator innerIt(): int {.closure.} =
yield 0
yield 1
yield 2
for x in innerIt(): yield x
for x in outerIt(): echo x
factory()
# the iterator's env outlives the proc that made it
proc keep(): iterator (): string =
let held = "kept"
result = iterator (): string =
yield held
yield held & "!"
for s in keep()(): echo s
#!STEP
# growing the captured state changes both env layouts
#!FILE main.nim
iterator foo(): int {.closure.} =
let x = 34
var log: seq[string] = @[]
proc bar() =
log.add "bar"
echo "bar sees ", x, " ", log.len
iterator bar2(): int {.closure.} =
bar()
bar()
yield x
for y in bar2():
yield y
for v in foo(): echo v
proc factory() =
iterator outerIt(): int {.closure.} =
var emitted = 0
iterator innerIt(): int {.closure.} =
yield 0
yield 1
yield 2
for x in innerIt():
inc emitted
yield x * emitted
for x in outerIt(): echo x
factory()
proc keep(): iterator (): string =
let held = "kept"
let extra = "+"
result = iterator (): string =
yield held & extra
yield held & "!" & extra
for s in keep()(): echo s
#!STEP

14
tests/ic/texportprivate.nim Normal file
View File

@@ -0,0 +1,14 @@
discard """
output: '''42'''
"""
# `export s` re-exports a symbol whose declaration has no `*`. It reaches the
# module interface through `reexportSym` alone, so a NIF writer that decides
# importability from `sfExported` ships it as private and the importer reports
# "undeclared identifier". `std/random` does exactly this
# (`proc initRand(): Rand` + `since (1, 5, 1): export initRand`), which made
# `--ic:on` unable to compile anything reaching `std/tempfiles`.
import mexportprivate
echo hidden()

17
tests/ic/timporthidden.nim Normal file
View File

@@ -0,0 +1,17 @@
discard """
output: '''42'''
"""
# `import x {.all.}` makes x's PRIVATE symbols visible. Under IC that means the
# hidden half of a loaded module's interface has to be there — and it is now
# built on demand rather than at load time, because almost nothing ever reads it
# (1.70M hidden stubs against 0.29M exported ones on a cold Atlas build).
#
# The trap the first attempt fell into: a module has TWO FileIndexes. `c.mods`
# in the decode context is keyed by the one `registerNifSuffix` mints for the
# NIF suffix; `g.ifaces` is indexed by the module's source file. Asking one with
# the other misses silently, and this test is what says so.
import mimporthidden {.all.}
echo secret() + hiddenToo(35)